alice.pds.demo.boltless.dev
This repository has no description
1package microvm
2
3import (
4 "errors"
5 "fmt"
6 "log/slog"
7 "os"
8 "path/filepath"
9 "regexp"
10 "strings"
11 "syscall"
12
13 cgroups "github.com/containerd/cgroups/v3"
14 "github.com/containerd/cgroups/v3/cgroup2"
15 "github.com/prometheus/procfs"
16)
17
18var (
19 cgroupInvalidChar = regexp.MustCompile(`[^a-zA-Z0-9\-_.]`)
20 cgroupConsecutiveSep = regexp.MustCompile(`[-_.]{2,}`)
21)
22
23const (
24 cgroupParentSelf = "self"
25 supervisorCgroupName = "supervisor"
26)
27
28type CgroupLimits struct {
29 Enabled bool
30 Parent *CgroupParent
31 Name string
32 MemoryMaxMiB int64
33 SwapMaxMiB *int64
34 PidsMax int64
35}
36
37type CgroupParent struct {
38 root *cgroup2.Manager
39 mountpoint string
40 group string
41}
42
43type CgroupHandle struct {
44 manager *cgroup2.Manager
45}
46
47func initCgroupParent(parent string, supervisorMemoryMinMiB int64, logger *slog.Logger) (*CgroupParent, error) {
48 if parent == "" {
49 parent = cgroupParentSelf
50 }
51 if cgroups.Mode() != cgroups.Unified {
52 return nil, fmt.Errorf("microVM cgroups require cgroup v2 unified mode")
53 }
54
55 mountpoint, group, err := resolveCgroupParent(parent)
56 if err != nil {
57 return nil, err
58 }
59 if _, err := os.Stat(filepath.Join(mountpoint, strings.TrimPrefix(group, "/"))); err != nil {
60 return nil, fmt.Errorf("stat cgroup parent %q:%q: %w", mountpoint, group, err)
61 }
62
63 root, err := cgroup2.Load(group, cgroup2.WithMountpoint(mountpoint))
64 if err != nil {
65 return nil, fmt.Errorf("load cgroup parent %q:%q: %w", mountpoint, group, err)
66 }
67
68 if group != "/" {
69 if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil {
70 return nil, err
71 }
72 } else if err := probeRootSubtreeControl(mountpoint); err != nil {
73 if !errors.Is(err, syscall.EBUSY) {
74 return nil, fmt.Errorf("enable controllers in subtree_control of cgroup root %q: %w", mountpoint, err)
75 }
76 // populated namespace root, not the real root: vacate it too
77 if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil {
78 return nil, err
79 }
80 }
81
82 if logger != nil {
83 logger.Info("initialized microVM cgroup parent", "mountpoint", mountpoint, "group", group)
84 }
85 return &CgroupParent{root: root, mountpoint: mountpoint, group: group}, nil
86}
87
88func prepareCgroup(limits CgroupLimits, logger *slog.Logger) (*CgroupHandle, error) {
89 if !limits.Enabled {
90 return nil, nil
91 }
92 if limits.Parent == nil || limits.Parent.root == nil {
93 return nil, fmt.Errorf("cgroup parent is not initialized")
94 }
95 name := sanitizeCgroupName(limits.Name)
96 if name == "" {
97 return nil, fmt.Errorf("cgroup name is empty")
98 }
99
100 manager, err := limits.Parent.root.NewChild(name, cgroupResources(limits))
101 if err != nil {
102 return nil, fmt.Errorf("create cgroup %q: %w", name, err)
103 }
104
105 if logger != nil {
106 logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group)
107 }
108 return &CgroupHandle{manager: manager}, nil
109}
110
111func cgroupResources(limits CgroupLimits) *cgroup2.Resources {
112 resources := &cgroup2.Resources{}
113 if limits.MemoryMaxMiB > 0 || limits.SwapMaxMiB != nil {
114 memory := &cgroup2.Memory{}
115 if limits.MemoryMaxMiB > 0 {
116 maxBytes := limits.MemoryMaxMiB * 1024 * 1024
117 memory.Max = &maxBytes
118 }
119 if limits.SwapMaxMiB != nil {
120 swapBytes := *limits.SwapMaxMiB * 1024 * 1024
121 memory.Swap = &swapBytes
122 }
123 oomGroup := true
124 memory.OOMGroup = &oomGroup
125 resources.Memory = memory
126 }
127 if limits.PidsMax > 0 {
128 resources.Pids = &cgroup2.Pids{Max: limits.PidsMax}
129 }
130 return resources
131}
132
133func supervisorResources(memoryMinMiB int64) *cgroup2.Resources {
134 if memoryMinMiB <= 0 {
135 return nil
136 }
137 minBytes := memoryMinMiB * 1024 * 1024
138 return &cgroup2.Resources{
139 Memory: &cgroup2.Memory{Min: &minBytes},
140 }
141}
142
143func (h *CgroupHandle) AddProcess(pid int, logger *slog.Logger) error {
144 if h == nil || h.manager == nil {
145 return nil
146 }
147 if pid <= 0 {
148 return fmt.Errorf("invalid pid %d", pid)
149 }
150 if err := h.manager.AddProc(uint64(pid)); err != nil {
151 return fmt.Errorf("add pid %d to cgroup: %w", pid, err)
152 }
153 if logger != nil {
154 logger.Info("added process to microVM cgroup", "pid", pid)
155 }
156 return nil
157}
158
159func (h *CgroupHandle) Close() error {
160 if h == nil || h.manager == nil {
161 return nil
162 }
163 return h.manager.Delete()
164}
165
166func (h *CgroupHandle) OOMKilled() bool {
167 if h == nil || h.manager == nil {
168 return false
169 }
170 metrics, err := h.manager.Stat()
171 if err != nil || metrics == nil || metrics.MemoryEvents == nil {
172 return false
173 }
174 return metrics.MemoryEvents.OomKill > 0
175}
176
177// probeRootSubtreeControl enables the domain controllers the engine needs
178// in the "/" parent's subtree. this fails EBUSY at a populated cgroup
179// namespace root (no-internal-process constraint); the real root is exempt.
180func probeRootSubtreeControl(mountpoint string) error {
181 return os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids"), 0)
182}
183
184func resolveCgroupParent(parent string) (string, string, error) {
185 mountpoint, err := cgroup2Mountpoint()
186 if err != nil {
187 return "", "", err
188 }
189
190 if parent == "" || parent == cgroupParentSelf {
191 group, err := selfCgroupV2Path()
192 if err != nil {
193 return "", "", err
194 }
195 return mountpoint, group, nil
196 }
197 if !filepath.IsAbs(parent) {
198 return "", "", fmt.Errorf("cgroup parent must be %q or an absolute delegated cgroupfs path: %q", cgroupParentSelf, parent)
199 }
200
201 cleanParent := filepath.Clean(parent)
202 rel, err := filepath.Rel(mountpoint, cleanParent)
203 if err != nil {
204 return "", "", fmt.Errorf("resolve cgroup parent %q relative to cgroup2 mount %q: %w", cleanParent, mountpoint, err)
205 }
206 if rel == ".." || strings.HasPrefix(rel, "../") {
207 return "", "", fmt.Errorf("cgroup parent %q is outside cgroup2 mount %q", cleanParent, mountpoint)
208 }
209 if rel == "." {
210 return mountpoint, "/", nil
211 }
212
213 group := "/" + filepath.ToSlash(rel)
214 if err := cgroup2.VerifyGroupPath(group); err != nil {
215 return "", "", fmt.Errorf("invalid cgroup parent path %q: %w", group, err)
216 }
217 return mountpoint, group, nil
218}
219
220func cgroup2Mountpoint() (string, error) {
221 mounts, err := procfs.GetMounts()
222 if err != nil {
223 return "", fmt.Errorf("read procfs mountinfo: %w", err)
224 }
225 for _, mount := range mounts {
226 if mount.FSType == "cgroup2" {
227 return mount.MountPoint, nil
228 }
229 }
230 return "", fmt.Errorf("cgroup v2 mountpoint not found")
231}
232
233func selfCgroupV2Path() (string, error) {
234 self, err := procfs.Self()
235 if err != nil {
236 return "", fmt.Errorf("open procfs self: %w", err)
237 }
238 groups, err := self.Cgroups()
239 if err != nil {
240 return "", fmt.Errorf("read procfs self cgroups: %w", err)
241 }
242 for _, group := range groups {
243 if group.HierarchyID != 0 {
244 continue
245 }
246 path := group.Path
247 if path == "" {
248 path = "/"
249 }
250 if err := cgroup2.VerifyGroupPath(path); err != nil {
251 return "", fmt.Errorf("invalid self cgroup path %q: %w", path, err)
252 }
253 return path, nil
254 }
255 return "", fmt.Errorf("current process has no cgroup v2 hierarchy entry")
256}
257
258func moveParentProcesses(parent *cgroup2.Manager, supervisorMemoryMinMiB int64, logger *slog.Logger) error {
259 procs, err := parent.Procs(false)
260 if err != nil {
261 return fmt.Errorf("list parent cgroup processes: %w", err)
262 }
263
264 // first create with empty resources
265 supervisor, err := parent.NewChild(supervisorCgroupName, &cgroup2.Resources{})
266 if err != nil {
267 return fmt.Errorf("create supervisor cgroup: %w", err)
268 }
269
270 // move procs
271 for _, pid := range procs {
272 if err := supervisor.AddProc(pid); err != nil {
273 return fmt.Errorf("move pid %d to supervisor cgroup: %w", pid, err)
274 }
275 }
276
277 // now apply resources. we can't do this while parent has procs still
278 if res := supervisorResources(supervisorMemoryMinMiB); res != nil {
279 // we use a "new" parent here, this is so we enable subtree_control.
280 // .Update() does not work here...
281 if _, err = parent.NewChild(supervisorCgroupName, res); err != nil {
282 return fmt.Errorf("apply supervisor cgroup resources: %w", err)
283 }
284 }
285
286 if logger != nil && len(procs) > 0 {
287 logger.Info("moved spindle processes to supervisor cgroup", "processes", len(procs))
288 }
289 return nil
290}
291
292func sanitizeCgroupName(name string) string {
293 name = cgroupInvalidChar.ReplaceAllLiteralString(name, "-")
294 name = cgroupConsecutiveSep.ReplaceAllLiteralString(name, "-")
295 return strings.Trim(name, "-_.")
296}