alice.pds.demo.boltless.dev
This repository has no description
1package microvm
2
3import (
4 "errors"
5 "fmt"
6 "log/slog"
7 "os"
8 "path/filepath"
9 "regexp"
10 "strconv"
11 "strings"
12 "syscall"
13
14 cgroups "github.com/containerd/cgroups/v3"
15 "github.com/containerd/cgroups/v3/cgroup2"
16 "github.com/prometheus/procfs"
17)
18
19var (
20 cgroupInvalidChar = regexp.MustCompile(`[^a-zA-Z0-9\-_.]`)
21 cgroupConsecutiveSep = regexp.MustCompile(`[-_.]{2,}`)
22)
23
24const (
25 cgroupParentSelf = "self"
26 supervisorCgroupName = "supervisor"
27)
28
29type CgroupLimits struct {
30 Enabled bool
31 Parent *CgroupParent
32 Name string
33 MemoryMaxMiB int64
34 SwapMaxMiB *int64
35 PidsMax int64
36 // cpu.max quota as a percentage of one core (100 = one core). <= 0
37 // leaves cpu unlimited
38 CPUQuotaPercent int64
39 // io.weight (1-10000). 0 leaves io unlimited, written directly since
40 // this cgroup2 lib only models io.bfq.weight/io.max
41 IOWeight uint64
42}
43
44type CgroupParent struct {
45 root *cgroup2.Manager
46 mountpoint string
47 group string
48}
49
50type CgroupHandle struct {
51 manager *cgroup2.Manager
52}
53
54func initCgroupParent(parent string, supervisorMemoryMinMiB int64, logger *slog.Logger) (*CgroupParent, error) {
55 if parent == "" {
56 parent = cgroupParentSelf
57 }
58 if cgroups.Mode() != cgroups.Unified {
59 return nil, fmt.Errorf("microVM cgroups require cgroup v2 unified mode")
60 }
61
62 mountpoint, group, err := resolveCgroupParent(parent)
63 if err != nil {
64 return nil, err
65 }
66 if _, err := os.Stat(filepath.Join(mountpoint, strings.TrimPrefix(group, "/"))); err != nil {
67 return nil, fmt.Errorf("stat cgroup parent %q:%q: %w", mountpoint, group, err)
68 }
69
70 root, err := cgroup2.Load(group, cgroup2.WithMountpoint(mountpoint))
71 if err != nil {
72 return nil, fmt.Errorf("load cgroup parent %q:%q: %w", mountpoint, group, err)
73 }
74
75 if group != "/" {
76 if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil {
77 return nil, err
78 }
79 } else if err := probeRootSubtreeControl(mountpoint); err != nil {
80 if !errors.Is(err, syscall.EBUSY) {
81 return nil, fmt.Errorf("enable controllers in subtree_control of cgroup root %q: %w", mountpoint, err)
82 }
83 // populated namespace root, not the real root: vacate it too
84 if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil {
85 return nil, err
86 }
87 }
88
89 if logger != nil {
90 logger.Info("initialized microVM cgroup parent", "mountpoint", mountpoint, "group", group)
91 }
92 return &CgroupParent{root: root, mountpoint: mountpoint, group: group}, nil
93}
94
95func prepareCgroup(limits CgroupLimits, logger *slog.Logger) (*CgroupHandle, error) {
96 if !limits.Enabled {
97 return nil, nil
98 }
99 if limits.Parent == nil || limits.Parent.root == nil {
100 return nil, fmt.Errorf("cgroup parent is not initialized")
101 }
102 name := sanitizeCgroupName(limits.Name)
103 if name == "" {
104 return nil, fmt.Errorf("cgroup name is empty")
105 }
106
107 manager, err := limits.Parent.root.NewChild(name, cgroupResources(limits))
108 if err != nil {
109 return nil, fmt.Errorf("create cgroup %q: %w", name, err)
110 }
111
112 if limits.IOWeight > 0 {
113 if err := manager.ToggleControllers([]string{"io"}, cgroup2.Enable); err != nil {
114 _ = manager.Delete()
115 return nil, fmt.Errorf("enable io controller for cgroup %q: %w", name, err)
116 }
117 ioWeightPath := filepath.Join(limits.Parent.mountpoint, strings.TrimPrefix(limits.Parent.group, "/"), name, "io.weight")
118 if err := os.WriteFile(ioWeightPath, []byte(strconv.FormatUint(limits.IOWeight, 10)), 0); err != nil {
119 _ = manager.Delete()
120 return nil, fmt.Errorf("write io.weight for cgroup %q: %w", name, err)
121 }
122 }
123
124 if logger != nil {
125 logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group)
126 }
127 return &CgroupHandle{manager: manager}, nil
128}
129
130func cgroupResources(limits CgroupLimits) *cgroup2.Resources {
131 resources := &cgroup2.Resources{}
132 if limits.MemoryMaxMiB > 0 || limits.SwapMaxMiB != nil {
133 memory := &cgroup2.Memory{}
134 if limits.MemoryMaxMiB > 0 {
135 maxBytes := limits.MemoryMaxMiB * 1024 * 1024
136 memory.Max = &maxBytes
137 }
138 if limits.SwapMaxMiB != nil {
139 swapBytes := *limits.SwapMaxMiB * 1024 * 1024
140 memory.Swap = &swapBytes
141 }
142 oomGroup := true
143 memory.OOMGroup = &oomGroup
144 resources.Memory = memory
145 }
146 if limits.PidsMax > 0 {
147 resources.Pids = &cgroup2.Pids{Max: limits.PidsMax}
148 }
149 if limits.CPUQuotaPercent > 0 {
150 quota := limits.CPUQuotaPercent * 1000 // 100% of one 100000us period
151 resources.CPU = &cgroup2.CPU{Max: cgroup2.NewCPUMax("a, nil)}
152 }
153 return resources
154}
155
156func supervisorResources(memoryMinMiB int64) *cgroup2.Resources {
157 if memoryMinMiB <= 0 {
158 return nil
159 }
160 minBytes := memoryMinMiB * 1024 * 1024
161 return &cgroup2.Resources{
162 Memory: &cgroup2.Memory{Min: &minBytes},
163 }
164}
165
166func (h *CgroupHandle) AddProcess(pid int, logger *slog.Logger) error {
167 if h == nil || h.manager == nil {
168 return nil
169 }
170 if pid <= 0 {
171 return fmt.Errorf("invalid pid %d", pid)
172 }
173 if err := h.manager.AddProc(uint64(pid)); err != nil {
174 return fmt.Errorf("add pid %d to cgroup: %w", pid, err)
175 }
176 if logger != nil {
177 logger.Info("added process to microVM cgroup", "pid", pid)
178 }
179 return nil
180}
181
182func (h *CgroupHandle) Close() error {
183 if h == nil || h.manager == nil {
184 return nil
185 }
186 return h.manager.Delete()
187}
188
189func (h *CgroupHandle) OOMKilled() bool {
190 if h == nil || h.manager == nil {
191 return false
192 }
193 metrics, err := h.manager.Stat()
194 if err != nil || metrics == nil || metrics.MemoryEvents == nil {
195 return false
196 }
197 return metrics.MemoryEvents.OomKill > 0
198}
199
200// probeRootSubtreeControl enables the domain controllers the engine needs
201// in the "/" parent's subtree. this fails EBUSY at a populated cgroup
202// namespace root (no-internal-process constraint); the real root is exempt.
203func probeRootSubtreeControl(mountpoint string) error {
204 // cpu/io may be unavailable (eg. no CONFIG_BLK_CGROUP), memory/pids
205 // are the hard requirement
206 if err := os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids +cpu +io"), 0); err == nil {
207 return nil
208 }
209 return os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids"), 0)
210}
211
212func resolveCgroupParent(parent string) (string, string, error) {
213 mountpoint, err := cgroup2Mountpoint()
214 if err != nil {
215 return "", "", err
216 }
217
218 if parent == "" || parent == cgroupParentSelf {
219 group, err := selfCgroupV2Path()
220 if err != nil {
221 return "", "", err
222 }
223 return mountpoint, group, nil
224 }
225 if !filepath.IsAbs(parent) {
226 return "", "", fmt.Errorf("cgroup parent must be %q or an absolute delegated cgroupfs path: %q", cgroupParentSelf, parent)
227 }
228
229 cleanParent := filepath.Clean(parent)
230 rel, err := filepath.Rel(mountpoint, cleanParent)
231 if err != nil {
232 return "", "", fmt.Errorf("resolve cgroup parent %q relative to cgroup2 mount %q: %w", cleanParent, mountpoint, err)
233 }
234 if rel == ".." || strings.HasPrefix(rel, "../") {
235 return "", "", fmt.Errorf("cgroup parent %q is outside cgroup2 mount %q", cleanParent, mountpoint)
236 }
237 if rel == "." {
238 return mountpoint, "/", nil
239 }
240
241 group := "/" + filepath.ToSlash(rel)
242 if err := cgroup2.VerifyGroupPath(group); err != nil {
243 return "", "", fmt.Errorf("invalid cgroup parent path %q: %w", group, err)
244 }
245 return mountpoint, group, nil
246}
247
248func cgroup2Mountpoint() (string, error) {
249 mounts, err := procfs.GetMounts()
250 if err != nil {
251 return "", fmt.Errorf("read procfs mountinfo: %w", err)
252 }
253 for _, mount := range mounts {
254 if mount.FSType == "cgroup2" {
255 return mount.MountPoint, nil
256 }
257 }
258 return "", fmt.Errorf("cgroup v2 mountpoint not found")
259}
260
261func selfCgroupV2Path() (string, error) {
262 self, err := procfs.Self()
263 if err != nil {
264 return "", fmt.Errorf("open procfs self: %w", err)
265 }
266 groups, err := self.Cgroups()
267 if err != nil {
268 return "", fmt.Errorf("read procfs self cgroups: %w", err)
269 }
270 for _, group := range groups {
271 if group.HierarchyID != 0 {
272 continue
273 }
274 path := group.Path
275 if path == "" {
276 path = "/"
277 }
278 if err := cgroup2.VerifyGroupPath(path); err != nil {
279 return "", fmt.Errorf("invalid self cgroup path %q: %w", path, err)
280 }
281 return path, nil
282 }
283 return "", fmt.Errorf("current process has no cgroup v2 hierarchy entry")
284}
285
286func moveParentProcesses(parent *cgroup2.Manager, supervisorMemoryMinMiB int64, logger *slog.Logger) error {
287 procs, err := parent.Procs(false)
288 if err != nil {
289 return fmt.Errorf("list parent cgroup processes: %w", err)
290 }
291
292 // first create with empty resources
293 supervisor, err := parent.NewChild(supervisorCgroupName, &cgroup2.Resources{})
294 if err != nil {
295 return fmt.Errorf("create supervisor cgroup: %w", err)
296 }
297
298 // move procs
299 for _, pid := range procs {
300 if err := supervisor.AddProc(pid); err != nil {
301 return fmt.Errorf("move pid %d to supervisor cgroup: %w", pid, err)
302 }
303 }
304
305 // now apply resources. we can't do this while parent has procs still
306 if res := supervisorResources(supervisorMemoryMinMiB); res != nil {
307 // we use a "new" parent here, this is so we enable subtree_control.
308 // .Update() does not work here...
309 if _, err = parent.NewChild(supervisorCgroupName, res); err != nil {
310 return fmt.Errorf("apply supervisor cgroup resources: %w", err)
311 }
312 }
313
314 if logger != nil && len(procs) > 0 {
315 logger.Info("moved spindle processes to supervisor cgroup", "processes", len(procs))
316 }
317 return nil
318}
319
320func sanitizeCgroupName(name string) string {
321 name = cgroupInvalidChar.ReplaceAllLiteralString(name, "-")
322 name = cgroupConsecutiveSep.ReplaceAllLiteralString(name, "-")
323 return strings.Trim(name, "-_.")
324}