alice.pds.demo.boltless.dev
This repository has no description
1{
2 config,
3 pkgs,
4 lib,
5 ...
6}: let
7 # these are modules / module trees we keep. everything else is pruned.
8 # this is a "cheap" way to save on what we ship, we don't have to recompile anything.
9 # this saves about 118mb!
10 keepTrees = [
11 "crypto"
12 "lib"
13 "arch"
14 "drivers/virtio"
15 # net: the firewall modprobes across netfilter/ipv4/ipv6; docker adds the
16 # bridge/llc/802(stp)/xfrm machinery + NAT targets in netfilter.
17 "net/core"
18 "net/netfilter"
19 "net/ipv4"
20 "net/ipv6"
21 "net/packet"
22 "net/sched"
23 "net/vmw_vsock"
24 "net/bridge"
25 "net/llc"
26 "net/802"
27 "net/xfrm"
28 "fs/configfs"
29 "fs/autofs"
30 "fs/nls"
31 "fs/unicode"
32 ];
33 keepMods = [
34 # boot + storage + common workflow filesystems
35 "erofs"
36 "ext4"
37 "jbd2"
38 "mbcache"
39 "overlay"
40 "fuse"
41 "loop"
42 # "btrfs"
43 # "xfs"
44 # "f2fs"
45 # "vfat"
46 # "exfat"
47 "squashfs"
48 "isofs"
49 "dm-mod"
50 "zram"
51 # virtio devices the runner exposes
52 "virtio"
53 "virtio_mmio"
54 "virtio_pci"
55 "virtio_blk"
56 "virtio_net"
57 "virtio_rng"
58 "virtio_console"
59 "vsock_loopback"
60 "vmw_vsock_virtio_transport"
61 "vmw_vsock_virtio_transport_common"
62 # container networking (docker default bridge + common custom networks)
63 "veth"
64 "tun"
65 "tap"
66 "bridge"
67 "br_netfilter"
68 "macvlan"
69 "ipvlan"
70 "vxlan"
71 "geneve"
72 "dummy"
73 "wireguard"
74 ];
75 keepTreesFile = pkgs.writeText "keep-trees" (lib.concatStringsSep "\n" keepTrees);
76 keepModsFile = pkgs.writeText "keep-mods" (lib.concatStringsSep "\n" keepMods);
77 slimModulesScript = pkgs.writeText "slim-modules.py" ''
78 import os, shutil, sys
79
80 src, dst, trees_file, mods_file = sys.argv[1:5]
81 KEEP_TREES = open(trees_file).read().split()
82 KEEP_MODS = open(mods_file).read().split()
83
84 def norm(name):
85 return name.replace("-", "_")
86
87 kerneldir = os.path.join(src, "kernel")
88
89 bypath, byname = {}, {}
90 for root, _, files in os.walk(kerneldir):
91 for f in files:
92 if ".ko" not in f:
93 continue
94 ap = os.path.join(root, f)
95 rel = os.path.relpath(ap, src)
96 bypath[rel] = ap
97 byname[norm(f.split(".ko")[0])] = rel
98
99 deps = {}
100 with open(os.path.join(src, "modules.dep")) as fh:
101 for line in fh:
102 if ":" in line:
103 mod, rest = line.split(":", 1)
104 deps[mod.strip()] = rest.split()
105
106 keep = set()
107 def add(rel):
108 if rel in keep or rel not in bypath:
109 return
110 keep.add(rel)
111 for dep in deps.get(rel, []):
112 add(dep)
113
114 for tree in KEEP_TREES:
115 for root, _, files in os.walk(os.path.join(kerneldir, tree)):
116 for f in files:
117 if ".ko" in f:
118 add(os.path.relpath(os.path.join(root, f), src))
119 for mod in KEEP_MODS:
120 rel = byname.get(norm(mod))
121 if rel:
122 add(rel)
123
124 for rel in keep:
125 target = os.path.join(dst, rel)
126 os.makedirs(os.path.dirname(target), exist_ok=True)
127 shutil.copy2(bypath[rel], target)
128 print(f"kept {len(keep)} of {len(bypath)} modules")
129 '';
130 slimKernelModules =
131 pkgs.runCommand "${config.boot.kernelPackages.kernel.name}-modules-microvm"
132 {nativeBuildInputs = [pkgs.python3 pkgs.kmod];}
133 ''
134 src=${lib.getOutput "modules" config.boot.kernelPackages.kernel}/lib/modules
135 ver=$(ls "$src")
136 mkdir -p "$out/lib/modules/$ver"
137 for f in "$src/$ver"/modules.builtin* "$src/$ver"/modules.order; do
138 [ -e "$f" ] && cp "$f" "$out/lib/modules/$ver/"
139 done
140 python3 ${slimModulesScript} "$src/$ver" "$out/lib/modules/$ver" ${keepTreesFile} ${keepModsFile}
141 # regen modules.dep
142 depmod -b "$out" "$ver"
143 '';
144in {
145 system.stateVersion = "26.05";
146
147 # actually use our slimmed down modules set
148 system.modulesTree = lib.mkForce ([slimKernelModules] ++ config.boot.extraModulePackages);
149
150 boot.initrd.includeDefaultModules = lib.mkForce false;
151 boot.initrd.availableKernelModules = lib.mkForce [];
152 boot.initrd.kernelModules = lib.mkForce [
153 "virtio_pci"
154 "virtio_mmio"
155 "virtio_blk"
156 "virtio_console"
157 "erofs"
158 "ext4"
159 "overlay"
160 ];
161 boot.kernelModules = ["loop"];
162
163 # some zram to help situations where burst memory usage causes OOM
164 zramSwap = {
165 enable = true;
166 algorithm = "zstd";
167 memoryPercent = 50;
168 };
169
170 programs.nano.enable = false;
171 # we are on a microvm we don't need the hardware map
172 environment.etc."udev/hwdb.bin".enable = lib.mkForce false;
173
174 networking.hostName = "spindle-microvm";
175 networking.useDHCP = false;
176 systemd.network.networks."40-eth0" = {
177 matchConfig.Name = "eth0";
178 address = ["10.0.3.15/24"];
179 gateway = ["10.0.3.2"];
180 dns = ["127.0.0.1"];
181 };
182 networking.nameservers = ["127.0.0.1"];
183
184 # this is disabled by microvm optimizations but we do need it
185 system.switch.enable = lib.mkForce true;
186
187 # don't install docs or any xdg things, not necessary
188 documentation.enable = false;
189 xdg.mime.enable = false;
190 xdg.icons.enable = false;
191 xdg.sounds.enable = false;
192
193 users.groups.spindle-workflow = {
194 gid = 970;
195 };
196 users.users.spindle-workflow = {
197 isSystemUser = true;
198 uid = 970;
199 group = "spindle-workflow";
200 home = "/workspace";
201 createHome = false;
202 };
203 users.users.spindle-workflow.extraGroups = lib.mkIf config.virtualisation.docker.enable [
204 "docker"
205 ];
206 virtualisation.docker.listenOptions = [
207 "/run/docker.sock"
208 "/var/run/docker.sock"
209 ];
210
211 nix = {
212 settings = {
213 experimental-features = [
214 "nix-command"
215 "flakes"
216 ];
217 trusted-users = ["root" "spindle-workflow"];
218 allowed-users = ["spindle-workflow"];
219 };
220 registry.nixpkgs.to = {
221 type = "path";
222 path = pkgs.path;
223 };
224 extraOptions = ''
225 extra-experimental-features = nix-command flakes
226 !include /run/spindle/nix.conf
227 '';
228 nixPath = ["nixpkgs=${config.nix.registry.nixpkgs.to.path}"];
229 };
230
231 systemd.tmpfiles.rules = [
232 "d /run/spindle 0755 root root -"
233 "d /workspace 0755 spindle-workflow spindle-workflow -"
234 "d /workspace/repo 0755 spindle-workflow spindle-workflow -"
235 ];
236
237 # nix flake cache should go on the persisted dir, not the root (which is on tmpfs)
238 systemd.services.shuttle = {
239 environment.XDG_CACHE_HOME = "/var/cache/shuttle";
240 serviceConfig.CacheDirectory = "shuttle";
241 };
242
243 # add any common packages / services here
244 environment.systemPackages = with pkgs; [
245 gitMinimal
246 curlMinimal
247 wget
248 coreutils-full
249 file
250 findutils
251 gnused
252 jq
253 yq
254 xxd
255 gnutar
256 zip
257 unzip
258 gz-utils
259 bzip2
260 lz4
261 p7zip
262 ];
263 # disable default nixos packages ([perl rsync strace])
264 environment.defaultPackages = [];
265 # this removed nixos-rebuild-ng and nixos-generate-config, which lets us
266 # remove python3 closure (~107MB)
267 system.disableInstallerTools = true;
268
269 # a single volume that will back /workspace, /var, and the nix store
270 microvm.storeOnDisk = true;
271 microvm.storeDiskType = "erofs";
272 # lz4hc, not zstd: the stock nixpkgs kernel builds erofs without
273 # CONFIG_EROFS_FS_ZIP_ZSTD, so a zstd image fails to mount at boot ("algorithm
274 # 3 isn't enabled on this kernel"); only lz4 is guaranteed. -Efragments and
275 # -Ededupe are omitted because microvm.nix falls back to single-threaded
276 # erofs-utils when either is present, which makes image builds really slow.
277 # for now, we take the compression hit, which isn't too much anyway.
278 # todo(dawn): the remaining big save needs a custom guest kernel (we'd want a
279 # binary cache first so downstream users don't rebuild it every time): enable
280 # EROFS_FS_ZIP_ZSTD for a better-compressing store-disk, build the essentials
281 # (virtio/erofs/ext4/overlay/netfilter) in as =y, and strip the kernel image
282 # itself. the modules tree is already pruned without a recompile, see
283 # slimKernelModules above.
284 microvm.storeDiskErofsFlags = [
285 "-zlz4hc"
286 "-Eztailpacking"
287 "-C131072" # bigger compression window lets lz4hc compress better (~47mb)
288 ];
289 microvm.writableStoreOverlay = "/persist/rw-store";
290 microvm.volumes = [
291 {
292 image = "persist.img";
293 mountPoint = "/persist";
294 size = 1024 * 24; # 24 GB
295 fsType = "ext4";
296 }
297 ];
298
299 # /persist must be mounted before the writable store overlay activates
300 fileSystems."/persist".neededForBoot = true;
301
302 fileSystems."/workspace" = {
303 device = "/persist/workspace";
304 fsType = "none";
305 options = ["bind"];
306 depends = ["/persist"];
307 };
308 # bind mounting /var is important since docker etc. can't use overlayfs
309 # (overlayfs on overlayfs does not work)
310 fileSystems."/var" = {
311 device = "/persist/var";
312 fsType = "none";
313 options = ["bind"];
314 depends = ["/persist"];
315 };
316 # bind mount /tmp so our tmp is disk backed...
317 fileSystems."/tmp" = {
318 device = "/persist/tmp";
319 fsType = "none";
320 options = ["bind"];
321 depends = ["/persist"];
322 };
323
324 # create bind sources before local-fs.target, which means we have to do this
325 # at initrd time
326 boot.initrd.systemd.enable = true;
327 boot.initrd.systemd.tmpfiles.settings."00-persist-layout" = {
328 "/sysroot/persist/rw-store".d = {
329 mode = "0755";
330 };
331 "/sysroot/persist/workspace".d = {
332 mode = "0755";
333 user = "spindle-workflow";
334 group = "spindle-workflow";
335 };
336 "/sysroot/persist/var".d = {
337 mode = "0755";
338 };
339 "/sysroot/persist/tmp".d = {
340 mode = "1777";
341 };
342 };
343}