alice.pds.demo.boltless.dev
This repository has no description
1{
2 pkgsStatic,
3 runCommand,
4 writeText,
5 squashfsTools,
6 shuttle,
7 binutils,
8 publicsuffix-list,
9 rootfs,
10 kernel,
11 initramfs,
12 modloop,
13 repositories,
14 arch ? "x86_64",
15}: let
16 nix = pkgsStatic.nixStatic;
17 bash = pkgsStatic.bashNonInteractive;
18 curl = pkgsStatic.curlMinimal;
19 jq = pkgsStatic.jq;
20 git =
21 (pkgsStatic.gitMinimal.override {
22 inherit curl;
23 pythonSupport = false;
24 withManual = false;
25 nlsSupport = false;
26 }).overrideAttrs (old: {
27 doCheck = false;
28 doInstallCheck = false;
29 configureFlags = (old.configureFlags or []) ++ ["ac_cv_lib_curl_curl_global_init=yes"];
30 });
31 # we don't include gnused, xxd etc. here because busybox has them
32 # we want to keep the image this image small!
33 guestTools = [nix bash git curl jq];
34
35 # run by busybox at sysinit
36 setupScript = writeText "spindle-setup" ''
37 #!/bin/sh
38
39 mountpoint -q /proc || mount -t proc proc /proc
40 mountpoint -q /sys || mount -t sysfs sys /sys
41 mountpoint -q /dev || mount -t devtmpfs dev /dev
42 mountpoint -q /dev/pts || {
43 install -d /dev/pts
44 mount -t devpts devpts /dev/pts
45 }
46 mountpoint -q /dev/shm || {
47 install -d /dev/shm
48 mount -t tmpfs -o mode=1777 shm /dev/shm
49 }
50 mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run
51 mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp
52
53 # setup xdg runtime dir, podman eg. needs it
54 install -d -m 0700 -o spindle-workflow -g spindle-workflow /run/user/970
55
56 # cgroup2 setup, normally we would do this with rc-service
57 # but minirootfs does not ship with those so we set it up ourselves.
58 mountpoint -q /sys/fs/cgroup || {
59 install -d /sys/fs/cgroup
60 mount -t cgroup2 -o nsdelegate cgroup2 /sys/fs/cgroup
61 chown -R spindle-workflow:spindle-workflow /sys/fs/cgroup 2>/dev/null || true
62 }
63
64 # the initramfs mdev leaves these 0660, which breaks non-root workflows
65 chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null
66
67 modprobe vmw_vsock_virtio_transport
68 # shuttle's cache enqueue listener binds a guest-local (CID 1) vsock
69 modprobe vsock_loopback
70 modprobe ext4
71
72 if [ -b /dev/vdb ]; then
73 # setup disk backed nix store
74 mount -t ext4 /dev/vdb /workspace
75 install -d -o spindle-workflow -g spindle-workflow /workspace /workspace/repo
76 install -d /workspace/.nix/rw-store /workspace/.nix/rw-store-work /workspace/.nix/build
77 mount -t overlay overlay \
78 -o lowerdir=/nix/store,upperdir=/workspace/.nix/rw-store,workdir=/workspace/.nix/rw-store-work \
79 /nix/store
80 fi
81
82 ip link set lo up
83 ip link set eth0 up
84 ip addr add 10.0.3.15/24 dev eth0
85 ip route add default via 10.0.3.2
86 hostname -F /etc/hostname
87 '';
88
89 inittab = writeText "inittab" ''
90 ::sysinit:/sbin/spindle-setup
91 ::respawn:env TMPDIR=/workspace/.nix/build /usr/local/bin/nix-daemon
92 ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle
93 ::ctrlaltdel:/sbin/reboot
94 '';
95
96 profileScript = writeText "spindle-profile" ''
97 export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
98 export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
99 export NIX_REMOTE=daemon
100 '';
101
102 # mirror nix/microvm/base.nix and nix/modules/shuttle.nix
103 nixConf = writeText "nix.conf" ''
104 experimental-features = nix-command flakes
105 trusted-users = root
106 allowed-users = spindle-workflow
107 post-build-hook = /usr/libexec/spindle-post-build-hook
108 # keep build sandboxes on the /workspace disk, not the RAM-backed root tmpfs
109 build-dir = /workspace/.nix/build
110 !include /run/spindle/nix.conf
111 '';
112
113 apkRepositories = writeText "apk-repositories" (builtins.concatStringsSep "\n" repositories + "\n");
114
115 postBuildHook = writeText "spindle-post-build-hook" ''
116 #!/bin/sh
117 set -f
118
119 if [ -z "''${OUT_PATHS:-}" ]; then
120 exit 0
121 fi
122
123 # OUT_PATHS is intentionally split into individual store paths
124 exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS
125 '';
126
127 imageSpecJSON = writeText "spec.json" (
128 builtins.toJSON {
129 inherit arch;
130 bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 root=/dev/vda rootfstype=squashfs modules=virtio_blk,virtio_net,virtio_console overlaytmpfs=yes init=/sbin/init";
131 kernel = "kernel";
132 initrd = "initrd";
133 runnerType = "qemu";
134 runnerConfig = {
135 cpu = "host,+x2apic,-sgx";
136 machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off";
137 console = "hvc0";
138 extraArgs = [];
139 };
140 memoryMiB = 4096;
141 storeDisk = "store-disk";
142 storeDiskType = "squashfs";
143 vcpus = 2;
144 shell = "/usr/local/bin/bash";
145 networkInterfaces = [
146 {
147 type = "slirp4netns";
148 id = "net0";
149 mac = "02:00:00:00:10:01";
150 }
151 ];
152 volumes = [
153 {
154 fsType = "ext4";
155 image = "workspace.img";
156 imageType = "raw";
157 mountPoint = "/workspace";
158 readOnly = false;
159 sizeMiB = 1024 * 16; # 16 GB
160 }
161 ];
162 }
163 );
164in
165 runCommand "spindle-alpine-image-${arch}" {
166 nativeBuildInputs = [squashfsTools binutils];
167 } ''
168 mkdir -p rootfs
169 tar -xzpf ${rootfs} -C rootfs
170
171 # kernel modules from modloop (ships its own modules.dep, no depmod needed)
172 unsquashfs -q -d modloop ${modloop}
173 mkdir -p rootfs/lib/modules
174 cp -a modloop/modules/* rootfs/lib/modules/
175
176 install -D -m 0755 ${shuttle}/bin/shuttle rootfs/usr/bin/shuttle
177 install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup
178 install -D -m 0644 ${inittab} rootfs/etc/inittab
179 install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh
180 install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf
181 install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook
182
183 # install dependencies
184 # we only copy binaries + libexec for minimal deps so the image size doesn't
185 # increase so much (if we copy the whole guestTools closure for example, it
186 # doubles the disk size)
187 mkdir -p rootfs/nix/store rootfs/usr/local/bin
188 for pkg in ${toString guestTools}; do
189 for bin in "$pkg/bin/"*; do
190 [[ -e "$bin" ]] || continue
191 name=$(basename "$bin")
192 # we resolve symlinks as to copy the actual binaries
193 if [[ -L "$bin" ]]; then
194 real=$(readlink "$bin")
195 else
196 real="$bin"
197 fi
198 # handle symlinks properly
199 if [[ "$real" != /nix/store* ]]; then
200 ln -vsf "$real" "rootfs/usr/local/bin/$name"
201 else
202 cp -v "$real" "rootfs/usr/local/bin/$name"
203 fi
204 done
205 # libexec has binaries used by packages even if statically compiled
206 if [[ -d "$pkg/libexec" ]]; then
207 mkdir -p "rootfs$pkg"
208 cp -av "$pkg/libexec" "rootfs$pkg/"
209 fi
210 done
211 # this is necessary for nix to work, it is not a library but nix hardcodes
212 # it in it's binary
213 cp -rv ${publicsuffix-list} rootfs/nix/store/
214
215 # scripts commonly hardcode #!/bin/bash
216 ln -sf ${bash}/bin/bash rootfs/bin/bash
217
218 echo "spindle-microvm" > rootfs/etc/hostname
219 printf 'nameserver 127.0.0.1\n' > rootfs/etc/resolv.conf
220 install -D -m 0644 ${apkRepositories} rootfs/etc/apk/repositories
221
222 echo "spindle-workflow:x:970:970:spindle workflow:/workspace:/bin/sh" >> rootfs/etc/passwd
223 echo "spindle-workflow:x:970:" >> rootfs/etc/group
224 echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow
225 mkdir -p rootfs/workspace
226
227 # subordinate id ranges so the workflow user can run rootless containers
228 # (podman/buildah): without these, user-namespace id mapping falls back to a
229 # single 970->0 map and any layer that chowns to another uid fails. the range
230 # is well clear of 970 and the 30000-block nixbld users.
231 echo "spindle-workflow:100000:65536" >> rootfs/etc/subuid
232 echo "spindle-workflow:100000:65536" >> rootfs/etc/subgid
233
234 # setup nix build users for the daemon
235 members=""
236 for i in $(seq 1 8); do
237 echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd
238 echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow
239 members="$members''${members:+,}nixbld$i"
240 done
241 echo "nixbld:x:30000:$members" >> rootfs/etc/group
242
243 mkdir -p "$out"
244 mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet \
245 -p '/sbin/apk m 4755 0 0' # suid apk so spindle-workflow can use it without having to doas or smth
246 cp ${kernel} "$out/kernel"
247 cp ${initramfs} "$out/initrd"
248 cp ${imageSpecJSON} "$out/spec.json"
249 ''