alice.pds.demo.boltless.dev
This repository has no description
1{
2 pkgsStatic,
3 runCommand,
4 writeText,
5 squashfsTools,
6 shuttle,
7 binutils,
8 publicsuffix-list,
9 rootfs,
10 kernel,
11 initramfs,
12 modloop,
13 repositories,
14 arch ? "x86_64",
15}: let
16 nix = pkgsStatic.nixStatic;
17 bash = pkgsStatic.bashNonInteractive;
18 curl = pkgsStatic.curlMinimal;
19 jq = pkgsStatic.jq;
20 git =
21 (pkgsStatic.gitMinimal.override {
22 inherit curl;
23 pythonSupport = false;
24 withManual = false;
25 nlsSupport = false;
26 }).overrideAttrs (old: {
27 doCheck = false;
28 doInstallCheck = false;
29 configureFlags = (old.configureFlags or []) ++ ["ac_cv_lib_curl_curl_global_init=yes"];
30 });
31 # we don't include gnused, xxd etc. here because busybox has them
32 # we want to keep the image this image small!
33 # zstd is not a busybox applet, and the spindle cache saves tar|zstd
34 guestTools = [nix bash git curl jq pkgsStatic.zstd];
35
36 # run by busybox at sysinit
37 setupScript = writeText "spindle-setup" ''
38 #!/bin/sh
39
40 mountpoint -q /proc || mount -t proc proc /proc
41 mountpoint -q /sys || mount -t sysfs sys /sys
42 mountpoint -q /dev || mount -t devtmpfs dev /dev
43 mountpoint -q /dev/pts || {
44 install -d /dev/pts
45 mount -t devpts devpts /dev/pts
46 }
47 mountpoint -q /dev/shm || {
48 install -d /dev/shm
49 mount -t tmpfs -o mode=1777 shm /dev/shm
50 }
51 mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run
52 mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp
53
54 # setup xdg runtime dir, podman eg. needs it
55 install -d -m 0700 -o spindle-workflow -g spindle-workflow /run/user/970
56
57 # cgroup2 setup, normally we would do this with rc-service
58 # but minirootfs does not ship with those so we set it up ourselves.
59 mountpoint -q /sys/fs/cgroup || {
60 install -d /sys/fs/cgroup
61 mount -t cgroup2 -o nsdelegate cgroup2 /sys/fs/cgroup
62 chown -R spindle-workflow:spindle-workflow /sys/fs/cgroup 2>/dev/null || true
63 }
64
65 # the initramfs mdev leaves these 0660, which breaks non-root workflows
66 chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null
67
68 modprobe vmw_vsock_virtio_transport
69 # shuttle's cache enqueue listener binds a guest-local (CID 1) vsock
70 modprobe vsock_loopback
71 modprobe ext4
72
73 if [ -b /dev/vdb ]; then
74 # setup disk backed nix store
75 mount -t ext4 /dev/vdb /workspace
76 install -d -o spindle-workflow -g spindle-workflow /workspace /workspace/repo
77 install -d /workspace/.nix/rw-store /workspace/.nix/rw-store-work /workspace/.nix/build
78 mount -t overlay overlay \
79 -o lowerdir=/nix/store,upperdir=/workspace/.nix/rw-store,workdir=/workspace/.nix/rw-store-work \
80 /nix/store
81 fi
82
83 ip link set lo up
84 ip link set eth0 up
85 ip addr add 10.0.3.15/24 dev eth0
86 ip route add default via 10.0.3.2
87 hostname -F /etc/hostname
88 '';
89
90 inittab = writeText "inittab" ''
91 ::sysinit:/sbin/spindle-setup
92 ::respawn:env TMPDIR=/workspace/.nix/build /usr/local/bin/nix-daemon
93 ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle
94 ::ctrlaltdel:/sbin/reboot
95 '';
96
97 profileScript = writeText "spindle-profile" ''
98 export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
99 export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
100 export NIX_REMOTE=daemon
101 '';
102
103 # mirror nix/microvm/base.nix and nix/modules/shuttle.nix
104 nixConf = writeText "nix.conf" ''
105 experimental-features = nix-command flakes
106 trusted-users = root spindle-workflow
107 allowed-users = spindle-workflow
108 post-build-hook = /usr/libexec/spindle-post-build-hook
109 # keep build sandboxes on the /workspace disk, not the RAM-backed root tmpfs
110 build-dir = /workspace/.nix/build
111 !include /run/spindle/nix.conf
112 '';
113
114 apkRepositories = writeText "apk-repositories" (builtins.concatStringsSep "\n" repositories + "\n");
115
116 postBuildHook = writeText "spindle-post-build-hook" ''
117 #!/bin/sh
118 set -f
119
120 if [ -z "''${OUT_PATHS:-}" ]; then
121 exit 0
122 fi
123
124 # OUT_PATHS is intentionally split into individual store paths
125 exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS
126 '';
127
128 imageSpecJSON = writeText "spec.json" (
129 builtins.toJSON {
130 inherit arch;
131 bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 root=/dev/vda rootfstype=squashfs modules=virtio_blk,virtio_net,virtio_console overlaytmpfs=yes init=/sbin/init";
132 kernel = "kernel";
133 initrd = "initrd";
134 runnerType = "qemu";
135 runnerConfig = {
136 cpu = "host,+x2apic,-sgx";
137 machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off";
138 console = "hvc0";
139 extraArgs = [];
140 };
141 memoryMiB = 4096;
142 storeDisk = "store-disk";
143 storeDiskType = "squashfs";
144 vcpus = 2;
145 shell = "/usr/local/bin/bash";
146 networkInterfaces = [
147 {
148 type = "slirp4netns";
149 id = "net0";
150 mac = "02:00:00:00:10:01";
151 }
152 ];
153 volumes = [
154 {
155 fsType = "ext4";
156 image = "workspace.img";
157 imageType = "raw";
158 mountPoint = "/workspace";
159 readOnly = false;
160 sizeMiB = 1024 * 16; # 16 GB
161 }
162 ];
163 }
164 );
165in
166 runCommand "spindle-alpine-image-${arch}" {
167 nativeBuildInputs = [squashfsTools binutils];
168 } ''
169 mkdir -p rootfs
170 tar -xzpf ${rootfs} -C rootfs
171
172 # kernel modules from modloop (ships its own modules.dep, no depmod needed)
173 unsquashfs -q -d modloop ${modloop}
174 mkdir -p rootfs/lib/modules
175 cp -a modloop/modules/* rootfs/lib/modules/
176
177 install -D -m 0755 ${shuttle}/bin/shuttle rootfs/usr/bin/shuttle
178 install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup
179 install -D -m 0644 ${inittab} rootfs/etc/inittab
180 install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh
181 install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf
182 install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook
183
184 # install dependencies
185 # we only copy binaries + libexec for minimal deps so the image size doesn't
186 # increase so much (if we copy the whole guestTools closure for example, it
187 # doubles the disk size)
188 mkdir -p rootfs/nix/store rootfs/usr/local/bin
189 for pkg in ${toString guestTools}; do
190 for bin in "$pkg/bin/"*; do
191 [[ -e "$bin" ]] || continue
192 name=$(basename "$bin")
193 # we resolve symlinks as to copy the actual binaries
194 if [[ -L "$bin" ]]; then
195 real=$(readlink "$bin")
196 else
197 real="$bin"
198 fi
199 # handle symlinks properly
200 if [[ "$real" != /nix/store* ]]; then
201 ln -vsf "$real" "rootfs/usr/local/bin/$name"
202 else
203 cp -v "$real" "rootfs/usr/local/bin/$name"
204 fi
205 done
206 # libexec has binaries used by packages even if statically compiled
207 if [[ -d "$pkg/libexec" ]]; then
208 mkdir -p "rootfs$pkg"
209 cp -av "$pkg/libexec" "rootfs$pkg/"
210 fi
211 done
212 # this is necessary for nix to work, it is not a library but nix hardcodes
213 # it in it's binary
214 cp -rv ${publicsuffix-list} rootfs/nix/store/
215
216 # scripts commonly hardcode #!/bin/bash
217 ln -sf ${bash}/bin/bash rootfs/bin/bash
218
219 echo "spindle-microvm" > rootfs/etc/hostname
220 printf 'nameserver 127.0.0.1\n' > rootfs/etc/resolv.conf
221 install -D -m 0644 ${apkRepositories} rootfs/etc/apk/repositories
222
223 echo "spindle-workflow:x:970:970:spindle workflow:/workspace:/bin/sh" >> rootfs/etc/passwd
224 echo "spindle-workflow:x:970:" >> rootfs/etc/group
225 echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow
226 mkdir -p rootfs/workspace
227
228 # subordinate id ranges so the workflow user can run rootless containers
229 # (podman/buildah): without these, user-namespace id mapping falls back to a
230 # single 970->0 map and any layer that chowns to another uid fails. the range
231 # is well clear of 970 and the 30000-block nixbld users.
232 echo "spindle-workflow:100000:65536" >> rootfs/etc/subuid
233 echo "spindle-workflow:100000:65536" >> rootfs/etc/subgid
234
235 # setup nix build users for the daemon
236 members=""
237 for i in $(seq 1 8); do
238 echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd
239 echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow
240 members="$members''${members:+,}nixbld$i"
241 done
242 echo "nixbld:x:30000:$members" >> rootfs/etc/group
243
244 mkdir -p "$out"
245 mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet \
246 -p '/sbin/apk m 4755 0 0' # suid apk so spindle-workflow can use it without having to doas or smth
247 cp ${kernel} "$out/kernel"
248 cp ${initramfs} "$out/initrd"
249 cp ${imageSpecJSON} "$out/spec.json"
250 ''