alice.pds.demo.boltless.dev
This repository has no description
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}: let
7 cfg = config.services.tangled.spindle;
8in
9 with lib; {
10 imports = [
11 (lib.mkRemovedOptionModule [ "services" "tangled" "spindle" "server" "owner" ] "The spindle owner setting is deprecated.")
12 ];
13 options = {
14 services.tangled.spindle = {
15 enable = mkOption {
16 type = types.bool;
17 default = false;
18 description = "Enable a tangled spindle";
19 };
20 package = mkOption {
21 type = types.package;
22 description = "Package to use for the spindle";
23 };
24
25 server = {
26 listenAddr = mkOption {
27 type = types.str;
28 default = "0.0.0.0:6555";
29 description = "Address to listen on";
30 };
31
32 dbPath = mkOption {
33 type = types.path;
34 default = "/var/lib/spindle/spindle.db";
35 description = "Path to the database file";
36 };
37
38 repoDir = mkOption {
39 type = types.path;
40 default = "/var/lib/spindle/repos";
41 description = "Path where synced git repositories live";
42 };
43
44 hostname = mkOption {
45 type = types.str;
46 example = "my.spindle.com";
47 description = "Hostname for the server (required)";
48 };
49
50 plcUrl = mkOption {
51 type = types.str;
52 default = "https://plc.directory";
53 description = "atproto PLC directory";
54 };
55
56 jetstreamEndpoint = mkOption {
57 type = types.str;
58 default = "wss://jetstream1.us-west.bsky.network/subscribe";
59 description = "Jetstream endpoint to subscribe to";
60 };
61
62 dev = mkOption {
63 type = types.bool;
64 default = false;
65 description = "Enable development mode (disables signature verification)";
66 };
67
68 maxJobCount = mkOption {
69 type = types.int;
70 default = 2;
71 example = 5;
72 description = "Maximum number of concurrent jobs to run";
73 };
74
75 queueSize = mkOption {
76 type = types.int;
77 default = 100;
78 example = 100;
79 description = "Maximum number of jobs queue up";
80 };
81
82 secrets = {
83 provider = mkOption {
84 type = types.str;
85 default = "sqlite";
86 description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'.";
87 };
88
89 openbao = {
90 proxyAddr = mkOption {
91 type = types.str;
92 default = "http://127.0.0.1:8200";
93 description = "Address of the OpenBAO proxy server";
94 };
95 mount = mkOption {
96 type = types.str;
97 default = "spindle";
98 description = "Mount path in OpenBAO to read secrets from";
99 };
100 };
101 };
102
103 tap = {
104 embed = mkOption {
105 type = types.bool;
106 default = true;
107 description = "Run an embedded tap inside the spindle process";
108 };
109
110 url = mkOption {
111 type = types.str;
112 default = "http://[::1]:2480";
113 description = "URL the spindle's tap client dials";
114 };
115
116 bind = mkOption {
117 type = types.str;
118 default = "[::1]:2480";
119 description = "Loopback address the embedded tap server listens on";
120 };
121
122 dbPath = mkOption {
123 type = types.path;
124 default = "/var/lib/spindle/tap.db";
125 description = "Path to the embedded tap sqlite database";
126 };
127
128 relayUrl = mkOption {
129 type = types.str;
130 default = "https://bsky.network";
131 description = "Relay used by the embedded tap firehose";
132 };
133 };
134 };
135
136 pipelines = {
137 logBucket = mkOption {
138 type = types.str;
139 default = "tangled-logs";
140 description = "S3 bucket for workflow logs";
141 };
142 workflowTimeout = mkOption {
143 type = types.str;
144 default = "5m";
145 description = "Timeout for each workflow step";
146 };
147
148 nixery = {
149 nixery = mkOption {
150 type = types.str;
151 default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet
152 description = "Nixery instance to use";
153 };
154
155 maxJobMemoryMb = mkOption {
156 type = types.int;
157 default = 6144;
158 description = "Memory limit per nixery workflow container in MiB (default 6 GiB)";
159 };
160 maxConcurrentWorkflows = mkOption {
161 type = types.int;
162 default = 8;
163 description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit.";
164 };
165 };
166
167 microvm = {
168 enableKVM = mkOption {
169 type = types.bool;
170 default = true;
171 description = "Enable KVM hardware acceleration";
172 };
173
174 imageDir = mkOption {
175 type = types.str;
176 default = "/var/lib/spindle/images";
177 description = "Directory containing microVM image spec JSONs or image spec directories";
178 };
179 overlayDir = mkOption {
180 type = types.str;
181 default = "/tmp";
182 description = "Directory to store microVM temporary overlay files";
183 };
184 defaultImage = mkOption {
185 type = types.str;
186 default = "nixos";
187 description = "Default microVM image spec to use if none is specified in workflow";
188 };
189 agentPort = mkOption {
190 type = types.port;
191 default = 10240;
192 description = "Host vsock port the microVM agent connects back to";
193 };
194
195 limits = {
196 total = {
197 memoryMiB = mkOption {
198 type = types.int;
199 default = 0;
200 description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit.";
201 };
202 vcpus = mkOption {
203 type = types.int;
204 default = 0;
205 description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit.";
206 };
207 diskMiB = mkOption {
208 type = types.int;
209 default = 0;
210 description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit.";
211 };
212 };
213
214 workflow = {
215 memoryMiB = mkOption {
216 type = types.int;
217 default = 0;
218 description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit.";
219 };
220 vcpus = mkOption {
221 type = types.int;
222 default = 0;
223 description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit.";
224 };
225 diskMiB = mkOption {
226 type = types.int;
227 default = 0;
228 description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit.";
229 };
230 };
231 };
232
233 cgroup = {
234 enable = mkOption {
235 type = types.bool;
236 default = false;
237 description = "Enable cgroup v2 containment for microVM processes.";
238 };
239 parent = mkOption {
240 type = types.str;
241 default = "self";
242 description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup.";
243 };
244 pidsMax = mkOption {
245 type = types.int;
246 default = 4096;
247 description = "Maximum number of processes allowed in each microVM workflow cgroup.";
248 };
249 swapMaxMiB = mkOption {
250 type = types.int;
251 default = 0;
252 description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap.";
253 };
254 supervisorMinMiB = mkOption {
255 type = types.int;
256 default = 512;
257 description = ''
258 Amount of memory in MiB that will be protected by the cgroup for the spindle
259 (allowing it to not get OOMed first.)
260 '';
261 };
262 };
263 };
264
265 nixCache = {
266 readUrls = mkOption {
267 type = types.listOf types.str;
268 default = [];
269 example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"];
270 description = "Nix binary cache URLs the Spindle guest should read from.";
271 };
272
273 trustedPublicKeys = mkOption {
274 type = types.listOf types.str;
275 default = [];
276 example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="];
277 description = "Public keys trusted for the configured Nix binary caches.";
278 };
279
280 uploadUrl = mkOption {
281 type = types.str;
282 default = "";
283 example = "local";
284 description = "Optional cache upload URL used by live cache import paths.";
285 };
286 };
287 };
288
289 environmentFile = mkOption {
290 type = with types; nullOr path;
291 default = null;
292 example = "/etc/spindle.env";
293 description = ''
294 Additional environment file as defined in {manpage}`systemd.exec(5)`.
295
296 Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`,
297 {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
298 may be passed to the service
299 without making them world readable in the nix store.
300 '';
301 };
302 };
303 };
304
305 config = let
306 deps = [
307 pkgs.git
308 pkgs.qemu
309 pkgs.e2fsprogs
310 pkgs.slirp4netns
311 pkgs.iproute2
312 pkgs.util-linux
313 config.nix.package
314 ];
315 in
316 mkIf cfg.enable {
317 environment.systemPackages = [
318 (pkgs.writeShellScriptBin "spindle" ''
319 export PATH="${lib.makeBinPath deps}:$PATH"
320 ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"}
321 ${lib.concatMapStringsSep "\n" (
322 e: "export ${e}"
323 )
324 config.systemd.services.spindle.serviceConfig.Environment}
325 exec ${cfg.package}/bin/spindle "$@"
326 '')
327 ];
328
329 virtualisation.docker.enable = true;
330
331 systemd.services.spindle = {
332 description = "spindle service";
333 after = [
334 "network.target"
335 "docker.service"
336 ];
337 wantedBy = ["multi-user.target"];
338 path = deps;
339 serviceConfig = {
340 LogsDirectory = "spindle";
341 StateDirectory = "spindle";
342 Delegate = cfg.pipelines.microvm.cgroup.enable;
343 EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
344
345 Environment = [
346 "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
347 "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
348 "SPINDLE_SERVER_REPO_DIR=${cfg.server.repoDir}"
349 "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}"
350 "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}"
351 "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}"
352 "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}"
353 "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}"
354 "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}"
355 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"
356 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"
357 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
358 "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}"
359 "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}"
360 "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}"
361 "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}"
362 "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}"
363 "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}"
364 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
365 "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}"
366 "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}"
367 "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}"
368 "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}"
369 "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}"
370 "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}"
371 "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}"
372 "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
373 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}"
374 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}"
375 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}"
376 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}"
377 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}"
378 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}"
379 "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}"
380 "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}"
381 "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}"
382 "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}"
383 "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}"
384 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}"
385 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}"
386 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}"
387 "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}"
388 ];
389 ExecStart = "${cfg.package}/bin/spindle";
390 Restart = "always";
391 };
392 };
393 };
394 }