alice.pds.demo.boltless.dev
This repository has no description
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}: let
7 cfg = config.services.tangled.spindle;
8in
9 with lib; {
10 options = {
11 services.tangled.spindle = {
12 enable = mkOption {
13 type = types.bool;
14 default = false;
15 description = "Enable a tangled spindle";
16 };
17 package = mkOption {
18 type = types.package;
19 description = "Package to use for the spindle";
20 };
21
22 server = {
23 listenAddr = mkOption {
24 type = types.str;
25 default = "0.0.0.0:6555";
26 description = "Address to listen on";
27 };
28
29 dbPath = mkOption {
30 type = types.path;
31 default = "/var/lib/spindle/spindle.db";
32 description = "Path to the database file";
33 };
34
35 repoDir = mkOption {
36 type = types.path;
37 default = "/var/lib/spindle/repos";
38 description = "Path where synced git repositories live";
39 };
40
41 hostname = mkOption {
42 type = types.str;
43 example = "my.spindle.com";
44 description = "Hostname for the server (required)";
45 };
46
47 plcUrl = mkOption {
48 type = types.str;
49 default = "https://plc.directory";
50 description = "atproto PLC directory";
51 };
52
53 jetstreamEndpoint = mkOption {
54 type = types.str;
55 default = "wss://jetstream1.us-west.bsky.network/subscribe";
56 description = "Jetstream endpoint to subscribe to";
57 };
58
59 dev = mkOption {
60 type = types.bool;
61 default = false;
62 description = "Enable development mode (disables signature verification)";
63 };
64
65 owner = mkOption {
66 type = types.str;
67 example = "did:plc:qfpnj4og54vl56wngdriaxug";
68 description = "DID of owner (required)";
69 };
70
71 maxJobCount = mkOption {
72 type = types.int;
73 default = 2;
74 example = 5;
75 description = "Maximum number of concurrent jobs to run";
76 };
77
78 queueSize = mkOption {
79 type = types.int;
80 default = 100;
81 example = 100;
82 description = "Maximum number of jobs queue up";
83 };
84
85 secrets = {
86 provider = mkOption {
87 type = types.str;
88 default = "sqlite";
89 description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'.";
90 };
91
92 openbao = {
93 proxyAddr = mkOption {
94 type = types.str;
95 default = "http://127.0.0.1:8200";
96 description = "Address of the OpenBAO proxy server";
97 };
98 mount = mkOption {
99 type = types.str;
100 default = "spindle";
101 description = "Mount path in OpenBAO to read secrets from";
102 };
103 };
104 };
105
106 tap = {
107 embed = mkOption {
108 type = types.bool;
109 default = true;
110 description = "Run an embedded tap inside the spindle process";
111 };
112
113 url = mkOption {
114 type = types.str;
115 default = "http://[::1]:2480";
116 description = "URL the spindle's tap client dials";
117 };
118
119 bind = mkOption {
120 type = types.str;
121 default = "[::1]:2480";
122 description = "Loopback address the embedded tap server listens on";
123 };
124
125 dbPath = mkOption {
126 type = types.path;
127 default = "/var/lib/spindle/tap.db";
128 description = "Path to the embedded tap sqlite database";
129 };
130
131 relayUrl = mkOption {
132 type = types.str;
133 default = "https://bsky.network";
134 description = "Relay used by the embedded tap firehose";
135 };
136 };
137 };
138
139 pipelines = {
140 logBucket = mkOption {
141 type = types.str;
142 default = "tangled-logs";
143 description = "S3 bucket for workflow logs";
144 };
145 workflowTimeout = mkOption {
146 type = types.str;
147 default = "5m";
148 description = "Timeout for each workflow step";
149 };
150
151 nixery = {
152 nixery = mkOption {
153 type = types.str;
154 default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet
155 description = "Nixery instance to use";
156 };
157
158 maxJobMemoryMb = mkOption {
159 type = types.int;
160 default = 6144;
161 description = "Memory limit per nixery workflow container in MiB (default 6 GiB)";
162 };
163 maxConcurrentWorkflows = mkOption {
164 type = types.int;
165 default = 8;
166 description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit.";
167 };
168 };
169
170 nix = {
171 enable = mkOption {
172 type = types.bool;
173 default = false;
174 description = "Enable the host Nix bubblewrap workflow engine. Enabling lets repository authors evaluate flakes and submit builds to the configured Nix daemon.";
175 };
176 workDirBase = mkOption {
177 type = types.str;
178 default = "/tmp";
179 description = "Directory for temporary Nix workflow workspaces";
180 };
181 maxConcurrentWorkflows = mkOption {
182 type = types.int;
183 default = 8;
184 description = "Maximum number of Nix workflows running simultaneously. Zero disables this limit.";
185 };
186 maxOutputs = mkOption {
187 type = types.int;
188 default = 100;
189 description = "Maximum number of build outputs per Nix workflow.";
190 };
191 maxEvalMemoryBytes = mkOption {
192 type = types.int;
193 default = 8589934592;
194 description = "Maximum memory limit in bytes during Nix evaluation.";
195 };
196 maxLogBytes = mkOption {
197 type = types.int;
198 default = 33554432;
199 description = "Maximum log output size in bytes for a Nix workflow step.";
200 };
201 sandboxUid = mkOption {
202 type = types.int;
203 default = 65534;
204 description = "User ID to run sandboxed Nix workflow processes as.";
205 };
206 sandboxGid = mkOption {
207 type = types.int;
208 default = 65534;
209 description = "Group ID to run sandboxed Nix workflow processes as.";
210 };
211 };
212
213 microvm = {
214 enableKVM = mkOption {
215 type = types.bool;
216 default = true;
217 description = "Enable KVM hardware acceleration";
218 };
219
220 imageDir = mkOption {
221 type = types.str;
222 default = "/var/lib/spindle/images";
223 description = "Directory containing microVM image spec JSONs or image spec directories";
224 };
225 overlayDir = mkOption {
226 type = types.str;
227 default = "/tmp";
228 description = "Directory to store microVM temporary overlay files";
229 };
230 defaultImage = mkOption {
231 type = types.str;
232 default = "nixos";
233 description = "Default microVM image spec to use if none is specified in workflow";
234 };
235 agentPort = mkOption {
236 type = types.port;
237 default = 10240;
238 description = "Host vsock port the microVM agent connects back to";
239 };
240
241 limits = {
242 total = {
243 memoryMiB = mkOption {
244 type = types.int;
245 default = 0;
246 description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit.";
247 };
248 vcpus = mkOption {
249 type = types.int;
250 default = 0;
251 description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit.";
252 };
253 diskMiB = mkOption {
254 type = types.int;
255 default = 0;
256 description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit.";
257 };
258 };
259
260 workflow = {
261 memoryMiB = mkOption {
262 type = types.int;
263 default = 0;
264 description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit.";
265 };
266 vcpus = mkOption {
267 type = types.int;
268 default = 0;
269 description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit.";
270 };
271 diskMiB = mkOption {
272 type = types.int;
273 default = 0;
274 description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit.";
275 };
276 };
277 };
278
279 cgroup = {
280 enable = mkOption {
281 type = types.bool;
282 default = false;
283 description = "Enable cgroup v2 containment for microVM processes.";
284 };
285 parent = mkOption {
286 type = types.str;
287 default = "self";
288 description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup.";
289 };
290 pidsMax = mkOption {
291 type = types.int;
292 default = 4096;
293 description = "Maximum number of processes allowed in each microVM workflow cgroup.";
294 };
295 swapMaxMiB = mkOption {
296 type = types.int;
297 default = 0;
298 description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap.";
299 };
300 supervisorMinMiB = mkOption {
301 type = types.int;
302 default = 512;
303 description = ''
304 Amount of memory in MiB that will be protected by the cgroup for the spindle
305 (allowing it to not get OOMed first.)
306 '';
307 };
308 };
309 };
310
311 nixCache = {
312 readUrls = mkOption {
313 type = types.listOf types.str;
314 default = [];
315 example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"];
316 description = "Nix binary cache URLs the Spindle guest should read from.";
317 };
318
319 trustedPublicKeys = mkOption {
320 type = types.listOf types.str;
321 default = [];
322 example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="];
323 description = "Public keys trusted for the configured Nix binary caches.";
324 };
325
326 uploadUrl = mkOption {
327 type = types.str;
328 default = "";
329 example = "local";
330 description = "Optional cache upload URL used by live cache import paths.";
331 };
332 };
333 };
334
335 environmentFile = mkOption {
336 type = with types; nullOr path;
337 default = null;
338 example = "/etc/spindle.env";
339 description = ''
340 Additional environment file as defined in {manpage}`systemd.exec(5)`.
341
342 Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`,
343 {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
344 may be passed to the service
345 without making them world readable in the nix store.
346 '';
347 };
348 };
349 };
350
351 config = let
352 deps = [
353 pkgs.bash
354 pkgs.git
355 pkgs.bubblewrap
356 pkgs.qemu
357 pkgs.e2fsprogs
358 pkgs.slirp4netns
359 pkgs.iproute2
360 pkgs.util-linux
361 config.nix.package
362 ];
363 in
364 mkIf cfg.enable {
365 environment.systemPackages = [
366 (pkgs.writeShellScriptBin "spindle" ''
367 export PATH="${lib.makeBinPath deps}:$PATH"
368 ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"}
369 ${lib.concatMapStringsSep "\n" (
370 e: "export ${e}"
371 )
372 config.systemd.services.spindle.serviceConfig.Environment}
373 exec ${cfg.package}/bin/spindle "$@"
374 '')
375 ];
376
377 virtualisation.docker.enable = true;
378
379 systemd.services.spindle = {
380 description = "spindle service";
381 after = [
382 "network.target"
383 "docker.service"
384 ];
385 wantedBy = ["multi-user.target"];
386 path = deps;
387 serviceConfig = {
388 LogsDirectory = "spindle";
389 StateDirectory = "spindle";
390 Delegate = cfg.pipelines.microvm.cgroup.enable;
391 EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
392
393 Environment = [
394 "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
395 "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
396 "SPINDLE_SERVER_REPO_DIR=${cfg.server.repoDir}"
397 "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}"
398 "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}"
399 "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}"
400 "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}"
401 "SPINDLE_SERVER_OWNER=${cfg.server.owner}"
402 "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}"
403 "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}"
404 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"
405 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"
406 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
407 "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}"
408 "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}"
409 "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}"
410 "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}"
411 "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}"
412 "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}"
413 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
414 "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}"
415 "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}"
416 "SPINDLE_NIX_PIPELINES_ENABLED=${lib.boolToString cfg.pipelines.nix.enable}"
417 "SPINDLE_NIX_PIPELINES_WORK_DIR_BASE=${cfg.pipelines.nix.workDirBase}"
418 "SPINDLE_NIX_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
419 "SPINDLE_NIX_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nix.maxConcurrentWorkflows}"
420 "SPINDLE_NIX_PIPELINES_MAX_OUTPUTS=${toString cfg.pipelines.nix.maxOutputs}"
421 "SPINDLE_NIX_PIPELINES_MAX_EVAL_MEMORY_BYTES=${toString cfg.pipelines.nix.maxEvalMemoryBytes}"
422 "SPINDLE_NIX_PIPELINES_MAX_LOG_BYTES=${toString cfg.pipelines.nix.maxLogBytes}"
423 "SPINDLE_NIX_PIPELINES_SANDBOX_UID=${toString cfg.pipelines.nix.sandboxUid}"
424 "SPINDLE_NIX_PIPELINES_SANDBOX_GID=${toString cfg.pipelines.nix.sandboxGid}"
425 "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}"
426 "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}"
427 "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}"
428 "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}"
429 "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}"
430 "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
431 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}"
432 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}"
433 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}"
434 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}"
435 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}"
436 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}"
437 "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}"
438 "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}"
439 "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}"
440 "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}"
441 "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}"
442 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}"
443 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}"
444 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}"
445 "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}"
446 ];
447 ExecStart = "${cfg.package}/bin/spindle";
448 Restart = "always";
449 };
450 };
451 };
452 }