alice.pds.demo.boltless.dev
This repository has no description
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}: let
7 cfg = config.services.tangled.spindle;
8in
9 with lib; {
10 options = {
11 services.tangled.spindle = {
12 enable = mkOption {
13 type = types.bool;
14 default = false;
15 description = "Enable a tangled spindle";
16 };
17 package = mkOption {
18 type = types.package;
19 description = "Package to use for the spindle";
20 };
21
22 server = {
23 listenAddr = mkOption {
24 type = types.str;
25 default = "0.0.0.0:6555";
26 description = "Address to listen on";
27 };
28
29 dbPath = mkOption {
30 type = types.path;
31 default = "/var/lib/spindle/spindle.db";
32 description = "Path to the database file";
33 };
34
35 repoDir = mkOption {
36 type = types.path;
37 default = "/var/lib/spindle/repos";
38 description = "Path where synced git repositories live";
39 };
40
41 hostname = mkOption {
42 type = types.str;
43 example = "my.spindle.com";
44 description = "Hostname for the server (required)";
45 };
46
47 plcUrl = mkOption {
48 type = types.str;
49 default = "https://plc.directory";
50 description = "atproto PLC directory";
51 };
52
53 jetstreamEndpoint = mkOption {
54 type = types.str;
55 default = "wss://jetstream1.us-west.bsky.network/subscribe";
56 description = "Jetstream endpoint to subscribe to";
57 };
58
59 dev = mkOption {
60 type = types.bool;
61 default = false;
62 description = "Enable development mode (disables signature verification)";
63 };
64
65 owner = mkOption {
66 type = types.str;
67 example = "did:plc:qfpnj4og54vl56wngdriaxug";
68 description = "DID of owner (required)";
69 };
70
71 maxJobCount = mkOption {
72 type = types.int;
73 default = 2;
74 example = 5;
75 description = "Maximum number of concurrent jobs to run";
76 };
77
78 queueSize = mkOption {
79 type = types.int;
80 default = 100;
81 example = 100;
82 description = "Maximum number of jobs queue up";
83 };
84
85 secrets = {
86 provider = mkOption {
87 type = types.str;
88 default = "sqlite";
89 description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'.";
90 };
91
92 openbao = {
93 proxyAddr = mkOption {
94 type = types.str;
95 default = "http://127.0.0.1:8200";
96 description = "Address of the OpenBAO proxy server";
97 };
98 mount = mkOption {
99 type = types.str;
100 default = "spindle";
101 description = "Mount path in OpenBAO to read secrets from";
102 };
103 };
104 };
105
106 tap = {
107 embed = mkOption {
108 type = types.bool;
109 default = true;
110 description = "Run an embedded tap inside the spindle process";
111 };
112
113 url = mkOption {
114 type = types.str;
115 default = "http://[::1]:2480";
116 description = "URL the spindle's tap client dials";
117 };
118
119 bind = mkOption {
120 type = types.str;
121 default = "[::1]:2480";
122 description = "Loopback address the embedded tap server listens on";
123 };
124
125 dbPath = mkOption {
126 type = types.path;
127 default = "/var/lib/spindle/tap.db";
128 description = "Path to the embedded tap sqlite database";
129 };
130
131 relayUrl = mkOption {
132 type = types.str;
133 default = "https://bsky.network";
134 description = "Relay used by the embedded tap firehose";
135 };
136 };
137 };
138
139 artifactStores = {
140 disk.dir = mkOption {
141 type = types.path;
142 default = "/var/log/spindle";
143 description = "Root directory for disk artifacts";
144 };
145
146 s3.bucket = mkOption {
147 type = types.str;
148 default = "tangled-logs";
149 description = "S3 bucket for artifacts";
150 };
151
152 s3.region = mkOption {
153 type = types.str;
154 default = "us-east-1";
155 description = "AWS region for the artifact bucket";
156 };
157 };
158
159 pipelines = {
160 workflowTimeout = mkOption {
161 type = types.str;
162 default = "5m";
163 description = "Timeout for each workflow step";
164 };
165
166 nixery = {
167 nixery = mkOption {
168 type = types.str;
169 default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet
170 description = "Nixery instance to use";
171 };
172
173 maxJobMemoryMb = mkOption {
174 type = types.int;
175 default = 6144;
176 description = "Memory limit per nixery workflow container in MiB (default 6 GiB)";
177 };
178 maxConcurrentWorkflows = mkOption {
179 type = types.int;
180 default = 8;
181 description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit.";
182 };
183 };
184
185 microvm = {
186 enableKVM = mkOption {
187 type = types.bool;
188 default = true;
189 description = "Enable KVM hardware acceleration";
190 };
191
192 imageDir = mkOption {
193 type = types.str;
194 default = "/var/lib/spindle/images";
195 description = "Directory containing microVM image spec JSONs or image spec directories";
196 };
197 overlayDir = mkOption {
198 type = types.str;
199 default = "/tmp";
200 description = "Directory to store microVM temporary overlay files";
201 };
202 defaultImage = mkOption {
203 type = types.str;
204 default = "nixos";
205 description = "Default microVM image spec to use if none is specified in workflow";
206 };
207 agentPort = mkOption {
208 type = types.port;
209 default = 10240;
210 description = "Host vsock port the microVM agent connects back to";
211 };
212
213 limits = {
214 total = {
215 memoryMiB = mkOption {
216 type = types.int;
217 default = 0;
218 description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit.";
219 };
220 vcpus = mkOption {
221 type = types.int;
222 default = 0;
223 description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit.";
224 };
225 diskMiB = mkOption {
226 type = types.int;
227 default = 0;
228 description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit.";
229 };
230 };
231
232 workflow = {
233 memoryMiB = mkOption {
234 type = types.int;
235 default = 0;
236 description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit.";
237 };
238 vcpus = mkOption {
239 type = types.int;
240 default = 0;
241 description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit.";
242 };
243 diskMiB = mkOption {
244 type = types.int;
245 default = 0;
246 description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit.";
247 };
248 };
249 };
250
251 cgroup = {
252 enable = mkOption {
253 type = types.bool;
254 default = false;
255 description = "Enable cgroup v2 containment for microVM processes.";
256 };
257 parent = mkOption {
258 type = types.str;
259 default = "self";
260 description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup.";
261 };
262 pidsMax = mkOption {
263 type = types.int;
264 default = 4096;
265 description = "Maximum number of processes allowed in each microVM workflow cgroup.";
266 };
267 swapMaxMiB = mkOption {
268 type = types.int;
269 default = 0;
270 description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap.";
271 };
272 supervisorMinMiB = mkOption {
273 type = types.int;
274 default = 512;
275 description = ''
276 Amount of memory in MiB that will be protected by the cgroup for the spindle
277 (allowing it to not get OOMed first.)
278 '';
279 };
280 };
281 };
282
283 nixCache = {
284 readUrls = mkOption {
285 type = types.listOf types.str;
286 default = [];
287 example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"];
288 description = "Nix binary cache URLs the Spindle guest should read from.";
289 };
290
291 trustedPublicKeys = mkOption {
292 type = types.listOf types.str;
293 default = [];
294 example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="];
295 description = "Public keys trusted for the configured Nix binary caches.";
296 };
297
298 uploadUrl = mkOption {
299 type = types.str;
300 default = "";
301 example = "local";
302 description = "Optional cache upload URL used by live cache import paths.";
303 };
304 };
305 };
306
307 environmentFile = mkOption {
308 type = with types; nullOr path;
309 default = null;
310 example = "/etc/spindle.env";
311 description = ''
312 Additional environment file as defined in {manpage}`systemd.exec(5)`.
313
314 Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`,
315 {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
316 may be passed to the service
317 without making them world readable in the nix store.
318 '';
319 };
320 };
321 };
322
323 config = let
324 deps = [
325 pkgs.git
326 pkgs.qemu
327 pkgs.e2fsprogs
328 pkgs.slirp4netns
329 pkgs.iproute2
330 pkgs.util-linux
331 config.nix.package
332 ];
333 in
334 mkIf cfg.enable {
335 environment.systemPackages = [
336 (pkgs.writeShellScriptBin "spindle" ''
337 export PATH="${lib.makeBinPath deps}:$PATH"
338 ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"}
339 ${lib.concatMapStringsSep "\n" (
340 e: "export ${e}"
341 )
342 config.systemd.services.spindle.serviceConfig.Environment}
343 exec ${cfg.package}/bin/spindle "$@"
344 '')
345 ];
346
347 virtualisation.docker.enable = true;
348
349 systemd.services.spindle = {
350 description = "spindle service";
351 after = [
352 "network.target"
353 "docker.service"
354 ];
355 wantedBy = ["multi-user.target"];
356 path = deps;
357 serviceConfig = {
358 LogsDirectory = "spindle";
359 StateDirectory = "spindle";
360 Delegate = cfg.pipelines.microvm.cgroup.enable;
361 EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
362
363 Environment = [
364 "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
365 "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
366 "SPINDLE_SERVER_REPO_DIR=${cfg.server.repoDir}"
367 "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}"
368 "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}"
369 "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}"
370 "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}"
371 "SPINDLE_SERVER_OWNER=${cfg.server.owner}"
372 "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}"
373 "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}"
374 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"
375 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"
376 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
377 "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}"
378 "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}"
379 "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}"
380 "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}"
381 "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}"
382 "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}"
383 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
384 "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}"
385 "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}"
386 "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}"
387 "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}"
388 "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}"
389 "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}"
390 "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}"
391 "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
392 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}"
393 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}"
394 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}"
395 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}"
396 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}"
397 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}"
398 "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}"
399 "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}"
400 "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}"
401 "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}"
402 "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}"
403 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}"
404 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}"
405 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}"
406 "SPINDLE_ARTIFACT_STORES_DISK_DIR=${cfg.artifactStores.disk.dir}"
407 "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${cfg.artifactStores.s3.bucket}"
408 "SPINDLE_ARTIFACT_STORES_S3_REGION=${cfg.artifactStores.s3.region}"
409 "SPINDLE_MILL_ARTIFACT_STORE=s3"
410 ];
411 ExecStart = "${cfg.package}/bin/spindle";
412 Restart = "always";
413 };
414 };
415 };
416 }