alice.pds.demo.boltless.dev
This repository has no description
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}: let
7 cfg = config.services.tangled.spindle;
8in
9 with lib; {
10 imports = [
11 (lib.mkRemovedOptionModule [ "services" "tangled" "spindle" "server" "owner" ] "The spindle owner setting is deprecated.")
12 ];
13 options = {
14 services.tangled.spindle = {
15 enable = mkOption {
16 type = types.bool;
17 default = false;
18 description = "Enable a tangled spindle";
19 };
20 package = mkOption {
21 type = types.package;
22 description = "Package to use for the spindle";
23 };
24
25 server = {
26 listenAddr = mkOption {
27 type = types.str;
28 default = "0.0.0.0:6555";
29 description = "Address to listen on";
30 };
31
32 dbPath = mkOption {
33 type = types.path;
34 default = "/var/lib/spindle/spindle.db";
35 description = "Path to the database file";
36 };
37
38 repoDir = mkOption {
39 type = types.path;
40 default = "/var/lib/spindle/repos";
41 description = "Path where synced git repositories live";
42 };
43
44 hostname = mkOption {
45 type = types.str;
46 example = "my.spindle.com";
47 description = "Hostname for the server (required)";
48 };
49
50 plcUrl = mkOption {
51 type = types.str;
52 default = "https://plc.directory";
53 description = "atproto PLC directory";
54 };
55
56 jetstreamEndpoint = mkOption {
57 type = types.str;
58 default = "wss://jetstream1.us-west.bsky.network/subscribe";
59 description = "Jetstream endpoint to subscribe to";
60 };
61
62 dev = mkOption {
63 type = types.bool;
64 default = false;
65 description = "Enable development mode (disables signature verification)";
66 };
67
68 inviteOnly = mkOption {
69 type = types.bool;
70 default = true;
71 description = "Only accept repos from members of this spindle";
72 };
73
74 maxJobCount = mkOption {
75 type = types.int;
76 default = 2;
77 example = 5;
78 description = "Maximum number of concurrent jobs to run";
79 };
80
81 queueSize = mkOption {
82 type = types.int;
83 default = 100;
84 example = 100;
85 description = "Maximum number of jobs queue up";
86 };
87
88 secrets = {
89 provider = mkOption {
90 type = types.str;
91 default = "sqlite";
92 description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'.";
93 };
94
95 openbao = {
96 proxyAddr = mkOption {
97 type = types.str;
98 default = "http://127.0.0.1:8200";
99 description = "Address of the OpenBAO proxy server";
100 };
101 mount = mkOption {
102 type = types.str;
103 default = "spindle";
104 description = "Mount path in OpenBAO to read secrets from";
105 };
106 };
107 };
108
109 tap = {
110 embed = mkOption {
111 type = types.bool;
112 default = true;
113 description = "Run an embedded tap inside the spindle process";
114 };
115
116 url = mkOption {
117 type = types.str;
118 default = "http://[::1]:2480";
119 description = "URL the spindle's tap client dials";
120 };
121
122 bind = mkOption {
123 type = types.str;
124 default = "[::1]:2480";
125 description = "Loopback address the embedded tap server listens on";
126 };
127
128 dbPath = mkOption {
129 type = types.path;
130 default = "/var/lib/spindle/tap.db";
131 description = "Path to the embedded tap sqlite database";
132 };
133
134 relayUrl = mkOption {
135 type = types.str;
136 default = "https://bsky.network";
137 description = "Relay used by the embedded tap firehose";
138 };
139 };
140 };
141
142 pipelines = {
143 logBucket = mkOption {
144 type = types.str;
145 default = "tangled-logs";
146 description = "S3 bucket for workflow logs";
147 };
148 workflowTimeout = mkOption {
149 type = types.str;
150 default = "5m";
151 description = "Timeout for each workflow step";
152 };
153
154 nixery = {
155 nixery = mkOption {
156 type = types.str;
157 default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet
158 description = "Nixery instance to use";
159 };
160
161 maxJobMemoryMb = mkOption {
162 type = types.int;
163 default = 6144;
164 description = "Memory limit per nixery workflow container in MiB (default 6 GiB)";
165 };
166 maxConcurrentWorkflows = mkOption {
167 type = types.int;
168 default = 8;
169 description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit.";
170 };
171 };
172
173 microvm = {
174 enableKVM = mkOption {
175 type = types.bool;
176 default = true;
177 description = "Enable KVM hardware acceleration";
178 };
179
180 imageDir = mkOption {
181 type = types.str;
182 default = "/var/lib/spindle/images";
183 description = "Directory containing microVM image spec JSONs or image spec directories";
184 };
185 overlayDir = mkOption {
186 type = types.str;
187 default = "/tmp";
188 description = "Directory to store microVM temporary overlay files";
189 };
190 defaultImage = mkOption {
191 type = types.str;
192 default = "nixos";
193 description = "Default microVM image spec to use if none is specified in workflow";
194 };
195 agentPort = mkOption {
196 type = types.port;
197 default = 10240;
198 description = "Host vsock port the microVM agent connects back to";
199 };
200
201 limits = {
202 total = {
203 memoryMiB = mkOption {
204 type = types.int;
205 default = 0;
206 description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit.";
207 };
208 vcpus = mkOption {
209 type = types.int;
210 default = 0;
211 description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit.";
212 };
213 diskMiB = mkOption {
214 type = types.int;
215 default = 0;
216 description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit.";
217 };
218 };
219
220 workflow = {
221 memoryMiB = mkOption {
222 type = types.int;
223 default = 0;
224 description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit.";
225 };
226 vcpus = mkOption {
227 type = types.int;
228 default = 0;
229 description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit.";
230 };
231 diskMiB = mkOption {
232 type = types.int;
233 default = 0;
234 description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit.";
235 };
236 };
237 };
238
239 cgroup = {
240 enable = mkOption {
241 type = types.bool;
242 default = false;
243 description = "Enable cgroup v2 containment for microVM processes.";
244 };
245 parent = mkOption {
246 type = types.str;
247 default = "self";
248 description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup.";
249 };
250 pidsMax = mkOption {
251 type = types.int;
252 default = 4096;
253 description = "Maximum number of processes allowed in each microVM workflow cgroup.";
254 };
255 swapMaxMiB = mkOption {
256 type = types.int;
257 default = 0;
258 description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap.";
259 };
260 supervisorMinMiB = mkOption {
261 type = types.int;
262 default = 512;
263 description = ''
264 Amount of memory in MiB that will be protected by the cgroup for the spindle
265 (allowing it to not get OOMed first.)
266 '';
267 };
268 };
269 };
270
271 nixCache = {
272 readUrls = mkOption {
273 type = types.listOf types.str;
274 default = [];
275 example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"];
276 description = "Nix binary cache URLs the Spindle guest should read from.";
277 };
278
279 trustedPublicKeys = mkOption {
280 type = types.listOf types.str;
281 default = [];
282 example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="];
283 description = "Public keys trusted for the configured Nix binary caches.";
284 };
285
286 uploadUrl = mkOption {
287 type = types.str;
288 default = "";
289 example = "local";
290 description = "Optional cache upload URL used by live cache import paths.";
291 };
292 };
293 };
294
295 environmentFile = mkOption {
296 type = with types; nullOr path;
297 default = null;
298 example = "/etc/spindle.env";
299 description = ''
300 Additional environment file as defined in {manpage}`systemd.exec(5)`.
301
302 Sensitive secrets such as {env}`SPINDLE_SERVER_ADMIN_PASSWORD`
303 (without it the `/admin` member management endpoints are not mounted),
304 {env}`AWS_SECRET_ACCESS_KEY`,
305 {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
306 may be passed to the service
307 without making them world readable in the nix store.
308 '';
309 };
310 };
311 };
312
313 config = let
314 deps = [
315 pkgs.git
316 pkgs.qemu
317 pkgs.e2fsprogs
318 pkgs.slirp4netns
319 pkgs.iproute2
320 pkgs.util-linux
321 config.nix.package
322 ];
323 in
324 mkIf cfg.enable {
325 environment.systemPackages = [
326 (pkgs.writeShellScriptBin "spindle" ''
327 export PATH="${lib.makeBinPath deps}:$PATH"
328 ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"}
329 ${lib.concatMapStringsSep "\n" (
330 e: "export ${e}"
331 )
332 config.systemd.services.spindle.serviceConfig.Environment}
333 exec ${cfg.package}/bin/spindle "$@"
334 '')
335 ];
336
337 virtualisation.docker.enable = true;
338
339 systemd.services.spindle = {
340 description = "spindle service";
341 after = [
342 "network.target"
343 "docker.service"
344 ];
345 wantedBy = ["multi-user.target"];
346 path = deps;
347 serviceConfig = {
348 LogsDirectory = "spindle";
349 StateDirectory = "spindle";
350 Delegate = cfg.pipelines.microvm.cgroup.enable;
351 EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
352
353 Environment = [
354 "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
355 "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
356 "SPINDLE_SERVER_REPO_DIR=${cfg.server.repoDir}"
357 "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}"
358 "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}"
359 "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}"
360 "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}"
361 "SPINDLE_SERVER_INVITE_ONLY=${lib.boolToString cfg.server.inviteOnly}"
362 "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}"
363 "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}"
364 "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"
365 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"
366 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
367 "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}"
368 "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}"
369 "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}"
370 "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}"
371 "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}"
372 "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}"
373 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
374 "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}"
375 "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}"
376 "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}"
377 "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}"
378 "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}"
379 "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}"
380 "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}"
381 "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
382 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}"
383 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}"
384 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}"
385 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}"
386 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}"
387 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}"
388 "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}"
389 "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}"
390 "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}"
391 "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}"
392 "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}"
393 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}"
394 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}"
395 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}"
396 "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}"
397 ];
398 ExecStart = "${cfg.package}/bin/spindle";
399 Restart = "always";
400 };
401 };
402 };
403 }