@cryptotaxi247 / infra-1 / commits / bb14833a

mimas: enable new hydra-queue-runner

Martin Weinelt committed Dec 22, 2025 at 18:54 UTC bb14833a80b46276b373cc278154ac9b497dfefd
11 files changed +144 -116
build/hydra-queue-runner.nix new
+96
@@ -0,0 +1,96 @@
1 +{
2 + config,
3 + lib,
4 + ...
5 +}:
6 +
7 +let
8 + machines = [
9 + "eager-heisenberg"
10 + "elated-minsky"
11 + "enormous-catfish"
12 + "goofy-hopcroft"
13 + "growing-jennet"
14 + "hopeful-rivest"
15 + "intense-heron"
16 + "kind-lumiere"
17 + "maximum-snail"
18 + "norwegian-blue"
19 + "sleepy-brown"
20 + "sweeping-filly"
21 + ];
22 +in
23 +{
24 + age.secrets = {
25 + hydra-aws-credentials = {
26 + file = ./secrets/hydra-aws-credentials.age;
27 + path = "/var/lib/hydra/queue-runner/.aws/credentials";
28 + owner = "hydra-queue-runner";
29 + group = "hydra";
30 + };
31 + }
32 + // lib.listToAttrs (
33 + map (
34 + machine:
35 + lib.nameValuePair "${machine}-queue-runner-token" {
36 + file = ./secrets/${machine}-queue-runner-token.age;
37 + }
38 + ) machines
39 + );
40 +
41 + services.nginx = {
42 + enable = true;
43 + virtualHosts."queue-runner.hydra.nixos.org" = {
44 + enableACME = true;
45 + forceSSL = true;
46 +
47 + locations."/".extraConfig = ''
48 + # This is necessary so that grpc connections do not get closed early
49 + # see https://stackoverflow.com/a/67805465
50 + client_body_timeout 31536000s;
51 + client_max_body_size 0;
52 +
53 + grpc_pass grpc://${config.services.hydra-queue-runner-dev.grpc.address}:${toString config.services.hydra-queue-runner-dev.grpc.port};
54 +
55 + grpc_read_timeout 31536000s; # 1 year in seconds
56 + grpc_send_timeout 31536000s; # 1 year in seconds
57 + grpc_socket_keepalive on;
58 +
59 + grpc_set_header Host $host;
60 + grpc_set_header X-Real-IP $remote_addr;
61 + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
62 + grpc_set_header X-Forwarded-Proto $scheme;
63 + '';
64 + };
65 + };
66 +
67 + services.hydra-queue-runner-dev = {
68 + enable = true;
69 + awsCredentialsFile = config.age.secrets."hydra-aws-credentials".path;
70 + settings = {
71 + dbUrl = "postgres://hydra@10.0.40.3:5432/hydra";
72 + machineFreeFn = "DynamicWithMaxJobLimit";
73 + stepSortFn = "WithCriticalPath";
74 + usePresignedUploads = true;
75 + # TODO: Expose dispatchTriggerTimerInS, defaults to 120s
76 + queueTriggerTimerInS = 60;
77 + concurrentUploadLimit = 48;
78 + maxConcurrentDownloads = 48;
79 + remoteStoreAddr = [
80 + "s3://nix-cache?${
81 + lib.concatStringsSep "&" [
82 + "secret-key=/var/lib/hydra/queue-runner/keys/cache.nixos.org-1/secret"
83 + "write-nar-listing=1"
84 + "compression=zstd"
85 + "compression-level=19"
86 + "ls-compression=zstd"
87 + "log-compression=zstd"
88 + "index-debug-info=true"
89 + ]
90 + }"
91 + ];
92 + rootsDir = "/nix/var/nix/gcroots/hydra";
93 + tokenPaths = map (machine: config.age.secrets."${machine}-queue-runner-token".path) machines;
94 + };
95 + };
96 +}
build/hydra.nix
+5 -9
@@ -61,13 +61,6 @@ in
61 # Don't rate-limit the journal.
62 services.journald.rateLimitBurst = 0;
63
64 - age.secrets.hydra-aws-credentials = {
65 - file = ./secrets/hydra-aws-credentials.age;
66 - path = "/var/lib/hydra/queue-runner/.aws/credentials";
67 - owner = "hydra-queue-runner";
68 - group = "hydra";
69 - };
70 -
64 age.secrets.hydra-github-client-secret = {
65 file = ./secrets/hydra-github-client-secret.age;
66 owner = "hydra-www";
@@ -75,7 +68,6 @@ in
68 };
69
70 services.hydra-dev.enable = true;
78 - services.hydra-dev.buildMachinesFiles = [ "/etc/nix/machines" ];
71 services.hydra-dev.dbi = "dbi:Pg:dbname=hydra;host=10.0.40.3;user=hydra;";
72 services.hydra-dev.logo = ./hydra-logo.png;
73 services.hydra-dev.hydraURL = "https://hydra.nixos.org";
@@ -91,7 +83,7 @@ in
83 github_client_id = Ov23liat892hkVARixsT
84 github_client_secret_file = ${config.age.secrets.hydra-github-client-secret.path}
85
94 - store_uri = s3://nix-cache?secret-key=/var/lib/hydra/queue-runner/keys/cache.nixos.org-1/secret&write-nar-listing=1&ls-compression=br&log-compression=br&index-debug-info=true
86 + store_uri = s3://nix-cache?secret-key=/var/lib/hydra/queue-runner/keys/cache.nixos.org-1/secret&write-nar-listing=1&compression=zstd&ls-compression=zstd&log-compression=zstd&index-debug-info=true
87 server_store_uri = https://cache.nixos.org?local-nar-cache=${narCache}
88 binary_cache_public_uri = https://cache.nixos.org
89
@@ -110,6 +102,8 @@ in
102 evaluator_workers = 16
103 evaluator_max_memory_size = 8192
104
105 + queue_runner_endpoint = http://${config.services.hydra-queue-runner-dev.rest.address}:${toString config.services.hydra-queue-runner-dev.rest.port}
106 +
107 max_concurrent_evals = 1
108
109 # increase the number of active compress slots (CPU is 48*2 on mimas)
@@ -151,7 +145,9 @@ in
145 # eats memory as if it was free
146 systemd.services.hydra-notify.enable = false;
147
148 + # replaced by hydra-queue-runner-dev
149 systemd.services.hydra-queue-runner = {
150 + enable = false;
151 # restarting the scheduler is very expensive
152 restartIfChanged = false;
153 serviceConfig = {
build/mimas/default.nix
+1
@@ -3,6 +3,7 @@
3 ../common.nix
4 ../hydra.nix
5 ../hydra-proxy.nix
6 + ../hydra-queue-runner.nix
7 ./boot.nix
8 ./firewall.nix
9 ./network.nix
builders/common/hydra-queue-builder.nix
+13 -12
@@ -1,25 +1,26 @@
1 {
2 config,
3 inputs,
4 - lib,
4 ...
5 }:
6
7 {
8 imports = [
10 - inputs.hydra-staging.nixosModules.builder
9 + inputs.hydra.nixosModules.builder
10 ];
11
13 - config = lib.mkIf false {
14 - age.secrets."queue-runner-token" = {
15 - file = ../../build/secrets/${config.networking.hostName}-queue-runner-token.age;
16 - owner = "hydra-queue-builder";
17 - };
12 + age.secrets."queue-runner-token" = {
13 + file = ../../build/secrets/${config.networking.hostName}-queue-runner-token.age;
14 + owner = "hydra-queue-builder";
15 + };
16
19 - services.hydra-queue-builder-dev = {
20 - enable = true;
21 - queueRunnerAddr = "https://queue-runner.hydra.nixos.org";
22 - authorizationFile = config.age.secrets."queue-runner-token".path;
23 - };
17 + services.hydra-queue-builder-dev = {
18 + enable = true;
19 + queueRunnerAddr = "https://queue-runner.hydra.nixos.org";
20 + authorizationFile = config.age.secrets."queue-runner-token".path;
21 + maxJobs = config.nix.settings.max-jobs;
22 + # Required for presigned uploads: builders fetch dependencies via
23 + # substitution and upload results to s3 directly.
24 + useSubstitutes = true;
25 };
26 }
flake.lock
+9 -70
@@ -310,52 +310,30 @@
310 }
311 },
312 "hydra": {
313 - "inputs": {
314 - "nix": [
315 - "nix"
316 - ],
317 - "nix-eval-jobs": "nix-eval-jobs",
318 - "nixpkgs": [
319 - "nixpkgs"
320 - ]
321 - },
322 - "locked": {
323 - "lastModified": 1773705119,
324 - "narHash": "sha256-8pttLK/JQiUL6EXJfjBtBggiLw+769JdQGrpM7klXdg=",
325 - "owner": "NixOS",
326 - "repo": "hydra",
327 - "rev": "a40d42862da88cce78a27dd594e1484a034aac4d",
328 - "type": "github"
329 - },
330 - "original": {
331 - "owner": "NixOS",
332 - "repo": "hydra",
333 - "rev": "a40d42862da88cce78a27dd594e1484a034aac4d",
334 - "type": "github"
335 - }
336 - },
337 - "hydra-staging": {
313 "inputs": {
314 "foreman": "foreman",
315 "nix": [
316 "nix"
317 ],
343 - "nix-eval-jobs": "nix-eval-jobs_2",
318 + "nix-eval-jobs": "nix-eval-jobs",
319 "nixpkgs": [
320 "nixpkgs"
321 ],
347 - "treefmt-nix": "treefmt-nix"
322 + "treefmt-nix": [
323 + "treefmt-nix"
324 + ]
325 },
326 "locked": {
350 - "lastModified": 1781197623,
351 - "narHash": "sha256-MUdFeKXzdLMvl0J+TAkVz8cxnQKWRcQcYqMSWLgbX8k=",
327 + "lastModified": 1781724387,
328 + "narHash": "sha256-FX6IznHKtaO+oPydvNIMeklV7785+e0sA18NFihKeG8=",
329 "owner": "NixOS",
330 "repo": "hydra",
354 - "rev": "f69a55cf3c73146ba4ac4d5e7d39377a226d85fb",
331 + "rev": "d001b8200bb9996ee5c0558fb82b9a4fff13b5e8",
332 "type": "github"
333 },
334 "original": {
335 "owner": "NixOS",
336 + "ref": "hydra.nixos.org",
337 "repo": "hydra",
338 "type": "github"
339 }
@@ -414,23 +392,6 @@
392 "type": "github"
393 }
394 },
417 - "nix-eval-jobs_2": {
418 - "flake": false,
419 - "locked": {
420 - "lastModified": 1773356729,
421 - "narHash": "sha256-OFGRoJOYhvZ3Enk5a8vMy0QNcG5ZxyzFhyHMrwKXde8=",
422 - "owner": "NixOS",
423 - "repo": "nix-eval-jobs",
424 - "rev": "65ebf5b7cd453a27af09cf02b1fc57b3568cc4b7",
425 - "type": "github"
426 - },
427 - "original": {
428 - "owner": "NixOS",
429 - "ref": "v2.34.1",
430 - "repo": "nix-eval-jobs",
431 - "type": "github"
432 - }
433 - },
395 "nix-github-actions": {
396 "inputs": {
397 "nixpkgs": [
@@ -591,7 +552,6 @@
552 "flake-utils": "flake-utils",
553 "freescout": "freescout",
554 "hydra": "hydra",
594 - "hydra-staging": "hydra-staging",
555 "nft-prefix-import": "nft-prefix-import",
556 "nix": "nix",
557 "nixos-channel-scripts": "nixos-channel-scripts",
@@ -604,7 +564,7 @@
564 "simple-nixos-mailserver": "simple-nixos-mailserver",
565 "sops-nix": "sops-nix",
566 "srvos": "srvos",
607 - "treefmt-nix": "treefmt-nix_2"
567 + "treefmt-nix": "treefmt-nix"
568 }
569 },
570 "simple-nixos-mailserver": {
@@ -702,27 +662,6 @@
662 }
663 },
664 "treefmt-nix": {
705 - "inputs": {
706 - "nixpkgs": [
707 - "hydra-staging",
708 - "nixpkgs"
709 - ]
710 - },
711 - "locked": {
712 - "lastModified": 1780220602,
713 - "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=",
714 - "owner": "numtide",
715 - "repo": "treefmt-nix",
716 - "rev": "db947814a175b7ca6ded66e21383d938df01c227",
717 - "type": "github"
718 - },
719 - "original": {
720 - "owner": "numtide",
721 - "repo": "treefmt-nix",
722 - "type": "github"
723 - }
724 - },
725 - "treefmt-nix_2": {
665 "inputs": {
666 "nixpkgs": [
667 "nixpkgs-unstable"
flake.nix
+2 -8
@@ -20,16 +20,10 @@
20 };
21
22 hydra = {
23 - url = "github:NixOS/hydra/a40d42862da88cce78a27dd594e1484a034aac4d";
23 + url = "github:NixOS/hydra/hydra.nixos.org";
24 inputs.nixpkgs.follows = "nixpkgs";
25 inputs.nix.follows = "nix";
26 - };
27 -
28 - hydra-staging = {
29 - url = "github:NixOS/hydra";
30 - inputs.nixpkgs.follows = "nixpkgs";
31 - # Can be kept in sync I suppose for now.
32 - inputs.nix.follows = "nix";
26 + inputs.treefmt-nix.follows = "treefmt-nix";
27 };
28
29 nixos-channel-scripts = {
macs/common/hydra-queue-builder.nix
+13 -12
@@ -8,20 +8,21 @@
8 {
9 imports = [
10 inputs.agenix.darwinModules.age
11 - inputs.hydra-staging.darwinModules.builder
11 + inputs.hydra.darwinModules.builder
12 ];
13
14 - config = lib.mkIf false {
15 - age.secrets."queue-runner-token" = {
16 - file = ../build/secrets/${config.networking.localHostName}-queue-runner-token.age;
17 - owner = "hydra-queue-builder";
18 - };
14 + age.secrets."queue-runner-token" = {
15 + file = ../../build/secrets/${config.networking.localHostName}-queue-runner-token.age;
16 + owner = "hydra-queue-builder";
17 + };
18
20 - services.hydra-queue-builder-dev = {
21 - enable = true;
22 - queueRunnerAddr = "https://queue-runner.hydra.nixos.org";
23 - authorizationFile = config.age.secrets."queue-runner-token".path;
24 - maxJobs = if lib.elem "big-parallel" (config.nix.settings.system-features or [ ]) then 2 else 4;
25 - };
19 + services.hydra-queue-builder-dev = {
20 + enable = true;
21 + queueRunnerAddr = "https://queue-runner.hydra.nixos.org";
22 + authorizationFile = config.age.secrets."queue-runner-token".path;
23 + maxJobs = if lib.elem "big-parallel" (config.nix.settings.system-features or [ ]) then 2 else 4;
24 + # Required for presigned uploads: builders fetch dependencies via
25 + # substitution and upload results to s3 directly.
26 + useSubstitutes = true;
27 };
28 }
macs/common/ofborg-queue-builder.nix
+1 -1
@@ -7,7 +7,7 @@
7
8 {
9 imports = [
10 - inputs.hydra-staging.darwinModules.builder
10 + inputs.hydra.darwinModules.builder
11 ];
12
13 services.hydra-queue-builder-dev = {
non-critical-infra/hosts/staging-hydra/default.nix
+1 -1
@@ -9,7 +9,7 @@
9 ];
10
11 nixpkgs.overlays = [
12 - inputs.hydra-staging.overlays.default
12 + inputs.hydra.overlays.default
13 ];
14
15 disko.devices = import ./disko.nix;
non-critical-infra/hosts/staging-hydra/hydra.nix
+2 -2
@@ -10,8 +10,8 @@ let
10 in
11 {
12 imports = [
13 - inputs.hydra-staging.nixosModules.web-app
14 - inputs.hydra-staging.nixosModules.queue-runner
13 + inputs.hydra.nixosModules.web-app
14 + inputs.hydra.nixosModules.queue-runner
15 ];
16
17 networking.firewall.allowedTCPPorts = [
non-critical-infra/modules/hydra/builder.nix
+1 -1
@@ -14,7 +14,7 @@ let
14 in
15 {
16 imports = [
17 - inputs.hydra-staging.nixosModules.builder
17 + inputs.hydra.nixosModules.builder
18 ];
19
20 services.hydra-queue-builder-dev = {