@cryptotaxi247 / infra-1 / commits / 8cfd161d

feat: init hydra-queue-runner-v2

Simon Hauser committed Jul 7, 2025 at 18:47 UTC 8cfd161df7d2acbeab4680a8f7f843ef8f444f58
13 files changed +994 -46
dns/nixos.org.js
+1
@@ -96,6 +96,7 @@ D("nixos.org",
96 // hydra staging area
97 A("staging-hydra", "157.180.25.203"),
98 AAAA("staging-hydra", "2a01:4f9:c012:d5d3::1"),
99 + CNAME("queue-runner.staging-hydra", "staging-hydra"),
100
101 // services infra
102 A("caliban", "65.109.26.213"),
non-critical-infra/hosts/staging-hydra/ca.crt new
+11
@@ -0,0 +1,11 @@
1 +-----BEGIN CERTIFICATE-----
2 +MIIBnTCCAU+gAwIBAgIUQpxYsPwAyTY70yYO9fcCmCaZreIwBQYDK2VwMEMxCzAJ
3 +BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4 +cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5 +WjBDMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExHjAcBgNVBAMM
6 +FWh5ZHJhLXF1ZXVlLXJ1bm5lci1jYTAqMAUGAytlcAMhAM+Mc/XSTXwJeWPxrpqo
7 +SPT5Xwi8/j85VO6TsfBlXFt4o1MwUTAdBgNVHQ4EFgQU0wQG6BxTKtYwlywuyD0a
8 +Vr/1r4gwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywuyD0aVr/1r4gwDwYDVR0TAQH/
9 +BAUwAwEB/zAFBgMrZXADQQA3BRP2+TkkDQPnPy6MQyDCxqfEeV6OQjtspSvCO0UL
10 +GWmfvzrlUQytwTFTPfVzaErbyVPbeYU5y8rmRoGPNSoI
11 +-----END CERTIFICATE-----
non-critical-infra/hosts/staging-hydra/client.crt new
+11
@@ -0,0 +1,11 @@
1 +-----BEGIN CERTIFICATE-----
2 +MIIBlDCCAUagAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYwwBQYDK2VwMEMxCzAJ
3 +BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4 +cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5 +WjBLMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExJjAkBgNVBAMM
6 +HWh5ZHJhLXF1ZXVlLWJ1aWxkZXItbG9jYWxob3N0MCowBQYDK2VwAyEAJqCvjdwY
7 +0l4z0cbwecONx3DfhpPo+SBymv7H2OQt4aejQjBAMB0GA1UdDgQWBBQw54bw+CrK
8 +O53J4qzB4Cknzf7CzTAfBgNVHSMEGDAWgBTTBAboHFMq1jCXLC7IPRpWv/WviDAF
9 +BgMrZXADQQB+XUCZQo0aX6IyEQDrds1d3SWF1rFBv5MjtZS25TYpnQQPgt9iiD6Q
10 +dtCwSVqyAgCG4XlkYBpU4AKVKqyMsSAC
11 +-----END CERTIFICATE-----
non-critical-infra/hosts/staging-hydra/default.nix
+2
@@ -5,6 +5,8 @@
5 inputs.srvos.nixosModules.server
6 inputs.srvos.nixosModules.hardware-hetzner-cloud-arm
7 ../../modules/common.nix
8 + ../../modules/hydra-queue-runner-v2.nix
9 + ../../modules/hydra-queue-builder-v2.nix
10 ./hydra-proxy.nix
11 ./hydra.nix
12 inputs.hydra.nixosModules.hydra
non-critical-infra/hosts/staging-hydra/genca.sh new
+55
@@ -0,0 +1,55 @@
1 +#!/usr/bin/env bash
2 +
3 +set -x
4 +
5 +hosts="localhost ofborg-eval02 ofborg-eval03 ofborg-eval04 ofborg-build01 ofborg-build02 ofborg-build03 ofborg-build04 ofborg-build05"
6 +
7 +C="DE"
8 +O="NixOS Infra"
9 +
10 +newDir="$(date '+%Y-%m-%dT%H:%M')"
11 +mkdir "${newDir}"
12 +cd "${newDir}" || exit
13 +
14 +openssl genpkey -algorithm Ed25519 -out ca.key
15 +openssl req -x509 -new -nodes -key ca.key -sha256 -days 18250 -out ca.crt \
16 + -subj "/C=${C}/O=${O}/CN=hydra-queue-runner-ca"
17 +
18 +cat <<EOF >server.cnf
19 +[req]
20 +prompt = no
21 +x509_extensions = v3_req
22 +req_extensions = v3_req
23 +default_md = sha256
24 +distinguished_name = req_distinguished_name
25 +
26 +[req_distinguished_name]
27 +C = ${C}
28 +O = ${O}
29 +CN = queue-runner.staging-hydra.nixos.org
30 +
31 +[v3_req]
32 +basicConstraints = CA:FALSE
33 +keyUsage = nonRepudiation, digitalSignature, keyEncipherment, keyAgreement
34 +extendedKeyUsage = critical, serverAuth
35 +subjectAltName = @alt_names
36 +
37 +[alt_names]
38 +DNS.1 = queue-runner.staging-hydra.nixos.org
39 +EOF
40 +
41 +openssl genpkey -algorithm Ed25519 -out server.key
42 +openssl req -new -key server.key -out server.csr -config server.cnf
43 +openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 18250 -sha256 -extfile server.cnf -extensions v3_req
44 +
45 +for host in ${hosts}; do
46 + openssl genpkey -algorithm Ed25519 -out "client-${host}.key"
47 + openssl req -new -key "client-${host}.key" -out "client-${host}.csr" \
48 + -subj "/C=${C}/O=${O}/CN=hydra-queue-builder-${host}"
49 + openssl x509 -req -in "client-${host}.csr" -CA ca.crt -CAkey ca.key -CAcreateserial -out "client-${host}.crt" -days 18250 -sha256
50 +done
51 +
52 +rm -rf -- *.csr *.srl
53 +rm server.cnf
54 +
55 +cd - || exit
non-critical-infra/hosts/staging-hydra/hydra.nix
+117 -46
@@ -44,67 +44,136 @@ in
44 };
45 };
46
47 - services.hydra-dev = {
48 - enable = true;
49 - package = pkgs.hydra;
50 - buildMachinesFiles = [
51 - (pkgs.writeText "local" ''
52 - localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
53 - '')
54 - ];
55 - logo = ../../../build/hydra-logo.png;
56 - hydraURL = "https://hydra.nixos.org";
57 - notificationSender = "edolstra@gmail.com";
58 - smtpHost = "localhost";
59 - useSubstitutes = true;
60 - extraConfig = ''
61 - max_servers 30
47 + services = {
48 + hydra-dev = {
49 + enable = true;
50 + package = pkgs.hydra;
51 + buildMachinesFiles = [
52 + (pkgs.writeText "local" ''
53 + localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
54 + '')
55 + ];
56 + logo = ../../../build/hydra-logo.png;
57 + hydraURL = "https://hydra.nixos.org";
58 + notificationSender = "edolstra@gmail.com";
59 + smtpHost = "localhost";
60 + useSubstitutes = true;
61 + extraConfig = ''
62 + max_servers 30
63
63 - store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
64 - server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
65 - binary_cache_public_uri = https://cache-staging.nixos.org
64 + store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
65 + server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
66 + binary_cache_public_uri = https://cache-staging.nixos.org
67
67 - <Plugin::Session>
68 - cache_size = 32m
69 - </Plugin::Session>
68 + <Plugin::Session>
69 + cache_size = 32m
70 + </Plugin::Session>
71
71 - # patchelf:master:3
72 - xxx-jobset-repeats = nixos:reproducibility:1
72 + # patchelf:master:3
73 + xxx-jobset-repeats = nixos:reproducibility:1
74
74 - upload_logs_to_binary_cache = true
75 - compress_build_logs = false # conflicts with upload_logs_to_binary_cache
75 + upload_logs_to_binary_cache = true
76 + compress_build_logs = false # conflicts with upload_logs_to_binary_cache
77
77 - log_prefix = https://cache.nixos.org/
78 + log_prefix = https://cache.nixos.org/
79
79 - evaluator_workers = 1
80 - evaluator_max_memory_size = 4096
80 + evaluator_workers = 1
81 + evaluator_max_memory_size = 4096
82
82 - max_concurrent_evals = 1
83 + queue_runner_endpoint = http://localhost:8080
84
84 - # increase the number of active compress slots (CPU is 48*2 on mimas)
85 - max_local_worker_threads = 144
85 + max_concurrent_evals = 1
86
87 - max_unsupported_time = 86400
87 + # increase the number of active compress slots (CPU is 48*2 on mimas)
88 + max_local_worker_threads = 144
89
89 - allow_import_from_derivation = false
90 + max_unsupported_time = 86400
91
91 - max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
92 - max_db_connections = 350
92 + allow_import_from_derivation = false
93
94 - queue_runner_metrics_address = [::]:9198
94 + max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
95 + max_db_connections = 350
96
96 - <hydra_notify>
97 - <prometheus>
98 - listen_address = 0.0.0.0
99 - port = 9199
100 - </prometheus>
101 - </hydra_notify>
102 - '';
97 + queue_runner_metrics_address = [::]:9198
98 +
99 + <hydra_notify>
100 + <prometheus>
101 + listen_address = 0.0.0.0
102 + port = 9199
103 + </prometheus>
104 + </hydra_notify>
105 + '';
106 + };
107 +
108 + hydra-queue-runner-v2 = {
109 + enable = true;
110 + settings.remoteStoreAddr = [
111 + "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
112 + ];
113 + };
114 +
115 + hydra-queue-builder-v2 = {
116 + enable = true;
117 + queueRunnerAddr = "https://queue-runner.staging-hydra.nixos.org";
118 + mtls = {
119 + serverRootCaCertPath = "${./ca.crt}";
120 + clientCertPath = "${./client.crt}";
121 + clientKeyPath = config.sops.secrets."queue-runner-client.key".path;
122 + domainName = "queue-runner.staging-hydra.nixos.org";
123 + };
124 + };
125 +
126 + nginx = {
127 + enable = true;
128 + virtualHosts."queue-runner.staging-hydra.nixos.org" = {
129 + extraConfig = ''
130 + ssl_client_certificate ${./ca.crt};
131 + ssl_verify_depth 2;
132 + ssl_verify_client on;
133 + '';
134 +
135 + sslCertificate = ./server.crt;
136 + sslCertificateKey = config.sops.secrets."queue-runner-server.key".path;
137 + onlySSL = true;
138 +
139 + locations."/".extraConfig = ''
140 + # This is necessary so that grpc connections do not get closed early
141 + # see https://stackoverflow.com/a/67805465
142 + client_body_timeout 31536000s;
143 +
144 + grpc_pass grpc://[::1]:50051;
145 +
146 + grpc_read_timeout 31536000s; # 1 year in seconds
147 + grpc_send_timeout 31536000s; # 1 year in seconds
148 + grpc_socket_keepalive on;
149 +
150 + grpc_set_header Host $host;
151 + grpc_set_header X-Real-IP $remote_addr;
152 + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
153 + grpc_set_header X-Forwarded-Proto $scheme;
154 +
155 + grpc_set_header X-Client-DN $ssl_client_s_dn;
156 + grpc_set_header X-Client-Cert $ssl_client_escaped_cert;
157 + '';
158 + };
159 + };
160 };
161
105 - sops.secrets.hydra-users = {
106 - sopsFile = ../../secrets/hydra-users.staging-hydra;
107 - format = "binary";
162 + sops.secrets = {
163 + "queue-runner-server.key" = {
164 + sopsFile = ../../secrets/queue-runner-server.key.staging-hydra;
165 + format = "binary";
166 + owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
167 + };
168 + "queue-runner-client.key" = {
169 + sopsFile = ../../secrets/queue-runner-client.key.staging-hydra;
170 + format = "binary";
171 + owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
172 + };
173 + hydra-users = {
174 + sopsFile = ../../secrets/hydra-users.staging-hydra;
175 + format = "binary";
176 + };
177 };
178
179 systemd = {
@@ -117,6 +186,8 @@ in
186 services = {
187 hydra-notify.enable = false;
188 hydra-queue-runner = {
189 + enable = false;
190 +
191 # restarting the scheduler is very expensive
192 restartIfChanged = false;
193 serviceConfig = {
non-critical-infra/hosts/staging-hydra/server.crt new
+13
@@ -0,0 +1,13 @@
1 +-----BEGIN CERTIFICATE-----
2 +MIIB/jCCAbCgAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYswBQYDK2VwMEMxCzAJ
3 +BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4 +cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5 +WjBSMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExLTArBgNVBAMM
6 +JHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAqMAUGAytlcAMh
7 +ANVnDi5rY0Ar4hPbqRJqS+Nw7b5GTg0QxL2DM7l1xTqHo4GkMIGhMAkGA1UdEwQC
8 +MAAwCwYDVR0PBAQDAgPoMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMBMC8GA1UdEQQo
9 +MCaCJHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAdBgNVHQ4E
10 +FgQU4ArR8rzVAt6dFkSXiMUlYYAzbwUwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywu
11 +yD0aVr/1r4gwBQYDK2VwA0EAScS72oaQ8PcYpH26FuRGnKaWe4e7fQ5RmKBUyC+5
12 +CiYIWu4D7fNGYJ15szCfh4nJIuyB0eXBv1ddAGAQMVdhDw==
13 +-----END CERTIFICATE-----
non-critical-infra/modules/hydra-queue-builder-v2.nix new
+275
@@ -0,0 +1,275 @@
1 +{
2 + config,
3 + pkgs,
4 + lib,
5 + ...
6 +}:
7 +let
8 + cfg = config.services.hydra-queue-builder-v2;
9 +in
10 +{
11 + options = {
12 + services.hydra-queue-builder-v2 = {
13 + enable = lib.mkEnableOption "QueueBuilder";
14 +
15 + queueRunnerAddr = lib.mkOption {
16 + description = "Queue Runner address to the grpc server";
17 + type = lib.types.singleLineStr;
18 + };
19 +
20 + pingInterval = lib.mkOption {
21 + description = "Interval in which pings are send to the runner";
22 + type = lib.types.ints.positive;
23 + default = 10;
24 + };
25 +
26 + speedFactor = lib.mkOption {
27 + description = "Additional Speed factor for this machine";
28 + type = lib.types.oneOf [
29 + lib.types.ints.positive
30 + lib.types.float
31 + ];
32 + default = 1;
33 + };
34 +
35 + maxJobs = lib.mkOption {
36 + description = "Maximum allowed of jobs. This only is used if the queue runner uses this metrics for determining free machines.";
37 + type = lib.types.ints.positive;
38 + default = 4;
39 + };
40 +
41 + tmpAvailThreshold = lib.mkOption {
42 + description = "Threshold in percent free for /tmp before jobs are no longer scheduled on the machine";
43 + type = lib.types.float;
44 + default = 10.0;
45 + };
46 +
47 + storeAvailThreshold = lib.mkOption {
48 + description = "Threshold in percent free for /nix/store before jobs are no longer scheduled on the machine";
49 + type = lib.types.float;
50 + default = 10.0;
51 + };
52 +
53 + load1Threshold = lib.mkOption {
54 + description = "Maximum Load1 threshold before we stop scheduling jobs on that node. Only used if PSI is not available.";
55 + type = lib.types.float;
56 + default = 8.0;
57 + };
58 +
59 + cpuPsiThreshold = lib.mkOption {
60 + description = "Maximum CPU PSI in the last 10s before we stop scheduling jobs on that node";
61 + type = lib.types.float;
62 + default = 75.0;
63 + };
64 +
65 + memPsiThreshold = lib.mkOption {
66 + description = "Maximum Memory PSI in the last 10s before we stop scheduling jobs on that node";
67 + type = lib.types.float;
68 + default = 80.0;
69 + };
70 +
71 + ioPsiThreshold = lib.mkOption {
72 + description = "Maximum IO PSI in the last 10s before we stop scheduling jobs on that node. If null then this pressure check is disabled.";
73 + type = lib.types.nullOr lib.types.float;
74 + default = null;
75 + };
76 +
77 + systems = lib.mkOption {
78 + description = "List of supported systems. If none are passed, system and extra-platforms are read from nix.";
79 + type = lib.types.listOf lib.types.singleLineStr;
80 + default = [ ];
81 + };
82 +
83 + supportedFeatures = lib.mkOption {
84 + description = "Pass supported features to the builder. If none are passed, system features will be used.";
85 + type = lib.types.listOf lib.types.singleLineStr;
86 + default = [ ];
87 + };
88 +
89 + mandatoryFeatures = lib.mkOption {
90 + description = "Pass mandatory features to the builder.";
91 + type = lib.types.listOf lib.types.singleLineStr;
92 + default = [ ];
93 + };
94 +
95 + useSubstitutes = lib.mkOption {
96 + description = "Use substitution for paths";
97 + type = lib.types.bool;
98 + default = true;
99 + };
100 +
101 + mtls = lib.mkOption {
102 + description = "mtls options";
103 + default = null;
104 + type = lib.types.nullOr (
105 + lib.types.submodule {
106 + options = {
107 + serverRootCaCertPath = lib.mkOption {
108 + description = "Server root ca certificate path";
109 + type = lib.types.path;
110 + };
111 + clientCertPath = lib.mkOption {
112 + description = "Client certificate path";
113 + type = lib.types.path;
114 + };
115 + clientKeyPath = lib.mkOption {
116 + description = "Client key path";
117 + type = lib.types.path;
118 + };
119 + domainName = lib.mkOption {
120 + description = "Domain name for mtls";
121 + type = lib.types.singleLineStr;
122 + };
123 + };
124 + }
125 + );
126 + };
127 +
128 + package = lib.mkOption {
129 + type = lib.types.package;
130 + default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).builder;
131 + };
132 + };
133 + };
134 +
135 + config = lib.mkIf cfg.enable {
136 + systemd.services.hydra-queue-builder-v2 = {
137 + description = "hydra-queue-builder-v2 main service";
138 +
139 + requires = [ "nix-daemon.socket" ];
140 + after = [ "network.target" ];
141 + wantedBy = [ "multi-user.target" ];
142 +
143 + environment = {
144 + NIX_REMOTE = "daemon";
145 + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2)
146 + RUST_BACKTRACE = "1";
147 +
148 + # Note: it's important to set this for nix-store, because it wants to use
149 + # $HOME in order to use a temporary cache dir. bizarre failures will occur
150 + # otherwise
151 + HOME = "/run/hydra-queue-builder-v2";
152 + };
153 +
154 + serviceConfig = {
155 + Type = "notify";
156 + Restart = "always";
157 + RestartSec = "5s";
158 +
159 + ExecStart = lib.escapeShellArgs (
160 + [
161 + (lib.getExe cfg.package)
162 + "--gateway-endpoint"
163 + cfg.queueRunnerAddr
164 + "--ping-interval"
165 + cfg.pingInterval
166 + "--speed-factor"
167 + cfg.speedFactor
168 + "--max-jobs"
169 + cfg.maxJobs
170 + "--tmp-avail-threshold"
171 + cfg.tmpAvailThreshold
172 + "--store-avail-threshold"
173 + cfg.storeAvailThreshold
174 + "--load1-threshold"
175 + cfg.load1Threshold
176 + "--cpu-psi-threshold"
177 + cfg.cpuPsiThreshold
178 + "--mem-psi-threshold"
179 + cfg.memPsiThreshold
180 + ]
181 + ++ lib.optionals (cfg.ioPsiThreshold != null) [
182 + "--io-psi-threshold"
183 + cfg.ioPsiThreshold
184 + ]
185 + ++ (builtins.concatMap (v: [
186 + "--systems"
187 + v
188 + ]) cfg.systems)
189 + ++ (builtins.concatMap (v: [
190 + "--supported-features"
191 + v
192 + ]) cfg.supportedFeatures)
193 + ++ (builtins.concatMap (v: [
194 + "--mandatory-features"
195 + v
196 + ]) cfg.mandatoryFeatures)
197 + ++ lib.optionals (cfg.useSubstitutes != null) [
198 + "--use-substitutes"
199 + ]
200 + ++ lib.optionals (cfg.mtls != null) [
201 + "--server-root-ca-cert-path"
202 + cfg.mtls.serverRootCaCertPath
203 + "--client-cert-path"
204 + cfg.mtls.clientCertPath
205 + "--client-key-path"
206 + cfg.mtls.clientKeyPath
207 + "--domain-name"
208 + cfg.mtls.domainName
209 + ]
210 + );
211 +
212 + User = "hydra-queue-builder";
213 + Group = "hydra";
214 +
215 + ReadWritePaths = [
216 + "/nix/var/nix/gcroots/"
217 + "/nix/var/nix/daemon-socket/socket"
218 + ];
219 + ReadOnlyPaths = [ "/nix/" ];
220 + RuntimeDirectory = "hydra-queue-builder-v2";
221 +
222 + PrivateNetwork = false;
223 + SystemCallFilter = [
224 + "@system-service"
225 + "~@privileged"
226 + "~@resources"
227 + ];
228 +
229 + ProtectSystem = "strict";
230 + ProtectHome = true;
231 + PrivateTmp = true;
232 + PrivateDevices = true;
233 + ProtectKernelTunables = true;
234 + ProtectControlGroups = true;
235 + RestrictSUIDSGID = true;
236 + PrivateMounts = true;
237 + RemoveIPC = true;
238 + UMask = "0077";
239 +
240 + CapabilityBoundingSet = "";
241 + NoNewPrivileges = true;
242 +
243 + ProtectKernelModules = true;
244 + SystemCallArchitectures = "native";
245 + ProtectKernelLogs = true;
246 + ProtectClock = true;
247 +
248 + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
249 +
250 + LockPersonality = true;
251 + ProtectHostname = true;
252 + RestrictRealtime = true;
253 + MemoryDenyWriteExecute = true;
254 + PrivateUsers = true;
255 + RestrictNamespaces = true;
256 + };
257 + };
258 + systemd.tmpfiles.rules = [
259 + "d /nix/var/nix/gcroots/per-user/hydra-queue-builder 0755 hydra-queue-builder hydra -"
260 + ];
261 + nix.settings = {
262 + allowed-users = [ "hydra-queue-builder" ];
263 + experimental-features = [ "nix-command" ];
264 + trusted-users = [ "hydra-queue-builder" ];
265 + };
266 +
267 + users = {
268 + groups.hydra = { };
269 + users.hydra-queue-builder = {
270 + group = "hydra";
271 + isSystemUser = true;
272 + };
273 + };
274 + };
275 +}
non-critical-infra/modules/hydra-queue-runner-v2.nix new
+300
@@ -0,0 +1,300 @@
1 +{
2 + config,
3 + pkgs,
4 + lib,
5 + ...
6 +}:
7 +let
8 + cfg = config.services.hydra-queue-runner-v2;
9 +
10 + format = pkgs.formats.toml { };
11 +in
12 +{
13 + options = {
14 + services.hydra-queue-runner-v2 = {
15 + enable = lib.mkEnableOption "QueueRunner";
16 +
17 + settings = lib.mkOption {
18 + description = "Reloadable settings for queue runner";
19 + type = lib.types.submodule {
20 + options = {
21 + hydraDataDir = lib.mkOption {
22 + description = "Hydra data directory";
23 + type = lib.types.path;
24 + default = "/var/lib/hydra";
25 + };
26 + dbUrl = lib.mkOption {
27 + description = "Postgresql database url";
28 + type = lib.types.singleLineStr;
29 + default = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra";
30 + };
31 + maxDbConnections = lib.mkOption {
32 + description = "Postgresql maximum db connections";
33 + type = lib.types.ints.positive;
34 + default = 128;
35 + };
36 + machineSortFn = lib.mkOption {
37 + description = "Function name for sorting machines";
38 + type = lib.types.enum [
39 + "SpeedFactorOnly"
40 + "CpuCoreCountWithSpeedFactor"
41 + "BogomipsWithSpeedFactor"
42 + ];
43 + default = "SpeedFactorOnly";
44 + };
45 + machineFreeFn = lib.mkOption {
46 + description = "Function name for determining \"idle\" machines";
47 + type = lib.types.enum [
48 + "Dynamic"
49 + "DynamicWithMaxJobLimit"
50 + "Static"
51 + ];
52 + default = "Static";
53 + };
54 + dispatchTriggerTimerInS = lib.mkOption {
55 + description = "Timer for triggering dispatch in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger the dispatcher if queue changes happend.";
56 + type = lib.types.int;
57 + default = 120;
58 + };
59 + queueTriggerTimerInS = lib.mkOption {
60 + description = "Timer for triggering queue in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger via pg notifications.";
61 + type = lib.types.int;
62 + default = -1;
63 + };
64 + remoteStoreAddr = lib.mkOption {
65 + description = "Remote store address";
66 + type = lib.types.listOf lib.types.singleLineStr;
67 + default = [ ];
68 + };
69 + useSubstitutes = lib.mkOption {
70 + description = "Use substitution for paths";
71 + type = lib.types.bool;
72 + default = false;
73 + };
74 + rootsDir = lib.mkOption {
75 + description = "Gcroots directory, defaults to /nix/var/nix/gcroots/per-user/$LOGNAME/hydra-roots";
76 + type = lib.types.nullOr lib.types.path;
77 + default = null;
78 + };
79 + maxRetries = lib.mkOption {
80 + description = "Number of maximum amount of retries for a build step.";
81 + type = lib.types.ints.positive;
82 + default = 5;
83 + };
84 + retryInterval = lib.mkOption {
85 + description = "Interval in which retires should be able to be attempted again.";
86 + type = lib.types.ints.positive;
87 + default = 60;
88 + };
89 + retryBackoff = lib.mkOption {
90 + description = "Additional backoff on top of the retry interval.";
91 + type = lib.types.float;
92 + default = 3.0;
93 + };
94 + maxUnsupportedTimeInS = lib.mkOption {
95 + description = "Time until unsupported steps are aborted.";
96 + type = lib.types.ints.unsigned;
97 + default = 120;
98 + };
99 + stopQueueRunAfterInS = lib.mkOption {
100 + description = "Seconds after which the queue run should be interupted early. Setting this to a value <= 0 will disable this feature and the queue run will never exit early.";
101 + type = lib.types.int;
102 + default = 60;
103 + };
104 + };
105 + };
106 + default = { };
107 + };
108 +
109 + grpc = lib.mkOption {
110 + description = "grpc options";
111 + default = { };
112 + type = lib.types.submodule {
113 + options = {
114 + address = lib.mkOption {
115 + type = lib.types.singleLineStr;
116 + default = "[::1]";
117 + description = "The IP address the grpc listener should bound to";
118 + };
119 +
120 + port = lib.mkOption {
121 + description = "Which grpc port this app should listen on";
122 + type = lib.types.port;
123 + default = 50051;
124 + };
125 + };
126 + };
127 + };
128 +
129 + rest = lib.mkOption {
130 + description = "rest options";
131 + default = { };
132 + type = lib.types.submodule {
133 + options = {
134 + address = lib.mkOption {
135 + type = lib.types.singleLineStr;
136 + default = "[::1]";
137 + description = "The IP address the rest listener should bound to";
138 + };
139 +
140 + port = lib.mkOption {
141 + description = "Which rest port this app should listen on";
142 + type = lib.types.port;
143 + default = 8080;
144 + };
145 + };
146 + };
147 + };
148 +
149 + mtls = lib.mkOption {
150 + description = "mtls options";
151 + default = null;
152 + type = lib.types.nullOr (
153 + lib.types.submodule {
154 + options = {
155 + serverCertPath = lib.mkOption {
156 + description = "Server certificate path";
157 + type = lib.types.path;
158 + };
159 + serverKeyPath = lib.mkOption {
160 + description = "Server key path";
161 + type = lib.types.path;
162 + };
163 + clientCaCertPath = lib.mkOption {
164 + description = "Client ca certificate path";
165 + type = lib.types.path;
166 + };
167 + };
168 + }
169 + );
170 + };
171 + package = lib.mkOption {
172 + type = lib.types.package;
173 + default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).runner;
174 + };
175 + };
176 + };
177 +
178 + config = lib.mkIf cfg.enable {
179 + systemd.services.hydra-queue-runner-v2 = {
180 + description = "hydra queue-runner-v2 main service";
181 +
182 + requires = [ "nix-daemon.socket" ];
183 + after = [
184 + "network.target"
185 + "postgresql.service"
186 + ];
187 + wantedBy = [ "multi-user.target" ];
188 + reloadTriggers = [ config.environment.etc."hydra/queue-runner.toml".source ];
189 +
190 + environment = {
191 + NIX_REMOTE = "daemon";
192 + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2)
193 + RUST_BACKTRACE = "1";
194 +
195 + # Note: it's important to set this for nix-store, because it wants to use
196 + # $HOME in order to use a temporary cache dir. bizarre failures will occur
197 + # otherwise
198 + HOME = "/var/lib/hydra/queue-runner";
199 + };
200 +
201 + serviceConfig = {
202 + Type = "notify";
203 + Restart = "always";
204 + RestartSec = "5s";
205 +
206 + ExecStart = lib.escapeShellArgs (
207 + [
208 + (lib.getExe cfg.package)
209 + "--rest-bind"
210 + "${cfg.rest.address}:${toString cfg.rest.port}"
211 + "--grpc-bind"
212 + "${cfg.grpc.address}:${toString cfg.grpc.port}"
213 + "--config-path"
214 + "/etc/hydra/queue-runner.toml"
215 + ]
216 + ++ lib.optionals (cfg.mtls != null) [
217 + "--server-cert-path"
218 + cfg.mtls.serverCertPath
219 + "--server-key-path"
220 + cfg.mtls.serverKeyPath
221 + "--client-ca-cert-path"
222 + cfg.mtls.clientCaCertPath
223 + ]
224 + );
225 + ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID";
226 +
227 + User = "hydra-queue-runner";
228 + Group = "hydra";
229 +
230 + StateDirectory = [ "hydra/queue-runner" ];
231 + StateDirectoryMode = "0700";
232 + ReadWritePaths = [
233 + "/nix/var/nix/gcroots/"
234 + "/run/postgresql/.s.PGSQL.${toString config.services.postgresql.port}"
235 + "/nix/var/nix/daemon-socket/socket"
236 + "/var/lib/hydra/build-logs/"
237 + ];
238 + ReadOnlyPaths = [ "/nix/" ];
239 + WorkingDirectory = "/var/lib/hydra/queue-runner";
240 +
241 + PrivateNetwork = false;
242 + SystemCallFilter = [
243 + "@system-service"
244 + "~@privileged"
245 + "~@resources"
246 + ];
247 +
248 + ProtectSystem = "strict";
249 + ProtectHome = true;
250 + PrivateTmp = true;
251 + PrivateDevices = true;
252 + ProtectKernelTunables = true;
253 + ProtectControlGroups = true;
254 + RestrictSUIDSGID = true;
255 + PrivateMounts = true;
256 + RemoveIPC = true;
257 + UMask = "0022";
258 +
259 + CapabilityBoundingSet = "";
260 + NoNewPrivileges = true;
261 +
262 + ProtectKernelModules = true;
263 + SystemCallArchitectures = "native";
264 + ProtectKernelLogs = true;
265 + ProtectClock = true;
266 +
267 + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
268 +
269 + LockPersonality = true;
270 + ProtectHostname = true;
271 + RestrictRealtime = true;
272 + MemoryDenyWriteExecute = true;
273 + PrivateUsers = true;
274 + RestrictNamespaces = true;
275 + };
276 + };
277 +
278 + environment.etc."hydra/queue-runner.toml".source = format.generate "queue-runner.toml" (
279 + lib.filterAttrsRecursive (_: v: v != null) cfg.settings
280 + );
281 + systemd.tmpfiles.rules = [
282 + "d /nix/var/nix/gcroots/per-user/hydra-queue-runner 0755 hydra-queue-runner hydra -"
283 + "d /var/lib/hydra/build-logs/ 0755 hydra-queue-runner hydra -"
284 + ];
285 +
286 + nix.settings = {
287 + allowed-users = [ "hydra-queue-runner" ];
288 + experimental-features = [ "nix-command" ];
289 + trusted-users = [ "hydra-queue-runner" ];
290 + };
291 +
292 + users = {
293 + groups.hydra = { };
294 + users.hydra-queue-runner = {
295 + group = "hydra";
296 + isSystemUser = true;
297 + };
298 + };
299 + };
300 +}
non-critical-infra/packages/hydra-queue-runner/default.nix new
+104
@@ -0,0 +1,104 @@
1 +{
2 + rustPackages,
3 + fetchFromGitHub,
4 + pkg-config,
5 + openssl,
6 + zlib,
7 + protobuf,
8 + lib,
9 + makeWrapper,
10 + nixVersions,
11 + nlohmann_json,
12 + libsodium,
13 + boost,
14 +}:
15 +let
16 + version = "unstable-2025-08-07";
17 + src = fetchFromGitHub {
18 + owner = "helsinki-systems";
19 + repo = "hydra-queue-runner";
20 + rev = "54b3c9351d2ae10be5c4d1b97cc0f86300cd70ca";
21 + hash = "sha256-gR2DzWkTykM9GdW3Nf/V8eRv68fl3aO+NW0zNPFSRT0=";
22 + };
23 + useFetchCargoVendor = true;
24 + cargoHash = "sha256-oNUMmFfts4rjBX0k5mzsxpYA2JqgsRu1nMRFf/2rZa8=";
25 + nativeBuildInputs = [
26 + pkg-config
27 + protobuf
28 + makeWrapper
29 + ];
30 + buildInputs = [
31 + openssl
32 + zlib
33 + protobuf
34 +
35 + nixVersions.nix_2_29
36 + nlohmann_json
37 + libsodium
38 + boost
39 + ];
40 + meta = {
41 + description = "Hydra Queue-Runner implemented in rust";
42 + homepage = "https://github.com/helsinki-systems/hydra-queue-runner";
43 + license = [ lib.licenses.gpl3 ];
44 + maintainers = [ lib.maintainers.conni2461 ];
45 + platforms = lib.platforms.all;
46 + };
47 +in
48 +{
49 + runner = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
50 + pname = "hydra-queue-runner";
51 + inherit version src;
52 + __structuredAttrs = true;
53 + strictDeps = true;
54 +
55 + inherit
56 + useFetchCargoVendor
57 + cargoHash
58 + nativeBuildInputs
59 + buildInputs
60 + ;
61 +
62 + cargoBuildFlags = [
63 + "-p"
64 + "queue-runner"
65 + ];
66 + cargoTestFlags = finalAttrs.cargoBuildFlags;
67 +
68 + postInstall = ''
69 + wrapProgram $out/bin/queue-runner --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
70 + '';
71 +
72 + meta = meta // {
73 + mainProgram = "queue-runner";
74 + };
75 + });
76 +
77 + builder = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
78 + pname = "hydra-queue-builder";
79 + inherit src version;
80 + __structuredAttrs = true;
81 + strictDeps = true;
82 +
83 + inherit
84 + useFetchCargoVendor
85 + cargoHash
86 + nativeBuildInputs
87 + buildInputs
88 + ;
89 +
90 + cargoBuildFlags = [
91 + "-p"
92 + "builder"
93 + ];
94 + cargoTestFlags = finalAttrs.cargoBuildFlags;
95 +
96 + postInstall = ''
97 + wrapProgram $out/bin/builder --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
98 + '';
99 +
100 + meta = meta // {
101 + mainProgram = "builder";
102 + };
103 + });
104 +}
non-critical-infra/secrets/queue-runner-ca.key.staging-hydra new
+35
@@ -0,0 +1,35 @@
1 +{
2 + "data": "ENC[AES256_GCM,data:pe1lPpTo2O6CjOwyW3Go673wo4kJJ8O7XGk3M89pcdaoHPr0gdvvsSg8tG98qng4E5vfYnQpzXNFPBjmv9DvFkm3LDqOSmJsNw6Vp4LAIpkxQmuehGjDnsu2WxWC/JlG2Qm+2FS2saxZrfaeHEiZDvPEm1rN3oA=,iv:rrKEmM0PLIS+ur+cjW5tBR3UqOftOi6FTaqrymr6OIg=,tag:zGZtHCxldMKeWoNICBe3+A==,type:str]",
3 + "sops": {
4 + "age": [
5 + {
6 + "recipient": "age1xj4dl6xdl5ztmetp9axa0epjj922hu6a2gut3rrs5rdc5xv85yjsq5ggpx",
7 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBETllNUWc0YUlEUW9KdmlE\nSnZNdW9kekxxUzJ0ZzBXTkxMWEpOcFhQK25RCi9iejVhTUk4cnQ5VU1rcTBiTGNw\nMXNnS1ducEhTWk14c3d2V1pWdzFQUm8KLS0tIGNXOU9rQmtVRExXZWpYTTlvbk9i\nYlZjUDVyRWZwSHlsWDhzbWZCdXJ2N3MK71/bfgXtdfcS/bJpf7pq0IViBpRPDzc6\n4Hi+E+aZodq6167TrYT26LV5pSKYER+dIvk02FSBvOxCDvk6u6NyCQ==\n-----END AGE ENCRYPTED FILE-----\n"
8 + },
9 + {
10 + "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
11 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGWDdPUWk1R2JCMW1pNDRI\nZFp3czA4YUdrekFMLzlFTHJtM083bDBSZUNNCjEwcUlSazBoUnZMM3BsWit1eTB4\nS3VEaFVad1ZRZDBXYk1vaHFKNklPZDQKLS0tIC9ucGpzd3NiRjEvN0dtQUtZZ3pj\nVE1zbHZvQ0R3T0tyL3dnZGU5dDVnMlUKjlH1sgor3H+zHzwAV9cZbdiGbcQo27pf\n7shoRdXedex44lm8+LlYQ9YfkyhZs3mR9/++0JsvijR7o7Lrb9w1gw==\n-----END AGE ENCRYPTED FILE-----\n"
12 + },
13 + {
14 + "recipient": "age1nnm255ah9wa4gpsaq0v023a75lnmlcxszt9lc6az3mtwzxgrucfq45rp7h",
15 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGSm8xaFBWTU5mdS9weFRG\nbnVkamVVMG1BbDgrZ3pXOVdWMmhJM3d2V2dnClozd0h1elNYSWN5SWZOK2s2WTI1\nTVROTFBCOWM5anJ4endoOWs4cmFjSlEKLS0tIGE4TURtcjFyclg1NEFnRlhiMnZ0\ncWQ3RHIyUnZUSEE5Q1hWaG9kR3k2eDAKJ5JgJzDBTv+6lb29mWhe9B9VtrncXGhG\nQbFqF9+U4Q23A/gHa+FBAqT1dzc7U+IE5WILmQa7wJg29W6A0aVt/A==\n-----END AGE ENCRYPTED FILE-----\n"
16 + },
17 + {
18 + "recipient": "age1j3mkgedmeru63vwww6m44zfw09tg8yw6xdzstaq7ejfkvgcau40qwakm8x",
19 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3TWRBemFSWk0yR2tJcXNM\nZzNBR3dTODRMa2JraGROTmphTHpjREdGQ2dFCkZudFF4Y05aOEZ2M3doR0NhY1V5\ndmtRV2s5ZVN4NEVmT2ZuNGdmQjhDY1kKLS0tICswZHpsc2RJMGZPc1hnd3FjZGRo\nVTFvdWFPeTlJa0p0RTdBaXlJb3hncDQKmO1/apc3mqJ4CyAWf3eVNlXdUhZCIRZF\nTK0HBh7xg8S/HkPJiBcMCdF/7slQeOLL3lzfeBjnAQ+mg8hoI2YCmA==\n-----END AGE ENCRYPTED FILE-----\n"
20 + },
21 + {
22 + "recipient": "age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h",
23 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlMjNQeGdheUVuMmNWSVN2\nekd5SVVkVTNnMVdmRnUwSTFFaDcrSE90bGxZCmJZRFp5NkFYUFdnOVJ4Y1NQZ1pk\nTHlja29waUxlWlF3alJwSXMyY3VDcGsKLS0tIE9LZWJnb1hRcTJBQlA2Z2NWSFhB\ndE9xSEUzVEI2UHVuQUw0cnZ5S052bEUKimqgYE85vdiTS3YV08bD/I3IpM1sPXcD\nEroNwJH8+boHOGZODLQUoezjHkajnIoo/wjK1gELEHlvShAlzNPQuQ==\n-----END AGE ENCRYPTED FILE-----\n"
24 + },
25 + {
26 + "recipient": "age17ez23w2kpxl0gcdx4ehcglkcfcfnv4qz0gq2n8ylxwx4yrtjpvjqxfasua",
27 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuZXVvSEJ5RFZBRGZCUFNS\nazVxcEpGTmNBaVd2OUVwRnU3Nm9zaUNPOG5zCmFwYkdlUWEweWptVjMxSDY4MERF\nbTNqTmUwSytWRjRiWFFHWmJKZE0waUkKLS0tIDZaNTZVRG5SWjF1Vm9QWm9ERDZ1\nZFBTSkRJVytKaVBlYkZibTRNSmwxZlUKdr9dALaKngENjcRYUa8KRKKfngc/BMqh\ngQrqhE2BxCKffGxsyuyqIozSm+nEw5ihQviWgWU78/XLraAWnubfRg==\n-----END AGE ENCRYPTED FILE-----\n"
28 + }
29 + ],
30 + "lastmodified": "2025-07-31T12:09:59Z",
31 + "mac": "ENC[AES256_GCM,data:EdssbCd38+X5Sr/qPSewtFgGyT0WFvR+0/ZkzpKGtj/7yS/Q9w0BoyZXbwuMF4R+n2aJ+NTmxPjCz9Ald0ENnXKXJcEHlKtTf9z2h+ft6rrFsBVBlrp3KfHY16EwcfBprvbbCkU6Bvxv5UGFGfJsw5l3jpL45LQQ7z528HiQz6c=,iv:02k7g9Fck1XFBor/TAHZGpgIiH2zPRDHsdRt4rjApeU=,tag:uJJ4J6znugGBQtARL6vwRg==,type:str]",
32 + "unencrypted_suffix": "_unencrypted",
33 + "version": "3.10.2"
34 + }
35 +}
non-critical-infra/secrets/queue-runner-client.key.staging-hydra new
+35
@@ -0,0 +1,35 @@
1 +{
2 + "data": "ENC[AES256_GCM,data:mER5O7YO36svhuSk8u5FLgBUkRaFs+k4sBccPqx0JZ5BWPhWDKQwlEKHeNuuMRPwbhtTEoKXFHN7PdcOKG3jtDNZuNuOvupNxeF8By47XpBFRJm7W9BhPdlN4UA8qoDdlWMZLBFjNSSGhV6+wGL/HGao/hpWDn0=,iv:JmAHNp6G4kVjhsLv5BgizEUatt+NbIyz8wi0YVdtN50=,tag:yAN1D0DMvOs0g4715HD6PQ==,type:str]",
3 + "sops": {
4 + "age": [
5 + {
6 + "recipient": "age1xj4dl6xdl5ztmetp9axa0epjj922hu6a2gut3rrs5rdc5xv85yjsq5ggpx",
7 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiUEkwL0xvV2trSUFUN1Ny\nNnBLOHhvaVFuNWxuQnE2Yys3MVAycjRCM0QwCmI1M0haRzJhUEw3aURhRmtWUG1C\nMWhpZHJiSnM4RXJvZ09WQWxxN3QyS28KLS0tIFo5Nm15U2hPSkxRUUZpRWdZTVJO\nV2p0QVlVbHV6am1LTTVoQzRLWkloUnMK6mK+TeQKok/XNH+wmGa1N1lVX3ME562o\nEev3kaAGWqz6f4bMacbCkgXmhOEryFiDCgok/TMQ6hSGa3COFXrzmQ==\n-----END AGE ENCRYPTED FILE-----\n"
8 + },
9 + {
10 + "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
11 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPcCtST1pMdk1vNFBLRlJN\nd2IvTmUwcGQ2cm5aaXgvQTdoMkFFbnJYVm00CjJmYS95Z2FrMUdrcDJwWDFYWXY0\nNGZkSkZMRDA3NUxjVG1NdExKTjNVM1kKLS0tIEN6eHJlazlRa3NIaUUvSWtTdTYw\nZXpmcXJpY1pwUzkva2hYM1R3THhIM00KiwOB/Twye2mzFzLUiF01DTaqb0OLybSF\njRYHaZ5KWZ756c2DEnnB8TbQsN2JDpfe8pYW7tdgyfqNKaYV/l8nxQ==\n-----END AGE ENCRYPTED FILE-----\n"
12 + },
13 + {
14 + "recipient": "age1nnm255ah9wa4gpsaq0v023a75lnmlcxszt9lc6az3mtwzxgrucfq45rp7h",
15 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6Q1htRVFycERhRHRSanFt\nTEhaTnIvMG1iYy9MZ0pLLzZqaVN0bVB1ZERNCjhkaVRXaXZoQjl6aUZYWnlvMGdG\nSDh1M09MU2I3aUF2YVgxUTVHZHYxazAKLS0tIFJtUzk1Wk9xazNUK3BJWTJMbGx6\nTzh5SlVBUkFIckQ3b0VYQWlyaGhuMEUKg+bdoltiqfMwrzytk4kwRKqUjS3XyUuN\ngX3ksj9ODZDvbwNXhgBLXkXpmefwUt0k2TJL/fTr4tqTU6hchkg7aA==\n-----END AGE ENCRYPTED FILE-----\n"
16 + },
17 + {
18 + "recipient": "age1j3mkgedmeru63vwww6m44zfw09tg8yw6xdzstaq7ejfkvgcau40qwakm8x",
19 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqS1pTY0Rzci94Ykd3Wm1t\nVVl1TWlQc2xjdTBBbjZCWGF3bWZXaTlNaWxjCjJ0OFd3bWRTYjF5cnRaSy8zeFFF\nSWd0bkpSbnVyNTZOMU0yR1dVcUtlQlkKLS0tICt2QkdsUDcyZ1J2YjRJRkIyYW5u\naDBPWUEwNkJ5dkxqaGhCcWlNZVNXOEkK6SEQCqKZ0OsWKs6vmG18ikaeSkS7Zo9+\npUc4e/QWsRDOd/frLWzKIo8JujckNpMbbalQfykW7s0xCvGnZ1NYfQ==\n-----END AGE ENCRYPTED FILE-----\n"
20 + },
21 + {
22 + "recipient": "age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h",
23 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwdXJ2bHIrdGdCSlRLL3Za\nVWZCUWRXajVmS2JCMkQ2NmZqMkVHcHAyTFc0Cngyd0QzbU9VMEVrOUxsSGVsWWNy\neGRNK3JleDFXN2p0dFYrYktVbzcrSGMKLS0tIFQ4WFc2RTFFNkx3ajUvQWJqWUpV\nUDBiOTFKQ0hIOWsxS2xYM1ptVDBwcGcK41AK30TtmKw9DCPCMnAZXl/C26t3NBNK\n3Sh4wL7yyhKmRUNutq5cnAw8vjzhRsgjJkZGmOuv5qN3s7nFTGdC5A==\n-----END AGE ENCRYPTED FILE-----\n"
24 + },
25 + {
26 + "recipient": "age17ez23w2kpxl0gcdx4ehcglkcfcfnv4qz0gq2n8ylxwx4yrtjpvjqxfasua",
27 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNFB6OW1sOHpseEdJSll4\neDlDN0xKQnJPMUFQcnpHaHFURUg4NmhNT3k0CmFJV2xSMnhocnlMQWI5ZThLK2R4\nZE1mRHBGbGM4UmxvalJ5TWpOUW1JR2cKLS0tIEd6UFNFc0E0QVI2bFBRUlFsWHh6\nLy9WYWpTK3E2ZjJEb01rNC84UG1DUTAKv1T3UPMBBYGHdmKCsSCUtBg/7SoQ7RjO\n0PyeOnGlvUUxj4H/z2sPRxuKj2QiqIiGqdSqOoGXNmp3omrQHtVfxg==\n-----END AGE ENCRYPTED FILE-----\n"
28 + }
29 + ],
30 + "lastmodified": "2025-07-31T12:14:02Z",
31 + "mac": "ENC[AES256_GCM,data:KWe6LDUZtpGR8btMzp4Arb4b3rooS+pGOOs0W0gZYLovh4RdvMSdOHbslIZt3HzNvOWDNz3BM77jdNiSctZpLK6sXxzXB0JH7dq3H9+7cPRGyEIUYRvysjmFF1HHEw0isVkV7mPHNAQBL7y716Gw5plAfyEgCqpknbnQpKMIGmI=,iv:rRfVfr30XfMuAgkeNWq6x3l4LVFArC3X2sPz8dCamfo=,tag:lKKy+PvsuwG99mEu0/OZ7w==,type:str]",
32 + "unencrypted_suffix": "_unencrypted",
33 + "version": "3.10.2"
34 + }
35 +}
non-critical-infra/secrets/queue-runner-server.key.staging-hydra new
+35
@@ -0,0 +1,35 @@
1 +{
2 + "data": "ENC[AES256_GCM,data:forqylxAxHHWDqqTG8nxgqnbuyCTtRiEZ+0VA6qYRhlRY7KAYmTM4GK3wDa8dZPBi0e+LXG5aaigyc9FFsl7hHKdtQo84OPHyJOSCZyBYjMg9qlBsy/USqxftzfMjMAxVk/ibAi5LDrNl9Jh/7w74BBtbdF3qiE=,iv:5YbKjIdd+EfVpMGzTzP0VDy0Wev22bRc9IO5e3StpA4=,tag:/jsQItUIschI6B0rs7dwQg==,type:str]",
3 + "sops": {
4 + "age": [
5 + {
6 + "recipient": "age1xj4dl6xdl5ztmetp9axa0epjj922hu6a2gut3rrs5rdc5xv85yjsq5ggpx",
7 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuajRobWMzdHhqSVUydlVZ\nanZKY2REQWJ5KzgxNmtwZ0d4Mm1mTWFYYTM4CjhIVG1COTZkbWp3SkxtS3VDeTRB\ndGcxY0hqUk1DeU9CQldkcTUwMys1OTAKLS0tIFBibTB6bWY2ZjRxL0JFUDZmYTJ2\nMG5IclV2T2twa0NhOGtrdXZQeXhqV1UK/9RyMKw8K3AOe8PntM+5BMSSGuWuYdt4\nkDMjS3NH7Ab1tM3O/4b6lxyW2nheBXnhhqbq5tfqOVafpoIqgjZgxg==\n-----END AGE ENCRYPTED FILE-----\n"
8 + },
9 + {
10 + "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
11 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArd3ZqdUhGREtLRlpla1Js\nK2JRRG1pYWx2NVNCT3F5dGlBc1EwOVo1Mm44CkNkSXQyK0VYZGYrMW05ZnhmbXFC\nOFpKSVVKZTE5WHkvN0hzVjRqbGV0eGMKLS0tIHJBNXNsSkcyd1EwckRTT2FFNDRk\nK1A5Q0dsZC80NWk3dE0rK2JKWG9GdjgK6exWpfYFkhSVZkU80rnLkHKqkZ6hrAGA\nNhiohmDfflTAurqEKzxPsVMiSWlmVEpKykzMrsE150evlZbj7YB9rw==\n-----END AGE ENCRYPTED FILE-----\n"
12 + },
13 + {
14 + "recipient": "age1nnm255ah9wa4gpsaq0v023a75lnmlcxszt9lc6az3mtwzxgrucfq45rp7h",
15 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQUjBtZFhER05McTJqNk1V\neVJIMEJLcnBoMVAxazduckhJWm13Y2drcUFVCm9vRlI2ZjJDbDU0Qk5icmI2dVRn\ndUlvbWFmMjZob0ZQclhFOHJ5T25vcXcKLS0tIGNSU3NNOTNFcmZRY3BUWHRYdUpz\nT2g0TlhnSjVuWlpCYTRWN1FUSUE2V2sKxiobZEtFJDOTmxjRcfTS7HwIXOZuySXt\nOxkJoFfhaFRFOIqP6Ai8/9MtYP2UypRzdTBLP6T8iLYbT0jeM2h93Q==\n-----END AGE ENCRYPTED FILE-----\n"
16 + },
17 + {
18 + "recipient": "age1j3mkgedmeru63vwww6m44zfw09tg8yw6xdzstaq7ejfkvgcau40qwakm8x",
19 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzTlBRb2lsV1ZYZ1htWUJ3\nS3dlQ1QrWXc3QVZFTWlxdzVGSXlNWFZ1aEdFCkpCSFB3WUlKa3ZUeGQ0SzFmZ0xP\nWGYwZk5MM3prMUUwbWovQzk1dGtvS3cKLS0tIC9jRlhLL3BtVyt4WS9rQisvc0Jo\ndjBHMVhGZlRlM3hsKzh2aU90ZzVhZTQKI1QZ9IhpNFC1hO8pl+2kjALVbTKnQjl7\nwsiwg7DUH8j16Cqe+BKqPLwV6wUZNcSgCc4b4kAdCeIme3Qc9VPofQ==\n-----END AGE ENCRYPTED FILE-----\n"
20 + },
21 + {
22 + "recipient": "age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h",
23 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBOXc1QklpenBjTGh3NFNS\nZW5vZFdDNHJPWGsrN0MzSU0yaHVxR0tkOEU0CnIwTGxuUUZxTUd4b0YxZ0h6QUVa\nV1FocWJKdlpLaUVJZUNwTi8zTmlDV2cKLS0tIHYzNVYzZHgyVnlLcmJtMmk1aWwy\nTDZBSkJLY001MGxYNFQ1a3hSdG4vaEUKOdJSdnxDBansg5boUQp9xLGFMkQzKDf9\n9D7W+cwqSQEUpSTofP5elJKOsg/tdpiZaiSck8/4EHhQt2/qncgbAg==\n-----END AGE ENCRYPTED FILE-----\n"
24 + },
25 + {
26 + "recipient": "age17ez23w2kpxl0gcdx4ehcglkcfcfnv4qz0gq2n8ylxwx4yrtjpvjqxfasua",
27 + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjcmJBbFJHbUJzYnpZWVBQ\nZjNtVTVmaUM2RzRYM1JoL2MrSmp4RFFMWkRFClBTcDdwaEMyMStNcEsrMzcwa2xG\nU083YnUvNStuNXNsOHR0TzB6NWROeVEKLS0tIFc5V0sxN1dUcksrdTFNdUZ4cyt2\nRy9rbFY2dEFZRlZmaitjUVZYZFZpK00KdLhQzJZLO4BMiWaJuXCxn/NDG5EJa/ee\nuNJj+AfwOzLk3M8rAapwAR7O0b9PL0gcU6oDybAnBlMo+UzdH8X9aQ==\n-----END AGE ENCRYPTED FILE-----\n"
28 + }
29 + ],
30 + "lastmodified": "2025-07-31T12:10:09Z",
31 + "mac": "ENC[AES256_GCM,data:QRLZ7S6CoHaYQ+m0npUo2ISRkke4JvYe/cN5nBM7ngIdKQ+yyLK1d8ucG3YwB6HaVZXDM4F0BAKkwl3VXTDujKk24BQhb4G50EL91UTcaQQ4kIOdaSVh17/D6cXV1Han8hSrxrsXQ9vvyR9EmCVbjZMIP+TyyTbdAkFfHDch2Fk=,iv:Wm7rVVZRUjUeb/SU2GF5TnOcNawf8rsndlReEUtrGeg=,tag:PsL1RufQyx+7wayK0IOSlg==,type:str]",
32 + "unencrypted_suffix": "_unencrypted",
33 + "version": "3.10.2"
34 + }
35 +}