| 1 | { |
| 2 | lib, |
| 3 | pkgs, |
| 4 | config, |
| 5 | inputs, |
| 6 | ... |
| 7 | }: |
| 8 | let |
| 9 | narCache = "/var/cache/hydra/nar-cache"; |
| 10 | in |
| 11 | { |
| 12 | imports = [ |
| 13 | inputs.hydra.nixosModules.web-app |
| 14 | inputs.hydra.nixosModules.queue-runner |
| 15 | ]; |
| 16 | |
| 17 | networking.firewall.allowedTCPPorts = [ |
| 18 | 9198 # queue-runnner metrics |
| 19 | 9199 # hydra-notify metrics |
| 20 | ]; |
| 21 | |
| 22 | services.postgresql.settings = { |
| 23 | log_min_duration_statement = 5000; |
| 24 | log_duration = "off"; |
| 25 | log_statement = "none"; |
| 26 | |
| 27 | max_connections = 500; |
| 28 | work_mem = "20MB"; |
| 29 | maintenance_work_mem = "2GB"; |
| 30 | }; |
| 31 | |
| 32 | # garbage collection |
| 33 | nix.gc = { |
| 34 | automatic = true; |
| 35 | options = ''--max-freed "$((400 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"''; |
| 36 | dates = "03,09,15,21:15"; |
| 37 | }; |
| 38 | |
| 39 | nix.settings = { |
| 40 | # gc outputs as well, since they are served from the cache |
| 41 | gc-keep-outputs = lib.mkForce false; |
| 42 | allowed-users = [ |
| 43 | "hydra" |
| 44 | "hydra-www" |
| 45 | ]; |
| 46 | }; |
| 47 | |
| 48 | # Don't rate-limit the journal. |
| 49 | services.journald.rateLimitBurst = 0; |
| 50 | |
| 51 | sops.secrets = { |
| 52 | signing-key = { |
| 53 | sopsFile = ../../secrets/signing-key.staging-hydra; |
| 54 | format = "binary"; |
| 55 | owner = config.systemd.services.hydra-queue-runner-dev.serviceConfig.User; |
| 56 | }; |
| 57 | hydra-aws-credentials = { |
| 58 | sopsFile = ../../secrets/hydra-aws-credentials.staging-hydra; |
| 59 | format = "binary"; |
| 60 | owner = config.systemd.services.hydra-queue-runner-dev.serviceConfig.User; |
| 61 | }; |
| 62 | }; |
| 63 | |
| 64 | services = { |
| 65 | hydra-dev = { |
| 66 | enable = true; |
| 67 | package = pkgs.hydra; |
| 68 | logo = ../../../build/hydra-logo.png; |
| 69 | hydraURL = "https://hydra.nixos.org"; |
| 70 | notificationSender = "edolstra@gmail.com"; |
| 71 | smtpHost = "localhost"; |
| 72 | useSubstitutes = true; |
| 73 | extraConfig = '' |
| 74 | max_servers 30 |
| 75 | |
| 76 | store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&compression=zstd&ls-compression=zstd&log-compression=zstd&narinfo-compression=zstd |
| 77 | server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache} |
| 78 | binary_cache_public_uri = https://cache-staging.nixos.org |
| 79 | |
| 80 | <Plugin::Session> |
| 81 | cache_size = 32m |
| 82 | </Plugin::Session> |
| 83 | |
| 84 | # patchelf:master:3 |
| 85 | xxx-jobset-repeats = nixos:reproducibility:1 |
| 86 | |
| 87 | upload_logs_to_binary_cache = true |
| 88 | compress_build_logs = false # conflicts with upload_logs_to_binary_cache |
| 89 | |
| 90 | log_prefix = https://cache.nixos.org/ |
| 91 | |
| 92 | evaluator_workers = 4 |
| 93 | evaluator_max_memory_size = 4096 |
| 94 | |
| 95 | queue_runner_endpoint = http://localhost:8080 |
| 96 | |
| 97 | max_concurrent_evals = 1 |
| 98 | |
| 99 | max_unsupported_time = 86400 |
| 100 | |
| 101 | allow_import_from_derivation = false |
| 102 | |
| 103 | max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB |
| 104 | max_db_connections = 50 |
| 105 | |
| 106 | queue_runner_metrics_address = [::]:9198 |
| 107 | |
| 108 | <hydra_notify> |
| 109 | <prometheus> |
| 110 | listen_address = 0.0.0.0 |
| 111 | port = 9199 |
| 112 | </prometheus> |
| 113 | </hydra_notify> |
| 114 | ''; |
| 115 | }; |
| 116 | |
| 117 | hydra-queue-runner-dev = { |
| 118 | enable = true; |
| 119 | awsCredentialsFile = config.sops.secrets.hydra-aws-credentials.path; |
| 120 | settings = { |
| 121 | queueTriggerTimerInS = 300; |
| 122 | concurrentUploadLimit = 2; |
| 123 | # bump from the 120s default: builder reconnects briefly drop their |
| 124 | # system and we'd abort buildable steps as unsupported (hydra#1805) |
| 125 | maxUnsupportedTimeInS = 86400; |
| 126 | remoteStoreAddr = [ |
| 127 | "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&write-nar-listing=1&compression=zstd&ls-compression=zstd&log-compression=zstd&narinfo-compression=zstd" |
| 128 | ]; |
| 129 | usePresignedUploads = true; |
| 130 | forcedSubstituters = [ "https://cache-staging.nixos.org" ]; |
| 131 | }; |
| 132 | }; |
| 133 | |
| 134 | nginx = { |
| 135 | enable = true; |
| 136 | virtualHosts."queue-runner.staging-hydra.nixos.org" = { |
| 137 | extraConfig = '' |
| 138 | ssl_client_certificate ${./ca.crt}; |
| 139 | ssl_verify_depth 2; |
| 140 | ssl_verify_client on; |
| 141 | ''; |
| 142 | |
| 143 | sslCertificate = ./server.crt; |
| 144 | sslCertificateKey = config.sops.secrets."queue-runner-server.key".path; |
| 145 | onlySSL = true; |
| 146 | |
| 147 | locations."/".extraConfig = '' |
| 148 | # This is necessary so that grpc connections do not get closed early |
| 149 | # see https://stackoverflow.com/a/67805465 |
| 150 | client_body_timeout 31536000s; |
| 151 | client_max_body_size 0; |
| 152 | |
| 153 | grpc_pass grpc://[::1]:50051; |
| 154 | |
| 155 | grpc_read_timeout 31536000s; # 1 year in seconds |
| 156 | grpc_send_timeout 31536000s; # 1 year in seconds |
| 157 | grpc_socket_keepalive on; |
| 158 | |
| 159 | grpc_set_header Host $host; |
| 160 | grpc_set_header X-Real-IP $remote_addr; |
| 161 | grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |
| 162 | grpc_set_header X-Forwarded-Proto $scheme; |
| 163 | |
| 164 | grpc_set_header X-Client-DN $ssl_client_s_dn; |
| 165 | grpc_set_header X-Client-Cert $ssl_client_escaped_cert; |
| 166 | ''; |
| 167 | }; |
| 168 | }; |
| 169 | }; |
| 170 | |
| 171 | sops.secrets = { |
| 172 | "queue-runner-server.key" = { |
| 173 | sopsFile = ../../secrets/queue-runner-server.key.staging-hydra; |
| 174 | format = "binary"; |
| 175 | owner = config.systemd.services.nginx.serviceConfig.User; |
| 176 | }; |
| 177 | hydra-users = { |
| 178 | sopsFile = ../../secrets/hydra-users.staging-hydra; |
| 179 | format = "binary"; |
| 180 | }; |
| 181 | }; |
| 182 | |
| 183 | systemd = { |
| 184 | tmpfiles.rules = [ |
| 185 | "d /var/cache/hydra 0755 hydra hydra - -" |
| 186 | "d ${narCache} 0775 hydra hydra 1d -" |
| 187 | ]; |
| 188 | |
| 189 | # eats memory as if it was free |
| 190 | services = { |
| 191 | hydra-notify.enable = false; |
| 192 | hydra-queue-runner = { |
| 193 | enable = false; |
| 194 | |
| 195 | # restarting the scheduler is very expensive |
| 196 | restartIfChanged = false; |
| 197 | serviceConfig = { |
| 198 | ManagedOOMPreference = "avoid"; |
| 199 | LimitNOFILE = 65535; |
| 200 | }; |
| 201 | }; |
| 202 | |
| 203 | hydra-prune-build-logs = { |
| 204 | description = "Clean up old build logs"; |
| 205 | startAt = "weekly"; |
| 206 | serviceConfig = { |
| 207 | User = "hydra-queue-runner"; |
| 208 | Group = "hydra"; |
| 209 | ExecStart = lib.concatStringsSep " " [ |
| 210 | (lib.getExe pkgs.findutils) |
| 211 | "/var/lib/hydra/build-logs/" |
| 212 | "-ignore_readdir_race" |
| 213 | "-type" |
| 214 | "f" |
| 215 | "-mtime" |
| 216 | "+${toString (3 * 365)}" # days |
| 217 | "-delete" |
| 218 | ]; |
| 219 | }; |
| 220 | }; |
| 221 | hydra-post-init = { |
| 222 | serviceConfig = { |
| 223 | Type = "oneshot"; |
| 224 | TimeoutStartSec = "60"; |
| 225 | }; |
| 226 | wantedBy = [ config.systemd.targets.multi-user.name ]; |
| 227 | after = [ config.systemd.services.hydra-server.name ]; |
| 228 | requires = [ config.systemd.services.hydra-server.name ]; |
| 229 | environment = { |
| 230 | inherit (config.systemd.services.hydra-init.environment) HYDRA_DBI; |
| 231 | }; |
| 232 | path = [ |
| 233 | config.services.hydra.package |
| 234 | pkgs.netcat |
| 235 | ]; |
| 236 | script = '' |
| 237 | set -e |
| 238 | while IFS=';' read -r user role passwordhash email fullname; do |
| 239 | opts=("$user" "--role" "$role" "--password-hash" "$passwordhash") |
| 240 | if [[ -n "$email" ]]; then |
| 241 | opts+=("--email-address" "$email") |
| 242 | fi |
| 243 | if [[ -n "$fullname" ]]; then |
| 244 | opts+=("--full-name" "$fullname") |
| 245 | fi |
| 246 | hydra-create-user "''${opts[@]}" |
| 247 | done < ${config.sops.secrets.hydra-users.path} |
| 248 | ''; |
| 249 | }; |
| 250 | }; |
| 251 | }; |
| 252 | |
| 253 | programs.ssh = { |
| 254 | hostKeyAlgorithms = [ |
| 255 | "rsa-sha2-512-cert-v01@openssh.com" |
| 256 | "ssh-ed25519" |
| 257 | "ssh-rsa" |
| 258 | "ecdsa-sha2-nistp256" |
| 259 | ]; |
| 260 | |
| 261 | extraConfig = lib.mkAfter '' |
| 262 | ServerAliveInterval 120 |
| 263 | TCPKeepAlive yes |
| 264 | ''; |
| 265 | }; |
| 266 | } |