| 1 | { |
| 2 | config, |
| 3 | lib, |
| 4 | ... |
| 5 | }: |
| 6 | |
| 7 | let |
| 8 | machines = [ |
| 9 | "eager-heisenberg" |
| 10 | "elated-minsky" |
| 11 | "enormous-catfish" |
| 12 | "goofy-hopcroft" |
| 13 | "growing-jennet" |
| 14 | "hopeful-rivest" |
| 15 | "intense-heron" |
| 16 | "kind-lumiere" |
| 17 | "maximum-snail" |
| 18 | "norwegian-blue" |
| 19 | "sleepy-brown" |
| 20 | "sweeping-filly" |
| 21 | ]; |
| 22 | in |
| 23 | { |
| 24 | age.secrets = { |
| 25 | hydra-aws-credentials = { |
| 26 | file = ./secrets/hydra-aws-credentials.age; |
| 27 | path = "/var/lib/hydra/queue-runner/.aws/credentials"; |
| 28 | owner = "hydra-queue-runner"; |
| 29 | group = "hydra"; |
| 30 | }; |
| 31 | } |
| 32 | // lib.listToAttrs ( |
| 33 | map ( |
| 34 | machine: |
| 35 | lib.nameValuePair "${machine}-queue-runner-token" { |
| 36 | file = ./secrets/${machine}-queue-runner-token.age; |
| 37 | owner = "hydra-queue-runner"; |
| 38 | group = "hydra"; |
| 39 | } |
| 40 | ) machines |
| 41 | ); |
| 42 | |
| 43 | services.nginx = { |
| 44 | enable = true; |
| 45 | virtualHosts."queue-runner.hydra.nixos.org" = { |
| 46 | enableACME = true; |
| 47 | forceSSL = true; |
| 48 | |
| 49 | # Expose the queue runner's prometheus metrics from its REST listener, |
| 50 | # which is otherwise only reachable on localhost. |
| 51 | locations."= /metrics".proxyPass = |
| 52 | "http://${config.services.hydra-queue-runner-dev.rest.address}:${toString config.services.hydra-queue-runner-dev.rest.port}/metrics"; |
| 53 | |
| 54 | locations."/".extraConfig = '' |
| 55 | # This is necessary so that grpc connections do not get closed early |
| 56 | # see https://stackoverflow.com/a/67805465 |
| 57 | client_body_timeout 31536000s; |
| 58 | client_max_body_size 0; |
| 59 | |
| 60 | grpc_pass grpc://${config.services.hydra-queue-runner-dev.grpc.address}:${toString config.services.hydra-queue-runner-dev.grpc.port}; |
| 61 | |
| 62 | grpc_read_timeout 31536000s; # 1 year in seconds |
| 63 | grpc_send_timeout 31536000s; # 1 year in seconds |
| 64 | grpc_socket_keepalive on; |
| 65 | |
| 66 | # Builders reuse one long-lived HTTP/2 channel for many RPCs. The |
| 67 | # default keepalive_requests (1000) makes nginx GOAWAY mid-stream, |
| 68 | # cancelling in-flight RPCs and aborting builds. |
| 69 | keepalive_requests 1000000; |
| 70 | keepalive_timeout 600s; |
| 71 | |
| 72 | grpc_set_header Host $host; |
| 73 | grpc_set_header X-Real-IP $remote_addr; |
| 74 | grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |
| 75 | grpc_set_header X-Forwarded-Proto $scheme; |
| 76 | |
| 77 | access_log /var/log/nginx/queue-runner.access.log; |
| 78 | error_log /var/log/nginx/queue-runner.error.log; |
| 79 | ''; |
| 80 | }; |
| 81 | }; |
| 82 | |
| 83 | services.hydra-queue-runner-dev = { |
| 84 | enable = true; |
| 85 | awsCredentialsFile = config.age.secrets."hydra-aws-credentials".path; |
| 86 | settings = { |
| 87 | dbUrl = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra"; |
| 88 | machineFreeFn = "DynamicWithMaxJobLimit"; |
| 89 | stepSortFn = "WithCriticalPath"; |
| 90 | usePresignedUploads = true; |
| 91 | maxOutputSize = 4294967295; # 4 GiB - 1 B, matches prod hydra.conf max_output_size |
| 92 | # TODO: Expose dispatchTriggerTimerInS, defaults to 120s |
| 93 | queueTriggerTimerInS = 60; |
| 94 | # bump from the 120s default: builder reconnects briefly drop their |
| 95 | # system and we'd abort buildable steps as unsupported (hydra#1805) |
| 96 | maxUnsupportedTimeInS = 86400; |
| 97 | maxSilentTime = 3 * 3600; |
| 98 | buildTimeout = 86400; |
| 99 | concurrentUploadLimit = 48; |
| 100 | maxConcurrentDownloads = 48; |
| 101 | remoteStoreAddr = [ |
| 102 | "s3://nix-cache?${ |
| 103 | lib.concatStringsSep "&" [ |
| 104 | "secret-key=/var/lib/hydra/queue-runner/keys/cache.nixos.org-1/secret" |
| 105 | "write-nar-listing=1" |
| 106 | "compression=zstd" |
| 107 | "compression-level=19" |
| 108 | "ls-compression=zstd" |
| 109 | "log-compression=zstd" |
| 110 | "index-debug-info=true" |
| 111 | ] |
| 112 | }" |
| 113 | ]; |
| 114 | rootsDir = "/nix/var/nix/gcroots/hydra"; |
| 115 | tokenPaths = map (machine: config.age.secrets."${machine}-queue-runner-token".path) machines; |
| 116 | }; |
| 117 | }; |
| 118 | } |