hydra-queue-runner: 54b3c935 -> 81e6209e
Simon Hauser committed
Aug 15, 2025 at 09:17 UTC
9abab547faf5fbd4637225ba9b9d37652d84ab32
4 files changed
+73
-28
non-critical-infra/hosts/staging-hydra/hydra.nix
+11
-4
@@ -37,7 +37,10 @@ in
37
nix.settings = {
38
# gc outputs as well, since they are served from the cache
39
gc-keep-outputs = lib.mkForce false;
40
- allowed-users = [ "hydra" "hydra-www" ];
40
+ allowed-users = [
41
+ "hydra"
42
+ "hydra-www"
43
+ ];
44
};
45
46
# Don't rate-limit the journal.
@@ -117,9 +120,13 @@ in
120
121
hydra-queue-runner-v2 = {
122
enable = true;
120
- settings.remoteStoreAddr = [
121
- "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
122
- ];
123
+ settings = {
124
+ queueTriggerTimerInS = 300;
125
+ concurrentUploadLimit = 2;
126
+ remoteStoreAddr = [
127
+ "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
128
+ ];
129
+ };
130
};
131
132
nginx = {
non-critical-infra/modules/hydra-queue-builder-v2.nix
+18
-1
@@ -2,10 +2,12 @@
2
config,
3
pkgs,
4
lib,
5
+ inputs,
6
...
7
}:
8
let
9
cfg = config.services.hydra-queue-builder-v2;
10
+ unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
11
in
12
{
13
options = {
@@ -98,6 +100,12 @@ in
100
default = true;
101
};
102
103
+ authorizationFile = lib.mkOption {
104
+ description = "Path to token authorization file if token auth should be used.";
105
+ type = lib.types.nullOr lib.types.path;
106
+ default = null;
107
+ };
108
+
109
mtls = lib.mkOption {
110
description = "mtls options";
111
default = null;
@@ -127,7 +135,12 @@ in
135
136
package = lib.mkOption {
137
type = lib.types.package;
130
- default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).builder;
138
+ default =
139
+ (pkgs.recurseIntoAttrs (
140
+ pkgs.callPackage ../packages/hydra-queue-runner {
141
+ inherit (unstable) nixVersions openssl;
142
+ }
143
+ )).builder;
144
};
145
};
146
};
@@ -197,6 +210,10 @@ in
210
++ lib.optionals (cfg.useSubstitutes != null) [
211
"--use-substitutes"
212
]
213
+ ++ lib.optionals (cfg.authorizationFile != null) [
214
+ "--authorization-file"
215
+ cfg.authorizationFile
216
+ ]
217
++ lib.optionals (cfg.mtls != null) [
218
"--server-root-ca-cert-path"
219
cfg.mtls.serverRootCaCertPath
non-critical-infra/modules/hydra-queue-runner-v2.nix
+23
-1
@@ -2,12 +2,14 @@
2
config,
3
pkgs,
4
lib,
5
+ inputs,
6
...
7
}:
8
let
9
cfg = config.services.hydra-queue-runner-v2;
10
11
format = pkgs.formats.toml { };
12
+ unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
13
in
14
{
15
options = {
@@ -101,6 +103,21 @@ in
103
type = lib.types.int;
104
default = 60;
105
};
106
+ maxConcurrentDownloads = lib.mkOption {
107
+ description = "Max count of concurrent downloads per build. Increasing this will increase memory usage of the queue runner.";
108
+ type = lib.types.ints.positive;
109
+ default = 5;
110
+ };
111
+ concurrentUploadLimit = lib.mkOption {
112
+ description = "Concurrent limit for uploading to s3.";
113
+ type = lib.types.ints.positive;
114
+ default = 5;
115
+ };
116
+ tokenListPath = lib.mkOption {
117
+ description = "Path to a list of allowed authentication tokens.";
118
+ type = lib.types.nullOr lib.types.path;
119
+ default = null;
120
+ };
121
};
122
};
123
default = { };
@@ -170,7 +187,12 @@ in
187
};
188
package = lib.mkOption {
189
type = lib.types.package;
173
- default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).runner;
190
+ default =
191
+ (pkgs.recurseIntoAttrs (
192
+ pkgs.callPackage ../packages/hydra-queue-runner {
193
+ inherit (unstable) nixVersions openssl;
194
+ }
195
+ )).runner;
196
};
197
};
198
};
non-critical-infra/packages/hydra-queue-runner/default.nix
+21
-22
@@ -1,5 +1,5 @@
1
{
2
- rustPackages,
2
+ rustPackages_1_88,
3
fetchFromGitHub,
4
pkg-config,
5
openssl,
@@ -11,17 +11,18 @@
11
nlohmann_json,
12
libsodium,
13
boost,
14
+ withOtel ? false,
15
}:
16
let
16
- version = "unstable-2025-08-07";
17
+ version = "unstable-2025-11-05";
18
src = fetchFromGitHub {
19
owner = "helsinki-systems";
20
repo = "hydra-queue-runner";
20
- rev = "54b3c9351d2ae10be5c4d1b97cc0f86300cd70ca";
21
- hash = "sha256-gR2DzWkTykM9GdW3Nf/V8eRv68fl3aO+NW0zNPFSRT0=";
21
+ rev = "81e6209e9107a62e9fba3d29f2ead8e9cf35bea1";
22
+ hash = "sha256-MGoC9bLa9Ih66UbHkxGuAGvj/GfPZzG9N6liiHX0Sos=";
23
};
24
useFetchCargoVendor = true;
24
- cargoHash = "sha256-oNUMmFfts4rjBX0k5mzsxpYA2JqgsRu1nMRFf/2rZa8=";
25
+ cargoHash = "sha256-eFfrwLUnI35WhzO+enr2dTe5blwY00cGQoT5mfs9ZTw=";
26
nativeBuildInputs = [
27
pkg-config
28
protobuf
@@ -32,7 +33,7 @@ let
33
zlib
34
protobuf
35
35
- nixVersions.nix_2_29
36
+ nixVersions.nix_2_31
37
nlohmann_json
38
libsodium
39
boost
@@ -46,7 +47,7 @@ let
47
};
48
in
49
{
49
- runner = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
50
+ runner = rustPackages_1_88.rustPlatform.buildRustPackage {
51
pname = "hydra-queue-runner";
52
inherit version src;
53
__structuredAttrs = true;
@@ -59,22 +60,21 @@ in
60
buildInputs
61
;
62
62
- cargoBuildFlags = [
63
- "-p"
64
- "queue-runner"
65
- ];
66
- cargoTestFlags = finalAttrs.cargoBuildFlags;
63
+ buildAndTestSubdir = "queue-runner";
64
+ buildFeatures = lib.optional withOtel "otel";
65
66
postInstall = ''
69
- wrapProgram $out/bin/queue-runner --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
67
+ wrapProgram $out/bin/queue-runner \
68
+ --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
69
+ --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
70
'';
71
72
meta = meta // {
73
mainProgram = "queue-runner";
74
};
75
- });
75
+ };
76
77
- builder = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
77
+ builder = rustPackages_1_88.rustPlatform.buildRustPackage {
78
pname = "hydra-queue-builder";
79
inherit src version;
80
__structuredAttrs = true;
@@ -87,18 +87,17 @@ in
87
buildInputs
88
;
89
90
- cargoBuildFlags = [
91
- "-p"
92
- "builder"
93
- ];
94
- cargoTestFlags = finalAttrs.cargoBuildFlags;
90
+ buildAndTestSubdir = "builder";
91
+ buildFeatures = lib.optional withOtel "otel";
92
93
postInstall = ''
97
- wrapProgram $out/bin/builder --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
94
+ wrapProgram $out/bin/builder \
95
+ --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
96
+ --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
97
'';
98
99
meta = meta // {
100
mainProgram = "builder";
101
};
103
- });
102
+ };
103
}