monitoring: scrape metrics of new hydra queue runner
The new queue runner only exposes its prometheus metrics on its localhost REST listener, so since the switch from the old queue runner its metrics are no longer collected; the existing scrape job still points at the old runner's now-dead 9198 port. Publish the metrics through the existing queue-runner.hydra.nixos.org nginx vhost and add a scrape job for it on pluto.
Jörg Thalheim committed
Jun 17, 2026 at 00:02 UTC
a30bf9559a2a835bb1f0a72ab6c3311a8ae20529
2 files changed
+11
build/hydra-queue-runner.nix
+5
@@ -44,6 +44,11 @@ in
44
enableACME = true;
45
forceSSL = true;
46
47
+ # Expose the queue runner's prometheus metrics from its REST listener,
48
+ # which is otherwise only reachable on localhost.
49
+ locations."= /metrics".proxyPass =
50
+ "http://${config.services.hydra-queue-runner-dev.rest.address}:${toString config.services.hydra-queue-runner-dev.rest.port}/metrics";
51
+
52
locations."/".extraConfig = ''
53
# This is necessary so that grpc connections do not get closed early
54
# see https://stackoverflow.com/a/67805465
build/pluto/prometheus/exporters/hydra.nix
+6
@@ -40,6 +40,12 @@
40
scheme = "http";
41
static_configs = [ { targets = [ "hydra.nixos.org:9198" ]; } ];
42
}
43
+ {
44
+ job_name = "hydra_queue_runner_dev";
45
+ metrics_path = "/metrics";
46
+ scheme = "https";
47
+ static_configs = [ { targets = [ "queue-runner.hydra.nixos.org:443" ]; } ];
48
+ }
49
{
50
job_name = "hydra-webserver";
51
metrics_path = "/metrics";