| 1 | { pkgs, ... }: |
| 2 | |
| 3 | { |
| 4 | services.prometheus = { |
| 5 | scrapeConfigs = [ |
| 6 | { |
| 7 | job_name = "hydra"; |
| 8 | metrics_path = "/prometheus"; |
| 9 | scheme = "https"; |
| 10 | static_configs = [ { targets = [ "hydra.nixos.org:443" ]; } ]; |
| 11 | } |
| 12 | { |
| 13 | job_name = "hydra_queue_runner"; |
| 14 | metrics_path = "/metrics"; |
| 15 | scheme = "https"; |
| 16 | static_configs = [ { targets = [ "queue-runner.hydra.nixos.org:443" ]; } ]; |
| 17 | } |
| 18 | { |
| 19 | job_name = "hydra-webserver"; |
| 20 | metrics_path = "/metrics"; |
| 21 | scheme = "https"; |
| 22 | static_configs = [ { targets = [ "hydra.nixos.org:443" ]; } ]; |
| 23 | } |
| 24 | ]; |
| 25 | |
| 26 | ruleFiles = [ |
| 27 | (pkgs.writeText "hydra-exporter.rules" ( |
| 28 | builtins.toJSON { |
| 29 | groups = [ |
| 30 | { |
| 31 | name = "hydra"; |
| 32 | rules = [ |
| 33 | { |
| 34 | alert = "BuildsStuckOverTwoDays"; |
| 35 | expr = ''hydra_machine_build_duration_bucket{le="+Inf"} - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0''; |
| 36 | for = "30m"; |
| 37 | labels.severity = "warning"; |
| 38 | annotations.summary = "{{ $labels.machine }} has {{ $value }} over-age jobs."; |
| 39 | annotations.grafana = "https://grafana.nixos.org/d/j0hJAY1Wk/in-progress-build-duration-heatmap"; |
| 40 | } |
| 41 | ]; |
| 42 | } |
| 43 | ]; |
| 44 | } |
| 45 | )) |
| 46 | ]; |
| 47 | }; |
| 48 | } |