Alert if any scraping job fails
My understanding is that we really ought to have something like this to avoid a scenario where (for whatever reason: network issue, accidentally stopped exporter process, etc) prometheus is no longer successfully scraping targets. Without this I believe many of our alerts would just happily continue to evaluate nothing at all.
Jeremy Fleischman committed
Feb 1, 2025 at 10:00 UTC
3778e540263e5db7c98e78414a396723b59bd686
3 files changed
+27
-7
build/pluto/prometheus/default.nix
+1
@@ -3,6 +3,7 @@
3
{
4
imports = [
5
./alertmanager.nix
6
+ ./exporters/up.nix
7
./exporters/blackbox.nix
8
./exporters/channel.nix
9
./exporters/domain.nix
build/pluto/prometheus/exporters/hydra.nix
-7
@@ -70,13 +70,6 @@
70
annotations.summary = "{{ $labels.machine }} has {{ $value }} over-age jobs.";
71
annotations.grafana = "https://grafana.nixos.org/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
72
}
73
- {
74
- alert = "HydraQueueRunnerUp";
75
- expr = ''up{job="hydra_queue_runner"} == 0'';
76
- for = "30m";
77
- labels.severity = "warning";
78
- annotations.summary = "hydra-queue-runner's prometheus exporter is not up";
79
- }
73
];
74
}
75
];
build/pluto/prometheus/exporters/up.nix
new
+26
@@ -0,0 +1,26 @@
1
+{ pkgs, ... }:
2
+
3
+{
4
+ services.prometheus.ruleFiles = [
5
+ (pkgs.writeText "up.rules" (
6
+ builtins.toJSON {
7
+ groups = [
8
+ {
9
+ name = "up";
10
+ rules = [
11
+ {
12
+ alert = "NotUp";
13
+ expr = ''
14
+ up == 0
15
+ '';
16
+ for = "10m";
17
+ labels.severity = "warning";
18
+ annotations.summary = "scrape job {{ $labels.job }} is failing on {{ $labels.instance }}";
19
+ }
20
+ ];
21
+ }
22
+ ];
23
+ }
24
+ ))
25
+ ];
26
+}