delft/eris: add alert for missing builder types
Martin Weinelt committed
Feb 13, 2022 at 12:40 UTC
9278ef46db50246000bef58ed755f4a3a4257323
1 file changed
+11
delft/eris.nix
+11
@@ -130,6 +130,17 @@ in {
130
annotations.summary = "{{ $labels.machine }} has {{ $value }} over-age jobs.";
131
annotations.grafana = "https://monitoring.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
132
}
133
+
134
+ {
135
+ alert = "BuilderTypeMissing";
136
+ expr = ''
137
+ hydra_machine_type_runnable > 0 and hydra_machine_type_running == 0 and (hydra_machine_type_last_active_total < time() - 60)
138
+ '';
139
+ for = "30m";
140
+ labels.severity = "warning";
141
+ annotations.summary = "Runnable jobs for {{ $labels.machineType }}, but no runners to execute.";
142
+ annotations.summary = "https://monitoring.nixos.org/grafana/d/MJw9PcAiz/hydra-jobs?orgId=1&from=now-7d&to=now&refresh=5m&var-machine=All&viewPanel=2";
143
+ }
144
];
145
}
146