eris: alert on failed systemd services
There might be some initial flakiness due to some badly behaved services, e.g. hydra-scale-equinix-metal, but nothing that we can't silence away and properly fix later. 15m is a very conservative threshold, I'm expecting to push it down at some point once things get more under control.
Pierre Bourdon committed
Feb 13, 2024 at 12:25 UTC
4ad5801069d9f589b64515d833ca1246cae0d0b3
1 file changed
+8
delft/eris.nix
+8
@@ -227,6 +227,14 @@ in
227
annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} GB free.";
228
annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
229
}
230
+
231
+ {
232
+ alert = "SystemdUnitFailed";
233
+ expr = ''node_systemd_unit_state{state="failed"} == 1'';
234
+ for = "15m";
235
+ labels.severity = "warning";
236
+ annotations.summary = "systemd unit {{ $labels.name }} on {{ $labels.instance }} has been down for more than 15 minutes.";
237
+ }
238
];
239
}
240