alertmanager: reduce alerts for flipping systemd states
Use the failed status over the last 5m instead of it's current value to avoid triggering multiple alerts when the status flips briefly.
Daiderd Jordan committed
Feb 3, 2020 at 19:27 UTC
833555065fdeb4ab937375bad5f23f47a3c80389
1 file changed
+2
-2
delft/eris.nix
+2
-2
@@ -141,14 +141,14 @@ in {
141
}
142
{
143
alert = "ChannelUpdateStuck";
144
- expr = ''sum(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"activating|failed"}) without(state) == 1'';
144
+ expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1'';
145
for = "30m";
146
labels.severity = "page";
147
annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
148
}
149
{
150
alert = "HomepageUpdateStuck";
151
- expr = ''sum(node_systemd_unit_state{name=~"^update-homepage.*.service$",state=~"activating|failed"}) without(state) == 1'';
151
+ expr = ''max_over_time(node_systemd_unit_state{name=~"^update-homepage.*.service$",state=~"failed"}[5m]) == 1'';
152
for = "30m";
153
labels.severity = "page";
154
annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";