alerts: hopefully make them less flappy
Graham Christensen committed
Nov 20, 2019 at 19:51 UTC
ad7ff5d9b5440c2198b6b07ef2c1aa11e56a0f02
1 file changed
+8
-4
delft/eris.nix
+8
-4
@@ -112,15 +112,17 @@ in {
112
name = "system";
113
rules = [
114
{
115
- alert = "RootPartitionNoFreeInodes4HrsAway";
116
- expr = ''predict_linear(node_filesystem_files_free{mountpoint="/"}[1h], 4 * 3600) <= 0'';
115
+ alert = "RootPartitionLowInodes";
116
+ expr = ''node_filesystem_files_free{mountpoint="/"} <= 10000'';
117
+ for = "30m";
118
labels.severity = "page";
119
annotations.summary = "https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
120
}
121
122
{
122
- alert = "RootPartitionNoFreeSpace4HrsAway";
123
- expr = ''predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 4 * 3600) <= 0'';
123
+ alert = "RootPartitionLowDiskSpace";
124
+ expr = ''node_filesystem_avail_bytes{mountpoint="/"} <= 10000000000'';
125
+ for = "30m";
126
labels.severity = "page";
127
annotations.summary = "https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
128
}
@@ -133,12 +135,14 @@ in {
135
{
136
alert = "ChannelUpdateStuck";
137
expr = ''node_systemd_unit_state{name=~"^update-nix.*.service$", state="failed"} == 1'';
138
+ for = "30m";
139
labels.severity = "page";
140
annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
141
}
142
{
143
alert = "HomepageUpdateStuck";
144
expr = ''node_systemd_unit_state{name=~"^update-homepage.service$", state="failed"} == 1'';
145
+ for = "30m";
146
labels.severity = "page";
147
annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
148
}