delft/eris: update disk alerts
We now round values to the nearest int, so we don't have an unnecessarily high decimal precision. Also we remove duplicate rule matches by reducing the number of labels we look at.
Martin Weinelt committed
Jan 7, 2022 at 13:53 UTC
e44b52bbe366758bd733b1c7b37de432823eedb3
1 file changed
+4
-3
delft/eris.nix
+4
-3
@@ -137,12 +137,13 @@ in {
137
name = "system";
138
rules = let
139
diskSelector = ''mountpoint=~"(/|/scratch)"'';
140
+ relevantLabels = "device,fstype,instance,mountpoint";
141
in
142
[
143
{
144
alert = "PartitionLowInodes";
145
expr = ''
145
- node_filesystem_files_free{${diskSelector}} <= 10000
146
+ avg (node_filesystem_files_free{${diskSelector}} <= 10000) by (${relevantLabels})
147
'';
148
for = "30m";
149
labels.severity = "warning";
@@ -153,9 +154,9 @@ in {
154
{
155
alert = "PartitionLowDiskSpace";
156
expr = ''
156
- (node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)
157
+ (avg (round(node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)) by (${relevantLabels}))
158
or
158
- (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10)
159
+ (avg (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10) by (${relevantLabels}))
160
'';
161
for = "30m";
162
labels.severity = "warning";