delft/eris: update disk monitoring alerts
Quite recently the `/scratch` partition on bastion ran out of free disk space. This is an update to the alerting rule to generalize and extend the existing disk fullness alerts. In addition to monitoring `/scratch` we are now sharing the filesystem selector between these rules and in addition monitor relative fullness on them.
Martin Weinelt committed
Dec 17, 2021 at 21:06 UTC
a9fe53c7f49394fe1aeff24c03101b2a13a02904
1 file changed
+15
-6
delft/eris.nix
+15
-6
@@ -130,10 +130,15 @@ in {
130
131
{
132
name = "system";
133
- rules = [
133
+ rules = let
134
+ diskSelector = ''mountpoint=~"(/|/scratch)"'';
135
+ in
136
+ [
137
{
135
- alert = "RootPartitionLowInodes";
136
- expr = ''node_filesystem_files_free{mountpoint="/"} <= 10000'';
138
+ alert = "PartitionLowInodes";
139
+ expr = ''
140
+ node_filesystem_files_free{${diskSelector}} <= 10000
141
+ '';
142
for = "30m";
143
labels.severity = "warning";
144
annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} inodes free.";
@@ -141,11 +146,15 @@ in {
146
}
147
148
{
144
- alert = "RootPartitionLowDiskSpace";
145
- expr = ''node_filesystem_avail_bytes{mountpoint="/"} <= 10000000000'';
149
+ alert = "PartitionLowDiskSpace";
150
+ expr = ''
151
+ (node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)
152
+ or
153
+ (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10)";
154
+ '';
155
for = "30m";
156
labels.severity = "warning";
148
- annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} bytes free.";
157
+ annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} GB free.";
158
annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
159
}
160
];