prometheus: update disk space monitoring alerts
Martin Weinelt committed
Jan 7, 2025 at 02:06 UTC
884f60afbf915c173e0cb7f9fbf3392e728d2082
1 file changed
+6
-8
build/pluto/prometheus/exporters/node.nix
+6
-8
@@ -56,7 +56,7 @@
56
57
ruleFiles =
58
let
59
- diskSelector = ''mountpoint=~"(/|/tmp)"'';
59
+ diskSelector = ''mountpoint="/"'';
60
relevantLabels = "device,fstype,instance,mountpoint";
61
in
62
[
@@ -69,24 +69,22 @@
69
{
70
alert = "PartitionLowInodes";
71
expr = ''
72
- avg (node_filesystem_files_free{${diskSelector}} <= 10000) by (${relevantLabels})
72
+ node_filesystem_files_free / node_filesystem_files{${diskSelector}} * 100 < 10
73
'';
74
for = "30m";
75
labels.severity = "warning";
76
- annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} inodes free.";
76
+ annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has only {{ $value }}% free inodes.";
77
annotations.grafana = "https://grafana.nixos.org/d/rYdddlPWk/node-exporter-full?orgId=1&var-job=node&var-node={{ $labels.instance }}";
78
}
79
{
80
alert = "PartitionLowDiskSpace";
81
expr = ''
82
- (avg (round(node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)) by (${relevantLabels}))
83
- or
84
- (avg (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10) by (${relevantLabels}))
82
+ round((node_filesystem_free_bytes{${diskSelector}} * 100) / node_filesystem_size_bytes{mountpoint="/"}) < 10
83
'';
84
for = "30m";
85
labels.severity = "warning";
88
- annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} GB free.";
89
- annotations.grafana = "https://grafana.nixos.org/d/rYdddlPWk/node-exporter-full?orgId=1&var-job=node&var-node=caliban.nixos.org:9100{{ $labels.instance }}";
86
+ annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }}% free.";
87
+ annotations.grafana = "https://grafana.nixos.org/d/rYdddlPWk/node-exporter-full?orgId=1&var-job=node&var-node={{ $labels.instance }}";
88
}
89
{
90
alert = "SystemdUnitFailed";