prom: add some alerts :)
Eelco Dolstra committed
Nov 13, 2019 at 22:22 UTC
645450276972d0cfcd13e3a5763a3306b699eb41
1 file changed
+53
delft/eris.nix
+53
@@ -57,6 +57,59 @@ in {
57
"--web.external-url=https://status.nixos.org/prometheus/"
58
];
59
60
+ rules = [ (builtins.toJSON {
61
+ groups = [
62
+ {
63
+ name = "hydra";
64
+ rules = [
65
+ {
66
+ alert = "BuildsStuckOverOneDay";
67
+ expr = ''hydra_machine_build_duration_bucket{le="172800"} - ignoring(le) hydra_machine_build_duration_bucket{le="86400"} > 0'';
68
+ labels.severity = "page";
69
+ annotations.summary = "https://status.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
70
+ }
71
+ ];
72
+ }
73
+
74
+ {
75
+ name = "system";
76
+ rules = [
77
+ {
78
+ alert = "RootPartitionNoFreeInodes4HrsAway";
79
+ expr = ''predict_linear(node_filesystem_files_free{mountpoint="/"}[1h], 4 * 3600) <= 0'';
80
+ labels.severity = "page";
81
+ annotations.summary = "https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
82
+ }
83
+
84
+ {
85
+ alert = "RootPartitionNoFreeSpace4HrsAway";
86
+ expr = ''predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 4 * 3600) <= 0'';
87
+ labels.severity = "page";
88
+ annotations.summary = "https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
89
+ }
90
+ ];
91
+ }
92
+
93
+ {
94
+ name = "scheduled-jobs";
95
+ rules = [
96
+ {
97
+ alert = "ChannelUpdateStuck";
98
+ expr = ''node_systemd_unit_state{name=~"^update-nix.*.service$", state="failed"} == 1'';
99
+ labels.severity = "page";
100
+ annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
101
+ }
102
+ {
103
+ alert = "HomepageUpdateStuck";
104
+ expr = ''node_systemd_unit_state{name=~"^update-homepage.service$", state="failed"} == 1'';
105
+ labels.severity = "page";
106
+ annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
107
+ }
108
+ ];
109
+ }
110
+ ];
111
+ }) ];
112
+
113
globalConfig.scrape_interval = "15s";
114
scrapeConfigs = [
115
{