delft/eris: drop alerts with severity!='page', ignore stuck channels
tilpner committed
Feb 4, 2020 at 08:39 UTC
8c520a01b72aefd87635de21ce138d5d141d0624
1 file changed
+14
-3
delft/eris.nix
+14
-3
@@ -73,15 +73,27 @@ in {
73
configuration = {
74
global = {};
75
route = {
76
- receiver = "default_receiver";
76
+ receiver = "ignore";
77
group_wait = "30s";
78
group_interval = "5m";
79
repeat_interval = "4h";
80
group_by = [ "alertname" ];
81
+
82
+ routes = [
83
+ {
84
+ receiver = "nixos_dev";
85
+ group_wait = "30s";
86
+ match.severity = "page";
87
+ }
88
+ ];
89
};
90
receivers = [
91
{
84
- name = "default_receiver";
92
+ # with no *_config, this will drop all alerts directed to it
93
+ name = "ignore";
94
+ }
95
+ {
96
+ name = "nixos_dev";
97
webhook_configs = [
98
{
99
url = "http://127.0.0.1:9080/?target_id=%23nixos-dev";
@@ -143,7 +155,6 @@ in {
155
alert = "ChannelUpdateStuck";
156
expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1'';
157
for = "30m";
146
- labels.severity = "page";
158
annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
159
}
160
{