eris: Disable the IRC forwarder, enable a Matrix forwarder (broken: no creds yet.)
Eelco Dolstra committed
Nov 21, 2021 at 15:40 UTC
e4d769531440a6449933b5af8b478340fdef4c09
4 files changed
+54
-80
delft/eris.nix
+4
-4
@@ -9,7 +9,7 @@ in {
9
../modules/prometheus
10
./eris/packet-spot-market-prices.nix
11
./eris/github-project-monitor.nix
12
- ./eris/alertmanager-irc-forwarder.nix
12
+ ./eris/alertmanager-matrix-forwarder.nix
13
./eris/channel-monitor.nix
14
];
15
deployment.targetEnv = "hetzner";
@@ -88,7 +88,7 @@ in {
88
89
routes = [
90
{
91
- receiver = "nixos_dev";
91
+ receiver = "go-neb";
92
group_wait = "30s";
93
match.severity = "page";
94
}
@@ -100,10 +100,10 @@ in {
100
name = "ignore";
101
}
102
{
103
- name = "nixos_dev";
103
+ name = "go-neb";
104
webhook_configs = [
105
{
106
- url = "http://127.0.0.1:9080/?target_id=%23nixos-dev";
106
+ url = "${config.services.go-neb.baseUrl}:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2UK";
107
send_resolved = true;
108
}
109
];
delft/eris/alertmanager-irc-forwarder.nix
deleted
-33
@@ -1,33 +0,0 @@
1
-{ pkgs, ... }:
2
-{
3
- users.extraUsers.alertmanager-notifier = {
4
- description = "Prometheus Alert Manager to IRC Forwarder";
5
- };
6
-
7
- deployment.keys."alertmanager-irc-forwarder.env" = {
8
- keyFile = /home/deploy/src/nixos-org-configurations/keys/alertmanager-irc-forwarder.env;
9
- user = "alertmanager-notifier";
10
- };
11
-
12
- systemd.services.prometheus-alertmanager-irc-notifier = {
13
- wantedBy = [ "multi-user.target" ];
14
-
15
- path = [
16
- (pkgs.python3.withPackages (p: with p; [
17
- flask pika
18
- ]))
19
- ];
20
- script = "exec python3 -m flask run --port 9080";
21
-
22
- environment = {
23
- FLASK_APP = ./prometheus-alertmanager-irc-notifier.py;
24
- EXTERNAL_URL = "https://monitoring.nixos.org/prometheus/alerts";
25
- };
26
-
27
- serviceConfig = {
28
- User = "alertmanager-notifier";
29
- Group = "keys";
30
- EnvironmentFile = "/run/keys/alertmanager-irc-forwarder.env";
31
- };
32
- };
33
-}
delft/eris/alertmanager-matrix-forwarder.nix
new
+50
@@ -0,0 +1,50 @@
1
+{
2
+ services.go-neb = {
3
+ enable = true;
4
+ baseUrl = "http://localhost";
5
+ config = {
6
+ clients = [
7
+ { UserId = "@bot:nixos.org";
8
+ AccessToken = "CHANGEME";
9
+ HomeServerUrl = "https://nixos.ems.host";
10
+ Sync = true;
11
+ AutoJoinRooms = true;
12
+ DisplayName = "Bot";
13
+ }
14
+ ];
15
+ services = [
16
+ { ID = "alertmanager_service";
17
+ Type = "alertmanager";
18
+ UserId = "@bot:nixos.org";
19
+ Config = {
20
+ webhook_url = "http://localhost:5040/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2UK";
21
+ rooms = {
22
+ "!QLQqibtFaVtDgurUAE:nixos.org" = { #bots:nixos.org
23
+ text_template = ''
24
+ {{range .Alerts -}} [{{ .Status }}] {{index .Labels "alertname" }}: {{index .Annotations "description"}} {{ end -}}
25
+ '';
26
+ html_template = ''
27
+ {{range .Alerts -}}
28
+ {{ $severity := index .Labels "severity" }}
29
+ {{ if eq .Status "firing" }}
30
+ {{ if eq $severity "critical"}}
31
+ <font color='red'><b>[FIRING - CRITICAL]</b></font>
32
+ {{ else if eq $severity "warning"}}
33
+ <font color='orange'><b>[FIRING - WARNING]</b></font>
34
+ {{ else }}
35
+ <b>[FIRING - {{ $severity }}]</b>
36
+ {{ end }}
37
+ {{ else }}
38
+ <font color='green'><b>[RESOLVED]</b></font>
39
+ {{ end }}
40
+ {{ index .Labels "alertname"}}: {{ index .Annotations "summary"}} (<a href="{{ .GeneratorURL }}">source</a>, <a href="{{ .SilenceURL }}">silence</a>)<br/>
41
+ {{end -}}'';
42
+ msg_type = "m.text"; # Must be either `m.text` or `m.notice`
43
+ };
44
+ };
45
+ };
46
+ }
47
+ ];
48
+ };
49
+ };
50
+}
delft/eris/prometheus-alertmanager-irc-notifier.py
deleted
-43
@@ -1,43 +0,0 @@
1
-
2
-
3
-import json, os
4
-from http import HTTPStatus
5
-from flask import Flask, request
6
-import pika
7
-
8
-app = Flask(__name__)
9
-parameters = pika.URLParameters(os.environ["AMQP_ENDPOINT"])
10
-
11
-externalUrl = os.environ["EXTERNAL_URL"]
12
-
13
-def send_notice(target, body):
14
- # alerts need to be reliable, so this is slow,
15
- # but has no problems with heartbeat timeouts
16
- connection = pika.BlockingConnection(parameters)
17
- channel = connection.channel()
18
- body = json.dumps({
19
- 'message_type': 'notice',
20
- 'target': target,
21
- 'body': body
22
- })
23
- channel.basic_publish(exchange='',
24
- routing_key='queue-publish',
25
- body=body)
26
-
27
-@app.route("/", methods=['POST'])
28
-def webhook():
29
- target_id = request.args["target_id"]
30
- send_notice(target_id, format_event(request.json))
31
- return ('', HTTPStatus.NO_CONTENT)
32
-
33
-def format_event(event):
34
- message = event.get("status", "unknown") + ": "
35
-
36
- commonLabels = event.get("commonLabels", {})
37
- alertname = commonLabels.get("alertname", "various alerts")
38
-
39
- message += alertname + ": "
40
- message += externalUrl
41
-
42
- return message
43
-