prometheus: send alerts to IRC
Graham Christensen committed
Nov 19, 2019 at 19:17 UTC
8e28e502a23dca75146014773a6481d1496c9a10
3 files changed
+113
delft/eris.nix
+37
@@ -7,6 +7,7 @@ in {
7
imports = [
8
../modules/prometheus
9
./eris/packet-spot-market-prices.nix
10
+ ./eris/alertmanager-irc-forwarder.nix
11
];
12
deployment.targetEnv = "hetzner";
13
deployment.hetzner.mainIPv4 = "138.201.32.77";
@@ -57,6 +58,42 @@ in {
58
"--web.external-url=https://status.nixos.org/prometheus/"
59
];
60
61
+ alertmanagers = [
62
+ {
63
+ scheme = "http";
64
+ static_configs = [
65
+ {
66
+ targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ];
67
+ }
68
+ ];
69
+ }
70
+ ];
71
+
72
+ alertmanager = {
73
+ enable = true;
74
+ configuration = {
75
+ global = {};
76
+ route = {
77
+ receiver = "default_receiver";
78
+ group_wait = "30s";
79
+ group_interval = "5m";
80
+ repeat_interval = "1m";
81
+ group_by = [ "alertname" ];
82
+ };
83
+ receivers = [
84
+ {
85
+ name = "default_receiver";
86
+ webhook_configs = [
87
+ {
88
+ url = "http://127.0.0.1:9080/?target_id=%23nixos-dev";
89
+ send_resolved = true;
90
+ }
91
+ ];
92
+ }
93
+ ];
94
+ };
95
+ };
96
+
97
rules = [ (builtins.toJSON {
98
groups = [
99
{
delft/eris/alertmanager-irc-forwarder.nix
new
+33
@@ -0,0 +1,33 @@
1
+{ pkgs, ... }:
2
+{
3
+ users.extraUsers.alertmanager-notifier = {
4
+ description = "Prometheus Alert Manager to IRC Forwarder";
5
+ };
6
+
7
+ deployment.keys."alertmanager-irc-forwarder.env" = {
8
+ keyFile = /home/deploy/src/nixos-org-configurations/keys/alertmanager-irc-forwarder.env;
9
+ user = "alertmanager-notifier";
10
+ };
11
+
12
+ systemd.services.prometheus-alertmanager-irc-notifier = {
13
+ wantedBy = [ "multi-user.target" ];
14
+
15
+ path = [
16
+ (pkgs.python3.withPackages (p: with p; [
17
+ flask pika
18
+ ]))
19
+ ];
20
+ script = "exec python3 -m flask run --port 9080";
21
+
22
+ environment = {
23
+ FLASK_APP = ./prometheus-alertmanager-irc-notifier.py;
24
+ EXTERNAL_URL = "https://status.nixos.org/prometheus/alerts";
25
+ };
26
+
27
+ serviceConfig = {
28
+ User = "alertmanager-notifier";
29
+ Group = "keys";
30
+ EnvironmentFile = "/run/keys/alertmanager-irc-forwarder.env";
31
+ };
32
+ };
33
+}
\ No newline at end of file
delft/eris/prometheus-alertmanager-irc-notifier.py
new
+43
@@ -0,0 +1,43 @@
1
+
2
+
3
+import json, os
4
+from http import HTTPStatus
5
+from flask import Flask, request
6
+import pika
7
+
8
+app = Flask(__name__)
9
+parameters = pika.URLParameters(os.environ["AMQP_ENDPOINT"])
10
+
11
+externalUrl = os.environ["EXTERNAL_URL"]
12
+
13
+def send_notice(target, body):
14
+ # alerts need to be reliable, so this is slow,
15
+ # but has no problems with heartbeat timeouts
16
+ connection = pika.BlockingConnection(parameters)
17
+ channel = connection.channel()
18
+ body = json.dumps({
19
+ 'message_type': 'notice',
20
+ 'target': target,
21
+ 'body': body
22
+ })
23
+ channel.basic_publish(exchange='',
24
+ routing_key='queue-publish',
25
+ body=body)
26
+
27
+@app.route("/", methods=['POST'])
28
+def webhook():
29
+ target_id = request.args["target_id"]
30
+ send_notice(target_id, format_event(request.json))
31
+ return ('', HTTPStatus.NO_CONTENT)
32
+
33
+def format_event(event):
34
+ message = event.get("status", "unknown") + ": "
35
+
36
+ commonLabels = event.get("commonLabels", {})
37
+ alertname = commonLabels.get("alertname", "various alerts")
38
+
39
+ message += alertname + ": "
40
+ message += externalUrl
41
+
42
+ return message
43
+