pluto: prometheus: add matrix federation monitoring/alerting
Fixes #410.
Pierre Bourdon committed
Apr 14, 2024 at 16:06 UTC
1993825cff522bfcd73304211420a91eddb919eb
2 files changed
+73
delft/pluto/prometheus/default.nix
+1
@@ -11,6 +11,7 @@
11
./exporters/fastly.nix
12
./exporters/github.nix
13
./exporters/hydra.nix
14
+ ./exporters/json.nix
15
./exporters/nixos.nix
16
./exporters/node.nix
17
./exporters/packet-sd.nix
delft/pluto/prometheus/exporters/json.nix
new
+72
@@ -0,0 +1,72 @@
1
+{ config, pkgs, ... }:
2
+
3
+{
4
+ services.prometheus = {
5
+ exporters.json = {
6
+ enable = true;
7
+ listenAddress = "localhost";
8
+
9
+ configFile = (pkgs.formats.yaml {}).generate "json-exporter-config.yml" {
10
+ modules.matrix-federation-checker = {
11
+ metrics = [
12
+ {
13
+ name = "matrix_homeserver_federation_ok";
14
+ path = "{.FederationOK}";
15
+ help = "False if there's any problem with federation reported.";
16
+ type = "value";
17
+ value_type = "gauge";
18
+ }
19
+ ];
20
+ };
21
+ };
22
+ };
23
+
24
+ scrapeConfigs = [
25
+ {
26
+ job_name = "matrix-federation-checker";
27
+ metrics_path = "/probe";
28
+ params = { module = [ "matrix-federation-checker" ]; };
29
+ relabel_configs = [
30
+ {
31
+ source_labels = [ "__address__" ];
32
+ target_label = "__param_target";
33
+ }
34
+ {
35
+ source_labels = [ "__address__" ];
36
+ target_label = "instance";
37
+ }
38
+ {
39
+ target_label = "__address__";
40
+ replacement = "localhost:${toString config.services.prometheus.exporters.json.port}";
41
+ }
42
+ ];
43
+
44
+ static_configs = [
45
+ {
46
+ targets = [ "https://federationtester.matrix.org/api/report?server_name=nixos.org" ];
47
+ labels.matrix_instance = "nixos.org";
48
+ }
49
+ ];
50
+ }
51
+ ];
52
+
53
+ ruleFiles = [
54
+ (pkgs.writeText "matrix-federation.rules" (builtins.toJSON {
55
+ groups = [
56
+ {
57
+ name = "matrix-federation";
58
+ rules = [
59
+ {
60
+ alert = "MatrixFederationFailure";
61
+ expr = "matrix_homeserver_federation_ok < 1";
62
+ for = "30m";
63
+ labels.severity = "warning";
64
+ annotations.summary = "Matrix federation for {{ $labels.matrix_instance }} appears to be failing.";
65
+ }
66
+ ];
67
+ }
68
+ ];
69
+ }))
70
+ ];
71
+ };
72
+}