Add a blackbox monitor of our mailserver
I re-locked in order to pull in <https://github.com/NixOS/nixpkgs/pull/383081> To avoid potential alerting noise: I'll wait until this is deployed and succeeding before declaring an additional alert. refs: https://github.com/NixOS/infra/issues/485
Jeremy Fleischman committed
Feb 16, 2025 at 22:50 UTC
ea244b0665ded9528d4cefb8b1ea0f6993e61e37
2 files changed
+64
-14
build/flake.lock
+9
-9
@@ -34,11 +34,11 @@
34
"stable": "stable"
35
},
36
"locked": {
37
- "lastModified": 1734897875,
38
- "narHash": "sha256-LLpiqfOGBippRax9F33kSJ/Imt8gJXb6o0JwSBiNHCk=",
37
+ "lastModified": 1739900653,
38
+ "narHash": "sha256-hPSLvw6AZQYrZyGI6Uq4XgST7benF/0zcCpugn/P0yM=",
39
"owner": "zhaofengli",
40
"repo": "colmena",
41
- "rev": "a6b51f5feae9bfb145daa37fd0220595acb7871e",
41
+ "rev": "2370d4336eda2a9ef29fce10fa7076ae011983ab",
42
"type": "github"
43
},
44
"original": {
@@ -76,11 +76,11 @@
76
]
77
},
78
"locked": {
79
- "lastModified": 1739634831,
80
- "narHash": "sha256-xFnU+uUl48Icas2wPQ+ZzlL2O3n8f6J2LrzNK9f2nng=",
79
+ "lastModified": 1739841949,
80
+ "narHash": "sha256-lSOXdgW/1zi/SSu7xp71v+55D5Egz8ACv0STkj7fhbs=",
81
"owner": "nix-community",
82
"repo": "disko",
83
- "rev": "fa5746ecea1772cf59b3f34c5816ab3531478142",
83
+ "rev": "15dbf8cebd8e2655a883b74547108e089f051bf0",
84
"type": "github"
85
},
86
"original": {
@@ -351,11 +351,11 @@
351
},
352
"nixpkgs": {
353
"locked": {
354
- "lastModified": 1739719496,
355
- "narHash": "sha256-emht/teBZSA+uod0i30X50l7Xt+7+ta09hIEPo3srTU=",
354
+ "lastModified": 1739923778,
355
+ "narHash": "sha256-BqUY8tz0AQ4to2Z4+uaKczh81zsGZSYxjgvtw+fvIfM=",
356
"owner": "NixOS",
357
"repo": "nixpkgs",
358
- "rev": "447e1660723ce6690f9250b45689e14c29d220ff",
358
+ "rev": "36864ed72f234b9540da4cf7a0c49e351d30d3f1",
359
"type": "github"
360
},
361
"original": {
build/pluto/prometheus/exporters/blackbox.nix
+55
-5
@@ -1,7 +1,7 @@
1
-{ pkgs, ... }:
1
+{ config, pkgs, ... }:
2
3
let
4
- mkProbe = module: targets: {
4
+ mkStaticProbe = module: targets: {
5
job_name = "blackbox-${module}";
6
metrics_path = "/probe";
7
params = {
@@ -19,12 +19,36 @@ let
19
}
20
{
21
target_label = "__address__";
22
- replacement = "localhost:9115";
22
+ replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
23
}
24
];
25
};
26
-in
26
27
+ mkDnsSdProbe = module: dns_sd_config: {
28
+ job_name = "blackbox-${module}";
29
+ metrics_path = "/probe";
30
+ params = {
31
+ module = [ module ];
32
+ };
33
+ dns_sd_configs = [
34
+ dns_sd_config
35
+ ];
36
+ relabel_configs = [
37
+ {
38
+ source_labels = [ "__address__" ];
39
+ target_label = "__param_target";
40
+ }
41
+ {
42
+ source_labels = [ "__meta_dns_name" ];
43
+ target_label = "instance";
44
+ }
45
+ {
46
+ target_label = "__address__";
47
+ replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
48
+ }
49
+ ];
50
+ };
51
+in
52
{
53
services.prometheus = {
54
exporters.blackbox = {
@@ -37,12 +61,31 @@ in
61
tcp.tls = true;
62
http.headers.User-Agent = "blackbox-exporter";
63
};
64
+
65
+ # From https://github.com/prometheus/blackbox_exporter/blob/53e78c2b3535ecedfd072327885eeba2e9e51ea2/example.yml#L120-L133
66
+ modules.smtp_starttls = {
67
+ prober = "tcp";
68
+ timeout = "5s";
69
+ tcp = {
70
+ query_response = [
71
+ { expect = "^220 ([^ ]+) ESMTP (.+)$"; }
72
+ { send = "EHLO prober\r"; }
73
+ { expect = "^250-STARTTLS"; }
74
+ { send = "STARTTLS\r"; }
75
+ { expect = "^220"; }
76
+ { starttls = true; }
77
+ { send = "EHLO prober\r"; }
78
+ { expect = "^250-AUTH"; }
79
+ { send = "QUIT\r"; }
80
+ ];
81
+ };
82
+ };
83
}
84
);
85
};
86
87
scrapeConfigs = [
45
- (mkProbe "https_success" [
88
+ (mkStaticProbe "https_success" [
89
"https://cache.nixos.org"
90
"https://channels.nixos.org"
91
"https://common-styles.nixos.org"
@@ -61,6 +104,13 @@ in
104
"https://www.nixos.org"
105
"https://tracker.security.nixos.org"
106
])
107
+ (mkDnsSdProbe "smtp_starttls" {
108
+ names = [
109
+ "mail-test.nixos.org"
110
+ ];
111
+ type = "MX";
112
+ port = 25;
113
+ })
114
];
115
116
ruleFiles = [