@cryptotaxi247 / infra-1
Code Commits Issues Pulls Sessions CI/CD
main
main
nix 170 lines 4.66 KB
Raw
1 { config, pkgs, ... }:
2
3 let
4 mkStaticProbe =
5 {
6 module,
7 targets,
8 job_suffix ? "",
9 }:
10 {
11 job_name = "blackbox-${module}${job_suffix}";
12 metrics_path = "/probe";
13 params = {
14 module = [ module ];
15 };
16 static_configs = [ { inherit targets; } ];
17 relabel_configs = [
18 {
19 source_labels = [ "__address__" ];
20 target_label = "__param_target";
21 }
22 {
23 source_labels = [ "__param_target" ];
24 target_label = "instance";
25 }
26 {
27 target_label = "__address__";
28 replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
29 }
30 ];
31 };
32
33 mkDnsSdProbe = module: dns_sd_config: {
34 job_name = "blackbox-${module}";
35 metrics_path = "/probe";
36 params = {
37 module = [ module ];
38 };
39 dns_sd_configs = [
40 dns_sd_config
41 ];
42 relabel_configs = [
43 {
44 source_labels = [ "__address__" ];
45 target_label = "__param_target";
46 }
47 {
48 source_labels = [ "__address__" ];
49 target_label = "host";
50 }
51 {
52 source_labels = [ "__meta_dns_name" ];
53 target_label = "instance";
54 }
55 {
56 target_label = "__address__";
57 replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
58 }
59 ];
60 };
61 in
62 {
63 services.prometheus = {
64 exporters.blackbox = {
65 enable = true;
66 listenAddress = "127.0.0.1";
67 configFile = pkgs.writeText "probes.yml" (
68 builtins.toJSON {
69 modules.https_success = {
70 prober = "http";
71 tcp.tls = true;
72 http.headers.User-Agent = "blackbox-exporter";
73 };
74
75 # From https://github.com/prometheus/blackbox_exporter/blob/53e78c2b3535ecedfd072327885eeba2e9e51ea2/example.yml#L120-L133
76 modules.smtp_starttls = {
77 prober = "tcp";
78 timeout = "10s";
79 tcp = {
80 query_response = [
81 { expect = "^220"; }
82 { send = "EHLO prober\r"; }
83 { expect = "^250-STARTTLS"; }
84 { send = "STARTTLS\r"; }
85 { expect = "^220"; }
86 { starttls = true; }
87 { send = "EHLO prober\r"; }
88 { expect = "^250-AUTH"; }
89 { send = "QUIT\r"; }
90 ];
91 };
92 };
93 }
94 );
95 };
96
97 scrapeConfigs = [
98 (mkStaticProbe {
99 module = "https_success";
100 targets = [
101 "https://cache.nixos.org"
102 "https://channels.nixos.org"
103 "https://common-styles.nixos.org"
104 "https://discourse.nixos.org"
105 "https://hydra.nixos.org"
106 "https://mobile.nixos.org"
107 "https://monitoring.nixos.org"
108 "https://nixos.org"
109 "https://planet.nixos.org"
110 "https://releases.nixos.org"
111 "https://status.nixos.org"
112 "https://survey.nixos.org"
113 "https://tarballs.nixos.org"
114 "https://weekly.nixos.org"
115 "https://wiki.nixos.org"
116 "https://www.nixos.org"
117 "https://tracker.security.nixos.org"
118 ];
119 })
120 (mkDnsSdProbe "smtp_starttls" {
121 names = [
122 "nixos.org"
123 ];
124 type = "MX";
125 port = 25;
126 })
127 ];
128
129 ruleFiles = [
130 (pkgs.writeText "blackbox-exporter.rules" (
131 builtins.toJSON {
132 groups = [
133 {
134 name = "blackbox";
135 rules = [
136 {
137 alert = "CertificateExpiry";
138 expr = ''
139 probe_ssl_earliest_cert_expiry - time() < 86400 * 14
140 '';
141 for = "15m";
142 labels.severity = "warning";
143 annotations.summary = "Certificate for {{ $labels.instance }} is expiring soon.";
144 }
145 {
146 alert = "HttpUnreachable";
147 expr = ''
148 probe_success{job="blackbox-https_success"} == 0
149 '';
150 for = "15m";
151 labels.severity = "warning";
152 annotations.summary = "Endpoint {{ $labels.instance }} is unreachable";
153 }
154 {
155 alert = "MxUnreachable";
156 expr = ''
157 probe_success{job=~"blackbox-smtp_starttls.*"} == 0
158 '';
159 for = "15m";
160 labels.severity = "warning";
161 annotations.summary = "Mail server {{ $labels.instance }} is unreachable";
162 }
163 ];
164 }
165 ];
166 }
167 ))
168 ];
169 };
170 }