@cryptotaxi247 / infra / commits / 5f7f3569

More mailserver probe fixes

A few things here: 1. Increase the `stmp_starttls` timeout. We're seeing probes sporadically fail against ImprovMX with "i/o timeout". (see logs below) 2. Add missing port for probing `umbriel` (oops). 3. Also alert on `umbriel` probe failures. ImprovMX probe logs: ``` ts=2025-03-10T16:27:37.785620787Z caller=main.go:190 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Beginning probe" probe=tcp timeout_seconds=5 ts=2025-03-10T16:27:37.785744902Z caller=tcp.go:40 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Resolving target address" target=mx2.improvmx.com ip_protocol=ip6 ts=2025-03-10T16:27:37.823984377Z caller=tcp.go:40 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Resolved target address" target=mx2.improvmx.com ip=2a05:d012:412:e202:e81e:cc44:3b53:8a 3d ts=2025-03-10T16:27:37.824035793Z caller=tcp.go:112 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Dialing TCP without TLS" ts=2025-03-10T16:27:37.980888559Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Successfully dialed" ts=2025-03-10T16:27:37.980965545Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Processing query response entry" entry_number=0 ts=2025-03-10T16:27:42.293492984Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=debug msg="Read line" line="220 mx2.improvmx.com Welcome to ImprovMX mail server v3. - Improv MX v2024.06.06" ts=2025-03-10T16:27:42.293596408Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Regexp matched" regexp=^220 line="220 mx2.improvmx.com Welcome to ImprovMX mail ser ver v3. - ImprovMX v2024.06.06" ts=2025-03-10T16:27:42.293617839Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Processing query response entry" entry_number=1 ts=2025-03-10T16:27:42.293631825Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=debug msg="Sending line" line="EHLO prober\r" ts=2025-03-10T16:27:42.293683932Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=info msg="Processing query response entry" entry_number=2 ts=2025-03-10T16:27:42.786519394Z caller=handler.go:119 module=smtp_starttls target=mx2.improvmx.com:25 level=error msg="Error reading from connection" err="read tcp6 [2a01:4ff:1f0:ad06::]:39860->[2a05:d 012:412:e202:e81e:cc44:3b53:8a3d]:25: i/o timeout" ts=2025-03-10T16:27:42.786645963Z caller=main.go:190 module=smtp_starttls target=mx2.improvmx.com:25 level=error msg="Probe failed" duration_seconds=5.000976894 ```

Jeremy Fleischman committed Mar 10, 2025 at 09:38 UTC 5f7f35693fe754809831b102fa07bf9bbb66c646
1 file changed +6 -6
build/pluto/prometheus/exporters/blackbox.nix
+6 -6
@@ -5,10 +5,10 @@ let
5 {
6 module,
7 targets,
8 - job_name ? "blackbox-${module}",
8 + job_suffix ? "",
9 }:
10 {
11 - inherit job_name;
11 + job_name = "blackbox-${module}${job_suffix}";
12 metrics_path = "/probe";
13 params = {
14 module = [ module ];
@@ -75,7 +75,7 @@ in
75 # From https://github.com/prometheus/blackbox_exporter/blob/53e78c2b3535ecedfd072327885eeba2e9e51ea2/example.yml#L120-L133
76 modules.smtp_starttls = {
77 prober = "tcp";
78 - timeout = "5s";
78 + timeout = "10s";
79 tcp = {
80 query_response = [
81 { expect = "^220"; }
@@ -122,8 +122,8 @@ in
122 # https://github.com/NixOS/infra/issues/485
123 (mkStaticProbe {
124 module = "smtp_starttls";
125 - job_name = "smtp_starttls_umbriel";
126 - targets = [ "umbriel.nixos.org" ];
125 + job_suffix = "_umbriel";
126 + targets = [ "umbriel.nixos.org:25" ];
127 })
128 (mkDnsSdProbe "smtp_starttls" {
129 names = [
@@ -162,7 +162,7 @@ in
162 {
163 alert = "MxUnreachable";
164 expr = ''
165 - probe_success{job="blackbox-smtp_starttls"} == 0
165 + probe_success{job=~"blackbox-smtp_starttls.*"} == 0
166 '';
167 for = "15m";
168 labels.severity = "warning";