@cryptotaxi247 / infra / commits / 47aeada2

Alert if scraping jobs fail

In addition to alerting on failing scraping jobs, we need to address our existing failures, which I researched in https://github.com/NixOS/infra/issues/551. Copy pasting analysis from there: - `r13y`: Last successful scrape: 2024-09-21. Website is down. @grahamc didn't respond to my query. - `rfc39`: IMO, this is mis-architected. This scrapes a job that only runs periodically, which means we regularly get scrape failures. I've filed https://github.com/NixOS/rfc39/issues/14 with the upstream project seeking advice. - Since we don't actually alert on these metrics, I propose doing the simplest thing and just disabling this scrape job for now. - `hydra_notify`: Last successful scrape: 2024-08-02. We disabled this service [in 2024](https://github.com/NixOS/infra/commit/66da5cfddb8c67b5e2f0b5bba62899f1f82eec42), there's no reason to keep scraping it.

Jeremy Fleischman committed Mar 13, 2025 at 14:40 UTC 47aeada2f90fdd3b66ffcd318c5d62c12f72ee4e
4 files changed +24 -60
build/pluto/prometheus/default.nix
+24 -3
@@ -1,4 +1,4 @@
1 -{ ... }:
1 +{ pkgs, ... }:
2
3 {
4 imports = [
@@ -16,8 +16,6 @@
16 ./exporters/owncast.nix
17 ./exporters/postgresql.nix
18 ./exporters/rasdaemon.nix
19 - ./exporters/r13y.nix
20 - ./exporters/rfc39.nix
19 ./exporters/zfs.nix
20 ];
21
@@ -40,5 +38,28 @@
38 "--web.external-url=https://prometheus.nixos.org/"
39 ];
40 globalConfig.scrape_interval = "15s";
41 +
42 + ruleFiles = [
43 + (pkgs.writeText "up.rules" (
44 + builtins.toJSON {
45 + groups = [
46 + {
47 + name = "up";
48 + rules = [
49 + {
50 + alert = "NotUp";
51 + expr = ''
52 + up == 0
53 + '';
54 + for = "10m";
55 + labels.severity = "warning";
56 + annotations.summary = "scrape job {{ $labels.job }} is failing on {{ $labels.instance }}";
57 + }
58 + ];
59 + }
60 + ];
61 + }
62 + ))
63 + ];
64 };
65 }
build/pluto/prometheus/exporters/hydra.nix
-6
@@ -36,12 +36,6 @@
36 scheme = "https";
37 static_configs = [ { targets = [ "hydra.nixos.org:443" ]; } ];
38 }
39 - {
40 - job_name = "hydra_notify";
41 - metrics_path = "/metrics";
42 - scheme = "http";
43 - static_configs = [ { targets = [ "hydra.nixos.org:9199" ]; } ];
44 - }
39 {
40 job_name = "hydra_queue_runner";
41 metrics_path = "/metrics";
build/pluto/prometheus/exporters/r13y.nix deleted
-10
@@ -1,10 +0,0 @@
1 -{
2 - services.prometheus.scrapeConfigs = [
3 - {
4 - job_name = "r13y";
5 - scheme = "https";
6 - metrics_path = "/metrics";
7 - static_configs = [ { targets = [ "r13y.com" ]; } ];
8 - }
9 - ];
10 -}
build/pluto/prometheus/exporters/rfc39.nix deleted
-41
@@ -1,41 +0,0 @@
1 -{ pkgs, ... }:
2 -
3 -{
4 - services.prometheus = {
5 - scrapeConfigs = [
6 - {
7 - job_name = "rfc39";
8 - metrics_path = "/";
9 - static_configs = [
10 - {
11 - targets = [
12 - # intermittently available, when the rfc39-sync.service runs
13 - "127.0.0.1:9190"
14 - ];
15 - }
16 - ];
17 - }
18 - ];
19 -
20 - ruleFiles = [
21 - (pkgs.writeText "rfc39-exporter.rules" (
22 - builtins.toJSON {
23 - groups = [
24 - {
25 - name = "rfc39";
26 - rules = [
27 - {
28 - alert = "RFC39MaintainerSync";
29 - expr = ''node_systemd_unit_state{name=~"^rfc39-sync.service$", state="failed"} == 1'';
30 - for = "30m";
31 - labels.severity = "warning";
32 - annotations.grafana = "https://grafana.nixos.org/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
33 - }
34 - ];
35 - }
36 - ];
37 - }
38 - ))
39 - ];
40 - };
41 -}