| 1 | { pkgs, ... }: |
| 2 | |
| 3 | { |
| 4 | services.prometheus = { |
| 5 | scrapeConfigs = [ |
| 6 | { |
| 7 | job_name = "rasdaemon"; |
| 8 | static_configs = [ |
| 9 | { |
| 10 | targets = [ |
| 11 | # build |
| 12 | "mimas.nixos.org:10029" |
| 13 | "haumea.nixos.org:10029" |
| 14 | "pluto.nixos.org:10029" |
| 15 | "titan.nixos.org:10029" |
| 16 | |
| 17 | # builders |
| 18 | "elated-minsky.builder.nixos.org:10029" |
| 19 | "sleepy-brown.builder.nixos.org:10029" |
| 20 | "goofy-hopcroft.builder.nixos.org:10029" |
| 21 | "hopeful-rivest.builder.nixos.org:10029" |
| 22 | |
| 23 | # non-critical |
| 24 | "caliban.nixos.org:10029" |
| 25 | ]; |
| 26 | } |
| 27 | ]; |
| 28 | } |
| 29 | ]; |
| 30 | |
| 31 | ruleFiles = [ |
| 32 | (pkgs.writeText "rasdaemon.rules" ( |
| 33 | builtins.toJSON { |
| 34 | groups = [ |
| 35 | { |
| 36 | name = "rasdaemon"; |
| 37 | rules = [ |
| 38 | { |
| 39 | alert = "MachineCheckError"; |
| 40 | expr = '' |
| 41 | increase(rasdaemon_mce_records_total{mce_msg!="Corrected error, no action required."}[1h]) > 0 |
| 42 | ''; |
| 43 | labels.severity = "warning"; |
| 44 | annotations.summary = "Machine check detected an error on {{ $labels.instance }}: {{ $labels.mce_msg }}"; |
| 45 | } |
| 46 | ]; |
| 47 | } |
| 48 | ]; |
| 49 | } |
| 50 | )) |
| 51 | ]; |
| 52 | }; |
| 53 | } |