| 1 | { pkgs, ... }: |
| 2 | |
| 3 | { |
| 4 | services.prometheus = { |
| 5 | scrapeConfigs = [ |
| 6 | { |
| 7 | job_name = "node"; |
| 8 | static_configs = [ |
| 9 | { |
| 10 | labels.role = "hydra"; |
| 11 | targets = [ |
| 12 | "mimas.nixos.org:9100" |
| 13 | ]; |
| 14 | } |
| 15 | { |
| 16 | labels.role = "database"; |
| 17 | targets = [ |
| 18 | "haumea.nixos.org:9100" |
| 19 | "titan.nixos.org:9100" |
| 20 | ]; |
| 21 | } |
| 22 | { |
| 23 | labels.role = "monitoring"; |
| 24 | targets = [ |
| 25 | "pluto.nixos.org:9100" |
| 26 | ]; |
| 27 | } |
| 28 | { |
| 29 | labels.role = "services"; |
| 30 | targets = [ |
| 31 | "caliban.nixos.org:9100" |
| 32 | "umbriel.nixos.org:9100" |
| 33 | "wiki.nixos.org:9100" |
| 34 | "tracker.security.nixos.org:9100" |
| 35 | "makemake.ngi.nixos.org:9100" |
| 36 | ]; |
| 37 | } |
| 38 | { |
| 39 | labels.role = "mac"; |
| 40 | targets = [ |
| 41 | # flying circus |
| 42 | "norwegian-blue.mac.nixos.org:9100" |
| 43 | # hetzner |
| 44 | "intense-heron.mac.nixos.org:9100" |
| 45 | "sweeping-filly.mac.nixos.org:9100" |
| 46 | "maximum-snail.mac.nixos.org:9100" |
| 47 | "growing-jennet.mac.nixos.org:9100" |
| 48 | "enormous-catfish.mac.nixos.org:9100" |
| 49 | # oakhost |
| 50 | "kind-lumiere.mac.nixos.org:9100" |
| 51 | "eager-heisenberg.mac.nixos.org:9100" |
| 52 | # macstadium |
| 53 | "mac01.ofborg.org:9100" |
| 54 | "mac02.ofborg.org:9100" |
| 55 | "mac03.ofborg.org:9100" |
| 56 | "mac04.ofborg.org:9100" |
| 57 | "mac05.ofborg.org:9100" |
| 58 | ]; |
| 59 | } |
| 60 | { |
| 61 | labels.role = "builders"; |
| 62 | targets = [ |
| 63 | "elated-minsky.builder.nixos.org:9100" |
| 64 | "sleepy-brown.builder.nixos.org:9100" |
| 65 | "goofy-hopcroft.builder.nixos.org:9100" |
| 66 | "hopeful-rivest.builder.nixos.org:9100" |
| 67 | ]; |
| 68 | } |
| 69 | { |
| 70 | labels.role = "ofborg"; |
| 71 | targets = [ |
| 72 | "build01.ofborg.org:9100" |
| 73 | "build02.ofborg.org:9100" |
| 74 | "build03.ofborg.org:9100" |
| 75 | "build04.ofborg.org:9100" |
| 76 | "build05.ofborg.org:9100" |
| 77 | "core01.ofborg.org:9100" |
| 78 | "eval01.ofborg.org:9100" |
| 79 | "eval02.ofborg.org:9100" |
| 80 | "eval03.ofborg.org:9100" |
| 81 | "eval04.ofborg.org:9100" |
| 82 | ]; |
| 83 | } |
| 84 | ]; |
| 85 | } |
| 86 | ]; |
| 87 | |
| 88 | ruleFiles = |
| 89 | let |
| 90 | diskSelector = ''mountpoint="/"''; |
| 91 | in |
| 92 | [ |
| 93 | (pkgs.writeText "node-exporter.rules" ( |
| 94 | builtins.toJSON { |
| 95 | groups = [ |
| 96 | { |
| 97 | name = "node"; |
| 98 | rules = [ |
| 99 | { |
| 100 | alert = "PartitionLowInodes"; |
| 101 | expr = '' |
| 102 | node_filesystem_files_free{${diskSelector}} / node_filesystem_files{${diskSelector}} * 100 < 10 |
| 103 | ''; |
| 104 | for = "60m"; |
| 105 | labels.severity = "warning"; |
| 106 | annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has only {{ $value }}% free inodes."; |
| 107 | annotations.grafana = "https://grafana.nixos.org/d/rYdddlPWk/node-exporter-full?orgId=1&var-job=node&var-node={{ $labels.instance }}"; |
| 108 | } |
| 109 | { |
| 110 | alert = "PartitionLowDiskSpace"; |
| 111 | expr = '' |
| 112 | round((node_filesystem_free_bytes{${diskSelector}} * 100) / node_filesystem_size_bytes{${diskSelector}}) < 10 and ON (instance, device, mountpoint) node_filesystem_free_bytes < 100 * 1024^3 |
| 113 | ''; |
| 114 | for = "60m"; |
| 115 | labels.severity = "warning"; |
| 116 | annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }}% free."; |
| 117 | annotations.grafana = "https://grafana.nixos.org/d/rYdddlPWk/node-exporter-full?orgId=1&var-job=node&var-node={{ $labels.instance }}"; |
| 118 | } |
| 119 | { |
| 120 | alert = "SystemdUnitFailed"; |
| 121 | expr = '' |
| 122 | node_systemd_unit_state{state="failed"} == 1 |
| 123 | ''; |
| 124 | for = "15m"; |
| 125 | labels.severity = "warning"; |
| 126 | annotations.summary = "systemd unit {{ $labels.name }} on {{ $labels.instance }} has been down for more than 15 minutes."; |
| 127 | } |
| 128 | ]; |
| 129 | } |
| 130 | { |
| 131 | name = "scheduled-jobs"; |
| 132 | rules = [ |
| 133 | { |
| 134 | alert = "ChannelUpdateStuck"; |
| 135 | expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1''; |
| 136 | for = "30m"; |
| 137 | labels.severity = "warning"; |
| 138 | annotations.summary = "{{ $labels.name }} on {{ $labels.instance }}"; |
| 139 | annotations.grafana = "https://grafana.nixos.org/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s"; |
| 140 | } |
| 141 | ]; |
| 142 | } |
| 143 | ]; |
| 144 | } |
| 145 | )) |
| 146 | ]; |
| 147 | }; |
| 148 | } |