@cryptotaxi247 / infra-1 / commits / 02316f27

eris: split up configuration into reusable parts

Martin Weinelt committed Feb 24, 2024 at 15:48 UTC 02316f27da8a81dde215926d6a10bd702db597c9
28 files changed +904 -913
delft/eris-physical.nix deleted
-59
@@ -1,59 +0,0 @@
1 -# Transitional. This is the auto-generated nixops config for eris, extracted
2 -# into a file that we can check in and import when evaluating outside of
3 -# nixops.
4 -
5 -{ config, lib, pkgs, modulesPath, ... }: {
6 - config = {
7 - boot.kernelModules = [];
8 - networking = {
9 - hostName = "eris";
10 - extraHosts = ''
11 - 138.201.32.77 eris eris-unencrypted
12 - 127.0.0.1 eris-encrypted
13 - 46.4.89.205 haumea haumea-unencrypted
14 - '';
15 - firewall.trustedInterfaces = [];
16 - };
17 - };
18 - imports = [
19 - {
20 - config = {
21 - users.extraUsers.root.openssh.authorizedKeys.keys = [
22 - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIXnddtPlqCgmGK3yE48/eoUke4u2O2SIin6kUp4T1eZ NixOps client key of eris"
23 - ];
24 - };
25 - imports = [
26 - ({
27 - swapDevices = [
28 - { label = "swap1"; }
29 - { label = "swap2"; }
30 - ];
31 - boot.loader.grub.devices = [
32 - "/dev/sda"
33 - "/dev/sdb"
34 - ];
35 - fileSystems = {
36 - "/" = {
37 - fsType = "ext4";
38 - label = "root";
39 - };
40 - };
41 - })
42 - ({ config, lib, pkgs, ... }:
43 -
44 - {
45 - imports =
46 - [ "${modulesPath}/installer/scan/not-detected.nix"
47 - ];
48 -
49 - boot.initrd.availableKernelModules = [ "ahci" "sd_mod" ];
50 - boot.kernelModules = [ "kvm-intel" ];
51 - boot.extraModulePackages = [ ];
52 -
53 - nix.settings.max-jobs = lib.mkDefault 8;
54 - powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
55 - })
56 - ];
57 - }
58 - ];
59 -}
delft/eris.nix deleted
-753
@@ -1,753 +0,0 @@
1 -{ options, config, lib, pkgs, ... }:
2 -let
3 - inherit (lib) filterAttrs flip mapAttrsToList;
4 -in
5 -{
6 - imports = [
7 - ./common.nix
8 - ../modules/wireguard.nix
9 - ./eris/packet-spot-market-prices.nix
10 - ./eris/github-project-monitor.nix
11 - ./eris/alertmanager-matrix-forwarder.nix
12 - ./eris/channel-monitor.nix
13 - ./eris/network.nix
14 - ];
15 -
16 - system.stateVersion = "18.03";
17 -
18 - users.users.root.openssh.authorizedKeys.keys =
19 - with import ../ssh-keys.nix; infra-core;
20 -
21 - networking.extraHosts = ''
22 - 10.254.1.1 bastion
23 - 10.254.1.5 rhea
24 - 10.254.1.6 pluto
25 -
26 - 10.254.1.9 haumea
27 -
28 - 10.254.3.1 webserver
29 - '';
30 -
31 - networking.firewall.allowedTCPPorts = [
32 - 443
33 - 80 # nginx
34 - 9090 # prometheus's web UI
35 - 9200 # hydra-queue-runner rexporter
36 - ];
37 -
38 - networking.firewall.interfaces.wg0.allowedTCPPorts = [
39 - 9093 # alertmanager
40 - ];
41 -
42 - systemd.units."mdmonitor.service".enable = false;
43 -
44 - services.fstrim.enable = true;
45 -
46 - zramSwap.enable = true; # Channel scripts can be memory hungry.
47 -
48 - services.nginx = {
49 - enable = true;
50 - recommendedProxySettings = true;
51 -
52 - eventsConfig = ''
53 - worker_connections 4096;
54 - '';
55 -
56 - virtualHosts."monitoring.nixos.org" = {
57 - enableACME = true;
58 - forceSSL = true;
59 - default = true;
60 - locations."/".return = "302 https://status.nixos.org";
61 - locations."/prometheus/".proxyPass = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
62 - locations."/grafana/".proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}/";
63 - };
64 - };
65 -
66 - services.prometheus = {
67 - enable = true;
68 - extraFlags = [
69 - "--storage.tsdb.retention=${toString (150 * 24)}h"
70 - "--web.external-url=https://monitoring.nixos.org/prometheus/"
71 - ];
72 -
73 - exporters.domain = {
74 - enable = true;
75 - listenAddress = "localhost";
76 - };
77 -
78 - exporters.blackbox = {
79 - enable = true;
80 - listenAddress = "127.0.0.1";
81 - configFile = pkgs.writeText "probes.yml" (builtins.toJSON {
82 - modules.https_success = {
83 - prober = "http";
84 - tcp.tls = true;
85 - http.headers.User-Agent = "blackbox-exporter";
86 - };
87 - });
88 - };
89 -
90 - alertmanagers = [
91 - {
92 - scheme = "http";
93 - static_configs = [
94 - {
95 - targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ];
96 - }
97 - ];
98 - }
99 - ];
100 -
101 - alertmanager = {
102 - enable = true;
103 -
104 - # Allow alertmanager to start even if it doesn't find an RFC1918 IP on
105 - # the machine's network interfaces.
106 - extraFlags = [ "--cluster.listen-address=''" ];
107 -
108 - webExternalUrl = "http://10.254.1.4:9093/";
109 - configuration = {
110 - global = { };
111 - route = {
112 - receiver = "ignore";
113 - group_wait = "30s";
114 - group_interval = "5m";
115 - repeat_interval = "4h";
116 - group_by = [ "alertname" ];
117 -
118 - routes = [
119 - {
120 - receiver = "go-neb";
121 - group_wait = "30s";
122 - match.severity = "warning";
123 - }
124 - ];
125 - };
126 - receivers = [
127 - {
128 - # with no *_config, this will drop all alerts directed to it
129 - name = "ignore";
130 - }
131 - {
132 - name = "go-neb";
133 - webhook_configs = [
134 - {
135 - url = "${config.services.go-neb.baseUrl}:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
136 - send_resolved = true;
137 - }
138 - ];
139 - }
140 - ];
141 - };
142 - };
143 -
144 - rules = [
145 - (builtins.toJSON {
146 - groups = [
147 - {
148 - name = "hydra";
149 - rules = [
150 - {
151 - alert = "BuildsStuckOverTwoDays";
152 - expr = ''hydra_machine_build_duration_bucket{le="+Inf"} - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0'';
153 - for = "30m";
154 - labels.severity = "warning";
155 - annotations.summary = "{{ $labels.machine }} has {{ $value }} over-age jobs.";
156 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
157 - }
158 - {
159 - alert = "HydraQueueRunnerUp";
160 - expr = ''up{job="hydra_queue_runner"} == 0'';
161 - for = "30m";
162 - labels.severity = "warning";
163 - annotations.summary = "hydra-queue-runner's prometheus exporter is not up";
164 - }
165 - ];
166 - }
167 -
168 - {
169 - name = "system";
170 - rules =
171 - let
172 - diskSelector = ''mountpoint=~"(/|/scratch)",instance!~".*packethost.net"'';
173 - relevantLabels = "device,fstype,instance,mountpoint";
174 - in
175 - [
176 - {
177 - alert = "PartitionLowInodes";
178 - expr = ''
179 - avg (node_filesystem_files_free{${diskSelector}} <= 10000) by (${relevantLabels})
180 - '';
181 - for = "30m";
182 - labels.severity = "warning";
183 - annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} inodes free.";
184 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
185 - }
186 -
187 - {
188 - alert = "PartitionLowDiskSpace";
189 - expr = ''
190 - (avg (round(node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)) by (${relevantLabels}))
191 - or
192 - (avg (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10) by (${relevantLabels}))
193 - '';
194 - for = "30m";
195 - labels.severity = "warning";
196 - annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} GB free.";
197 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
198 - }
199 -
200 - {
201 - alert = "SystemdUnitFailed";
202 - expr = ''node_systemd_unit_state{state="failed"} == 1'';
203 - for = "15m";
204 - labels.severity = "warning";
205 - annotations.summary = "systemd unit {{ $labels.name }} on {{ $labels.instance }} has been down for more than 15 minutes.";
206 - }
207 - ];
208 - }
209 -
210 - {
211 - name = "scheduled-jobs";
212 - rules = [
213 - {
214 - alert = "RFC39MaintainerSync";
215 - expr = ''node_systemd_unit_state{name=~"^rfc39-sync.service$", state="failed"} == 1'';
216 - for = "30m";
217 - labels.severity = "warning";
218 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
219 - }
220 - {
221 - alert = "ChannelUpdateStuck";
222 - expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1'';
223 - for = "30m";
224 - labels.severity = "warning";
225 - annotations.summary = "{{ $labels.name }} on {{ $labels.instance }}";
226 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
227 - }
228 - ];
229 - }
230 -
231 - {
232 - name = "blackbox";
233 - rules = [
234 - {
235 - alert = "CertificateExpiry";
236 - expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 14";
237 - for = "10m";
238 - labels.severity = "warning";
239 - annotations.summary = "Certificate for {{ $labels.instance }} is expiring soon.";
240 - }
241 - ];
242 - }
243 -
244 - {
245 - name = "domain";
246 - rules = [
247 - {
248 - alert = "DomainExpiry";
249 - expr = "domain_expiry_days < 30";
250 - for = "1h";
251 - labels.severity = "warning";
252 - annotations.summary = "Domain {{ $labels.domain }} will expire in less than 30 days";
253 - }
254 - ];
255 - }
256 - ];
257 - })
258 - ];
259 -
260 - globalConfig.scrape_interval = "15s";
261 - scrapeConfigs = [
262 - {
263 - job_name = "node";
264 - static_configs = [
265 - {
266 - targets = [
267 - "rhea:9100"
268 - ];
269 - labels.role = "hydra";
270 - }
271 - {
272 - targets = [
273 - "eris:9100"
274 - "pluto:9100"
275 - ];
276 - labels.role = "monitoring";
277 - }
278 - {
279 - targets = [
280 - "haumea:9100"
281 - ];
282 - labels.role = "database";
283 - }
284 - {
285 - targets = [
286 - "bastion:9100"
287 - ];
288 - labels.role = "bastion";
289 - }
290 - {
291 - targets = [
292 - "intense-heron.mac.nixos.org:9100"
293 - "sweeping-filly.mac.nixos.org:9100"
294 - "maximum-snail.mac.nixos.org:9100"
295 - "growing-jennet.mac.nixos.org:9100"
296 - "enormous-catfish.mac.nixos.org:9100"
297 - ];
298 - labels.role = "mac";
299 - }
300 - ];
301 - }
302 - {
303 - job_name = "nixos";
304 - static_configs = [
305 - {
306 - targets = [
307 - "rhea:9300"
308 - ];
309 - labels.role = "hydra";
310 - }
311 - {
312 - targets = [
313 - "eris:9300"
314 - "pluto:9300"
315 - ];
316 - labels.role = "monitoring";
317 - }
318 - {
319 - targets = [
320 - "haumea:9300"
321 - ];
322 - labels.role = "database";
323 - }
324 - {
325 - targets = [
326 - "bastion:9300"
327 - ];
328 - labels.role = "bastion";
329 - }
330 - ];
331 - }
332 - {
333 - job_name = "packet_nodes";
334 - file_sd_configs = [
335 - {
336 - files = [ "/var/lib/packet-sd/packet-sd.json" ];
337 - refresh_interval = "30s";
338 - }
339 - ];
340 - relabel_configs = [
341 - {
342 - source_labels = [ "__meta_packet_public_ipv4" ];
343 - target_label = "__address__";
344 - replacement = "\${1}:9100";
345 - action = "replace";
346 - }
347 - {
348 - source_labels = [ "__meta_packet_facility" ];
349 - target_label = "facility";
350 - }
351 - {
352 - source_labels = [ "__meta_packet_facility" ];
353 - target_label = "packet_facility";
354 - }
355 - {
356 - source_labels = [ "__meta_packet_plan" ];
357 - target_label = "plan";
358 - }
359 - {
360 - source_labels = [ "__meta_packet_plan" ];
361 - target_label = "packet_plan";
362 - }
363 - {
364 - # todo: change from _id to _uuid
365 - source_labels = [ "__meta_packet_switch_id" ];
366 - target_label = "packet_switch_id";
367 - }
368 - {
369 - source_labels = [ "__meta_packet_device_id" ];
370 - target_label = "packet_device_id";
371 - }
372 - {
373 - source_labels = [ "__meta_packet_state" ];
374 - target_label = "packet_device_state";
375 - }
376 - {
377 - source_labels = [ "__meta_packet_short_id" ];
378 - target_label = "instance";
379 - replacement = "\${1}.packethost.net";
380 - action = "replace";
381 - }
382 - {
383 - source_labels = [ "__meta_packet_tags" ];
384 - target_label = "role";
385 - regex = ".*hydra.*";
386 - replacement = "builder";
387 - action = "replace";
388 - }
389 - {
390 - source_labels = [ "__meta_packet_tags" ];
391 - regex = ".*prometheus-scraping-disabled.*";
392 - action = "drop";
393 - }
394 - ];
395 - }
396 - {
397 - job_name = "haumea-postgresql";
398 - metrics_path = "/metrics";
399 - static_configs = [
400 - {
401 - targets = [
402 - "haumea:9187"
403 - ];
404 - }
405 - ];
406 - }
407 - {
408 - # https://github.com/caarlos0/domain_exporter#configuration
409 - job_name = "domain";
410 - metrics_path = "/probe";
411 - relabel_configs = [
412 - {
413 - source_labels = [ "__address__" ];
414 - target_label = "__param_target";
415 - }
416 - {
417 - target_label = "__address__";
418 - replacement = "localhost:9222";
419 - }
420 - ];
421 - static_configs = [
422 - {
423 - targets = [
424 - "nix.ci"
425 - "nix.dev"
426 - "nixos.org"
427 - "ofborg.org"
428 - ];
429 - }
430 - ];
431 - }
432 - {
433 - job_name = "fastly";
434 - metrics_path = "/metrics";
435 - static_configs = [
436 - {
437 - targets = [
438 - "127.0.0.1:9118"
439 - ];
440 - }
441 - ];
442 - }
443 - {
444 - job_name = "rfc39";
445 - metrics_path = "/";
446 - static_configs = [
447 - {
448 - targets = [
449 - "127.0.0.1:9190"
450 - ];
451 - }
452 - ];
453 - }
454 - {
455 - job_name = "hydra-reexport";
456 - metrics_path = "/";
457 - static_configs = [
458 - {
459 - targets = [
460 - "monitoring.nixos.org:9200"
461 - ];
462 - }
463 - ];
464 - }
465 - {
466 - job_name = "hydra";
467 - metrics_path = "/prometheus";
468 - scheme = "https";
469 - static_configs = [
470 - {
471 - targets = [
472 - "hydra.nixos.org:443"
473 - ];
474 - }
475 - ];
476 - }
477 - {
478 - job_name = "hydra_notify";
479 - metrics_path = "/metrics";
480 - scheme = "http";
481 - static_configs = [
482 - {
483 - targets = [
484 - "hydra.nixos.org:9199"
485 - ];
486 - }
487 - ];
488 - }
489 - {
490 - job_name = "hydra_queue_runner";
491 - metrics_path = "/metrics";
492 - scheme = "http";
493 - static_configs = [
494 - {
495 - targets = [
496 - "hydra.nixos.org:9198"
497 - ];
498 - }
499 - ];
500 - }
501 -
502 - {
503 - job_name = "hydra-webserver";
504 - metrics_path = "/metrics";
505 - scheme = "https";
506 - static_configs = [
507 - {
508 - targets = [
509 - "hydra.nixos.org:443"
510 - ];
511 - }
512 - ];
513 - }
514 -
515 -
516 - {
517 - job_name = "prometheus-packet-sd";
518 - metrics_path = "/metrics";
519 - static_configs = [
520 - {
521 - targets = [
522 - "127.0.0.1:9465"
523 - ];
524 - }
525 - ];
526 - }
527 -
528 - {
529 - job_name = "prometheus-packet-spot-price-exporter";
530 - metrics_path = "/metrics";
531 - static_configs = [
532 - {
533 - targets = [
534 - "127.0.0.1:9400"
535 - ];
536 - }
537 - ];
538 - }
539 -
540 - {
541 - job_name = "prometheus-github-exporter";
542 - metrics_path = "/";
543 - static_configs = [
544 - {
545 - targets = [
546 - "127.0.0.1:9401"
547 - ];
548 - }
549 - ];
550 - }
551 -
552 - {
553 - job_name = "r13y";
554 - scheme = "https";
555 - metrics_path = "/metrics";
556 - static_configs = [
557 - {
558 - targets = [
559 - "r13y.com"
560 - ];
561 - }
562 - ];
563 - }
564 -
565 - {
566 - job_name = "channel-updates";
567 - metrics_path = "/";
568 - static_configs = [
569 - {
570 - targets = [
571 - "127.0.0.1:9402"
572 - ];
573 - }
574 - ];
575 - }
576 - ]
577 - ++ (
578 - let
579 - mkProbe = module: targets: {
580 - job_name = "blackbox-${module}";
581 - metrics_path = "/probe";
582 - params = {
583 - module = [ module ];
584 - };
585 - static_configs = [{ inherit targets; }];
586 - relabel_configs = [{
587 - source_labels = [ "__address__" ];
588 - target_label = "__param_target";
589 - }
590 - {
591 - source_labels = [ "__param_target" ];
592 - target_label = "instance";
593 - }
594 - {
595 - target_label = "__address__";
596 - replacement = "localhost:9115";
597 - }];
598 - };
599 - in
600 - [
601 - (mkProbe "https_success" [
602 - "https://cache.nixos.org"
603 - "https://channels.nixos.org"
604 - "https://common-styles.nixos.org"
605 - "https://conf.nixos.org"
606 - "https://discourse.nixos.org"
607 - "https://hydra.nixos.org"
608 - "https://mobile.nixos.org"
609 - "https://monitoring.nixos.org"
610 - "https://nixos.org"
611 - "https://planet.nixos.org"
612 - "https://releases.nixos.org"
613 - "https://status.nixos.org"
614 - "https://survey.nixos.org"
615 - "https://tarballs.nixos.org"
616 - "https://weekly.nixos.org"
617 - "https://www.nixos.org"
618 - "https://netboot.nixos.org"
619 - ])
620 - ]
621 - )
622 - ++ lib.mapAttrsToList
623 - (name: value: {
624 - job_name = "channel-job-${name}";
625 - scheme = "https";
626 - scrape_interval = "5m";
627 - metrics_path = "/job/${value.job}/prometheus";
628 - static_configs = [{
629 - labels = {
630 - current = if value.status != "unmaintained" then "1" else "0";
631 - channel = name;
632 - };
633 - targets = [ "hydra.nixos.org:443" ];
634 - }];
635 - })
636 - (import ../channels.nix).channels;
637 - };
638 -
639 - systemd.services.pull-nixos-metrics = {
640 - description = "Pull nixos metrics from github:NixOS/nixos-metrics and push to local VictoriaMetrics";
641 - script =
642 - let
643 - inherit (config.services.victoriametrics) listenAddress;
644 - importURL = "http://localhost${listenAddress}/api/v1/import";
645 - resetURL = "http://localhost${listenAddress}/internal/resetRollupResultCache";
646 - dataURL = "https://raw.githubusercontent.com/NixOS/nixos-metrics/data/victoriametrics.jsonl";
647 - curl = "${pkgs.curl}/bin/curl";
648 - in
649 - ''
650 - ${curl} ${dataURL} | ${curl} -X POST --data-binary @- ${importURL}
651 - ${curl} -G ${resetURL}
652 - '';
653 - serviceConfig = {
654 - Type = "oneshot";
655 - User = "nobody";
656 - };
657 - };
658 -
659 - systemd.timers.pull-nixos-metrics = {
660 - description = "Pull nixos metrics, timed for after they're done updating each day.";
661 - wantedBy = [ "timers.target" ];
662 - timerConfig.OnCalendar = "12:00:00";
663 - };
664 -
665 - services.victoriametrics = {
666 - enable = true;
667 - retentionPeriod = 1200; # 100 years
668 - };
669 -
670 - services.grafana = {
671 - enable = true;
672 - auth.anonymous.enable = true;
673 - users.allowSignUp = true;
674 - addr = "0.0.0.0";
675 - domain = "monitoring.nixos.org";
676 - rootUrl = "https://monitoring.nixos.org/grafana/";
677 - };
678 -
679 - systemd.services.prometheus-hydra-exporter = {
680 - wantedBy = [ "multi-user.target" ];
681 - after = [ "network.target" ];
682 - serviceConfig = {
683 - Restart = "always";
684 - RestartSec = "60s";
685 - PrivateTmp = true;
686 - WorkingDirectory = "/tmp";
687 - ExecStart =
688 - let
689 - python = pkgs.python3.withPackages (p: [
690 - p.requests
691 - p.prometheus_client
692 - ]);
693 - in
694 - ''
695 - ${python}/bin/python ${./prometheus/hydra-queue-runner-reexporter.py}
696 - '';
697 - };
698 - };
699 -
700 - age.secrets.packet-sd-env = {
701 - file = ./secrets/packet-sd-env.age;
702 - owner = "packet-sd";
703 - };
704 -
705 - users.users.packet-sd = {
706 - description = "Prometheus Packet Service Discovery";
707 - isSystemUser = true;
708 - group = "packet-sd";
709 - };
710 - users.groups.packet-sd = { };
711 -
712 - systemd.tmpfiles.rules = [
713 - "d /var/lib/packet-sd 0755 packet-sd - -"
714 - "f /var/lib/packet-sd/packet-sd.json 0644 packet-sd - -"
715 - ];
716 -
717 - systemd.services.prometheus-packet-sd =
718 - let
719 - sd = pkgs.callPackage ./prometheus/packet-sd.nix { };
720 - in
721 - {
722 - wantedBy = [ "multi-user.target" "prometheus.service" ];
723 - after = [ "network.target" ];
724 -
725 - serviceConfig = {
726 - User = "packet-sd";
727 - Group = "keys";
728 - ExecStart = "${sd}/bin/prometheus-packet-sd --output.file=/var/lib/packet-sd/packet-sd.json";
729 - EnvironmentFile = config.age.secrets.packet-sd-env.path;
730 - Restart = "always";
731 - RestartSec = "60s";
732 - };
733 - };
734 -
735 - age.secrets.fastly-read-only-api-token.file = ./secrets/fastly-read-only-api-token.age;
736 -
737 - systemd.services.prometheus-fastly-exporter = {
738 - # module script is outdated; https://github.com/NixOS/nixpkgs/pull/287348
739 - script = with config.services.prometheus.exporters.fastly; lib.mkForce ''
740 - export FASTLY_API_TOKEN=$(cat ${tokenPath})
741 - ${pkgs.prometheus-fastly-exporter}/bin/fastly-exporter \
742 - -listen ${listenAddress}:${toString port}
743 - '';
744 - serviceConfig.LoadCredential = "fastyl-api-token:${config.age.secrets.fastly-read-only-api-token.path}";
745 - };
746 -
747 - services.prometheus.exporters.fastly = {
748 - enable = true;
749 - listenAddress = "127.0.0.1";
750 - tokenPath = "/run/credentials/prometheus-fastly-exporter.service/fastyl-api-token";
751 - };
752 -
753 -}
delft/eris/alertmanager-matrix-forwarder.nix deleted
-77
@@ -1,77 +0,0 @@
1 -{ config, pkgs, ... }:
2 -{
3 - age.secrets.alertmanager-matrix-forwarder = {
4 - file = ../secrets/alertmanager-matrix-forwarder.age;
5 - owner = config.systemd.services.go-neb.serviceConfig.User;
6 - };
7 -
8 - # Create user so that we can set the ownership of the key to
9 - # it. DynamicUser will not take full effect as a result of this.
10 - users.users.go-neb = {
11 - isSystemUser = true;
12 - group = "go-neb";
13 - };
14 - users.groups.go-neb = {};
15 -
16 - systemd.services.go-neb.serviceConfig.SupplementaryGroups = [ "keys" ];
17 -
18 - services.go-neb = {
19 - enable = true;
20 - baseUrl = "http://localhost";
21 - secretFile = config.age.secrets.alertmanager-matrix-forwarder.path;
22 - config = {
23 - clients = [
24 - {
25 - UserId = "@bot:nixos.org";
26 - AccessToken = "$CHANGEME";
27 - HomeServerUrl = "https://nixos.ems.host";
28 - Sync = true;
29 - AutoJoinRooms = true;
30 - DisplayName = "Bot";
31 - }
32 - ];
33 - services = [
34 - {
35 - ID = "alertmanager_service";
36 - Type = "alertmanager";
37 - UserId = "@bot:nixos.org";
38 - Config = {
39 - webhook_url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
40 - rooms = {
41 - "!QLQqibtFaVtDgurUAE:nixos.org" = {
42 - #bots:nixos.org
43 - text_template = ''
44 - {{range .Alerts -}} [{{ .Status }}] {{index .Labels "alertname" }}: {{index .Annotations "description"}} {{ end -}}
45 - '';
46 -
47 - # $$severity otherwise envsubst replaces $severity with an empty string
48 - html_template = ''
49 - {{range .Alerts -}}
50 - {{ $$severity := index .Labels "severity" }}
51 - {{ if eq .Status "firing" }}
52 - {{ if eq $$severity "critical"}}
53 - <font color='red'><b>[FIRING - CRITICAL]</b></font>
54 - {{ else if eq $$severity "warning"}}
55 - <font color='orange'><b>[FIRING - WARNING]</b></font>
56 - {{ else }}
57 - <b>[FIRING - {{ $$severity }}]</b>
58 - {{ end }}
59 - {{ else }}
60 - <font color='green'><b>[RESOLVED]</b></font>
61 - {{ end }}
62 - {{ index .Labels "alertname"}}: {{ index .Annotations "summary"}}
63 - (
64 - <a href="{{ index .Annotations "grafana" }}">📈 Grafana</a>,
65 - <a href="{{ .GeneratorURL }}">🔥 Prometheus</a>,
66 - <a href="{{ .SilenceURL }}">🔕 Silence</a>
67 - )<br/>
68 - {{end -}}'';
69 - msg_type = "m.text"; # Must be either `m.text` or `m.notice`
70 - };
71 - };
72 - };
73 - }
74 - ];
75 - };
76 - };
77 -}
delft/eris/boot.nix new
+10
@@ -0,0 +1,10 @@
1 +{
2 + boot.loader.grub = {
3 + devices = [
4 + "/dev/sda"
5 + "/dev/sdb"
6 + ];
7 + };
8 + boot.initrd.availableKernelModules = [ "ahci" "sd_mod" ];
9 + boot.kernelModules = [ "kvm-intel" ];
10 +}
delft/eris/channel-monitor.nix deleted
-14
@@ -1,14 +0,0 @@
1 -{ pkgs, ... }:
2 -let
3 - channels = pkgs.writeText "channels.json" (builtins.toJSON (import ../../channels.nix).channels);
4 -in {
5 - systemd.services.channel-update-exporter = {
6 - description = "Check all active channels' last-update times";
7 - path = [ (pkgs.python3.withPackages (pypkgs: with pypkgs; [ requests dateutil prometheus_client ])) ];
8 - wantedBy = [ "multi-user.target" ];
9 - serviceConfig = {
10 - DynamicUser = true;
11 - ExecStart = "${./channel-exporter.py} ${channels}";
12 - };
13 - };
14 -}
\ No newline at end of file
delft/eris/default.nix new
+44
@@ -0,0 +1,44 @@
1 +{ lib
2 +, modulesPath
3 +, ...
4 +}:
5 +
6 +{
7 + imports = [
8 + "${modulesPath}/installer/scan/not-detected.nix"
9 + ../common.nix
10 + ./boot.nix
11 + ./grafana.nix
12 + ./network.nix
13 + ./nginx.nix
14 + ./nixos-metrics.nix
15 + ./prometheus
16 + ];
17 +
18 + networking = {
19 + hostName = "eris";
20 + domain = "nixos.org";
21 + };
22 +
23 + fileSystems = {
24 + "/" = {
25 + fsType = "ext4";
26 + label = "root";
27 + };
28 + };
29 +
30 + swapDevices = [
31 + { label = "swap1"; }
32 + { label = "swap2"; }
33 + ];
34 +
35 + systemd.units."mdmonitor.service".enable = false;
36 +
37 + services.fstrim.enable = true;
38 +
39 + nix.settings.max-jobs = lib.mkDefault 8;
40 +
41 + powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
42 +
43 + system.stateVersion = "18.03";
44 +}
delft/eris/grafana.nix new
+10
@@ -0,0 +1,10 @@
1 +{
2 + services.grafana = {
3 + enable = true;
4 + auth.anonymous.enable = true;
5 + users.allowSignUp = true;
6 + addr = "0.0.0.0";
7 + domain = "monitoring.nixos.org";
8 + rootUrl = "https://monitoring.nixos.org/grafana/";
9 + };
10 +}
delft/eris/nginx.nix new
+25
@@ -0,0 +1,25 @@
1 +{ config
2 +, ...
3 +}:
4 +
5 +{
6 + networking.firewall.allowedTCPPorts = [ 80 443 ];
7 +
8 + services.nginx = {
9 + enable = true;
10 + recommendedProxySettings = true;
11 +
12 + eventsConfig = ''
13 + worker_connections 4096;
14 + '';
15 +
16 + virtualHosts."monitoring.nixos.org" = {
17 + enableACME = true;
18 + forceSSL = true;
19 + default = true;
20 + locations."/".return = "302 https://status.nixos.org";
21 + locations."/prometheus/".proxyPass = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
22 + locations."/grafana/".proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}/";
23 + };
24 + };
25 +}
delft/eris/nixos-metrics.nix new
+37
@@ -0,0 +1,37 @@
1 +{ config
2 +, pkgs
3 +, ...
4 +}:
5 +
6 +{
7 + systemd.services.pull-nixos-metrics = {
8 + description = "Pull nixos metrics from github:NixOS/nixos-metrics and push to local VictoriaMetrics";
9 + script =
10 + let
11 + inherit (config.services.victoriametrics) listenAddress;
12 + importURL = "http://localhost${listenAddress}/api/v1/import";
13 + resetURL = "http://localhost${listenAddress}/internal/resetRollupResultCache";
14 + dataURL = "https://raw.githubusercontent.com/NixOS/nixos-metrics/data/victoriametrics.jsonl";
15 + curl = "${pkgs.curl}/bin/curl";
16 + in
17 + ''
18 + ${curl} ${dataURL} | ${curl} -X POST --data-binary @- ${importURL}
19 + ${curl} -G ${resetURL}
20 + '';
21 + serviceConfig = {
22 + Type = "oneshot";
23 + User = "nobody";
24 + };
25 + };
26 +
27 + systemd.timers.pull-nixos-metrics = {
28 + description = "Pull nixos metrics, timed for after they're done updating each day.";
29 + wantedBy = [ "timers.target" ];
30 + timerConfig.OnCalendar = "12:00:00";
31 + };
32 +
33 + services.victoriametrics = {
34 + enable = true;
35 + retentionPeriod = 1200; # 100 years
36 + };
37 +}
delft/eris/prometheus/alertmanager.nix new
+125
@@ -0,0 +1,125 @@
1 +{ config
2 +, ...
3 +}:
4 +
5 +{
6 + networking.firewall.interfaces.wg0.allowedTCPPorts = [
7 + 9093
8 + ];
9 +
10 + services.prometheus = {
11 + alertmanagers = [ {
12 + scheme = "http";
13 + static_configs = [ {
14 + targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ];
15 + } ];
16 + } ];
17 +
18 + alertmanager = {
19 + enable = true;
20 +
21 + # Allow alertmanager to start even if it doesn't find an RFC1918 IP on
22 + # the machine's network interfaces.
23 + extraFlags = [ "--cluster.listen-address=''" ];
24 +
25 + webExternalUrl = "http://10.254.1.4:9093/";
26 + configuration = {
27 + global = { };
28 + route = {
29 + receiver = "ignore";
30 + group_wait = "30s";
31 + group_interval = "5m";
32 + repeat_interval = "4h";
33 + group_by = [ "alertname" ];
34 +
35 + routes = [ {
36 + receiver = "go-neb";
37 + group_wait = "30s";
38 + match.severity = "warning";
39 + } ];
40 + };
41 + receivers = [ {
42 + # with no *_config, this will drop all alerts directed to it
43 + name = "ignore";
44 + } {
45 + name = "go-neb";
46 + webhook_configs = [ {
47 + url = "${config.services.go-neb.baseUrl}:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
48 + send_resolved = true;
49 + } ];
50 + } ];
51 + };
52 + };
53 + };
54 +
55 + age.secrets.alertmanager-matrix-forwarder = {
56 + file = ../../secrets/alertmanager-matrix-forwarder.age;
57 + owner = config.systemd.services.go-neb.serviceConfig.User;
58 + };
59 +
60 + # Create user so that we can set the ownership of the key to
61 + # it. DynamicUser will not take full effect as a result of this.
62 + users.users.go-neb = {
63 + isSystemUser = true;
64 + group = "go-neb";
65 + };
66 + users.groups.go-neb = {};
67 +
68 + systemd.services.go-neb.serviceConfig.SupplementaryGroups = [ "keys" ];
69 +
70 + services.go-neb = {
71 + enable = true;
72 + baseUrl = "http://localhost";
73 + secretFile = config.age.secrets.alertmanager-matrix-forwarder.path;
74 + config = {
75 + clients = [ {
76 + UserId = "@bot:nixos.org";
77 + AccessToken = "$CHANGEME";
78 + HomeServerUrl = "https://nixos.ems.host";
79 + Sync = true;
80 + AutoJoinRooms = true;
81 + DisplayName = "Bot";
82 + } ];
83 + services = [ {
84 + ID = "alertmanager_service";
85 + Type = "alertmanager";
86 + UserId = "@bot:nixos.org";
87 + Config = {
88 + webhook_url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
89 + rooms = {
90 + # infra-alerts:nixos.org
91 + "!QLQqibtFaVtDgurUAE:nixos.org" = {
92 + text_template = ''
93 + {{range .Alerts -}} [{{ .Status }}] {{index .Labels "alertname" }}: {{index .Annotations "description"}} {{ end -}}
94 + '';
95 +
96 + # $$severity otherwise envsubst replaces $severity with an empty string
97 + html_template = ''
98 + {{range .Alerts -}}
99 + {{ $$severity := index .Labels "severity" }}
100 + {{ if eq .Status "firing" }}
101 + {{ if eq $$severity "critical"}}
102 + <font color='red'><b>[FIRING - CRITICAL]</b></font>
103 + {{ else if eq $$severity "warning"}}
104 + <font color='orange'><b>[FIRING - WARNING]</b></font>
105 + {{ else }}
106 + <b>[FIRING - {{ $$severity }}]</b>
107 + {{ end }}
108 + {{ else }}
109 + <font color='green'><b>[RESOLVED]</b></font>
110 + {{ end }}
111 + {{ index .Labels "alertname"}}: {{ index .Annotations "summary"}}
112 + (
113 + <a href="{{ index .Annotations "grafana" }}">📈 Grafana</a>,
114 + <a href="{{ .GeneratorURL }}">🔥 Prometheus</a>,
115 + <a href="{{ .SilenceURL }}">🔕 Silence</a>
116 + )<br/>
117 + {{end -}}'';
118 + msg_type = "m.text"; # Must be either `m.text` or `m.notice`
119 + };
120 + };
121 + };
122 + } ];
123 + };
124 + };
125 +}
delft/eris/prometheus/default.nix new
+41
@@ -0,0 +1,41 @@
1 +{
2 + imports = [
3 + ./alertmanager.nix
4 + ./exporters/blackbox.nix
5 + ./exporters/channel.nix
6 + ./exporters/domain.nix
7 + ./exporters/fastly.nix
8 + ./exporters/github.nix
9 + ./exporters/hydra.nix
10 + ./exporters/nixos.nix
11 + ./exporters/node.nix
12 + ./exporters/packet-sd.nix
13 + ./exporters/packet-spot-market.nix
14 + ./exporters/postgresql.nix
15 + ./exporters/r13y.nix
16 + ./exporters/rfc39.nix
17 + ];
18 +
19 + networking.extraHosts = ''
20 + 10.254.1.1 bastion
21 + 10.254.1.5 rhea
22 + 10.254.1.6 pluto
23 +
24 + 10.254.1.9 haumea
25 +
26 + 10.254.3.1 webserver
27 + '';
28 +
29 + networking.firewall.allowedTCPPorts = [
30 + 9090
31 + ];
32 +
33 + services.prometheus = {
34 + enable = true;
35 + extraFlags = [
36 + "--storage.tsdb.retention=${toString (150 * 24)}h"
37 + "--web.external-url=https://monitoring.nixos.org/prometheus/"
38 + ];
39 + globalConfig.scrape_interval = "15s";
40 + };
41 +}
delft/eris/prometheus/exporters/blackbox.nix new
+77
@@ -0,0 +1,77 @@
1 +{ pkgs
2 +, ...
3 +}:
4 +
5 +let
6 + mkProbe = module: targets: {
7 + job_name = "blackbox-${module}";
8 + metrics_path = "/probe";
9 + params = {
10 + module = [ module ];
11 + };
12 + static_configs = [ {
13 + inherit targets;
14 + } ];
15 + relabel_configs = [ {
16 + source_labels = [ "__address__" ];
17 + target_label = "__param_target";
18 + } {
19 + source_labels = [ "__param_target" ];
20 + target_label = "instance";
21 + } {
22 + target_label = "__address__";
23 + replacement = "localhost:9115";
24 + } ];
25 + };
26 +in
27 +
28 +{
29 + services.prometheus = {
30 + exporters.blackbox = {
31 + enable = true;
32 + listenAddress = "127.0.0.1";
33 + configFile = pkgs.writeText "probes.yml" (builtins.toJSON {
34 + modules.https_success = {
35 + prober = "http";
36 + tcp.tls = true;
37 + http.headers.User-Agent = "blackbox-exporter";
38 + };
39 + });
40 + };
41 +
42 + scrapeConfigs = [
43 + (mkProbe "https_success" [
44 + "https://cache.nixos.org"
45 + "https://channels.nixos.org"
46 + "https://common-styles.nixos.org"
47 + "https://conf.nixos.org"
48 + "https://discourse.nixos.org"
49 + "https://hydra.nixos.org"
50 + "https://mobile.nixos.org"
51 + "https://monitoring.nixos.org"
52 + "https://nixos.org"
53 + "https://planet.nixos.org"
54 + "https://releases.nixos.org"
55 + "https://status.nixos.org"
56 + "https://survey.nixos.org"
57 + "https://tarballs.nixos.org"
58 + "https://weekly.nixos.org"
59 + "https://www.nixos.org"
60 + "https://netboot.nixos.org"
61 + ])
62 + ];
63 +
64 + rules = [
65 + (builtins.toJSON {
66 + name = "blackbox";
67 + rules = [ {
68 + alert = "CertificateExpiry";
69 + expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 14";
70 + for = "10m";
71 + labels.severity = "warning";
72 + annotations.summary = "Certificate for {{ $labels.instance }} is expiring soon.";
73 + } ];
74 + })
75 + ];
76 + };
77 +}
delft/eris/prometheus/exporters/channel-exporter.py renamed
delft/eris/prometheus/exporters/channel.nix new
+42
@@ -0,0 +1,42 @@
1 +{ lib
2 +, pkgs
3 +, ...
4 +}:
5 +
6 +let
7 + channels = pkgs.writeText "channels.json" (builtins.toJSON (import ../../../../channels.nix).channels);
8 +in {
9 + systemd.services.channel-update-exporter = {
10 + description = "Check all active channels' last-update times";
11 + path = [ (pkgs.python3.withPackages (pypkgs: with pypkgs; [ requests dateutil prometheus_client ])) ];
12 + wantedBy = [ "multi-user.target" ];
13 + serviceConfig = {
14 + DynamicUser = true;
15 + ExecStart = "${./channel-exporter.py} ${channels}";
16 + };
17 + };
18 +
19 + services.prometheus.scrapeConfigs = [ {
20 + job_name = "channel-updates";
21 + metrics_path = "/";
22 + static_configs = [ {
23 + targets = [
24 + "127.0.0.1:9402"
25 + ];
26 + } ];
27 + } ] ++ lib.mapAttrsToList
28 + (name: value: {
29 + job_name = "channel-job-${name}";
30 + scheme = "https";
31 + scrape_interval = "5m";
32 + metrics_path = "/job/${value.job}/prometheus";
33 + static_configs = [{
34 + labels = {
35 + current = if value.status != "unmaintained" then "1" else "0";
36 + channel = name;
37 + };
38 + targets = [ "hydra.nixos.org:443" ];
39 + }];
40 + })
41 + (import ../../../../channels.nix).channels;
42 +}
delft/eris/prometheus/exporters/domain.nix new
+44
@@ -0,0 +1,44 @@
1 +{
2 + services.prometheus = {
3 + exporters.domain = {
4 + enable = true;
5 + listenAddress = "localhost";
6 + };
7 +
8 + scrapeConfigs = [ {
9 + # https://github.com/caarlos0/domain_exporter#configuration
10 + job_name = "domain";
11 + metrics_path = "/probe";
12 + relabel_configs = [ {
13 + source_labels = [ "__address__" ];
14 + target_label = "__param_target";
15 + } {
16 + target_label = "__address__";
17 + replacement = "localhost:9222";
18 + } ];
19 + static_configs = [ {
20 + targets = [
21 + "nix.ci"
22 + "nix.dev"
23 + "nixos.org"
24 + "ofborg.org"
25 + ];
26 + } ];
27 + } ];
28 +
29 + rules = [
30 + (builtins.toJSON {
31 + groups = [ {
32 + name = "domain";
33 + rules = [ {
34 + alert = "DomainExpiry";
35 + expr = "domain_expiry_days < 30";
36 + for = "1h";
37 + labels.severity = "warning";
38 + annotations.summary = "Domain {{ $labels.domain }} will expire in less than 30 days";
39 + } ];
40 + } ];
41 + })
42 + ];
43 + };
44 +}
delft/eris/prometheus/exporters/fastly.nix new
+37
@@ -0,0 +1,37 @@
1 +{ config
2 +, lib
3 +, pkgs
4 +, ...
5 +}:
6 +
7 +{
8 + age.secrets.fastly-read-only-api-token.file = ../../../secrets/fastly-read-only-api-token.age;
9 +
10 + systemd.services.prometheus-fastly-exporter = {
11 + # module script is outdated; https://github.com/NixOS/nixpkgs/pull/287348
12 + script = with config.services.prometheus.exporters.fastly; lib.mkForce ''
13 + export FASTLY_API_TOKEN=$(cat ${tokenPath})
14 + ${pkgs.prometheus-fastly-exporter}/bin/fastly-exporter \
15 + -listen ${listenAddress}:${toString port}
16 + '';
17 + serviceConfig.LoadCredential = "fastyl-api-token:${config.age.secrets.fastly-read-only-api-token.path}";
18 + };
19 +
20 + services.prometheus = {
21 + exporters.fastly = {
22 + enable = true;
23 + listenAddress = "127.0.0.1";
24 + tokenPath = "/run/credentials/prometheus-fastly-exporter.service/fastyl-api-token";
25 + };
26 +
27 + scrapeConfigs = [ {
28 + job_name = "fastly";
29 + metrics_path = "/metrics";
30 + static_configs = [ {
31 + targets = [
32 + "127.0.0.1:9118"
33 + ];
34 + } ];
35 + } ];
36 + };
37 +}
delft/eris/prometheus/exporters/github.nix renamed
+24 -4
@@ -1,4 +1,7 @@
1 -{ config, pkgs, ... }:
1 +{ pkgs
2 +, ...
3 +}:
4 +
5 let
6 exporter = pkgs.fetchFromGitHub {
7 owner = "grahamc";
@@ -23,8 +26,12 @@ in {
26 users.groups.github-exporter = {};
27
28 systemd.services.prometheus-github-exporter = {
26 - wantedBy = [ "multi-user.target" ];
27 - after = [ "network.target" ];
29 + wantedBy = [
30 + "multi-user.target"
31 + ];
32 + after = [
33 + "network.target"
34 + ];
35 serviceConfig = {
36 User = "github-exporter";
37 Restart = "always";
@@ -33,9 +40,22 @@ in {
40 };
41
42 path = [
36 - (pkgs.python3.withPackages (p: [ p.prometheus_client p.requests ]))
43 + (pkgs.python3.withPackages (ps: with ps; [
44 + prometheus_client
45 + requests
46 + ]))
47 ];
48
49 script = "exec python3 ${exporter}/scrape.py ${config}";
50 };
51 +
52 + services.prometheus.scrapeConfigs = [ {
53 + job_name = "prometheus-github-exporter";
54 + metrics_path = "/";
55 + static_configs = [ {
56 + targets = [
57 + "127.0.0.1:9401"
58 + ];
59 + } ];
60 + } ];
61 }
delft/eris/prometheus/exporters/hydra-queue-runner-reexporter.py renamed
delft/eris/prometheus/exporters/hydra.nix new
+104
@@ -0,0 +1,104 @@
1 +{ pkgs
2 +, ...
3 +}:
4 +
5 +{
6 + networking.firewall.allowedTCPPorts = [ 9200 ];
7 +
8 + systemd.services.prometheus-hydra-queue-runner-exporter = {
9 + wantedBy = [
10 + "multi-user.target"
11 + ];
12 + after = [
13 + "network.target"
14 + ];
15 + wants = [
16 + "network.target"
17 + ];
18 + serviceConfig = {
19 + Restart = "always";
20 + RestartSec = "60s";
21 + PrivateTmp = true;
22 + WorkingDirectory = "/tmp";
23 + ExecStart = let
24 + python = pkgs.python3.withPackages (ps: with ps; [
25 + requests
26 + prometheus_client
27 + ]);
28 + in
29 + ''
30 + ${python.interpreter} ${./hydra-queue-runner-reexporter.py}
31 + '';
32 + };
33 + };
34 +
35 + services.prometheus = {
36 + scrapeConfigs = [ {
37 + job_name = "hydra";
38 + metrics_path = "/prometheus";
39 + scheme = "https";
40 + static_configs = [ {
41 + targets = [
42 + "hydra.nixos.org:443"
43 + ];
44 + } ];
45 + } {
46 + job_name = "hydra_notify";
47 + metrics_path = "/metrics";
48 + scheme = "http";
49 + static_configs = [ {
50 + targets = [
51 + "hydra.nixos.org:9199"
52 + ];
53 + } ];
54 + } {
55 + job_name = "hydra_queue_runner";
56 + metrics_path = "/metrics";
57 + scheme = "http";
58 + static_configs = [ {
59 + targets = [
60 + "hydra.nixos.org:9198"
61 + ];
62 + } ];
63 + } {
64 + job_name = "hydra-webserver";
65 + metrics_path = "/metrics";
66 + scheme = "https";
67 + static_configs = [ {
68 + targets = [
69 + "hydra.nixos.org:443"
70 + ];
71 + } ];
72 + } {
73 + job_name = "hydra-reexport";
74 + metrics_path = "/";
75 + static_configs = [ {
76 + targets = [
77 + "monitoring.nixos.org:9200"
78 + ];
79 + } ];
80 + } ];
81 +
82 + rules = [
83 + (builtins.toJSON {
84 + groups = [ {
85 + name = "hydra";
86 + rules = [ {
87 + alert = "BuildsStuckOverTwoDays";
88 + expr = ''hydra_machine_build_duration_bucket{le="+Inf"} - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0'';
89 + for = "30m";
90 + labels.severity = "warning";
91 + annotations.summary = "{{ $labels.machine }} has {{ $value }} over-age jobs.";
92 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
93 + } {
94 + alert = "HydraQueueRunnerUp";
95 + expr = ''up{job="hydra_queue_runner"} == 0'';
96 + for = "30m";
97 + labels.severity = "warning";
98 + annotations.summary = "hydra-queue-runner's prometheus exporter is not up";
99 + } ];
100 + } ];
101 + })
102 + ];
103 + };
104 +}
delft/eris/prometheus/exporters/nixos.nix new
+27
@@ -0,0 +1,27 @@
1 +{
2 + services.prometheus.scrapeConfigs = [ {
3 + job_name = "nixos";
4 + static_configs = [ {
5 + labels.role = "hydra";
6 + targets = [
7 + "rhea:9300"
8 + ];
9 + } {
10 + labels.role = "monitoring";
11 + targets = [
12 + "eris:9300"
13 + "pluto:9300"
14 + ];
15 + } {
16 + labels.role = "database";
17 + targets = [
18 + "haumea:9300"
19 + ];
20 + } {
21 + labels.role = "bastion";
22 + targets = [
23 + "bastion:9300"
24 + ];
25 + } ];
26 + } ];
27 +}
delft/eris/prometheus/exporters/node.nix new
+86
@@ -0,0 +1,86 @@
1 +{
2 + services.prometheus = {
3 + scrapeConfigs = [ {
4 + job_name = "node";
5 + static_configs = [ {
6 + labels.role = "hydra";
7 + targets = [
8 + "rhea:9100"
9 + ];
10 + } {
11 + labels.role = "database";
12 + targets = [
13 + "haumea:9100"
14 + ];
15 + } {
16 + labels.role = "monitoring";
17 + targets = [
18 + "eris:9100"
19 + "pluto:9100"
20 + ];
21 + } {
22 + labels.role = "bastion";
23 + targets = [
24 + "bastion:9100"
25 + ];
26 + } {
27 + labels.role = "mac";
28 + targets = [
29 + "intense-heron.mac.nixos.org:9100"
30 + "sweeping-filly.mac.nixos.org:9100"
31 + "maximum-snail.mac.nixos.org:9100"
32 + "growing-jennet.mac.nixos.org:9100"
33 + "enormous-catfish.mac.nixos.org:9100"
34 + ];
35 + } ];
36 + } ];
37 +
38 + rules = let
39 + diskSelector = ''mountpoint=~"(/|/scratch)",instance!~".*packethost.net"'';
40 + relevantLabels = "device,fstype,instance,mountpoint";
41 + in [
42 + (builtins.toJSON {
43 + groups = [ {
44 + name = "node";
45 + rules = [ {
46 + alert = "PartitionLowInodes";
47 + expr = ''
48 + avg (node_filesystem_files_free{${diskSelector}} <= 10000) by (${relevantLabels})
49 + '';
50 + for = "30m";
51 + labels.severity = "warning";
52 + annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} inodes free.";
53 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
54 + } {
55 + alert = "PartitionLowDiskSpace";
56 + expr = ''
57 + (avg (round(node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)) by (${relevantLabels}))
58 + or
59 + (avg (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10) by (${relevantLabels}))
60 + '';
61 + for = "30m";
62 + labels.severity = "warning";
63 + annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} GB free.";
64 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
65 + } {
66 + alert = "SystemdUnitFailed";
67 + expr = ''node_systemd_unit_state{state="failed"} == 1'';
68 + for = "15m";
69 + labels.severity = "warning";
70 + annotations.summary = "systemd unit {{ $labels.name }} on {{ $labels.instance }} has been down for more than 15 minutes.";
71 + } ];
72 + } {
73 + name = "scheduled-jobs";
74 + rules = [ {
75 + alert = "ChannelUpdateStuck";
76 + expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1'';
77 + for = "30m";
78 + labels.severity = "warning";
79 + annotations.summary = "{{ $labels.name }} on {{ $labels.instance }}";
80 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
81 + } ];
82 + } ];
83 + })
84 + ];
85 + };
86 +}
delft/eris/prometheus/exporters/packages/packet-sd.nix renamed
delft/eris/prometheus/exporters/packet-sd.nix new
+107
@@ -0,0 +1,107 @@
1 +{ config
2 +, pkgs
3 +, ...
4 +}:
5 +
6 +let
7 + packet-sd = pkgs.callPackage ./packages/packet-sd.nix { };
8 +in
9 +
10 +{
11 + age.secrets.packet-sd-env = {
12 + file = ../../../secrets/packet-sd-env.age;
13 + owner = "packet-sd";
14 + };
15 +
16 + users.users.packet-sd = {
17 + description = "Prometheus Packet Service Discovery";
18 + isSystemUser = true;
19 + group = "packet-sd";
20 + };
21 + users.groups.packet-sd = { };
22 +
23 + systemd.tmpfiles.rules = [
24 + "d /var/lib/packet-sd 0755 packet-sd - -"
25 + "f /var/lib/packet-sd/packet-sd.json 0644 packet-sd - -"
26 + ];
27 +
28 + systemd.services.prometheus-packet-sd = {
29 + wantedBy = [
30 + "multi-user.target"
31 + "prometheus.service"
32 + ];
33 + after = [
34 + "network.target"
35 + ];
36 +
37 + serviceConfig = {
38 + User = "packet-sd";
39 + Group = "keys";
40 + ExecStart = "${packet-sd}/bin/prometheus-packet-sd --output.file=/var/lib/packet-sd/packet-sd.json";
41 + EnvironmentFile = config.age.secrets.packet-sd-env.path;
42 + Restart = "always";
43 + RestartSec = "60s";
44 + };
45 + };
46 +
47 + services.prometheus = {
48 + scrapeConfigs = [ {
49 + job_name = "prometheus-packet-sd";
50 + metrics_path = "/metrics";
51 + static_configs = [ {
52 + targets = [
53 + "127.0.0.1:9465"
54 + ];
55 + } ];
56 + } {
57 + job_name = "packet_nodes";
58 + file_sd_configs = [ {
59 + files = [ "/var/lib/packet-sd/packet-sd.json" ];
60 + refresh_interval = "30s";
61 + } ];
62 + relabel_configs = [ {
63 + source_labels = [ "__meta_packet_public_ipv4" ];
64 + target_label = "__address__";
65 + replacement = "\${1}:9100";
66 + action = "replace";
67 + } {
68 + source_labels = [ "__meta_packet_facility" ];
69 + target_label = "facility";
70 + } {
71 + source_labels = [ "__meta_packet_facility" ];
72 + target_label = "packet_facility";
73 + } {
74 + source_labels = [ "__meta_packet_plan" ];
75 + target_label = "plan";
76 + } {
77 + source_labels = [ "__meta_packet_plan" ];
78 + target_label = "packet_plan";
79 + } {
80 + # todo: change from _id to _uuid
81 + source_labels = [ "__meta_packet_switch_id" ];
82 + target_label = "packet_switch_id";
83 + } {
84 + source_labels = [ "__meta_packet_device_id" ];
85 + target_label = "packet_device_id";
86 + } {
87 + source_labels = [ "__meta_packet_state" ];
88 + target_label = "packet_device_state";
89 + } {
90 + source_labels = [ "__meta_packet_short_id" ];
91 + target_label = "instance";
92 + replacement = "\${1}.packethost.net";
93 + action = "replace";
94 + } {
95 + source_labels = [ "__meta_packet_tags" ];
96 + target_label = "role";
97 + regex = ".*hydra.*";
98 + replacement = "builder";
99 + action = "replace";
100 + } {
101 + source_labels = [ "__meta_packet_tags" ];
102 + regex = ".*prometheus-scraping-disabled.*";
103 + action = "drop";
104 + } ];
105 + } ];
106 + };
107 +}
delft/eris/prometheus/exporters/packet-spot-market.nix renamed
+11 -4
@@ -8,7 +8,7 @@ let
8 };
9 in {
10 age.secrets.prometheus-packet-spot-market-price-exporter = {
11 - file = ../secrets/prometheus-packet-spot-market-price-exporter.age;
11 + file = ../../../secrets/prometheus-packet-spot-market-price-exporter.age;
12 owner = "spot-price-exporter";
13 };
14
@@ -20,8 +20,12 @@ in {
20 users.groups.spot-price-exporter = {};
21
22 systemd.services.prometheus-packet-spot-market-price-exporter = {
23 - wantedBy = [ "multi-user.target" ];
24 - after = [ "network.target" ];
23 + wantedBy = [
24 + "multi-user.target"
25 + ];
26 + after = [
27 + "network.target"
28 + ];
29 serviceConfig = {
30 User = "spot-price-exporter";
31 Group = "keys";
@@ -31,7 +35,10 @@ in {
35 };
36
37 path = [
34 - (pkgs.python3.withPackages (p: [ p.prometheus_client p.requests ]))
38 + (pkgs.python3.withPackages (ps: with ps; [
39 + prometheus_client
40 + requests
41 + ]))
42 ];
43
44 script = "exec python3 ${exporter}/scrape.py ${config.age.secrets.prometheus-packet-spot-market-price-exporter.path}";
delft/eris/prometheus/exporters/postgresql.nix new
+11
@@ -0,0 +1,11 @@
1 +{
2 + services.prometheus.scrapeConfigs = [ {
3 + job_name = "haumea-postgresql";
4 + metrics_path = "/metrics";
5 + static_configs = [ {
6 + targets = [
7 + "haumea:9187"
8 + ];
9 + } ];
10 + } ];
11 +}
delft/eris/prometheus/exporters/r13y.nix new
+12
@@ -0,0 +1,12 @@
1 +{
2 + services.prometheus.scrapeConfigs = [ {
3 + job_name = "r13y";
4 + scheme = "https";
5 + metrics_path = "/metrics";
6 + static_configs = [ {
7 + targets = [
8 + "r13y.com"
9 + ];
10 + } ];
11 + } ];
12 +}
delft/eris/prometheus/exporters/rfc39.nix new
+29
@@ -0,0 +1,29 @@
1 +{
2 + services.prometheus = {
3 + scrapeConfigs = [ {
4 + job_name = "rfc39";
5 + metrics_path = "/";
6 + static_configs = [ {
7 + targets = [
8 + # intermittently available, when the rfc39-sync.service runs
9 + "127.0.0.1:9190"
10 + ];
11 + } ];
12 + } ];
13 +
14 + rules = [
15 + (builtins.toJSON {
16 + groups = [ {
17 + name = "rfc39";
18 + rules = [ {
19 + alert = "RFC39MaintainerSync";
20 + expr = ''node_systemd_unit_state{name=~"^rfc39-sync.service$", state="failed"} == 1'';
21 + for = "30m";
22 + labels.severity = "warning";
23 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
24 + } ];
25 + } ];
26 + })
27 + ];
28 + };
29 +}
delft/flake.nix
+1 -2
@@ -45,8 +45,7 @@
45
46 modules = [
47 flakesModule
48 - ./eris.nix
49 - ./eris-physical.nix
48 + ./eris
49 ];
50 };
51