@cryptotaxi247 / infra / commits / 732025be

reformat a few files

Graham Christensen committed Sep 26, 2022 at 08:16 UTC 732025be746a90fca4bc33e39f70b26c6c482f3c
3 files changed +187 -164
delft/eris.nix
+175 -155
@@ -2,9 +2,10 @@
2 let
3 inherit (lib) filterAttrs flip mapAttrsToList;
4
5 - macs = filterAttrs (_: v: (v.macosGuest or {}).enable or false) resources.machines;
6 -in {
7 - imports = [
5 + macs = filterAttrs (_: v: (v.macosGuest or { }).enable or false) resources.machines;
6 +in
7 +{
8 + imports = [
9 ../modules/rfc39.nix
10 ../modules/prometheus
11 ./eris/packet-spot-market-prices.nix
@@ -24,12 +25,13 @@ in {
25
26 10.254.3.1 webserver
27
27 - '' + (toString (flip mapAttrsToList macs (machine: v: ''
28 + '' + (toString (flip mapAttrsToList macs (machine: v: ''
29 ${v.deployment.targetHost} ${machine}
29 - '')));
30 + '')));
31
32 networking.firewall.allowedTCPPorts = [
32 - 443 80 # nginx
33 + 443
34 + 80 # nginx
35 9090 # prometheus's web UI
36 9200 # hydra-queue-runner rexporter
37 ];
@@ -84,7 +86,7 @@ in {
86 enable = true;
87 webExternalUrl = "http://10.254.1.4:9093/";
88 configuration = {
87 - global = {};
89 + global = { };
90 route = {
91 receiver = "ignore";
92 group_wait = "30s";
@@ -118,97 +120,100 @@ in {
120 };
121 };
122
121 - rules = [ (builtins.toJSON {
122 - groups = [
123 - {
124 - name = "hydra";
125 - rules = [
126 - {
127 - alert = "BuildsStuckOverTwoDays";
128 - expr = ''hydra_machine_build_duration_bucket{le="+Inf"} - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0'';
129 - for = "30m";
130 - labels.severity = "warning";
131 - annotations.summary = "{{ $labels.machine }} has {{ $value }} over-age jobs.";
132 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
133 - }
134 - {
135 - alert = "HydraQueueRunnerUp";
136 - expr = ''up{job="hydra_queue_runner"} == 0'';
137 - for = "30m";
138 - labels.severity = "warning";
139 - annotations.summary = "hydra-queue-runner's prometheus exporter is not up";
140 - }
141 - ];
142 - }
143 -
144 - {
145 - name = "system";
146 - rules = let
147 - diskSelector = ''mountpoint=~"(/|/scratch)"'';
148 - relevantLabels = "device,fstype,instance,mountpoint";
149 - in
150 - [
151 - {
152 - alert = "PartitionLowInodes";
153 - expr = ''
154 - avg (node_filesystem_files_free{${diskSelector}} <= 10000) by (${relevantLabels})
155 - '';
156 - for = "30m";
157 - labels.severity = "warning";
158 - annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} inodes free.";
159 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
160 - }
161 -
162 - {
163 - alert = "PartitionLowDiskSpace";
164 - expr = ''
165 - (avg (round(node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)) by (${relevantLabels}))
166 - or
167 - (avg (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10) by (${relevantLabels}))
168 - '';
169 - for = "30m";
170 - labels.severity = "warning";
171 - annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} GB free.";
172 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
173 - }
174 - ];
175 - }
123 + rules = [
124 + (builtins.toJSON {
125 + groups = [
126 + {
127 + name = "hydra";
128 + rules = [
129 + {
130 + alert = "BuildsStuckOverTwoDays";
131 + expr = ''hydra_machine_build_duration_bucket{le="+Inf"} - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0'';
132 + for = "30m";
133 + labels.severity = "warning";
134 + annotations.summary = "{{ $labels.machine }} has {{ $value }} over-age jobs.";
135 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
136 + }
137 + {
138 + alert = "HydraQueueRunnerUp";
139 + expr = ''up{job="hydra_queue_runner"} == 0'';
140 + for = "30m";
141 + labels.severity = "warning";
142 + annotations.summary = "hydra-queue-runner's prometheus exporter is not up";
143 + }
144 + ];
145 + }
146
177 - {
178 - name = "scheduled-jobs";
179 - rules = [
180 - {
181 - alert = "RFC39MaintainerSync";
182 - expr = ''node_systemd_unit_state{name=~"^rfc39-sync.service$", state="failed"} == 1'';
183 - for = "30m";
184 - labels.severity = "warning";
185 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
186 - }
187 - {
188 - alert = "ChannelUpdateStuck";
189 - expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1'';
190 - for = "30m";
191 - labels.severity = "warning";
192 - annotations.summary = "{{ $labels.name }} on {{ $labels.instance }}";
193 - annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
194 - }
195 - ];
196 - }
147 + {
148 + name = "system";
149 + rules =
150 + let
151 + diskSelector = ''mountpoint=~"(/|/scratch)"'';
152 + relevantLabels = "device,fstype,instance,mountpoint";
153 + in
154 + [
155 + {
156 + alert = "PartitionLowInodes";
157 + expr = ''
158 + avg (node_filesystem_files_free{${diskSelector}} <= 10000) by (${relevantLabels})
159 + '';
160 + for = "30m";
161 + labels.severity = "warning";
162 + annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} inodes free.";
163 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
164 + }
165 +
166 + {
167 + alert = "PartitionLowDiskSpace";
168 + expr = ''
169 + (avg (round(node_filesystem_avail_bytes{${diskSelector}} * 10^(-9) <= 10)) by (${relevantLabels}))
170 + or
171 + (avg (((node_filesystem_avail_bytes{${diskSelector}} / node_filesystem_size_bytes) * 100) <= 10) by (${relevantLabels}))
172 + '';
173 + for = "30m";
174 + labels.severity = "warning";
175 + annotations.summary = "{{ $labels.device }} mounted to {{ $labels.mountpoint }} ({{ $labels.fstype }}) on {{ $labels.instance }} has {{ $value }} GB free.";
176 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
177 + }
178 + ];
179 + }
180 +
181 + {
182 + name = "scheduled-jobs";
183 + rules = [
184 + {
185 + alert = "RFC39MaintainerSync";
186 + expr = ''node_systemd_unit_state{name=~"^rfc39-sync.service$", state="failed"} == 1'';
187 + for = "30m";
188 + labels.severity = "warning";
189 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
190 + }
191 + {
192 + alert = "ChannelUpdateStuck";
193 + expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1'';
194 + for = "30m";
195 + labels.severity = "warning";
196 + annotations.summary = "{{ $labels.name }} on {{ $labels.instance }}";
197 + annotations.grafana = "https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
198 + }
199 + ];
200 + }
201
198 - {
199 - name = "blackbox";
200 - rules = [
201 - {
202 - alert = "CertificateExpiry";
203 - expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 14";
204 - for = "10m";
205 - labels.severity = "warning";
206 - annotations.summary = "Certificate for {{ $labels.instance }} is expiring soon.";
207 - }
208 - ];
209 - }
210 - ];
211 - }) ];
202 + {
203 + name = "blackbox";
204 + rules = [
205 + {
206 + alert = "CertificateExpiry";
207 + expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 14";
208 + for = "10m";
209 + labels.severity = "warning";
210 + annotations.summary = "Certificate for {{ $labels.instance }} is expiring soon.";
211 + }
212 + ];
213 + }
214 + ];
215 + })
216 + ];
217
218 globalConfig.scrape_interval = "15s";
219 scrapeConfigs = [
@@ -300,7 +305,8 @@ in {
305 source_labels = [ "__meta_packet_plan" ];
306 target_label = "packet_plan";
307 }
303 - { # todo: change from _id to _uuid
308 + {
309 + # todo: change from _id to _uuid
310 source_labels = [ "__meta_packet_switch_id" ];
311 target_label = "packet_switch_id";
312 }
@@ -488,25 +494,30 @@ in {
494 ];
495 }
496 ]
491 - ++ (let
492 - mkProbe = module: targets: {
493 - job_name = "blackbox-${module}";
494 - metrics_path = "/probe";
495 - params = {
496 - module = [ module ];
497 + ++ (
498 + let
499 + mkProbe = module: targets: {
500 + job_name = "blackbox-${module}";
501 + metrics_path = "/probe";
502 + params = {
503 + module = [ module ];
504 + };
505 + static_configs = [{ inherit targets; }];
506 + relabel_configs = [{
507 + source_labels = [ "__address__" ];
508 + target_label = "__param_target";
509 + }
510 + {
511 + source_labels = [ "__param_target" ];
512 + target_label = "instance";
513 + }
514 + {
515 + target_label = "__address__";
516 + replacement = "localhost:9115";
517 + }];
518 };
498 - static_configs = [{ inherit targets; }];
499 - relabel_configs = [ {
500 - source_labels = [ "__address__" ];
501 - target_label = "__param_target";
502 - } {
503 - source_labels = [ "__param_target" ];
504 - target_label = "instance";
505 - } {
506 - target_label = "__address__";
507 - replacement = "localhost:9115";
508 - } ];
509 - }; in [
519 + in
520 + [
521 (mkProbe "https_success" [
522 "https://cache.nixos.org"
523 "https://channels.nixos.org"
@@ -527,19 +538,21 @@ in {
538 ])
539 ]
540 )
530 - ++ lib.mapAttrsToList (name: value: {
541 + ++ lib.mapAttrsToList
542 + (name: value: {
543 job_name = "channel-job-${name}";
544 scheme = "https";
545 scrape_interval = "5m";
546 metrics_path = "/job/${value.job}/prometheus";
535 - static_configs = [ {
547 + static_configs = [{
548 labels = {
549 current = if value.status != "unmaintained" then "1" else "0";
550 channel = name;
551 };
552 targets = [ "hydra.nixos.org:443" ];
541 - } ];
542 - }) (import ../channels.nix).channels;
553 + }];
554 + })
555 + (import ../channels.nix).channels;
556 };
557
558 services.grafana = {
@@ -557,15 +570,18 @@ in {
570 serviceConfig = {
571 Restart = "always";
572 RestartSec = "60s";
560 - PrivateTmp = true;
573 + PrivateTmp = true;
574 WorkingDirectory = "/tmp";
562 - ExecStart = let
575 + ExecStart =
576 + let
577 python = pkgs.python3.withPackages (p: [
564 - p.requests p.prometheus_client
578 + p.requests
579 + p.prometheus_client
580 ]);
566 - in ''
581 + in
582 + ''
583 ${python}/bin/python ${./prometheus/hydra-queue-runner-reexporter.py}
568 - '';
584 + '';
585 };
586 };
587
@@ -579,49 +595,53 @@ in {
595 isSystemUser = true;
596 group = "packet-sd";
597 };
582 - users.groups.packet-sd = {};
598 + users.groups.packet-sd = { };
599
600 systemd.tmpfiles.rules = [
601 "d /var/lib/packet-sd 0755 packet-sd - -"
602 "f /var/lib/packet-sd/packet-sd.json 0644 packet-sd - -"
603 ];
604
589 - systemd.services.prometheus-packet-sd = let
590 - sd = pkgs.callPackage ./prometheus/packet-sd.nix {};
591 - in {
592 - wantedBy = [ "multi-user.target" "prometheus.service" ];
593 - after = [ "network.target" ];
594 -
595 - serviceConfig = {
596 - User = "packet-sd";
597 - Group = "keys";
598 - ExecStart = "${sd}/bin/prometheus-packet-sd --output.file=/var/lib/packet-sd/packet-sd.json";
599 - EnvironmentFile = "/run/keys/packet-sd-env";
600 - Restart = "always";
601 - RestartSec = "60s";
605 + systemd.services.prometheus-packet-sd =
606 + let
607 + sd = pkgs.callPackage ./prometheus/packet-sd.nix { };
608 + in
609 + {
610 + wantedBy = [ "multi-user.target" "prometheus.service" ];
611 + after = [ "network.target" ];
612 +
613 + serviceConfig = {
614 + User = "packet-sd";
615 + Group = "keys";
616 + ExecStart = "${sd}/bin/prometheus-packet-sd --output.file=/var/lib/packet-sd/packet-sd.json";
617 + EnvironmentFile = "/run/keys/packet-sd-env";
618 + Restart = "always";
619 + RestartSec = "60s";
620 + };
621 };
603 - };
622
623 deployment.keys."fastly-read-only-api-token" = {
624 keyFile = /home/deploy/src/nixos-org-configurations/fastly-read-only-api-token;
625 };
626
609 - systemd.services.prometheus-fastly-exporter = let
610 - fastly = pkgs.callPackage ./prometheus/fastly.nix {};
611 - in {
612 - wantedBy = [ "multi-user.target" "prometheus.service" ];
613 - after = [ "network.target" ];
614 -
615 - script = ''
616 - export FASTLY_API_TOKEN=$(cat /run/keys/fastly-read-only-api-token)
617 - ${fastly}/bin/fastly-exporter \
618 - -endpoint http://127.0.0.1:9118/metrics
619 - '';
627 + systemd.services.prometheus-fastly-exporter =
628 + let
629 + fastly = pkgs.callPackage ./prometheus/fastly.nix { };
630 + in
631 + {
632 + wantedBy = [ "multi-user.target" "prometheus.service" ];
633 + after = [ "network.target" ];
634 +
635 + script = ''
636 + export FASTLY_API_TOKEN=$(cat /run/keys/fastly-read-only-api-token)
637 + ${fastly}/bin/fastly-exporter \
638 + -endpoint http://127.0.0.1:9118/metrics
639 + '';
640
621 - serviceConfig = {
622 - Group = "keys";
623 - Restart = "always";
624 - RestartSec = "60s";
641 + serviceConfig = {
642 + Group = "keys";
643 + Restart = "always";
644 + RestartSec = "60s";
645 + };
646 };
626 - };
647 }
delft/hydra.nix
+5 -4
@@ -71,15 +71,16 @@ in
71 '';
72
73 systemd.tmpfiles.rules =
74 - [ "d /var/cache/hydra 0755 hydra hydra - -"
74 + [
75 + "d /var/cache/hydra 0755 hydra hydra - -"
76 "d ${narCache} 0775 hydra hydra 1d -"
77 ];
78
79 # users.extraUsers.hydra.home = mkForce "/home/hydra";
80
81 systemd.services.hydra-queue-runner.restartIfChanged = false;
81 - systemd.services.hydra-queue-runner.wantedBy = mkForce [];
82 - systemd.services.hydra-queue-runner.requires = mkForce [];
82 + systemd.services.hydra-queue-runner.wantedBy = mkForce [ ];
83 + systemd.services.hydra-queue-runner.requires = mkForce [ ];
84 systemd.services.hydra-queue-runner.serviceConfig.LimitNOFILE = 65535;
85
86 programs.ssh.hostKeyAlgorithms = [ "rsa-sha2-512-cert-v01@openssh.com" "ssh-ed25519" "ssh-rsa" "ecdsa-sha2-nistp256" ];
@@ -196,7 +197,7 @@ in
197 Host hetzner-m1-1643228
198 Hostname 142.132.140.199
199 Compression yes
199 - '';
200 + '';
201
202 services.openssh.knownHosts = {
203 "*.cloudscalehydra.detsys.dev" = { certAuthority = true; publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC6HhovazfX+rqm2YuOwgM1X16Z7bJpLLj4DXWUsuGGqG/OlwIyioVodKPd3dYwsltQD8W4YrWQCOXlqyV50xpngKJX6OXdDt0IWwwgHdW0bT/l+4Yd/B57PbSnXGKLwa7slBwMCGkxpivMwnkQm+1zfQEdVNX5UPiH6xwZSwgsaRHCpumpVUrLNWlWxI5+g3alez/mfp29bgSvMdnIu72Ykb8u0uKOvGVQY7UD9sVU00NSQ16m+NhvvFvVomIF6OXMinBkATuSsoa4jOIg4UTsS5mo8Up8RdZ1qyzxvqf874osn5sYkMVnRZ5G/0bmwdwyYs7mjKh3agt37Fnaj8obyfVRm9aRlKT+Gwc5U2XZF/AdhOq+TdRL2HgaNYwJspHYUQ2jm5YilOgcEfTOgunxDfMIGueqnM6nZoGe7EHA6affr8QLrOXEUVA9uwIMInpLWiDZXk74owDGhIpg8WBpWch+x3SqaLDwLlUYDseJR0BdH9al+UZW1eBinAiEVl6H7KzVLpLqYss38CWT9c7Cq/gltUuwIqgziXFCR4skXfpN5Ozg0Sr9OmkJbxHjGdOmMVT0VKC05KsUEkWW9J18WX3uN1O3Mqu7vWgK9VOqbsnsknP0oBSznniFZYblK0vRgcrKPMAGTZ7RMdTBbys28sj3HKY/CQPv08KV0xgOJQ=="; };
delft/network.nix
+7 -5
@@ -2,7 +2,7 @@ flakes:
2
3 let
4 networkoverlay = self: super: {
5 - prometheus-postgres-exporter = self.callPackage ./prometheus/postgres-exporter.nix {};
5 + prometheus-postgres-exporter = self.callPackage ./prometheus/postgres-exporter.nix { };
6 };
7
8 makeMac = { ip, extra, useCatalina ? false }: {
@@ -37,7 +37,7 @@ let
37 {
38 zvolName = lib.mkForce "rpool/catalina";
39 guestConfigDir = lib.mkForce ../macs/guest-catalina;
40 - cloverImage = (pkgs.callPackage ../macs/dist/clover-catalina {}).clover-image;
40 + cloverImage = (pkgs.callPackage ../macs/dist/clover-catalina { }).clover-image;
41 }
42 else
43 {
@@ -49,7 +49,8 @@ let
49 })
50 ];
51 };
52 -in {
52 +in
53 +{
54 defaults = {
55 documentation.nixos.enable = false;
56
@@ -60,7 +61,8 @@ in {
61 ../modules/wireguard.nix
62 ../modules/prometheus
63 # flakes.dwarffs.nixosModules.dwarffs # broken by Nix 2.6
63 - { system.configurationRevision = flakes.self.rev
64 + {
65 + system.configurationRevision = flakes.self.rev
66 or (throw "Cannot deploy from an unclean source tree!");
67 nixpkgs.overlays = [
68 flakes.nix.overlays.default
@@ -96,7 +98,7 @@ in {
98 guest = {
99 zvolName = lib.mkForce "rpool/catalina";
100 guestConfigDir = lib.mkForce ../macs/guest-catalina;
99 - cloverImage = (pkgs.callPackage ../macs/dist/clover-catalina {}).clover-image;
101 + cloverImage = (pkgs.callPackage ../macs/dist/clover-catalina { }).clover-image;
102 };
103 };
104 };