@cryptotaxi247 / infra-1 / commits / 4c4256b7

haumea: split up transitional configurations

Martin Weinelt committed Feb 23, 2024 at 12:49 UTC 4c4256b7a8ee94462dfe679956cde5037975cbf8
7 files changed +278 -293
delft/flake.nix
+1 -2
@@ -55,8 +55,7 @@
55
56 modules = [
57 flakesModule
58 - ./haumea.nix
59 - ./haumea-physical.nix
58 + ./haumea
59 ];
60 };
61
delft/haumea-physical.nix deleted
-45
@@ -1,45 +0,0 @@
1 -# Transitional. This is the auto-generated nixops config for haumea, extracted
2 -# into a file that we can check in and import when evaluating outside of
3 -# nixops.
4 -
5 -{ config, lib, pkgs, modulesPath, ... }: {
6 - config = {
7 - boot.kernelModules = [];
8 - networking = {
9 - hostName = "haumea";
10 - extraHosts = ''
11 - 138.201.32.77 eris eris-unencrypted
12 - 46.4.89.205 haumea haumea-unencrypted
13 - 127.0.0.1 haumea-encrypted
14 - '';
15 - firewall.trustedInterfaces = [];
16 - };
17 - };
18 - imports = [
19 - {
20 - config = {
21 - users.extraUsers.root.openssh.authorizedKeys.keys = [
22 - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+xcwa7Oj8At7n8gHQu7UXArxCJSQZgMaspfkyLbP1j NixOps client key of haumea"
23 - ];
24 - };
25 - imports = [
26 - ({})
27 - ({ config, lib, pkgs, ... }:
28 -
29 - {
30 - imports =
31 - [ "${modulesPath}/installer/scan/not-detected.nix"
32 - ];
33 -
34 - boot.initrd.availableKernelModules = [ "ahci" "nvme" "usbhid" ];
35 - boot.initrd.kernelModules = [ ];
36 - boot.kernelModules = [ "kvm-amd" ];
37 - boot.extraModulePackages = [ ];
38 -
39 - nix.settings.max-jobs = lib.mkDefault 16;
40 - powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
41 - })
42 - ];
43 - }
44 - ];
45 -}
delft/haumea.nix deleted
-246
@@ -1,246 +0,0 @@
1 -{ config, lib, pkgs, ...}:
2 -
3 -{
4 - imports =
5 - [ ./common.nix
6 - ./haumea/network.nix
7 - ];
8 -
9 - system.stateVersion = "14.12";
10 - environment.systemPackages = [ pkgs.lz4 ];
11 -
12 - users.users.root.openssh.authorizedKeys.keys =
13 - with import ../ssh-keys.nix; infra-core;
14 -
15 - fileSystems."/" =
16 - { device = "rpool/safe/root";
17 - fsType = "zfs";
18 - };
19 -
20 - fileSystems."/boot" =
21 - { device = "/dev/disk/by-label/boot0";
22 - fsType = "ext4";
23 - };
24 -
25 - fileSystems."/nix" =
26 - { device = "rpool/local/nix";
27 - fsType = "zfs";
28 - };
29 -
30 - fileSystems."/var/db/postgresql" =
31 - { device = "rpool/safe/postgres";
32 - fsType = "zfs";
33 - };
34 -
35 - networking.hostId = "83c81a23";
36 -
37 - boot.loader.grub.devices = [ "/dev/nvme0n1" "/dev/nvme1n1" ];
38 - boot.loader.grub.copyKernels = true;
39 -
40 - systemd.services.postgresql = {
41 - after = [ "wireguard-wg0.service" ];
42 - requires = [ "wireguard-wg0.service" ];
43 - };
44 - services.postgresql = {
45 - enable = true;
46 - package = pkgs.postgresql_16;
47 - dataDir = "/var/db/postgresql/16";
48 - # https://pgtune.leopard.in.ua/#/
49 - logLinePrefix = "user=%u,db=%d,app=%a,client=%h ";
50 - settings = {
51 - listen_addresses = lib.mkForce "10.254.1.9";
52 -
53 - checkpoint_completion_target = "0.9";
54 - default_statistics_target = 100;
55 -
56 - log_duration = "off";
57 - log_statement = "none";
58 -
59 - # pgbadger-compatible logging
60 - log_transaction_sample_rate = 0.01;
61 - log_min_duration_statement = 5000;
62 - log_checkpoints = "on";
63 - log_connections = "on";
64 - log_disconnections = "on";
65 - log_lock_waits = "on";
66 - log_temp_files = 0;
67 - log_autovacuum_min_duration = 0;
68 -
69 - max_connections = 500;
70 - work_mem = "20MB";
71 - maintenance_work_mem = "2GB";
72 -
73 - # 25% of memory
74 - shared_buffers = "16GB";
75 -
76 - # Checkpoint every 1GB. (default)
77 - # increased after seeing many warninsg about frequent checkpoints
78 - min_wal_size = "1GB";
79 - max_wal_size = "2GB";
80 - wal_buffers = "16MB";
81 -
82 - max_worker_processes = 16;
83 - max_parallel_workers_per_gather = 8;
84 - max_parallel_workers = 16;
85 -
86 - # NVMe related performance tuning
87 - effective_io_concurrency = 200;
88 - random_page_cost = "1.1";
89 -
90 - # We can risk losing some transactions.
91 - synchronous_commit = "off";
92 -
93 - effective_cache_size = "16GB";
94 -
95 - # Enable JIT compilation if possible.
96 - jit = "on";
97 -
98 - # autovacuum and autoanalyze much more frequently:
99 - # at these values vacuum should run approximately
100 - # every 2 mass rebuilds, or a couple times a day
101 - # on the builds table. Some of those queries really
102 - # benefit from frequent vacuums, so this should
103 - # help. In particular, I'm thinking the jobsets
104 - # pages.
105 - autovacuum_vacuum_scale_factor = 0.002;
106 - autovacuum_analyze_scale_factor = 0.001;
107 -
108 - shared_preload_libraries = "pg_stat_statements";
109 - compute_query_id = "on";
110 - };
111 -
112 - # FIXME: don't use 'trust'.
113 - authentication = ''
114 - host hydra all 10.254.1.3/32 trust
115 - host hydra all 10.254.1.5/32 trust
116 - local all root peer map=prometheus
117 - '';
118 -
119 - identMap = ''
120 - prometheus root root
121 - prometheus postgres-exporter root
122 - '';
123 - };
124 -
125 - networking = {
126 - firewall.interfaces.wg0.allowedTCPPorts = [ 5432 ];
127 - firewall.allowPing = true;
128 - firewall.logRefusedConnections = true;
129 - };
130 -
131 - services.prometheus.exporters.postgres = {
132 - enable = true;
133 - dataSourceName = "user=root database=hydra host=/run/postgresql sslmode=disable";
134 - firewallFilter = "-i wg0 -p tcp -m tcp --dport 9187";
135 - openFirewall = true;
136 - extraFlags = [ "--extend.query-path" "${pkgs.prometheus-postgres-exporter.src}/queries.yaml" ];
137 - };
138 -
139 - programs.ssh = {
140 - knownHosts = {
141 - rsync-net = {
142 - hostNames = [ "zh2543b.rsync.net" "2001:1620:2019::324" ];
143 - publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKlIcNwmx7id/XdYKZzVX2KtZQ4PAsEa9KVQ9N43L3PX";
144 - };
145 - delroth-backup-server = {
146 - hostNames = [ "smol.delroth.net" ];
147 - publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID9Ta4DYE3YxXzV57s6TX6KEbIa3O4re+J4NzATCOiXb";
148 - };
149 - hexa-backup-server = {
150 - hostNames = [ "meduna.hexa-home.v6.army" ];
151 - publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDUe5BqMDt562KOIcUm4RqZC5ejmd62elkYKkqExUYsl";
152 - };
153 - };
154 - };
155 -
156 - services.zfs.autoScrub.enable = true;
157 -
158 - services.zrepl = let
159 - defaultBackupJob = {
160 - type = "push";
161 - filesystems."rpool/safe<" = true;
162 - snapshotting = {
163 - type = "periodic";
164 - interval = "5m";
165 - prefix = "zrepl_snap_";
166 - };
167 - pruning = {
168 - keep_sender = [
169 - {
170 - type = "grid";
171 - regex = "^zrepl_snap_.*";
172 - grid = lib.concatStringsSep " | " [
173 - "3x5m"
174 - "4x15m"
175 - "24x1h"
176 - "4x1d"
177 - "3x1w"
178 - ];
179 - }
180 - ];
181 - keep_receiver = [
182 - { type = "grid";
183 - regex = "^zrepl_snap_.*";
184 - grid = lib.concatStringsSep " | " [
185 - "20x5m"
186 - "96x1h"
187 - "12x4h"
188 - "7x1d"
189 - "52x1w"
190 - "120x3w"
191 - ];
192 - }
193 - ];
194 - };
195 - };
196 - in {
197 - enable = true;
198 - settings = {
199 - global = {
200 - logging = [
201 - {
202 - type = "syslog";
203 - level = "info";
204 - format = "human";
205 - }
206 - ];
207 - };
208 -
209 - jobs = [
210 - # XXX: Broken since 2024-01-10?
211 - # (defaultBackupJob // {
212 - # name = "rsyncnet";
213 - # connect = {
214 - # identity_file = "/root/.ssh/id_ed25519";
215 - # type = "ssh+stdinserver";
216 - # host = "zh2543b.rsync.net";
217 - # user = "root";
218 - # port = 22;
219 - # };
220 - # })
221 -
222 - (defaultBackupJob // {
223 - name = "delroth";
224 - connect = {
225 - identity_file = "/root/.ssh/id_ed25519";
226 - type = "ssh+stdinserver";
227 - host = "smol.delroth.net";
228 - user = "zrepl";
229 - port = 22;
230 - };
231 - })
232 -
233 - (defaultBackupJob // {
234 - name = "hexa";
235 - connect = {
236 - identity_file = "/root/.ssh/id_ed25519";
237 - type = "ssh+stdinserver";
238 - host = "meduna.hexa-home.v6.army";
239 - user = "zrepl";
240 - port = 22;
241 - };
242 - })
243 - ];
244 - };
245 - };
246 -}
delft/haumea/boot.nix new
+11
@@ -0,0 +1,11 @@
1 +{
2 + boot.loader.grub = {
3 + devices = [
4 + "/dev/nvme0n1"
5 + "/dev/nvme1n1"
6 + ];
7 + copyKernels = true;
8 + };
9 + boot.initrd.availableKernelModules = [ "ahci" "nvme" "usbhid" ];
10 + boot.kernelModules = [ "kvm-amd" ];
11 +}
delft/haumea/default.nix new
+52
@@ -0,0 +1,52 @@
1 +{ lib
2 +, modulesPath
3 +, pkgs
4 +, ...
5 +}:
6 +
7 +{
8 + imports = [
9 + "${modulesPath}/installer/scan/not-detected.nix"
10 + ../common.nix
11 + ./boot.nix
12 + ./network.nix
13 + ./postgresql.nix
14 + ./zrepl.nix
15 + ];
16 +
17 + networking = {
18 + hostId = "83c81a23";
19 + hostName = "haumea";
20 + domain = "nixos.org";
21 + };
22 +
23 + environment.systemPackages = [ pkgs.lz4 ];
24 +
25 + fileSystems."/" =
26 + { device = "rpool/safe/root";
27 + fsType = "zfs";
28 + };
29 +
30 + fileSystems."/boot" =
31 + { device = "/dev/disk/by-label/boot0";
32 + fsType = "ext4";
33 + };
34 +
35 + fileSystems."/nix" =
36 + { device = "rpool/local/nix";
37 + fsType = "zfs";
38 + };
39 +
40 + fileSystems."/var/db/postgresql" =
41 + { device = "rpool/safe/postgres";
42 + fsType = "zfs";
43 + };
44 +
45 + services.zfs.autoScrub.enable = true;
46 +
47 + nix.settings.max-jobs = lib.mkDefault 16;
48 +
49 + powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
50 +
51 + system.stateVersion = "14.12";
52 +}
delft/haumea/postgresql.nix new
+103
@@ -0,0 +1,103 @@
1 +{ lib
2 +, pkgs
3 +, ...
4 +}:
5 +
6 +{
7 + systemd.services.postgresql = {
8 + after = [ "wireguard-wg0.service" ];
9 + requires = [ "wireguard-wg0.service" ];
10 + };
11 +
12 + services.prometheus.exporters.postgres = {
13 + enable = true;
14 + dataSourceName = "user=root database=hydra host=/run/postgresql sslmode=disable";
15 + firewallFilter = "-i wg0 -p tcp -m tcp --dport 9187";
16 + openFirewall = true;
17 + extraFlags = [
18 + "--extend.query-path" "${pkgs.prometheus-postgres-exporter.src}/queries.yaml"
19 + ];
20 + };
21 +
22 + networking.firewall.interfaces.wg0.allowedTCPPorts = [ 5432 ];
23 +
24 + services.postgresql = {
25 + enable = true;
26 + package = pkgs.postgresql_16;
27 + dataDir = "/var/db/postgresql/16";
28 + # https://pgtune.leopard.in.ua/#/
29 + logLinePrefix = "user=%u,db=%d,app=%a,client=%h ";
30 + settings = {
31 + listen_addresses = lib.mkForce "10.254.1.9";
32 +
33 + checkpoint_completion_target = "0.9";
34 + default_statistics_target = 100;
35 +
36 + log_duration = "off";
37 + log_statement = "none";
38 +
39 + # pgbadger-compatible logging
40 + log_transaction_sample_rate = 0.01;
41 + log_min_duration_statement = 5000;
42 + log_checkpoints = "on";
43 + log_connections = "on";
44 + log_disconnections = "on";
45 + log_lock_waits = "on";
46 + log_temp_files = 0;
47 + log_autovacuum_min_duration = 0;
48 +
49 + max_connections = 500;
50 + work_mem = "20MB";
51 + maintenance_work_mem = "2GB";
52 +
53 + # 25% of memory
54 + shared_buffers = "16GB";
55 +
56 + # Checkpoint every 1GB. (default)
57 + # increased after seeing many warninsg about frequent checkpoints
58 + min_wal_size = "1GB";
59 + max_wal_size = "2GB";
60 + wal_buffers = "16MB";
61 +
62 + max_worker_processes = 16;
63 + max_parallel_workers_per_gather = 8;
64 + max_parallel_workers = 16;
65 +
66 + # NVMe related performance tuning
67 + effective_io_concurrency = 200;
68 + random_page_cost = "1.1";
69 +
70 + # We can risk losing some transactions.
71 + synchronous_commit = "off";
72 +
73 + effective_cache_size = "16GB";
74 +
75 + # Enable JIT compilation if possible.
76 + jit = "on";
77 +
78 + # autovacuum and autoanalyze much more frequently:
79 + # at these values vacuum should run approximately
80 + # every 2 mass rebuilds, or a couple times a day
81 + # on the builds table. Some of those queries really
82 + # benefit from frequent vacuums, so this should
83 + # help. In particular, I'm thinking the jobsets
84 + # pages.
85 + autovacuum_vacuum_scale_factor = 0.002;
86 + autovacuum_analyze_scale_factor = 0.001;
87 +
88 + shared_preload_libraries = "pg_stat_statements";
89 + compute_query_id = "on";
90 + };
91 +
92 + # FIXME: don't use 'trust'.
93 + authentication = ''
94 + host hydra all 10.254.1.3/32 trust
95 + host hydra all 10.254.1.5/32 trust
96 + local all root peer map=prometheus
97 + '';
98 +
99 + identMap = ''
100 + prometheus root root
101 + prometheus postgres-exporter root
102 + '';
103 + };}
delft/haumea/zrepl.nix new
+111
@@ -0,0 +1,111 @@
1 +{ lib
2 +, ...
3 +}:
4 +
5 +{
6 + programs.ssh = {
7 + knownHosts = {
8 + rsync-net = {
9 + hostNames = [ "zh2543b.rsync.net" "2001:1620:2019::324" ];
10 + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKlIcNwmx7id/XdYKZzVX2KtZQ4PAsEa9KVQ9N43L3PX";
11 + };
12 + delroth-backup-server = {
13 + hostNames = [ "smol.delroth.net" ];
14 + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID9Ta4DYE3YxXzV57s6TX6KEbIa3O4re+J4NzATCOiXb";
15 + };
16 + hexa-backup-server = {
17 + hostNames = [ "meduna.hexa-home.v6.army" ];
18 + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDUe5BqMDt562KOIcUm4RqZC5ejmd62elkYKkqExUYsl";
19 + };
20 + };
21 + };
22 +
23 + services.zrepl = let
24 + defaultBackupJob = {
25 + type = "push";
26 + filesystems."rpool/safe<" = true;
27 + snapshotting = {
28 + type = "periodic";
29 + interval = "5m";
30 + prefix = "zrepl_snap_";
31 + };
32 + pruning = {
33 + keep_sender = [
34 + {
35 + type = "grid";
36 + regex = "^zrepl_snap_.*";
37 + grid = lib.concatStringsSep " | " [
38 + "3x5m"
39 + "4x15m"
40 + "24x1h"
41 + "4x1d"
42 + "3x1w"
43 + ];
44 + }
45 + ];
46 + keep_receiver = [
47 + { type = "grid";
48 + regex = "^zrepl_snap_.*";
49 + grid = lib.concatStringsSep " | " [
50 + "20x5m"
51 + "96x1h"
52 + "12x4h"
53 + "7x1d"
54 + "52x1w"
55 + "120x3w"
56 + ];
57 + }
58 + ];
59 + };
60 + };
61 + in {
62 + enable = true;
63 + settings = {
64 + global = {
65 + logging = [
66 + {
67 + type = "syslog";
68 + level = "info";
69 + format = "human";
70 + }
71 + ];
72 + };
73 +
74 + jobs = [
75 + # XXX: Broken since 2024-01-10?
76 + # (defaultBackupJob // {
77 + # name = "rsyncnet";
78 + # connect = {
79 + # identity_file = "/root/.ssh/id_ed25519";
80 + # type = "ssh+stdinserver";
81 + # host = "zh2543b.rsync.net";
82 + # user = "root";
83 + # port = 22;
84 + # };
85 + # })
86 +
87 + (defaultBackupJob // {
88 + name = "delroth";
89 + connect = {
90 + identity_file = "/root/.ssh/id_ed25519";
91 + type = "ssh+stdinserver";
92 + host = "smol.delroth.net";
93 + user = "zrepl";
94 + port = 22;
95 + };
96 + })
97 +
98 + (defaultBackupJob // {
99 + name = "hexa";
100 + connect = {
101 + identity_file = "/root/.ssh/id_ed25519";
102 + type = "ssh+stdinserver";
103 + host = "meduna.hexa-home.v6.army";
104 + user = "zrepl";
105 + port = 22;
106 + };
107 + })
108 + ];
109 + };
110 + };
111 +}