haumea: split up transitional configurations
Martin Weinelt committed
Feb 23, 2024 at 12:49 UTC
4c4256b7a8ee94462dfe679956cde5037975cbf8
7 files changed
+278
-293
delft/flake.nix
+1
-2
@@ -55,8 +55,7 @@
55
56
modules = [
57
flakesModule
58
- ./haumea.nix
59
- ./haumea-physical.nix
58
+ ./haumea
59
];
60
};
61
delft/haumea-physical.nix
deleted
-45
@@ -1,45 +0,0 @@
1
-# Transitional. This is the auto-generated nixops config for haumea, extracted
2
-# into a file that we can check in and import when evaluating outside of
3
-# nixops.
4
-
5
-{ config, lib, pkgs, modulesPath, ... }: {
6
- config = {
7
- boot.kernelModules = [];
8
- networking = {
9
- hostName = "haumea";
10
- extraHosts = ''
11
- 138.201.32.77 eris eris-unencrypted
12
- 46.4.89.205 haumea haumea-unencrypted
13
- 127.0.0.1 haumea-encrypted
14
- '';
15
- firewall.trustedInterfaces = [];
16
- };
17
- };
18
- imports = [
19
- {
20
- config = {
21
- users.extraUsers.root.openssh.authorizedKeys.keys = [
22
- "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+xcwa7Oj8At7n8gHQu7UXArxCJSQZgMaspfkyLbP1j NixOps client key of haumea"
23
- ];
24
- };
25
- imports = [
26
- ({})
27
- ({ config, lib, pkgs, ... }:
28
-
29
- {
30
- imports =
31
- [ "${modulesPath}/installer/scan/not-detected.nix"
32
- ];
33
-
34
- boot.initrd.availableKernelModules = [ "ahci" "nvme" "usbhid" ];
35
- boot.initrd.kernelModules = [ ];
36
- boot.kernelModules = [ "kvm-amd" ];
37
- boot.extraModulePackages = [ ];
38
-
39
- nix.settings.max-jobs = lib.mkDefault 16;
40
- powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
41
- })
42
- ];
43
- }
44
- ];
45
-}
delft/haumea.nix
deleted
-246
@@ -1,246 +0,0 @@
1
-{ config, lib, pkgs, ...}:
2
-
3
-{
4
- imports =
5
- [ ./common.nix
6
- ./haumea/network.nix
7
- ];
8
-
9
- system.stateVersion = "14.12";
10
- environment.systemPackages = [ pkgs.lz4 ];
11
-
12
- users.users.root.openssh.authorizedKeys.keys =
13
- with import ../ssh-keys.nix; infra-core;
14
-
15
- fileSystems."/" =
16
- { device = "rpool/safe/root";
17
- fsType = "zfs";
18
- };
19
-
20
- fileSystems."/boot" =
21
- { device = "/dev/disk/by-label/boot0";
22
- fsType = "ext4";
23
- };
24
-
25
- fileSystems."/nix" =
26
- { device = "rpool/local/nix";
27
- fsType = "zfs";
28
- };
29
-
30
- fileSystems."/var/db/postgresql" =
31
- { device = "rpool/safe/postgres";
32
- fsType = "zfs";
33
- };
34
-
35
- networking.hostId = "83c81a23";
36
-
37
- boot.loader.grub.devices = [ "/dev/nvme0n1" "/dev/nvme1n1" ];
38
- boot.loader.grub.copyKernels = true;
39
-
40
- systemd.services.postgresql = {
41
- after = [ "wireguard-wg0.service" ];
42
- requires = [ "wireguard-wg0.service" ];
43
- };
44
- services.postgresql = {
45
- enable = true;
46
- package = pkgs.postgresql_16;
47
- dataDir = "/var/db/postgresql/16";
48
- # https://pgtune.leopard.in.ua/#/
49
- logLinePrefix = "user=%u,db=%d,app=%a,client=%h ";
50
- settings = {
51
- listen_addresses = lib.mkForce "10.254.1.9";
52
-
53
- checkpoint_completion_target = "0.9";
54
- default_statistics_target = 100;
55
-
56
- log_duration = "off";
57
- log_statement = "none";
58
-
59
- # pgbadger-compatible logging
60
- log_transaction_sample_rate = 0.01;
61
- log_min_duration_statement = 5000;
62
- log_checkpoints = "on";
63
- log_connections = "on";
64
- log_disconnections = "on";
65
- log_lock_waits = "on";
66
- log_temp_files = 0;
67
- log_autovacuum_min_duration = 0;
68
-
69
- max_connections = 500;
70
- work_mem = "20MB";
71
- maintenance_work_mem = "2GB";
72
-
73
- # 25% of memory
74
- shared_buffers = "16GB";
75
-
76
- # Checkpoint every 1GB. (default)
77
- # increased after seeing many warninsg about frequent checkpoints
78
- min_wal_size = "1GB";
79
- max_wal_size = "2GB";
80
- wal_buffers = "16MB";
81
-
82
- max_worker_processes = 16;
83
- max_parallel_workers_per_gather = 8;
84
- max_parallel_workers = 16;
85
-
86
- # NVMe related performance tuning
87
- effective_io_concurrency = 200;
88
- random_page_cost = "1.1";
89
-
90
- # We can risk losing some transactions.
91
- synchronous_commit = "off";
92
-
93
- effective_cache_size = "16GB";
94
-
95
- # Enable JIT compilation if possible.
96
- jit = "on";
97
-
98
- # autovacuum and autoanalyze much more frequently:
99
- # at these values vacuum should run approximately
100
- # every 2 mass rebuilds, or a couple times a day
101
- # on the builds table. Some of those queries really
102
- # benefit from frequent vacuums, so this should
103
- # help. In particular, I'm thinking the jobsets
104
- # pages.
105
- autovacuum_vacuum_scale_factor = 0.002;
106
- autovacuum_analyze_scale_factor = 0.001;
107
-
108
- shared_preload_libraries = "pg_stat_statements";
109
- compute_query_id = "on";
110
- };
111
-
112
- # FIXME: don't use 'trust'.
113
- authentication = ''
114
- host hydra all 10.254.1.3/32 trust
115
- host hydra all 10.254.1.5/32 trust
116
- local all root peer map=prometheus
117
- '';
118
-
119
- identMap = ''
120
- prometheus root root
121
- prometheus postgres-exporter root
122
- '';
123
- };
124
-
125
- networking = {
126
- firewall.interfaces.wg0.allowedTCPPorts = [ 5432 ];
127
- firewall.allowPing = true;
128
- firewall.logRefusedConnections = true;
129
- };
130
-
131
- services.prometheus.exporters.postgres = {
132
- enable = true;
133
- dataSourceName = "user=root database=hydra host=/run/postgresql sslmode=disable";
134
- firewallFilter = "-i wg0 -p tcp -m tcp --dport 9187";
135
- openFirewall = true;
136
- extraFlags = [ "--extend.query-path" "${pkgs.prometheus-postgres-exporter.src}/queries.yaml" ];
137
- };
138
-
139
- programs.ssh = {
140
- knownHosts = {
141
- rsync-net = {
142
- hostNames = [ "zh2543b.rsync.net" "2001:1620:2019::324" ];
143
- publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKlIcNwmx7id/XdYKZzVX2KtZQ4PAsEa9KVQ9N43L3PX";
144
- };
145
- delroth-backup-server = {
146
- hostNames = [ "smol.delroth.net" ];
147
- publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID9Ta4DYE3YxXzV57s6TX6KEbIa3O4re+J4NzATCOiXb";
148
- };
149
- hexa-backup-server = {
150
- hostNames = [ "meduna.hexa-home.v6.army" ];
151
- publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDUe5BqMDt562KOIcUm4RqZC5ejmd62elkYKkqExUYsl";
152
- };
153
- };
154
- };
155
-
156
- services.zfs.autoScrub.enable = true;
157
-
158
- services.zrepl = let
159
- defaultBackupJob = {
160
- type = "push";
161
- filesystems."rpool/safe<" = true;
162
- snapshotting = {
163
- type = "periodic";
164
- interval = "5m";
165
- prefix = "zrepl_snap_";
166
- };
167
- pruning = {
168
- keep_sender = [
169
- {
170
- type = "grid";
171
- regex = "^zrepl_snap_.*";
172
- grid = lib.concatStringsSep " | " [
173
- "3x5m"
174
- "4x15m"
175
- "24x1h"
176
- "4x1d"
177
- "3x1w"
178
- ];
179
- }
180
- ];
181
- keep_receiver = [
182
- { type = "grid";
183
- regex = "^zrepl_snap_.*";
184
- grid = lib.concatStringsSep " | " [
185
- "20x5m"
186
- "96x1h"
187
- "12x4h"
188
- "7x1d"
189
- "52x1w"
190
- "120x3w"
191
- ];
192
- }
193
- ];
194
- };
195
- };
196
- in {
197
- enable = true;
198
- settings = {
199
- global = {
200
- logging = [
201
- {
202
- type = "syslog";
203
- level = "info";
204
- format = "human";
205
- }
206
- ];
207
- };
208
-
209
- jobs = [
210
- # XXX: Broken since 2024-01-10?
211
- # (defaultBackupJob // {
212
- # name = "rsyncnet";
213
- # connect = {
214
- # identity_file = "/root/.ssh/id_ed25519";
215
- # type = "ssh+stdinserver";
216
- # host = "zh2543b.rsync.net";
217
- # user = "root";
218
- # port = 22;
219
- # };
220
- # })
221
-
222
- (defaultBackupJob // {
223
- name = "delroth";
224
- connect = {
225
- identity_file = "/root/.ssh/id_ed25519";
226
- type = "ssh+stdinserver";
227
- host = "smol.delroth.net";
228
- user = "zrepl";
229
- port = 22;
230
- };
231
- })
232
-
233
- (defaultBackupJob // {
234
- name = "hexa";
235
- connect = {
236
- identity_file = "/root/.ssh/id_ed25519";
237
- type = "ssh+stdinserver";
238
- host = "meduna.hexa-home.v6.army";
239
- user = "zrepl";
240
- port = 22;
241
- };
242
- })
243
- ];
244
- };
245
- };
246
-}
delft/haumea/boot.nix
new
+11
@@ -0,0 +1,11 @@
1
+{
2
+ boot.loader.grub = {
3
+ devices = [
4
+ "/dev/nvme0n1"
5
+ "/dev/nvme1n1"
6
+ ];
7
+ copyKernels = true;
8
+ };
9
+ boot.initrd.availableKernelModules = [ "ahci" "nvme" "usbhid" ];
10
+ boot.kernelModules = [ "kvm-amd" ];
11
+}
delft/haumea/default.nix
new
+52
@@ -0,0 +1,52 @@
1
+{ lib
2
+, modulesPath
3
+, pkgs
4
+, ...
5
+}:
6
+
7
+{
8
+ imports = [
9
+ "${modulesPath}/installer/scan/not-detected.nix"
10
+ ../common.nix
11
+ ./boot.nix
12
+ ./network.nix
13
+ ./postgresql.nix
14
+ ./zrepl.nix
15
+ ];
16
+
17
+ networking = {
18
+ hostId = "83c81a23";
19
+ hostName = "haumea";
20
+ domain = "nixos.org";
21
+ };
22
+
23
+ environment.systemPackages = [ pkgs.lz4 ];
24
+
25
+ fileSystems."/" =
26
+ { device = "rpool/safe/root";
27
+ fsType = "zfs";
28
+ };
29
+
30
+ fileSystems."/boot" =
31
+ { device = "/dev/disk/by-label/boot0";
32
+ fsType = "ext4";
33
+ };
34
+
35
+ fileSystems."/nix" =
36
+ { device = "rpool/local/nix";
37
+ fsType = "zfs";
38
+ };
39
+
40
+ fileSystems."/var/db/postgresql" =
41
+ { device = "rpool/safe/postgres";
42
+ fsType = "zfs";
43
+ };
44
+
45
+ services.zfs.autoScrub.enable = true;
46
+
47
+ nix.settings.max-jobs = lib.mkDefault 16;
48
+
49
+ powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
50
+
51
+ system.stateVersion = "14.12";
52
+}
delft/haumea/postgresql.nix
new
+103
@@ -0,0 +1,103 @@
1
+{ lib
2
+, pkgs
3
+, ...
4
+}:
5
+
6
+{
7
+ systemd.services.postgresql = {
8
+ after = [ "wireguard-wg0.service" ];
9
+ requires = [ "wireguard-wg0.service" ];
10
+ };
11
+
12
+ services.prometheus.exporters.postgres = {
13
+ enable = true;
14
+ dataSourceName = "user=root database=hydra host=/run/postgresql sslmode=disable";
15
+ firewallFilter = "-i wg0 -p tcp -m tcp --dport 9187";
16
+ openFirewall = true;
17
+ extraFlags = [
18
+ "--extend.query-path" "${pkgs.prometheus-postgres-exporter.src}/queries.yaml"
19
+ ];
20
+ };
21
+
22
+ networking.firewall.interfaces.wg0.allowedTCPPorts = [ 5432 ];
23
+
24
+ services.postgresql = {
25
+ enable = true;
26
+ package = pkgs.postgresql_16;
27
+ dataDir = "/var/db/postgresql/16";
28
+ # https://pgtune.leopard.in.ua/#/
29
+ logLinePrefix = "user=%u,db=%d,app=%a,client=%h ";
30
+ settings = {
31
+ listen_addresses = lib.mkForce "10.254.1.9";
32
+
33
+ checkpoint_completion_target = "0.9";
34
+ default_statistics_target = 100;
35
+
36
+ log_duration = "off";
37
+ log_statement = "none";
38
+
39
+ # pgbadger-compatible logging
40
+ log_transaction_sample_rate = 0.01;
41
+ log_min_duration_statement = 5000;
42
+ log_checkpoints = "on";
43
+ log_connections = "on";
44
+ log_disconnections = "on";
45
+ log_lock_waits = "on";
46
+ log_temp_files = 0;
47
+ log_autovacuum_min_duration = 0;
48
+
49
+ max_connections = 500;
50
+ work_mem = "20MB";
51
+ maintenance_work_mem = "2GB";
52
+
53
+ # 25% of memory
54
+ shared_buffers = "16GB";
55
+
56
+ # Checkpoint every 1GB. (default)
57
+ # increased after seeing many warninsg about frequent checkpoints
58
+ min_wal_size = "1GB";
59
+ max_wal_size = "2GB";
60
+ wal_buffers = "16MB";
61
+
62
+ max_worker_processes = 16;
63
+ max_parallel_workers_per_gather = 8;
64
+ max_parallel_workers = 16;
65
+
66
+ # NVMe related performance tuning
67
+ effective_io_concurrency = 200;
68
+ random_page_cost = "1.1";
69
+
70
+ # We can risk losing some transactions.
71
+ synchronous_commit = "off";
72
+
73
+ effective_cache_size = "16GB";
74
+
75
+ # Enable JIT compilation if possible.
76
+ jit = "on";
77
+
78
+ # autovacuum and autoanalyze much more frequently:
79
+ # at these values vacuum should run approximately
80
+ # every 2 mass rebuilds, or a couple times a day
81
+ # on the builds table. Some of those queries really
82
+ # benefit from frequent vacuums, so this should
83
+ # help. In particular, I'm thinking the jobsets
84
+ # pages.
85
+ autovacuum_vacuum_scale_factor = 0.002;
86
+ autovacuum_analyze_scale_factor = 0.001;
87
+
88
+ shared_preload_libraries = "pg_stat_statements";
89
+ compute_query_id = "on";
90
+ };
91
+
92
+ # FIXME: don't use 'trust'.
93
+ authentication = ''
94
+ host hydra all 10.254.1.3/32 trust
95
+ host hydra all 10.254.1.5/32 trust
96
+ local all root peer map=prometheus
97
+ '';
98
+
99
+ identMap = ''
100
+ prometheus root root
101
+ prometheus postgres-exporter root
102
+ '';
103
+ };}
delft/haumea/zrepl.nix
new
+111
@@ -0,0 +1,111 @@
1
+{ lib
2
+, ...
3
+}:
4
+
5
+{
6
+ programs.ssh = {
7
+ knownHosts = {
8
+ rsync-net = {
9
+ hostNames = [ "zh2543b.rsync.net" "2001:1620:2019::324" ];
10
+ publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKlIcNwmx7id/XdYKZzVX2KtZQ4PAsEa9KVQ9N43L3PX";
11
+ };
12
+ delroth-backup-server = {
13
+ hostNames = [ "smol.delroth.net" ];
14
+ publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID9Ta4DYE3YxXzV57s6TX6KEbIa3O4re+J4NzATCOiXb";
15
+ };
16
+ hexa-backup-server = {
17
+ hostNames = [ "meduna.hexa-home.v6.army" ];
18
+ publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDUe5BqMDt562KOIcUm4RqZC5ejmd62elkYKkqExUYsl";
19
+ };
20
+ };
21
+ };
22
+
23
+ services.zrepl = let
24
+ defaultBackupJob = {
25
+ type = "push";
26
+ filesystems."rpool/safe<" = true;
27
+ snapshotting = {
28
+ type = "periodic";
29
+ interval = "5m";
30
+ prefix = "zrepl_snap_";
31
+ };
32
+ pruning = {
33
+ keep_sender = [
34
+ {
35
+ type = "grid";
36
+ regex = "^zrepl_snap_.*";
37
+ grid = lib.concatStringsSep " | " [
38
+ "3x5m"
39
+ "4x15m"
40
+ "24x1h"
41
+ "4x1d"
42
+ "3x1w"
43
+ ];
44
+ }
45
+ ];
46
+ keep_receiver = [
47
+ { type = "grid";
48
+ regex = "^zrepl_snap_.*";
49
+ grid = lib.concatStringsSep " | " [
50
+ "20x5m"
51
+ "96x1h"
52
+ "12x4h"
53
+ "7x1d"
54
+ "52x1w"
55
+ "120x3w"
56
+ ];
57
+ }
58
+ ];
59
+ };
60
+ };
61
+ in {
62
+ enable = true;
63
+ settings = {
64
+ global = {
65
+ logging = [
66
+ {
67
+ type = "syslog";
68
+ level = "info";
69
+ format = "human";
70
+ }
71
+ ];
72
+ };
73
+
74
+ jobs = [
75
+ # XXX: Broken since 2024-01-10?
76
+ # (defaultBackupJob // {
77
+ # name = "rsyncnet";
78
+ # connect = {
79
+ # identity_file = "/root/.ssh/id_ed25519";
80
+ # type = "ssh+stdinserver";
81
+ # host = "zh2543b.rsync.net";
82
+ # user = "root";
83
+ # port = 22;
84
+ # };
85
+ # })
86
+
87
+ (defaultBackupJob // {
88
+ name = "delroth";
89
+ connect = {
90
+ identity_file = "/root/.ssh/id_ed25519";
91
+ type = "ssh+stdinserver";
92
+ host = "smol.delroth.net";
93
+ user = "zrepl";
94
+ port = 22;
95
+ };
96
+ })
97
+
98
+ (defaultBackupJob // {
99
+ name = "hexa";
100
+ connect = {
101
+ identity_file = "/root/.ssh/id_ed25519";
102
+ type = "ssh+stdinserver";
103
+ host = "meduna.hexa-home.v6.army";
104
+ user = "zrepl";
105
+ port = 22;
106
+ };
107
+ })
108
+ ];
109
+ };
110
+ };
111
+}