rhea: move the files to a subdir
Graham Christensen committed
Mar 30, 2022 at 18:23 UTC
bdafe887d11f3b542b324719e1a74fee7fede0d4
6 files changed
+418
-79
delft/network.nix
+1
-1
@@ -86,7 +86,7 @@ in {
86
};
87
88
rhea = {
89
- imports = [ ./rhea.nix ];
89
+ imports = [ ./rhea ];
90
};
91
92
mac1 = makeMac {
delft/rhea.nix
deleted
-78
@@ -1,78 +0,0 @@
1
-{ nodes, config, lib, pkgs, ... }:
2
-
3
-{
4
- imports =
5
- [ ./common.nix
6
-# ./hydra.nix
7
-# ./hydra-proxy.nix
8
-# ./fstrim.nix
9
-# ./packet-importer.nix
10
- ];
11
-
12
- deployment.targetEnv = "hetzner";
13
- deployment.hetzner.mainIPv4 = "5.9.122.43";
14
-
15
- deployment.hetzner.partitions = ''
16
- set -eux
17
-
18
- if ! [ -e /usr/local/sbin/zfs ]; then
19
- echo "installing zfs..."
20
- bash -i -c 'echo y | zfsonlinux_install'
21
- fi
22
-
23
- umount -R /mnt || true
24
-
25
- zpool destroy rpool || true
26
-
27
-
28
- for disk in /dev/nvme0n1 /dev/nvme1n1; do
29
- echo "partitioning $disk..."
30
- index="''${disk: -3:1}"
31
- parted -s $disk "mklabel gpt"
32
- parted -a optimal -s $disk "mkpart primary fat32 1m 512m"
33
- parted -a optimal -s $disk "mkpart primary zfs 512m 100%"
34
- parted -s $disk "set 1 esp on"
35
- udevadm settle
36
- mkfs.vfat -n BOOT$index ''${disk}p1
37
- done
38
-
39
- zpool create -f -o ashift=12 -o autotrim=on \
40
- -O mountpoint=legacy -O atime=off -O compression=on \
41
- rpool mirror /dev/nvme0n1p2 /dev/nvme1n1p2
42
-
43
- zfs create rpool/local
44
- zfs create rpool/local/nix
45
- zfs create -o recordsize=4k rpool/local/nix/db
46
- zfs create -o xattr=sa -o acltype=posix rpool/local/var
47
- zfs create rpool/safe
48
- zfs create rpool/safe/root
49
-
50
- mkdir -p /mnt
51
- mount -t zfs rpool/safe/root /mnt
52
-
53
- mkdir -p /mnt/nix
54
- mount -t zfs rpool/local/nix /mnt/nix
55
-
56
- mkdir -p /mnt/nix/var/nix/db
57
- mount -t zfs rpool/local/nix/db /mnt/nix/var/nix/db
58
-
59
- mkdir -p /mnt/var
60
- mount -t zfs rpool/local/var /mnt/var
61
-
62
- mkdir -p /mnt/boot
63
- mount /dev/disk/by-label/BOOT0 /mnt/boot
64
- '';
65
-
66
- networking = {
67
- firewall.allowedTCPPorts = [
68
- 80 443
69
- 9199 # hydra-notify's prometheus
70
- ];
71
- firewall.allowPing = true;
72
- firewall.logRefusedConnections = false;
73
- };
74
-
75
- time.timeZone = lib.mkForce "UTC";
76
- system.stateVersion = lib.mkForce "21.11";
77
-}
78
-
delft/rhea/default.nix
new
+36
@@ -0,0 +1,36 @@
1
+{ nodes, config, lib, pkgs, ... }:
2
+{
3
+ imports =
4
+ [ ./hardware-configuration.nix
5
+ ./hetzner.nix
6
+ ../common.nix
7
+# ../hydra.nix
8
+# ../hydra-proxy.nix
9
+# ../fstrim.nix
10
+# ../packet-importer.nix
11
+ ];
12
+
13
+ # This is a Hetzner machine, but when trying to set this machine up
14
+ # I found the Hetzner NixOps plugin isn't able to create robot
15
+ # sub-accounts, and even if I can get past that with
16
+ # `createSubAccount = false`, the bootstrap tarball doesn't work.
17
+ #
18
+ # See: ./rhea/install.md for documentation about how I set it up by
19
+ # hand.
20
+ #deployment.targetEnv = "hetzner";
21
+ #deployment.hetzner.mainIPv4 = "5.9.122.43";
22
+ deployment.targetHost = "5.9.122.43";
23
+
24
+ networking = {
25
+ firewall.allowedTCPPorts = [
26
+ 80 443
27
+ 9199 # hydra-notify's prometheus
28
+ ];
29
+ firewall.allowPing = true;
30
+ firewall.logRefusedConnections = false;
31
+ };
32
+
33
+ time.timeZone = lib.mkForce "UTC";
34
+ system.stateVersion = lib.mkForce "21.11";
35
+}
36
+
delft/rhea/hardware-configuration.nix
new
+43
@@ -0,0 +1,43 @@
1
+{ config, lib, pkgs, modulesPath, ... }:
2
+
3
+{
4
+ imports =
5
+ [ (modulesPath + "/installer/scan/not-detected.nix")
6
+ ];
7
+
8
+ boot.initrd.availableKernelModules = [ "ahci" "nvme" ];
9
+ boot.initrd.kernelModules = [ ];
10
+ boot.kernelModules = [ "kvm-amd" ];
11
+ boot.extraModulePackages = [ ];
12
+
13
+ fileSystems."/" =
14
+ { device = "rpool/safe/root";
15
+ fsType = "zfs";
16
+ };
17
+
18
+ fileSystems."/nix" =
19
+ { device = "rpool/local/nix";
20
+ fsType = "zfs";
21
+ };
22
+
23
+ fileSystems."/nix/var/nix/db" =
24
+ { device = "rpool/local/nix/db";
25
+ fsType = "zfs";
26
+ neededForBoot = true;
27
+ };
28
+
29
+ fileSystems."/var" =
30
+ { device = "rpool/local/var";
31
+ fsType = "zfs";
32
+ };
33
+
34
+ fileSystems."/boot" =
35
+ { device = "/dev/disk/by-uuid/073C-DDED";
36
+ fsType = "vfat";
37
+ };
38
+
39
+ swapDevices = [ ];
40
+
41
+ powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
42
+ hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
43
+}
delft/rhea/hetzner.nix
new
+33
@@ -0,0 +1,33 @@
1
+{
2
+ networking.hostId = "9cd372da";
3
+ networking.useNetworkd = true;
4
+ systemd.network.networks."40-enp7s0" = {
5
+ matchConfig = {
6
+ MACAddress = "50:eb:f6:22:f0:3a";
7
+ };
8
+
9
+ addresses = [
10
+ {
11
+ addressConfig.Address = "5.9.122.43/27";
12
+ }
13
+ {
14
+ addressConfig.Address = "2a01:4f8:162:71eb::/64";
15
+ }
16
+ ];
17
+ routes = [
18
+ {
19
+ routeConfig.Gateway = "5.9.122.33";
20
+ }
21
+ {
22
+ routeConfig.Gateway = "fe80::1";
23
+ }
24
+ ];
25
+
26
+ dns = [
27
+ "185.12.64.1"
28
+ "185.12.64.2"
29
+ "2a01:4ff:ff00::add:1"
30
+ "2a01:4ff:ff00::add:2"
31
+ ];
32
+ };
33
+}
delft/rhea/install.md
new
+305
@@ -0,0 +1,305 @@
1
+# Setup
2
+
3
+## Switch to UEFI
4
+First submit a support ticket asking them to enable UEFI.
5
+See: https://docs.hetzner.com/robot/dedicated-server/operating-systems/uefi/
6
+
7
+# Correct the NVMe namespace's block size
8
+
9
+Verify the NVMe disks are formatted at the namespace level with 4096 blocks.
10
+See https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Hardware.html#nvme-low-level-formatting
11
+
12
+This disk's LBA is 512:
13
+
14
+```console
15
+root@rescue ~ # smartctl -a /dev/nvme1n1
16
+smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.16.5] (local build)
17
+Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org
18
+
19
+=== START OF INFORMATION SECTION ===
20
+Model Number: SAMSUNG MZQL23T8HCLS-00A07
21
+Serial Number: S64HNE0T226681
22
+Firmware Version: GDC5602Q
23
+PCI Vendor/Subsystem ID: 0x144d
24
+IEEE OUI Identifier: 0x002538
25
+Total NVM Capacity: 3,840,755,982,336 [3.84 TB]
26
+Unallocated NVM Capacity: 0
27
+Controller ID: 6
28
+NVMe Version: 1.4
29
+Number of Namespaces: 32
30
+Namespace 1 Size/Capacity: 3,840,755,982,336 [3.84 TB]
31
+Namespace 1 Utilization: 4,309,307,392 [4.30 GB]
32
+Namespace 1 Formatted LBA Size: 512
33
+Local Time is: Wed Mar 30 03:28:16 2022 CEST
34
+Firmware Updates (0x17): 3 Slots, Slot 1 R/O, no Reset required
35
+Optional Admin Commands (0x005f): Security Format Frmw_DL NS_Mngmt Self_Test MI_Snd/Rec
36
+Optional NVM Commands (0x005f): Comp Wr_Unc DS_Mngmt Wr_Zero Sav/Sel_Feat Timestmp
37
+Log Page Attributes (0x0e): Cmd_Eff_Lg Ext_Get_Lg Telmtry_Lg
38
+Maximum Data Transfer Size: 512 Pages
39
+Warning Comp. Temp. Threshold: 80 Celsius
40
+Critical Comp. Temp. Threshold: 83 Celsius
41
+Namespace 1 Features (0x1a): NA_Fields No_ID_Reuse NP_Fields
42
+
43
+Supported Power States
44
+St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat
45
+ 0 + 25.00W 14.00W - 0 0 0 0 70 70
46
+ 1 + 8.00W 8.00W - 1 1 1 1 70 70
47
+
48
+Supported LBA Sizes (NSID 0x1)
49
+Id Fmt Data Metadt Rel_Perf
50
+ 0 + 512 0 0
51
+ 1 - 4096 0 0
52
+
53
+=== START OF SMART DATA SECTION ===
54
+SMART overall-health self-assessment test result: PASSED
55
+
56
+SMART/Health Information (NVMe Log 0x02)
57
+Critical Warning: 0x00
58
+Temperature: 43 Celsius
59
+Available Spare: 100%
60
+Available Spare Threshold: 10%
61
+Percentage Used: 0%
62
+Data Units Read: 187 [95.7 MB]
63
+Data Units Written: 8,423 [4.31 GB]
64
+Host Read Commands: 2,591
65
+Host Write Commands: 3,438
66
+Controller Busy Time: 0
67
+Power Cycles: 5
68
+Power On Hours: 203
69
+Unsafe Shutdowns: 0
70
+Media and Data Integrity Errors: 0
71
+Error Information Log Entries: 0
72
+Warning Comp. Temperature Time: 0
73
+Critical Comp. Temperature Time: 0
74
+Temperature Sensor 1: 43 Celsius
75
+Temperature Sensor 2: 53 Celsius
76
+
77
+Error Information (NVMe Log 0x01, 16 of 64 entries)
78
+No Errors Logged
79
+```
80
+
81
+and correctable with:
82
+
83
+```sh
84
+nvme format /dev/nvme0n1 -l 1
85
+```
86
+
87
+which yields a corrected formatting:
88
+
89
+```console
90
+root@rescue ~ # smartctl -a /dev/nvme1n1
91
+smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.16.5] (local build)
92
+Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org
93
+
94
+=== START OF INFORMATION SECTION ===
95
+Model Number: SAMSUNG MZQL23T8HCLS-00A07
96
+Serial Number: S64HNE0T226681
97
+Firmware Version: GDC5602Q
98
+PCI Vendor/Subsystem ID: 0x144d
99
+IEEE OUI Identifier: 0x002538
100
+Total NVM Capacity: 3,840,755,982,336 [3.84 TB]
101
+Unallocated NVM Capacity: 0
102
+Controller ID: 6
103
+NVMe Version: 1.4
104
+Number of Namespaces: 32
105
+Namespace 1 Size/Capacity: 3,840,755,982,336 [3.84 TB]
106
+Namespace 1 Utilization: 4,309,307,392 [4.30 GB]
107
+Namespace 1 Formatted LBA Size: 512
108
+Local Time is: Wed Mar 30 03:29:46 2022 CEST
109
+Firmware Updates (0x17): 3 Slots, Slot 1 R/O, no Reset required
110
+Optional Admin Commands (0x005f): Security Format Frmw_DL NS_Mngmt Self_Test MI_Snd/Rec
111
+Optional NVM Commands (0x005f): Comp Wr_Unc DS_Mngmt Wr_Zero Sav/Sel_Feat Timestmp
112
+Log Page Attributes (0x0e): Cmd_Eff_Lg Ext_Get_Lg Telmtry_Lg
113
+Maximum Data Transfer Size: 512 Pages
114
+Warning Comp. Temp. Threshold: 80 Celsius
115
+Critical Comp. Temp. Threshold: 83 Celsius
116
+Namespace 1 Features (0x1a): NA_Fields No_ID_Reuse NP_Fields
117
+
118
+Supported Power States
119
+St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat
120
+ 0 + 25.00W 14.00W - 0 0 0 0 70 70
121
+ 1 + 8.00W 8.00W - 1 1 1 1 70 70
122
+
123
+Supported LBA Sizes (NSID 0x1)
124
+Id Fmt Data Metadt Rel_Perf
125
+ 0 + 512 0 0
126
+ 1 - 4096 0 0
127
+
128
+=== START OF SMART DATA SECTION ===
129
+SMART overall-health self-assessment test result: PASSED
130
+
131
+SMART/Health Information (NVMe Log 0x02)
132
+Critical Warning: 0x00
133
+Temperature: 43 Celsius
134
+Available Spare: 100%
135
+Available Spare Threshold: 10%
136
+Percentage Used: 0%
137
+Data Units Read: 187 [95.7 MB]
138
+Data Units Written: 8,423 [4.31 GB]
139
+Host Read Commands: 2,591
140
+Host Write Commands: 3,438
141
+Controller Busy Time: 0
142
+Power Cycles: 5
143
+Power On Hours: 203
144
+Unsafe Shutdowns: 0
145
+Media and Data Integrity Errors: 0
146
+Error Information Log Entries: 0
147
+Warning Comp. Temperature Time: 0
148
+Critical Comp. Temperature Time: 0
149
+Temperature Sensor 1: 43 Celsius
150
+Temperature Sensor 2: 53 Celsius
151
+
152
+Error Information (NVMe Log 0x01, 16 of 64 entries)
153
+No Errors Logged
154
+```
155
+
156
+We can now use an ashift of 12 (2^12 = 4096) without a performance penalty.
157
+
158
+## Partitioning
159
+
160
+The following script can be, and was run fully automatically:
161
+
162
+```sh
163
+set -eux
164
+
165
+if ! [ -e /usr/local/sbin/zfs ]; then
166
+echo "installing zfs..."
167
+bash -i -c 'echo y | zfsonlinux_install'
168
+fi
169
+
170
+umount -R /mnt || true
171
+
172
+zpool destroy rpool || true
173
+
174
+
175
+for disk in /dev/nvme0n1 /dev/nvme1n1; do
176
+echo "partitioning $disk..."
177
+index="${disk: -3:1}"
178
+parted -s $disk "mklabel gpt"
179
+parted -a optimal -s $disk "mkpart primary fat32 1m 512m"
180
+parted -a optimal -s $disk "mkpart primary zfs 512m 100%"
181
+parted -s $disk "set 1 esp on"
182
+udevadm settle
183
+mkfs.vfat -n BOOT$index ''${disk}p1
184
+done
185
+
186
+zpool create -f -o ashift=12 -o autotrim=on \
187
+-O mountpoint=legacy -O atime=off -O compression=on \
188
+rpool mirror /dev/nvme0n1p2 /dev/nvme1n1p2
189
+
190
+zfs create rpool/local
191
+zfs create rpool/local/nix
192
+zfs create -o recordsize=4k rpool/local/nix/db
193
+zfs create -o xattr=sa -o acltype=posix rpool/local/var
194
+zfs create rpool/safe
195
+zfs create rpool/safe/root
196
+
197
+mkdir -p /mnt
198
+mount -t zfs rpool/safe/root /mnt
199
+
200
+mkdir -p /mnt/nix
201
+mount -t zfs rpool/local/nix /mnt/nix
202
+
203
+mkdir -p /mnt/nix/var/nix/db
204
+mount -t zfs rpool/local/nix/db /mnt/nix/var/nix/db
205
+
206
+mkdir -p /mnt/var
207
+mount -t zfs rpool/local/var /mnt/var
208
+
209
+mkdir -p /mnt/boot
210
+mount /dev/disk/by-label/BOOT0 /mnt/boot
211
+```
212
+
213
+## Installing Nix
214
+
215
+Install Nix into the rescue system as root:
216
+
217
+```sh
218
+groupadd -g 30000 nixbld
219
+useradd --system --groups nixbld nixbld1
220
+useradd --system --groups nixbld nixbld2
221
+useradd --system --groups nixbld nixbld3
222
+useradd --system --groups nixbld nixbld4
223
+useradd --system --groups nixbld nixbld5
224
+mkdir -m 0755 /nix && chown root /nix
225
+sh <(curl -L https://nixos.org/nix/install) --no-daemon
226
+```
227
+
228
+## Configure NixOS
229
+
230
+```sh
231
+nix-shell -p nixos-install-tools -I nixpkgs=channel:nixos-21.11
232
+
233
+nixos-generate-config --root /mnt
234
+```
235
+
236
+In the `configuration.nix`:
237
+
238
+1. Add `hetzner.nix` to the list of `imports` at the top.
239
+2. Add an authorized key and enable SSH. This will be removed later when it is imported into NixOps, so it is just for bootstrapping:
240
+
241
+```
242
+services.openssh.enable = true;
243
+users.users.root.openssh.authorizedKeys.keys = [ "ssh-..." ];
244
+```
245
+
246
+### Hardware Configuration Changes
247
+
248
+Edit `hardware-configuration.nix` and change the fileSystems value for `/nix/var/nix` to make it required for boot:
249
+
250
+```nix
251
+ fileSystems."/nix/var/nix/db" =
252
+ { device = "rpool/local/nix/db";
253
+ fsType = "zfs";
254
+ neededForBoot = true;
255
+ };
256
+```
257
+
258
+### Hetzner.nix
259
+
260
+Then create a file, `hetzner.nix`.
261
+
262
+* The all-zeros hostId is fine, though I generated one with `head -c4 /dev/urandom | od -A none -t x4`
263
+* The `enp7s0` and `MACAddress` value I got from `ip addr`
264
+* The IP addresses and gateways I got from the Robot webpage under the IPs tab, hovering over the IPv4 and IPv6 addresses.
265
+* Thee DNS resolvers I got from https://docs.hetzner.com/dns-console/dns/general/recursive-name-servers/
266
+
267
+```nix
268
+{
269
+ networking.hostId = "00000000";
270
+ networking.useNetworkd = true;
271
+ systemd.network.networks."40-enp7s0" = {
272
+ matchConfig.MACAddress = "50:eb:f6:22:f0:3a";
273
+
274
+ addresses = [
275
+ {
276
+ addressConfig.Address = "5.9.122.43/27";
277
+ }
278
+ {
279
+ addressConfig.Address = "2a01:4f8:162:71eb::/64";
280
+ }
281
+ ];
282
+ routes = [
283
+ {
284
+ routeConfig.Gateway = "5.9.122.33";
285
+ }
286
+ {
287
+ routeConfig.Gateway = "fe80::1";
288
+ }
289
+ ];
290
+
291
+ dns = [
292
+ "185.12.64.1"
293
+ "185.12.64.2"
294
+ "2a01:4ff:ff00::add:1"
295
+ "2a01:4ff:ff00::add:2"
296
+ ];
297
+ };
298
+}
299
+```
300
+
301
+Then run:
302
+
303
+```
304
+nixos-install -I nixpkgs=channel:nixos-21.11
305
+```
\ No newline at end of file