titan: init
New machine to replace haumea, which exhibits correctable ECC errors. But instead of taking it out of production for maintenance we pick up a more potent replacement machine that can better serve our needs.
Martin Weinelt committed
Nov 26, 2025 at 22:57 UTC
2272cf5b6499d934f4b7bd0b45e9023a338b45a6
13 files changed
+462
build/flake-module.nix
+11
@@ -23,6 +23,7 @@ in
23
haumea = { };
24
pluto = { };
25
mimas = { };
26
+ titan = { };
27
};
28
29
flake = {
@@ -55,6 +56,16 @@ in
56
./mimas
57
];
58
};
59
+
60
+ nixosConfigurations.titan = lib.nixosSystem {
61
+ system = "x86_64-linux";
62
+
63
+ specialArgs = { inherit inputs; };
64
+ modules = [
65
+ flakesModule
66
+ ./titan
67
+ ];
68
+ };
69
};
70
71
perSystem =
build/pluto/prometheus/exporters/nixos.nix
+1
@@ -19,6 +19,7 @@
19
labels.role = "database";
20
targets = [
21
"haumea.nixos.org:9300"
22
+ "titan.nixos.org:9300"
23
];
24
}
25
];
build/pluto/prometheus/exporters/node.nix
+1
@@ -16,6 +16,7 @@
16
labels.role = "database";
17
targets = [
18
"haumea.nixos.org:9100"
19
+ "titan.nixos.org:9100"
20
];
21
}
22
{
build/pluto/prometheus/exporters/postgresql.nix
+1
@@ -7,6 +7,7 @@
7
{
8
targets = [
9
"haumea.nixos.org:9187"
10
+ "titan.nixos.org:9187"
11
"tracker.security.nixos.org:9187"
12
];
13
}
build/pluto/prometheus/exporters/rasdaemon.nix
+1
@@ -12,6 +12,7 @@
12
"mimas.nixos.org:10029"
13
"haumea.nixos.org:10029"
14
"pluto.nixos.org:10029"
15
+ "titan.nixos.org:10029"
16
17
# builders
18
"elated-minsky.builder.nixos.org:10029"
build/pluto/prometheus/exporters/zfs.nix
+1
@@ -13,6 +13,7 @@
13
"haumea.nixos.org:9134"
14
"mimas.nixos.org:9134"
15
"pluto.nixos.org:9134"
16
+ "titan.nixos.org:9134"
17
];
18
}
19
];
build/titan/boot.nix
new
+30
@@ -0,0 +1,30 @@
1
+{
2
+ boot = {
3
+ initrd.availableKernelModules = [
4
+ "ahci"
5
+ "xhci_pci"
6
+ "nvme"
7
+ "usbhid"
8
+ ];
9
+ kernelModules = [ "kvm-amd" ];
10
+ supportedFilesystems.zfs = true;
11
+ loader = {
12
+ efi.canTouchEfiVariables = false;
13
+ grub = {
14
+ enable = true;
15
+ efiSupport = true;
16
+ efiInstallAsRemovable = true;
17
+ mirroredBoots = [
18
+ {
19
+ devices = [ "nodev" ];
20
+ path = "/efi/a";
21
+ }
22
+ {
23
+ devices = [ "nodev" ];
24
+ path = "/efi/b";
25
+ }
26
+ ];
27
+ };
28
+ };
29
+ };
30
+}
build/titan/default.nix
new
+21
@@ -0,0 +1,21 @@
1
+{
2
+ imports = [
3
+ ../common.nix
4
+ ./boot.nix
5
+ ./network.nix
6
+ ./postgresql.nix
7
+ ./zrepl.nix
8
+ ];
9
+
10
+ disko.devices = import ./disko.nix;
11
+
12
+ networking = {
13
+ hostId = "e1ce6466";
14
+ hostName = "titan";
15
+ domain = "nixos.org";
16
+ };
17
+
18
+ services.zfs.autoScrub.enable = true;
19
+
20
+ system.stateVersion = "25.11";
21
+}
build/titan/disko.nix
new
+78
@@ -0,0 +1,78 @@
1
+let
2
+ layout = id: {
3
+ type = "gpt";
4
+ partitions = {
5
+ esp = {
6
+ type = "EF00";
7
+ size = "1G";
8
+ content = {
9
+ type = "filesystem";
10
+ format = "vfat";
11
+ mountpoint = "/efi/${id}";
12
+ };
13
+ };
14
+ zfs = {
15
+ size = "100%";
16
+ content = {
17
+ type = "zfs";
18
+ pool = "zroot";
19
+ };
20
+ };
21
+ };
22
+ };
23
+in
24
+{
25
+ disk = {
26
+ nvme0n1 = {
27
+ type = "disk";
28
+ device = "/dev/disk/by-id/nvme-MTFDKCC1T9TGP-1BK1DABYY_0925109FB623";
29
+ content = layout "a";
30
+ };
31
+ nvme1n1 = {
32
+ type = "disk";
33
+ device = "/dev/disk/by-id/nvme-MTFDKCC1T9TGP-1BK1DABYY_0925109FB922";
34
+ content = layout "b";
35
+ };
36
+ };
37
+
38
+ zpool.zroot = {
39
+ type = "zpool";
40
+ mode = "mirror";
41
+ options.ashift = "12";
42
+
43
+ rootFsOptions = {
44
+ acltype = "posixacl";
45
+ atime = "off";
46
+ compression = "zstd-3";
47
+ mountpoint = "none";
48
+ xattr = "sa";
49
+ };
50
+
51
+ datasets = {
52
+ "root" = {
53
+ type = "zfs_fs";
54
+ mountpoint = "/";
55
+ };
56
+ "nix" = {
57
+ type = "zfs_fs";
58
+ mountpoint = "/nix";
59
+ };
60
+ "pg" = {
61
+ type = "zfs_fs";
62
+ mountpoint = "/var/lib/postgresql";
63
+ options = {
64
+ logbias = "latency";
65
+ recordsize = "16K";
66
+ redundant_metadata = "most";
67
+ };
68
+ };
69
+ "reserved" = {
70
+ type = "zfs_fs";
71
+ options = {
72
+ canmount = "off";
73
+ refreservation = "16G"; # roughly one system closure
74
+ };
75
+ };
76
+ };
77
+ };
78
+}
build/titan/network.nix
new
+49
@@ -0,0 +1,49 @@
1
+{
2
+ systemd.network = {
3
+ enable = true;
4
+ netdevs = {
5
+ "20-vlan4000" = {
6
+ netdevConfig = {
7
+ Kind = "vlan";
8
+ Name = "vlan4000";
9
+ };
10
+ vlanConfig.Id = 4000;
11
+ };
12
+ };
13
+ networks = {
14
+ "30-enp35s0" = {
15
+ matchConfig = {
16
+ MACAddress = "9c:6b:00:1f:aa:fd";
17
+ Type = "ether";
18
+ };
19
+ address = [
20
+ "159.69.62.224/26"
21
+ "2a01:4f8:231:e53::1/64"
22
+ ];
23
+ routes = [
24
+ { Gateway = "159.69.62.193"; }
25
+ { Gateway = "fe80::1"; }
26
+ ];
27
+ vlan = [
28
+ "vlan4000"
29
+ ];
30
+ networkConfig.Description = "WAN";
31
+ linkConfig.RequiredForOnline = true;
32
+ };
33
+ "30-vlan4000" = {
34
+ matchConfig.Name = "vlan4000";
35
+ networkConfig = {
36
+ DHCP = false;
37
+ IPv6AcceptRA = false;
38
+ };
39
+ linkConfig = {
40
+ MTUBytes = "1400";
41
+ RequiredForOnline = "routable";
42
+ };
43
+ address = [
44
+ "10.0.40.3/31"
45
+ ];
46
+ };
47
+ };
48
+ };
49
+}
build/titan/postgresql.nix
new
+109
@@ -0,0 +1,109 @@
1
+{
2
+ config,
3
+ lib,
4
+ pkgs,
5
+ ...
6
+}:
7
+
8
+{
9
+ services.prometheus.exporters.postgres = {
10
+ enable = true;
11
+ dataSourceName = "user=root database=hydra host=/run/postgresql sslmode=disable";
12
+ openFirewall = true;
13
+ firewallRules = ''
14
+ ip6 saddr $prometheus_inet6 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept
15
+ ip saddr $prometheus_inet4 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept
16
+ '';
17
+ };
18
+
19
+ networking.firewall.interfaces."vlan4000".allowedTCPPorts = [ 5432 ];
20
+
21
+ services.postgresql = {
22
+ enable = false; # TODO: enable after data migration
23
+ enableJIT = true;
24
+ package = pkgs.postgresql_16;
25
+ # https://pgtune.leopard.in.ua/#/
26
+ settings = {
27
+ listen_addresses = lib.mkForce "10.254.1.9";
28
+
29
+ # https://vadosware.io/post/everything-ive-seen-on-optimizing-postgres-on-zfs-on-linux/#zfs-related-tunables-on-the-postgres-side
30
+ full_page_writes = "off";
31
+
32
+ wal_init_zero = "off";
33
+ wal_recycle = "off";
34
+
35
+ checkpoint_completion_target = "0.9";
36
+ default_statistics_target = 100;
37
+
38
+ log_duration = "off";
39
+ log_statement = "none";
40
+
41
+ # pgbadger-compatible logging
42
+ log_transaction_sample_rate = 0.01;
43
+ log_min_duration_statement = 5000;
44
+ log_checkpoints = "on";
45
+ log_connections = "on";
46
+ log_disconnections = "on";
47
+ log_lock_waits = "on";
48
+ log_temp_files = 0;
49
+ log_autovacuum_min_duration = 0;
50
+ log_line_prefix = "user=%u,db=%d,app=%a,client=%h ";
51
+
52
+ max_connections = 500;
53
+ work_mem = "20MB";
54
+ maintenance_work_mem = "2GB";
55
+
56
+ # 25% of memory
57
+ shared_buffers = "32GB";
58
+
59
+ # Checkpoint every 1GB. (default)
60
+ # increased after seeing many warnings about frequent checkpoints
61
+ min_wal_size = "1GB";
62
+ max_wal_size = "4GB";
63
+ wal_buffers = "16MB";
64
+
65
+ max_worker_processes = 32;
66
+ max_parallel_workers_per_gather = 4;
67
+ max_parallel_workers = 32;
68
+
69
+ # NVMe related performance tuning
70
+ effective_io_concurrency = 200;
71
+ random_page_cost = "1.1";
72
+
73
+ # We can risk losing some transactions.
74
+ synchronous_commit = "off";
75
+
76
+ effective_cache_size = "64GB";
77
+
78
+ # try to allocate huge pages, if possible
79
+ huge_pages = "try";
80
+
81
+ # Enable JIT compilation if possible.
82
+ jit = "on";
83
+
84
+ # autovacuum and autoanalyze much more frequently:
85
+ # at these values vacuum should run approximately
86
+ # every 2 mass rebuilds, or a couple times a day
87
+ # on the builds table. Some of those queries really
88
+ # benefit from frequent vacuums, so this should
89
+ # help. In particular, I'm thinking the jobsets
90
+ # pages.
91
+ autovacuum_vacuum_scale_factor = 0.02;
92
+ autovacuum_analyze_scale_factor = 0.01;
93
+
94
+ shared_preload_libraries = "pg_stat_statements";
95
+ compute_query_id = "on";
96
+ };
97
+
98
+ # FIXME: don't use 'trust'.
99
+ authentication = ''
100
+ host hydra all 10.254.1.1/32 trust
101
+ local all root peer map=prometheus
102
+ '';
103
+
104
+ identMap = ''
105
+ prometheus root root
106
+ prometheus postgres-exporter root
107
+ '';
108
+ };
109
+}
build/titan/zrepl.nix
new
+135
@@ -0,0 +1,135 @@
1
+{
2
+ config,
3
+ lib,
4
+ ...
5
+}:
6
+
7
+{
8
+ age.secrets."zrepl-ssh-key" = {
9
+ file = ../secrets/zrepl-ssh-key.age;
10
+ mode = "0400";
11
+ };
12
+
13
+ programs.ssh = {
14
+ knownHosts = {
15
+ rsync-net = {
16
+ hostNames = [
17
+ "zh2543b.rsync.net"
18
+ "2001:1620:2019::324"
19
+ ];
20
+ publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKlIcNwmx7id/XdYKZzVX2KtZQ4PAsEa9KVQ9N43L3PX";
21
+ };
22
+ };
23
+ };
24
+
25
+ services.zrepl =
26
+ let
27
+ defaultBackupJob = {
28
+ type = "push";
29
+ filesystems."zroot/pg<" = true;
30
+ snapshotting = {
31
+ type = "periodic";
32
+ interval = "30m";
33
+ prefix = "zrepl_snap_";
34
+ hooks = [
35
+ {
36
+ # https://zrepl.github.io/master/configuration/snapshotting.html#postgres-checkpoint-hook
37
+ type = "postgres-checkpoint";
38
+ dsn = "host=/run/postgresql dbname=hydra user=root sslmode=disable";
39
+ filesystems."zroot/pg" = true;
40
+ }
41
+ ];
42
+ };
43
+
44
+ # The current pruning setup is an exponentially growing scheme, at both sides.
45
+ pruning = {
46
+ keep_sender = [
47
+ { type = "not_replicated"; }
48
+ {
49
+ type = "grid";
50
+ regex = "^zrepl_snap_.*";
51
+ grid = lib.concatStringsSep " | " [
52
+ "1x1h(keep=all)"
53
+ "1x1h"
54
+ "1x2h"
55
+ "1x4h"
56
+ # "grid" acts weird if an interval isn't a whole-number multiple
57
+ # of the previous one, so we jump from 8h to 24h
58
+ "2x8h"
59
+ "1x1d"
60
+ "1x2d"
61
+ "1x4d"
62
+ "1x8d"
63
+ # At this point we keep ~10 snapshots spanning 8--16 days (depends on moment),
64
+ # with exponentially increasing spacing (almost).
65
+ ];
66
+ }
67
+ ];
68
+ keep_receiver = [
69
+ {
70
+ type = "grid";
71
+ regex = "^zrepl_snap_.*";
72
+ grid = lib.concatStringsSep " | " [
73
+ "2x1h(keep=all)"
74
+ "2x1h"
75
+ "2x2h"
76
+ "2x4h"
77
+ "4x8h"
78
+ # At this point the grid spans 2 days by ~13 snapshots.
79
+ # (See note above about 8h -> 24h.)
80
+ "2x1d"
81
+ "2x2d"
82
+ "2x4d"
83
+ "2x8d"
84
+ "2x16d"
85
+ "2x32d"
86
+ "2x64d"
87
+ "2x128d"
88
+ # At this point we keep ~29 snapshots spanning 384--512 days (depends on moment),
89
+ # with exponentially increasing spacing (almost).
90
+ ];
91
+ }
92
+ ];
93
+ };
94
+ };
95
+ in
96
+ {
97
+ enable = false; # TODO: enable post migration
98
+ settings = {
99
+ global = {
100
+ logging = [
101
+ {
102
+ type = "syslog";
103
+ level = "info";
104
+ format = "human";
105
+ }
106
+ ];
107
+ };
108
+
109
+ jobs = [
110
+ # Covers 20240629+
111
+ (
112
+ defaultBackupJob
113
+ // {
114
+ name = "rsyncnet";
115
+ connect = {
116
+ identity_file = config.age.secrets."zrepl-ssh-key".path;
117
+ type = "ssh+stdinserver";
118
+ host = "zh4461b.rsync.net";
119
+ user = "root";
120
+ port = 22;
121
+ };
122
+ }
123
+ )
124
+ /*
125
+ rsync.net provides a VM with FreeBSD
126
+ - almost nothing is preserved on upgrades except this "data1" zpool
127
+ $ scp ./zrepl.yml root@zh4461b.rsync.net:/usr/local/etc/zrepl/zrepl.yml
128
+ # pkg install zrepl
129
+ # service zrepl enable
130
+ # service zrepl start
131
+ */
132
+ ];
133
+ };
134
+ };
135
+}
build/titan/zrepl.yml
new
+24
@@ -0,0 +1,24 @@
1
+# root@zh4461b.rsync.net:/usr/local/etc/zrepl/zrepl.yml
2
+# zrepl main configuration file.
3
+# For documentation, refer to https://zrepl.github.io/
4
+#
5
+global:
6
+ logging:
7
+ - type: "stdout"
8
+ level: "error"
9
+ format: "human"
10
+ - type: "syslog"
11
+ level: "info"
12
+ format: "logfmt"
13
+
14
+# mostly from https://blog.lenny.ninja/zrepl-on-rsync-net.html
15
+jobs:
16
+ - name: sink
17
+ type: sink
18
+ serve:
19
+ type: stdinserver
20
+ client_identities: [titan]
21
+ recv:
22
+ placeholder:
23
+ encryption: off
24
+ root_fs: "data1"