hydra: migrate database to mimas
In the current setup mimas is not the central bottleneck for sending inputs and compressing outputs, so we have some spare cycles that we can use to make database traffic local again. This will allow us to reduce lateny, increase throughput and allow bandwidth bursts beyond the 1 Gbps network link we used before.
Martin Weinelt committed
Jul 15, 2026 at 16:53 UTC
157b915fc3d97265f1c830a4b6fb49a0b51d52b3
11 files changed
+295
-3
build/hydra-queue-runner.nix
+1
-1
@@ -84,7 +84,7 @@ in
84
enable = true;
85
awsCredentialsFile = config.age.secrets."hydra-aws-credentials".path;
86
settings = {
87
- dbUrl = "postgres://hydra@10.0.40.3:5432/hydra";
87
+ dbUrl = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra";
88
machineFreeFn = "DynamicWithMaxJobLimit";
89
stepSortFn = "WithCriticalPath";
90
usePresignedUploads = true;
build/hydra.nix
-1
@@ -62,7 +62,6 @@ in
62
};
63
64
services.hydra-dev.enable = true;
65
- services.hydra-dev.dbi = "dbi:Pg:dbname=hydra;host=10.0.40.3;user=hydra;";
65
services.hydra-dev.logo = ./hydra-logo.png;
66
services.hydra-dev.hydraURL = "https://hydra.nixos.org";
67
services.hydra-dev.notificationSender = "edolstra@gmail.com";
build/mimas/default.nix
+2
@@ -6,6 +6,8 @@
6
../hydra-queue-runner.nix
7
./boot.nix
8
./network.nix
9
+ ./postgresql.nix
10
+ ./zrepl.nix
11
];
12
13
disko.devices = import ./disko.nix;
build/mimas/disko.nix
+9
@@ -69,6 +69,15 @@ in
69
type = "zfs_fs";
70
mountpoint = "/var/lib/hydra";
71
};
72
+ "pg" = {
73
+ type = "zfs_fs";
74
+ mountpoint = "/var/lib/postgresql";
75
+ options = {
76
+ logbias = "latency";
77
+ recordsize = "16K";
78
+ redundant_metadata = "most";
79
+ };
80
+ };
81
"reserved" = {
82
type = "zfs_fs";
83
options = {
build/mimas/postgresql.nix
new
+102
@@ -0,0 +1,102 @@
1
+{
2
+ config,
3
+ pkgs,
4
+ ...
5
+}:
6
+
7
+{
8
+ services.prometheus.exporters.postgres = {
9
+ enable = true;
10
+ dataSourceName = "user=root database=hydra host=/run/postgresql sslmode=disable";
11
+ openFirewall = true;
12
+ firewallRules = ''
13
+ ip6 saddr $prometheus_inet6 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept
14
+ ip saddr $prometheus_inet4 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept
15
+ '';
16
+ };
17
+
18
+ services.postgresql = {
19
+ enable = true;
20
+ enableJIT = true;
21
+ package = pkgs.postgresql_18;
22
+ # https://pgtune.leopard.in.ua/#/
23
+ settings = {
24
+ # https://vadosware.io/post/everything-ive-seen-on-optimizing-postgres-on-zfs-on-linux/#zfs-related-tunables-on-the-postgres-side
25
+ full_page_writes = "off";
26
+
27
+ wal_init_zero = "off";
28
+ wal_recycle = "off";
29
+
30
+ checkpoint_completion_target = "0.9";
31
+ default_statistics_target = 100;
32
+
33
+ log_duration = "off";
34
+ log_statement = "none";
35
+
36
+ # pgbadger-compatible logging
37
+ log_transaction_sample_rate = 0.01;
38
+ log_min_duration_statement = 5000;
39
+ log_checkpoints = "on";
40
+ log_connections = "on";
41
+ log_disconnections = "on";
42
+ log_lock_waits = "on";
43
+ log_temp_files = 0;
44
+ log_autovacuum_min_duration = 0;
45
+ log_line_prefix = "user=%u,db=%d,app=%a,client=%h ";
46
+
47
+ max_connections = 500;
48
+ work_mem = "20MB";
49
+ maintenance_work_mem = "2GB";
50
+
51
+ # 25% of memory
52
+ shared_buffers = "32GB";
53
+
54
+ # Checkpoint every 1GB. (default)
55
+ # increased after seeing many warnings about frequent checkpoints
56
+ min_wal_size = "1GB";
57
+ max_wal_size = "4GB";
58
+ wal_buffers = "16MB";
59
+
60
+ max_worker_processes = 32;
61
+ max_parallel_workers_per_gather = 4;
62
+ max_parallel_workers = 32;
63
+
64
+ # NVMe related performance tuning
65
+ effective_io_concurrency = 200;
66
+ random_page_cost = "1.1";
67
+
68
+ # We can risk losing some transactions.
69
+ synchronous_commit = "off";
70
+
71
+ effective_cache_size = "64GB";
72
+
73
+ # try to allocate huge pages, if possible
74
+ huge_pages = "try";
75
+
76
+ # Enable JIT compilation if possible.
77
+ jit = "on";
78
+
79
+ # autovacuum and autoanalyze much more frequently:
80
+ # at these values vacuum should run approximately
81
+ # every 2 mass rebuilds, or a couple times a day
82
+ # on the builds table. Some of those queries really
83
+ # benefit from frequent vacuums, so this should
84
+ # help. In particular, I'm thinking the jobsets
85
+ # pages.
86
+ autovacuum_vacuum_scale_factor = 0.02;
87
+ autovacuum_analyze_scale_factor = 0.01;
88
+
89
+ shared_preload_libraries = "pg_stat_statements";
90
+ compute_query_id = "on";
91
+ };
92
+
93
+ authentication = ''
94
+ local all root peer map=prometheus
95
+ '';
96
+
97
+ identMap = ''
98
+ prometheus root root
99
+ prometheus postgres-exporter root
100
+ '';
101
+ };
102
+}
build/mimas/zrepl.nix
new
+151
@@ -0,0 +1,151 @@
1
+{
2
+ config,
3
+ lib,
4
+ ...
5
+}:
6
+
7
+let
8
+ metricsPort = 9811;
9
+in
10
+{
11
+ age.secrets."zrepl-ssh-key" = {
12
+ file = ../secrets/zrepl-ssh-key.age;
13
+ mode = "0400";
14
+ };
15
+
16
+ programs.ssh = {
17
+ knownHosts = {
18
+ rsync-net = {
19
+ hostNames = [
20
+ "zh4461b.rsync.net"
21
+ "2001:1620:2019::336"
22
+ ];
23
+ publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILtF46LwRn+hC9vuw0vedXBKGNPMSIqrXdxl+EQOI/8J";
24
+ };
25
+ };
26
+ };
27
+
28
+ services.zrepl =
29
+ let
30
+ defaultBackupJob = {
31
+ type = "push";
32
+ filesystems."zroot/pg<" = true;
33
+ snapshotting = {
34
+ type = "periodic";
35
+ interval = "30m";
36
+ prefix = "zrepl_snap_";
37
+ hooks = [
38
+ {
39
+ # https://zrepl.github.io/configuration/snapshotting.html#postgres-checkpoint-hook
40
+ type = "postgres-checkpoint";
41
+ dsn = "host=/run/postgresql dbname=hydra user=root sslmode=disable";
42
+ filesystems."zroot/pg" = true;
43
+ }
44
+ ];
45
+ };
46
+
47
+ # The current pruning setup is an exponentially growing scheme, at both sides.
48
+ pruning = {
49
+ keep_sender = [
50
+ { type = "not_replicated"; }
51
+ {
52
+ type = "grid";
53
+ regex = "^zrepl_snap_.*";
54
+ grid = lib.concatStringsSep " | " [
55
+ "1x1h(keep=all)"
56
+ "1x1h"
57
+ "1x2h"
58
+ "1x4h"
59
+ # "grid" acts weird if an interval isn't a whole-number multiple
60
+ # of the previous one, so we jump from 8h to 24h
61
+ "2x8h"
62
+ "1x1d"
63
+ "1x2d"
64
+ "1x4d"
65
+ "1x8d"
66
+ # At this point we keep ~10 snapshots spanning 8--16 days (depends on moment),
67
+ # with exponentially increasing spacing (almost).
68
+ ];
69
+ }
70
+ ];
71
+ keep_receiver = [
72
+ {
73
+ type = "grid";
74
+ regex = "^zrepl_snap_.*";
75
+ grid = lib.concatStringsSep " | " [
76
+ "2x1h(keep=all)"
77
+ "2x1h"
78
+ "2x2h"
79
+ "2x4h"
80
+ "4x8h"
81
+ # At this point the grid spans 2 days by ~13 snapshots.
82
+ # (See note above about 8h -> 24h.)
83
+ "2x1d"
84
+ "2x2d"
85
+ "2x4d"
86
+ "2x8d"
87
+ "2x16d"
88
+ "2x32d"
89
+ "2x64d"
90
+ "2x128d"
91
+ # At this point we keep ~29 snapshots spanning 384--512 days (depends on moment),
92
+ # with exponentially increasing spacing (almost).
93
+ ];
94
+ }
95
+ ];
96
+ };
97
+ };
98
+ in
99
+ {
100
+ enable = true;
101
+ settings = {
102
+ global = {
103
+ logging = [
104
+ {
105
+ type = "syslog";
106
+ level = "info";
107
+ format = "human";
108
+ }
109
+ ];
110
+
111
+ # https://zrepl.github.io/configuration/monitoring.html
112
+ monitoring = [
113
+ {
114
+ type = "prometheus";
115
+ listen = ":${toString metricsPort}";
116
+ }
117
+ ];
118
+ };
119
+
120
+ jobs = [
121
+ # Covers 20240629+
122
+ (
123
+ defaultBackupJob
124
+ // {
125
+ name = "rsyncnet";
126
+ connect = {
127
+ identity_file = config.age.secrets."zrepl-ssh-key".path;
128
+ type = "ssh+stdinserver";
129
+ host = "zh4461b.rsync.net";
130
+ user = "root";
131
+ port = 22;
132
+ };
133
+ }
134
+ )
135
+ /*
136
+ rsync.net provides a VM with FreeBSD
137
+ - almost nothing is preserved on upgrades except this "data1" zpool
138
+ $ scp ./zrepl.yml root@zh4461b.rsync.net:/usr/local/etc/zrepl/zrepl.yml
139
+ # pkg install zrepl
140
+ # service zrepl enable
141
+ # service zrepl start
142
+ */
143
+ ];
144
+ };
145
+ };
146
+
147
+ networking.firewall.extraInputRules = ''
148
+ ip6 saddr $prometheus_inet6 tcp dport ${toString metricsPort} accept
149
+ ip saddr $prometheus_inet4 tcp dport ${toString metricsPort} accept
150
+ '';
151
+}
build/mimas/zrepl.yml
new
+24
@@ -0,0 +1,24 @@
1
+# root@zh4461b.rsync.net:/usr/local/etc/zrepl/zrepl.yml
2
+# zrepl main configuration file.
3
+# For documentation, refer to https://zrepl.github.io/
4
+#
5
+global:
6
+ logging:
7
+ - type: "stdout"
8
+ level: "error"
9
+ format: "human"
10
+ - type: "syslog"
11
+ level: "info"
12
+ format: "logfmt"
13
+
14
+# mostly from https://blog.lenny.ninja/zrepl-on-rsync-net.html
15
+jobs:
16
+ - name: sink
17
+ type: sink
18
+ serve:
19
+ type: stdinserver
20
+ client_identities: [hydra]
21
+ recv:
22
+ placeholder:
23
+ encryption: off
24
+ root_fs: "data1"
build/pluto/prometheus/exporters/postgresql.nix
+1
@@ -8,6 +8,7 @@
8
targets = [
9
"haumea.nixos.org:9187"
10
"titan.nixos.org:9187"
11
+ "mimas.nixos.org:9187"
12
"tracker.security.nixos.org:9187"
13
];
14
}
build/pluto/prometheus/exporters/zrepl.nix
+1
@@ -9,6 +9,7 @@
9
{
10
labels.role = "database";
11
targets = [
12
+ "mimas.nixos.org:9811"
13
"titan.nixos.org:9811"
14
];
15
}
build/secrets.nix
+4
-1
@@ -19,7 +19,10 @@ let
19
rfc39-record-push = [ pluto ];
20
storagebox-exporter-token = [ pluto ];
21
tarball-mirror-aws-credentials = [ pluto ];
22
- zrepl-ssh-key = [ titan ];
22
+ zrepl-ssh-key = [
23
+ titan
24
+ mimas
25
+ ];
26
27
# builders/
28
elated-minsky-queue-runner-token = [
build/secrets/zrepl-ssh-key.age
Binary files a/build/secrets/zrepl-ssh-key.age and b/build/secrets/zrepl-ssh-key.age differ