Move Hydra DB to haumea
Eelco Dolstra committed
Jan 8, 2020 at 10:23 UTC
396ad4524d209811f64058a18215950bbf169f21
4 files changed
+88
-5
delft/ceres.nix
+1
-1
@@ -43,7 +43,7 @@
43
firewall.logRefusedConnections = true;
44
};
45
46
- services.hydra-dev.dbi = "dbi:Pg:dbname=hydra;host=10.254.1.2;user=hydra;";
46
+ services.hydra-dev.dbi = "dbi:Pg:dbname=hydra;host=10.254.1.9;user=hydra;";
47
#systemd.services.hydra-init.wants = [ "sys-devices-virtual-net-wg0.device" ];
48
49
services.hydra-dev.buildMachinesFiles = [ "/etc/nix/machines" ];
delft/chef.nix
+1
@@ -38,6 +38,7 @@
38
# FIXME: don't use 'trust'.
39
authentication = ''
40
host hydra all 10.254.1.3/32 trust
41
+ host hydra all 10.254.1.9/32 trust
42
local all root peer map=prometheus
43
'';
44
delft/eris.nix
+14
-1
@@ -25,6 +25,7 @@ in {
25
10.254.1.6 hydra
26
10.254.1.7 lucifer
27
10.254.1.8 wendy
28
+ 10.254.1.9 haumea
29
30
10.254.3.1 webserver
31
@@ -169,6 +170,7 @@ in {
170
{
171
targets = [
172
"chef:9100"
173
+ "haumea:9100"
174
];
175
labels.role = "database";
176
}
@@ -280,7 +282,18 @@ in {
282
static_configs = [
283
{
284
targets = [
283
- "10.254.1.2:9187"
285
+ "10.254.1.2:9187" # chef but stupidly named
286
+ ];
287
+ }
288
+ ];
289
+ }
290
+ {
291
+ job_name = "haumea-postgresql";
292
+ metrics_path = "/metrics";
293
+ static_configs = [
294
+ {
295
+ targets = [
296
+ "haumea:9187"
297
];
298
}
299
];
delft/haumea.nix
+72
-3
@@ -7,6 +7,8 @@
7
./fstrim.nix
8
];
9
10
+ environment.systemPackages = [ pkgs.lz4 ];
11
+
12
deployment.targetEnv = "hetzner";
13
deployment.hetzner.mainIPv4 = "46.4.89.205";
14
@@ -49,8 +51,8 @@
51
mount -t zfs rpool/safe/root /mnt
52
mkdir -p /mnt/nix
53
mount -t zfs rpool/local/nix /mnt/nix
52
- mkdir -p /mnt/var/lib/postgresql
53
- mount -t zfs rpool/safe/postgres /mnt/var/lib/postgresql
54
+ mkdir -p /mnt/var/db/postgresql
55
+ mount -t zfs rpool/safe/postgres /mnt/var/db/postgresql
56
mkdir -p /mnt/boot
57
mount /dev/disk/by-label/boot0 /mnt/boot
58
'';
@@ -70,7 +72,7 @@
72
fsType = "zfs";
73
};
74
73
- fileSystems."/var/lib/postgresql" =
75
+ fileSystems."/var/db/postgresql" =
76
{ device = "rpool/safe/postgres";
77
fsType = "zfs";
78
};
@@ -79,4 +81,71 @@
81
82
boot.loader.grub.devices = [ "/dev/nvme0n1" "/dev/nvme1n1" ];
83
boot.loader.grub.copyKernels = true;
84
+
85
+ services.postgresql = {
86
+ enable = true;
87
+ package = pkgs.postgresql_11;
88
+ # https://pgtune.leopard.in.ua/#/
89
+ extraConfig = ''
90
+ listen_addresses = '10.254.1.9'
91
+ max_connections = 50
92
+
93
+ effective_cache_size = 48GB
94
+ checkpoint_completion_target = 0.9
95
+ default_statistics_target = 100
96
+
97
+ log_min_duration_statement = 5000
98
+ log_duration = off
99
+ log_statement = 'none'
100
+ max_connections = 250
101
+ work_mem = 20MB
102
+ maintenance_work_mem = 2GB
103
+
104
+ # 25% of memory
105
+ shared_buffers = 16GB
106
+
107
+ # Checkpoint every 1GB. (default)
108
+ # increased after seeing many warninsg about frequent checkpoints
109
+ min_wal_size = 1GB
110
+ max_wal_size = 2GB
111
+ wal_buffers = 16MB
112
+
113
+ max_worker_processes = 16
114
+ max_parallel_workers_per_gather = 8
115
+ max_parallel_workers = 16
116
+
117
+ # NVMe related performance tuning
118
+ effective_io_concurrency = 200
119
+ random_page_cost = 1.1
120
+
121
+ # We can risk losing some transactions.
122
+ synchronous_commit = off
123
+
124
+ effective_cache_size = 16GB
125
+ '';
126
+
127
+ # FIXME: don't use 'trust'.
128
+ authentication = ''
129
+ host hydra all 10.254.1.3/32 trust
130
+ local all root peer map=prometheus
131
+ '';
132
+
133
+ identMap = ''
134
+ prometheus root root
135
+ prometheus postgres-exporter root
136
+ '';
137
+ };
138
+
139
+ networking = {
140
+ firewall.interfaces.wg0.allowedTCPPorts = [ 5432 ];
141
+ firewall.allowPing = true;
142
+ firewall.logRefusedConnections = true;
143
+ };
144
+
145
+ services.prometheus.exporters.postgres = {
146
+ enable = true;
147
+ dataSourceName = "user=root database=hydra host=/run/postgresql sslmode=disable";
148
+ firewallFilter = "-i wg0 -p tcp -m tcp --dport 9187";
149
+ openFirewall = true;
150
+ };
151
}