staging-hydra: init
Michael Schneider committed
Feb 12, 2025 at 10:54 UTC
1bf5d241bd58fc0bad3bc473fe7dc75807601106
10 files changed
+366
-19
non-critical-infra/.sops.yaml
+17
@@ -3,6 +3,9 @@ keys:
3
- &zimbatm age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h
4
- &caliban age1sv307kkrxwgjah8pjpap5kzl4j2r6fqr3vg234n7m32chlchs9lsey7nlq
5
- &umbriel age15vcp7875xwtf64j4yshyld0a3hpgzv6n2kxky493s3q0swr9hdaqxugpv6
6
+ - &staging-hydra age13emk4xkrde0qhgnuu24jl7vt6mdq99w56c4ngse9mxh4j0pxvuvq3zsppt
7
+ - &m1-s age1j3spleg4m7rtjww4zqsws6qaj69v5j0rc3asaxxlpwnu7fmx8p9qvmvjn8
8
+ - &mic92 age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz
9
10
creation_rules:
11
- path_regex: secrets/[^/]+.caliban
@@ -18,3 +21,17 @@ creation_rules:
21
- *umbriel
22
- *hexa
23
- *zimbatm
24
+
25
+ # ssh keys used to bootstrap new machines
26
+ - path_regex: secrets/[^/]+.hostkeys
27
+ key_groups:
28
+ - age:
29
+ - *m1-s
30
+ - *mic92
31
+
32
+ - path_regex: secrets/[^/]+.staging-hydra
33
+ key_groups:
34
+ - age:
35
+ - *staging-hydra
36
+ - *m1-s
37
+ - *mic92
non-critical-infra/flake-module.nix
+21
-19
@@ -1,7 +1,7 @@
1
{
2
- inputs,
3
- lib,
4
- ...
2
+ inputs
3
+, lib
4
+, ...
5
}:
6
{
7
colmena.hosts = {
@@ -17,23 +17,25 @@
17
));
18
in
19
{
20
- nixosConfigurations = builtins.mapAttrs (
21
- _name: value:
22
- inputs.nixpkgs.lib.nixosSystem {
23
- inherit lib;
24
- system = "x86_64-linux";
25
- specialArgs = {
26
- inherit inputs;
27
- };
28
- modules = [
29
- value
30
- inputs.disko.nixosModules.disko
31
- inputs.sops-nix.nixosModules.sops
32
- ];
33
- extraModules = [ inputs.colmena.nixosModules.deploymentOptions ];
20
+ nixosConfigurations = builtins.mapAttrs
21
+ (
22
+ _name: value:
23
+ inputs.nixpkgs.lib.nixosSystem {
24
+ inherit lib;
25
+ system = "x86_64-linux";
26
+ specialArgs = {
27
+ inherit inputs;
28
+ };
29
+ modules = [
30
+ value
31
+ inputs.disko.nixosModules.disko
32
+ inputs.sops-nix.nixosModules.sops
33
+ ];
34
+ extraModules = [ inputs.colmena.nixosModules.deploymentOptions ];
35
35
- }
36
- ) (importConfig ./hosts);
36
+ }
37
+ )
38
+ (importConfig ./hosts);
39
40
};
41
non-critical-infra/hosts/staging-hydra/default.nix
new
+37
@@ -0,0 +1,37 @@
1
+{ inputs, lib, ... }:
2
+{
3
+ imports = [
4
+ ./hardware.nix
5
+ inputs.srvos.nixosModules.server
6
+ inputs.srvos.nixosModules.hardware-hetzner-cloud-arm
7
+ ../../modules/common.nix
8
+ ./hydra.nix
9
+ inputs.hydra.nixosModules.hydra
10
+ ];
11
+
12
+ boot = {
13
+ loader = {
14
+ systemd-boot.enable = true;
15
+ timeout = lib.mkForce 5;
16
+ efi.efiSysMountPoint = "/efi";
17
+ };
18
+ kernelParams = [ "console=tty" ];
19
+ };
20
+ networking = {
21
+ hostName = "staging-hydra";
22
+ domain = "nixos.org";
23
+ };
24
+
25
+ systemd.network.networks."10-uplink".networkConfig.Address = "2a01:4f9:c012:d5d3::1/128";
26
+
27
+ disko.devices = import ./disko.nix;
28
+
29
+ networking.firewall.allowedTCPPorts = [
30
+ 80
31
+ 443
32
+ ];
33
+ networking.firewall.allowedUDPPorts = [ ];
34
+
35
+ system.stateVersion = "24.11";
36
+ users.users.root.openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFq+rXslVKnGlJKlSmuenBaZtVUZCL2rtFgmDmcbLQyT" ];
37
+}
non-critical-infra/hosts/staging-hydra/disko.nix
new
+61
@@ -0,0 +1,61 @@
1
+{
2
+ disk = {
3
+ main = {
4
+ device = "/dev/sda";
5
+ type = "disk";
6
+ content = {
7
+ type = "gpt";
8
+ partitions = {
9
+ esp = {
10
+ type = "EF00";
11
+ size = "1024M";
12
+ content = {
13
+ type = "filesystem";
14
+ format = "vfat";
15
+ mountpoint = "/efi";
16
+ };
17
+ };
18
+ root = {
19
+ size = "100%";
20
+ content = {
21
+ type = "zfs";
22
+ pool = "zroot";
23
+ };
24
+ };
25
+ };
26
+ };
27
+ };
28
+ };
29
+
30
+ zpool.zroot = {
31
+ type = "zpool";
32
+ options = {
33
+ # smartctl --all /dev/sda
34
+ # Logical block size: 512 bytes
35
+ ashift = "9";
36
+ };
37
+ rootFsOptions = {
38
+ acltype = "posixacl";
39
+ compression = "zstd";
40
+ mountpoint = "none";
41
+ xattr = "sa";
42
+ };
43
+ datasets = {
44
+ "root" = {
45
+ type = "zfs_fs";
46
+ mountpoint = "/";
47
+ };
48
+ "nix" = {
49
+ type = "zfs_fs";
50
+ mountpoint = "/nix";
51
+ };
52
+ "reserved" = {
53
+ type = "zfs_fs";
54
+ options = {
55
+ canmount = "off";
56
+ refreservation = "1G";
57
+ };
58
+ };
59
+ };
60
+ };
61
+}
non-critical-infra/hosts/staging-hydra/hardware.nix
new
+15
@@ -0,0 +1,15 @@
1
+{ lib, ... }:
2
+{
3
+
4
+ boot.initrd = {
5
+ availableKernelModules = [
6
+ "xhci_pci"
7
+ "virtio_pci"
8
+ "usbhid"
9
+ "sr_mod"
10
+ ];
11
+ kernelModules = [ "virtio_gpu" ];
12
+ };
13
+
14
+ nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
15
+}
non-critical-infra/hosts/staging-hydra/hydra.nix
new
+155
@@ -0,0 +1,155 @@
1
+{
2
+ lib,
3
+ pkgs,
4
+ config,
5
+ ...
6
+}:
7
+let
8
+ narCache = "/var/cache/hydra/nar-cache";
9
+ localSystems = [
10
+ "builtin"
11
+ config.nixpkgs.hostPlatform.system
12
+ ];
13
+in
14
+{
15
+ networking.firewall.allowedTCPPorts = [
16
+ 9198 # queue-runnner metrics
17
+ 9199 # hydra-notify metrics
18
+ ];
19
+
20
+ # garbage collection
21
+ nix.gc = {
22
+ automatic = true;
23
+ options = ''--max-freed "$((400 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
24
+ dates = "03,09,15,21:15";
25
+ };
26
+
27
+ # gc outputs as well, since they are served from the cache
28
+ nix.settings.gc-keep-outputs = lib.mkForce false;
29
+
30
+ # Don't rate-limit the journal.
31
+ services.journald.rateLimitBurst = 0;
32
+
33
+ # age.secrets.hydra-aws-credentials = {
34
+ # file = ./secrets/hydra-aws-credentials.age;
35
+ # path = "/var/lib/hydra/queue-runner/.aws/credentials";
36
+ # owner = "hydra-queue-runner";
37
+ # group = "hydra";
38
+ # };
39
+
40
+ sops.secrets.signing-key = {
41
+ sopsFile = ../../secrets/signing-key.staging-hydra;
42
+ format = "binary";
43
+ };
44
+
45
+ services.hydra-dev = {
46
+ enable = true;
47
+ package = pkgs.hydra;
48
+ buildMachinesFiles = [
49
+ (pkgs.writeText "local" ''
50
+ localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
51
+ '')
52
+ ];
53
+ logo = ../../../build/hydra-logo.png;
54
+ hydraURL = "https://hydra.nixos.org";
55
+ notificationSender = "edolstra@gmail.com";
56
+ smtpHost = "localhost";
57
+ useSubstitutes = false;
58
+ extraConfig = ''
59
+ max_servers 30
60
+
61
+ store_uri = s3://nixos-cache-staging?secret-key=${config.sops.secrets.signing-key.path}=1&ls-compression=br&log-compression=br
62
+ server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
63
+ binary_cache_public_uri = https://cache-staging.nixos.org
64
+
65
+ <Plugin::Session>
66
+ cache_size = 32m
67
+ </Plugin::Session>
68
+
69
+ # patchelf:master:3
70
+ xxx-jobset-repeats = nixos:reproducibility:1
71
+
72
+ upload_logs_to_binary_cache = true
73
+ compress_build_logs = false # conflicts with upload_logs_to_binary_cache
74
+
75
+ log_prefix = https://cache.nixos.org/
76
+
77
+ evaluator_workers = 16
78
+ evaluator_max_memory_size = 8192
79
+
80
+ max_concurrent_evals = 1
81
+
82
+ # increase the number of active compress slots (CPU is 48*2 on mimas)
83
+ max_local_worker_threads = 144
84
+
85
+ max_unsupported_time = 86400
86
+
87
+ allow_import_from_derivation = false
88
+
89
+ max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
90
+ max_db_connections = 350
91
+
92
+ queue_runner_metrics_address = [::]:9198
93
+
94
+ <hydra_notify>
95
+ <prometheus>
96
+ listen_address = 0.0.0.0
97
+ port = 9199
98
+ </prometheus>
99
+ </hydra_notify>
100
+ '';
101
+ };
102
+
103
+ systemd = {
104
+ tmpfiles.rules = [
105
+ "d /var/cache/hydra 0755 hydra hydra - -"
106
+ "d ${narCache} 0775 hydra hydra 1d -"
107
+ ];
108
+
109
+ # eats memory as if it was free
110
+ services = {
111
+ hydra-notify.enable = false;
112
+ hydra-queue-runner = {
113
+ # restarting the scheduler is very expensive
114
+ restartIfChanged = false;
115
+ serviceConfig = {
116
+ ManagedOOMPreference = "avoid";
117
+ LimitNOFILE = 65535;
118
+ };
119
+ };
120
+
121
+ hydra-prune-build-logs = {
122
+ description = "Clean up old build logs";
123
+ startAt = "weekly";
124
+ serviceConfig = {
125
+ User = "hydra-queue-runner";
126
+ Group = "hydra";
127
+ ExecStart = lib.concatStringsSep " " [
128
+ (lib.getExe pkgs.findutils)
129
+ "/var/lib/hydra/build-logs/"
130
+ "-ignore_readdir_race"
131
+ "-type"
132
+ "f"
133
+ "-mtime"
134
+ "+${toString (3 * 365)}" # days
135
+ "-delete"
136
+ ];
137
+ };
138
+ };
139
+ };
140
+ };
141
+
142
+ programs.ssh = {
143
+ hostKeyAlgorithms = [
144
+ "rsa-sha2-512-cert-v01@openssh.com"
145
+ "ssh-ed25519"
146
+ "ssh-rsa"
147
+ "ecdsa-sha2-nistp256"
148
+ ];
149
+
150
+ extraConfig = lib.mkAfter ''
151
+ ServerAliveInterval 120
152
+ TCPKeepAlive yes
153
+ '';
154
+ };
155
+}
non-critical-infra/secrets/signing-key.staging-hydra
new
+28
@@ -0,0 +1,28 @@
1
+{
2
+ "data": "ENC[AES256_GCM,data:r5OuTSn3XvJnDotfbVuCC/CmMJM30GTpvi9xYd824nc8RwoFG5ivV29mUo8kj2A/C8I1sgmxHbRSM3zCDQ5nDV6ul9XFqsFlK+9l5O2hZXSEq7crE6IEPJKXlYvQGTbZH9VcIrigDOCUiZgbfDFTnWIQ,iv:DcMMvNoINfUwCp4kKcQt3Ya5iOD1rQ08ft0blz7QuoA=,tag:APdvyc+0L9rz+5vhtothtQ==,type:str]",
3
+ "sops": {
4
+ "kms": null,
5
+ "gcp_kms": null,
6
+ "azure_kv": null,
7
+ "hc_vault": null,
8
+ "age": [
9
+ {
10
+ "recipient": "age13emk4xkrde0qhgnuu24jl7vt6mdq99w56c4ngse9mxh4j0pxvuvq3zsppt",
11
+ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1U0xMRmlPV2ROMnFOVk9X\nZC9NWjJ5a2ZoTWlWVTNwUTVxWlFhZHN1bFd3Cnl5TmgwRE5aVkdjMVluYmI5VHJD\nMVhyQTM0eVVpejVRMjV1WWxyTGRzQkEKLS0tIG1IZnM4cmo5NW9ha3h2cFFOdWRQ\nb25yUmVOdkZ6NCt4QW9LNHpDSWNJZXcKbvtROz30bj6DYn8bRX++TpZWYTc6LPBl\nu3WgIKIq1nucagf+tAsIO5ZTdNByK3DGYKFdp9+RLFxHLu6XCTePZg==\n-----END AGE ENCRYPTED FILE-----\n"
12
+ },
13
+ {
14
+ "recipient": "age1j3spleg4m7rtjww4zqsws6qaj69v5j0rc3asaxxlpwnu7fmx8p9qvmvjn8",
15
+ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrNHJCcmUzT2pFL3ZrT2FF\nN1grblQvL3Y2eWNXb2pJS0JXeEVOMUlPVFZvCmgyMDRwamFaSFk5TzE4cFJGaHlF\ncStOaUplOUZOTVBWUCtnWVNseFpoOHcKLS0tIDRIUGtpd1dWV2pqQnliK3NBcHVQ\nQnU2MTJNeUtHMEpmcmI0K0lzMjhZaHMKgGzuhwZqGLzufcGAYD/Io7rRO7EMmegq\ntKik/ZPqCRSw8XIX+Iiqp1miDV2MzgmLwQXm8IAT/9kWJmVK4j8ILg==\n-----END AGE ENCRYPTED FILE-----\n"
16
+ },
17
+ {
18
+ "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
19
+ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWb2ljNWl5SUhGUXZmQXhP\nTDF4bzhVYnpSdUFFVWVzeURnS1gxSVUrNWtVClR0S05HWUtpSU5ETWh0YVhJcS9C\nMmtwS1ZZSnRSMlVYQWtMMk5GVGhWVjAKLS0tIGdnYmEyaDBMbDhaUVpOUzN5OWk3\nKzkvSU1wQWxETDJiMWxVYUhuV1V4aEkKwjiLNhN2WvjV1WC648tl2bUIgcthFo/r\nwGF2G+J7ueAp3WGWRtd1a2kQNn8MLqCZksUMeksy4+3570kKmdCOrg==\n-----END AGE ENCRYPTED FILE-----\n"
20
+ }
21
+ ],
22
+ "lastmodified": "2025-02-12T07:28:02Z",
23
+ "mac": "ENC[AES256_GCM,data:Kk+FO9fjspAPkzs1orp5iXuVc1iLXbMr5PRmM9lDULjGZ2gpgS2JJF/Uf+4OeWrmLRmUwqa0dPPr3gR4/BEQmPk7FKqlr66iFun8Y/ExV38hWKGzBOvDxZIGdzT5i0S5SrohjhjQN2qzM8l6YANN21wogv/I0Mb/KKUNlBK/z9E=,iv:2fN19YDoZ7+0BVBo9FQJkPpjyGr2AB9GtYp4kvX+tUw=,tag:KrYOLR38mxv1eYOEoG9pNg==,type:str]",
24
+ "pgp": null,
25
+ "unencrypted_suffix": "_unencrypted",
26
+ "version": "3.9.4"
27
+ }
28
+}
\ No newline at end of file
non-critical-infra/secrets/staging-hydra.hostkeys
new
+24
@@ -0,0 +1,24 @@
1
+{
2
+ "data": "ENC[AES256_GCM,data:6b/6T5yhrAHKFORpq0JM0ge/lI0LUTdT+IcyYu5oDwy4LEhIKfICpziYqKeRm1JWHf+DAjCweJlTsK53HT31h6wcC7eA9j4J2pEk2ws7UYKKAeeqaAdaQO0EKaZvcwcVq48tY2gvdVhsaIS9s7/rvZEWXrrAOaNn9h3hM/HWLgfZTo9HZm87A+EKMPnQ75LRYZubZFLgrkKU/t82ocSIQD4VGk8ne58ncLY8BT/BdXOdwauyho6BQ8VKMK4JzXEK4ISvOwW+xrnOjuOz0R5QwPHyewBF9hLx6Wd8JT1fDqLdpTsWTi2uPlGHgkvBFcrbBxHpioJ5Z5wQVIWkKG21+7cRhLK1TJCi0O+z0czxS2BMNaPxrKiE5QfL+vtmnyX2xfYTcxrWdg3pqkemaM7T54kIa3O1SIQxitGXlgBsFQxr9Nw/iiz+Ys05ObhWPmENQIZlslUUg3ounnXmTA7eU7ogoH05feNOawPDWcUdyLX9eQ2JfhC9SlfQnKSJ7egNZd1592ybFPJkTnQeE5QFCxUOzISSV33BTnObFPSXYOEHYixvyPm8ALHZQdL9xbWu7qSADouc4xgR9vWm85BxJwjSqUNAM/L2CCDJtJqYT+O8k8M0DLHqrYeuMlnL5zLQ4ZcDYF2agezZStf7Cdm8xVHdd+Tfi1AePU9op3MeHmYMHzIIkc/eLGNOhBlQ5pZZ3S0gjqbD+0av+qSLTdx6fjaqnUCp7eS44QRSVUbzqO8mYWgXbRlfh0ewpBIHaTM=,iv:5z+8/wJR2tFuZF5aRKUX7xa89QqHVx0YVCjsPfIoC8Y=,tag:sC4/gyabxyFhdhWc7jAFIg==,type:str]",
3
+ "sops": {
4
+ "kms": null,
5
+ "gcp_kms": null,
6
+ "azure_kv": null,
7
+ "hc_vault": null,
8
+ "age": [
9
+ {
10
+ "recipient": "age1j3spleg4m7rtjww4zqsws6qaj69v5j0rc3asaxxlpwnu7fmx8p9qvmvjn8",
11
+ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTSzY3WFc2dDhPRVNTOFJH\ncVlkWnQ2L1V4dGNhZllvZVhlYnlVZXQvVzFnCnBiWXkwS2FFbVMrdjNZOGhIU001\nL3VmOGp1QTRpMWlRY2s5UGFqN01tc0UKLS0tIHV1b2FwWEhTY09SdGlGc3NhbnVZ\nUmd5dVdIZE9jWUV2VW8vb3dNK0JDYlkKElQj+C53jgVk2QTKAhsHCFMzKAwXWozP\ngSI8j4osvza/GqEM6XvQl/tW/TYcxIPDazK9tXOJEfbHmGh8ADyCaA==\n-----END AGE ENCRYPTED FILE-----\n"
12
+ },
13
+ {
14
+ "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
15
+ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMakVLWHFSaDVwTnpud2ZF\nZExnSDY1VGI1YjF4ZGl0NWdLQi9udGlwS2d3CnhwVEtIU1ZLMGdFaGhLQlNMSjVJ\nTjZXTUo4SzJiVUZiM2tjQVJJT3ZiL2MKLS0tIDkxUlhTZnZmVEN6RVdSTndwazdn\nZU1QcWlWL0gwTWIvWXhtQVFOb2VYaWsKEZX9COQbwRP/OvimWL6ijR2lsQBihssd\n7lZwH5xcKNWXH/u4lLxXQx7ARtEBX9kMKGtF8uGq6NCt2Gjlt/lJWg==\n-----END AGE ENCRYPTED FILE-----\n"
16
+ }
17
+ ],
18
+ "lastmodified": "2025-02-12T07:19:21Z",
19
+ "mac": "ENC[AES256_GCM,data:ZS6cY/umbUO7Aj1cnmLiCMLyhiiyAupEBL8/qDkO5JYZ4ufQN3WfiKebbndpLyl+35bMBL9C640BWTFh90Y0AwaZ2IxbAmTLGiqaeMG7ESzNXv1XYWF1MJdIa5i7hsl6DdKn26aHocmDIMMQ1tglZfk42fucJTWncUbio4trPfg=,iv:9dPqNw6utDO9boVzbRdtd1CtKSTG84upG0QQDPfH5Ow=,tag:CJm4QzxRi0SwCuGHxxw/Ew==,type:str]",
20
+ "pgp": null,
21
+ "unencrypted_suffix": "_unencrypted",
22
+ "version": "3.9.1"
23
+ }
24
+}
\ No newline at end of file
non-critical-infra/secrets/staging-hydra.hostkeys.pub.tmp
new
+1
@@ -0,0 +1 @@
1
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK81IHfeA/GXIgb9N5hl/qtCSsSBqs+FkppGklxworUD m1-s@thinkbook
non-critical-infra/secrets/staging-hydra.hostkeys.tmp
new
+7
@@ -0,0 +1,7 @@
1
+-----BEGIN OPENSSH PRIVATE KEY-----
2
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
3
+QyNTUxOQAAACCvNSB33gPxlyIG/TeYZf6rQkrEgarPhZKaRpJccKK1AwAAAJhqdU+1anVP
4
+tQAAAAtzc2gtZWQyNTUxOQAAACCvNSB33gPxlyIG/TeYZf6rQkrEgarPhZKaRpJccKK1Aw
5
+AAAEBsDJIMqVf7Y3FoTZk42DsRyLaBz0fGJUjazwskqATxxq81IHfeA/GXIgb9N5hl/qtC
6
+SsSBqs+FkppGklxworUDAAAADm0xLXNAdGhpbmtib29rAQIDBAUGBw==
7
+-----END OPENSSH PRIVATE KEY-----