Migration to TU Delft datacenter
Eelco Dolstra committed
Sep 13, 2013 at 15:54 UTC
5cb71fb9ce8bab2bf9ecae6ca9a64b73cde7bf49
9 files changed
+98
-134
delft/build-machines-common.nix
+1
-3
@@ -12,7 +12,7 @@ let
12
in
13
{
14
require = [ ./common.nix ];
15
-
15
+
16
boot.loader.grub.device = "/dev/sda";
17
boot.loader.grub.copyKernels = true;
18
@@ -27,8 +27,6 @@ in
27
nix.gc.dates = "03,09,15,21:15";
28
nix.gc.options = ''--max-freed "$((100 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
29
30
- networking.hostName = ""; # obtain from DHCP server
31
-
30
services.cron.systemCronJobs = [
31
"0,30 * * * * ${m3chown}"
32
];
delft/common.nix
+4
-2
@@ -3,6 +3,8 @@
3
with pkgs.lib;
4
5
{
6
+ imports = [ ./static-net-config.nix ];
7
+
8
users.extraUsers.root.openssh.authorizedKeys.keys =
9
[ # ~/.ssh/id_mass_update
10
"ssh-dss AAAAB3NzaC1kc3MAAACBAN/f/VlDwxI0T51Kqen4WLz0ittuJFAgPZ6VwbwPPyHRpmKY/m5Zd2nycY8zDTDF1JJGlFpDC3wsoOlaYr4/AlJvRy/0SUvlnDcocXHs1BM1ZLWV2MdUuG6dCHNUYDsQat8bKm4YdjLmfL1p/PKpKS83+0S59u1PCkPWsoL0Wqc7AAAAFQCU4FSXrHs9GHEKuXQ2zpmsKcx2kwAAAIB40t8aJlEipcDtLPax3wfPxqAtbzDsYPuYrX5VF48tdbH4f/kZPm1qKaU7vq+m5n0uuT3mxsBFzuQpDcPhL7ZXJJEHRDMJgvq3dOCk0ejrXTTdnYHDMWUdC9S2f8kYTJ0lf7Jwro5R97PsTpsjfDRGLWoXUfpF6NARANQ0q+tM3wAAAIA14dh6XTX2NBsh+Cew8YYSX5ZK76zNREEbXxuzecXuP2VP14ZR3fMLXI201QyWP+U1Kj8QsS1v2XQ2MtNnXW3HOCb5C0L2Qs0AIV5YQ+UhXUen2RgA8tITUBBV6hLvdhnrmZ8Odrmf0+iAGXBxTgXwpWqW6X9W3CbXyA1Ncs0ZSQ== root@buildfarm"
@@ -35,7 +37,7 @@ with pkgs.lib;
37
environment.systemPackages =
38
[ pkgs.emacs pkgs.subversion pkgs.sysstat pkgs.hdparm pkgs.sdparm # pkgs.lsiutil
39
pkgs.htop pkgs.sqlite pkgs.iotop pkgs.lm_sensors pkgs.gitFull pkgs.hwloc
38
- pkgs.lsof pkgs.numactl pkgs.gcc pkgs.smartmontools
40
+ pkgs.lsof pkgs.numactl pkgs.gcc pkgs.smartmontools pkgs.tcpdump
41
];
42
43
services.sshd.enable = true;
@@ -53,7 +55,7 @@ with pkgs.lib;
55
'';
56
57
services.zabbixAgent.enable = true;
56
- services.zabbixAgent.server = "192.168.1.26,127.0.0.1,130.161.158.181";
58
+ services.zabbixAgent.server = "131.180.119.77";
59
services.zabbixAgent.extraConfig =
60
''
61
UserParameter=hardware.temp.cpu.average,shopt -s nullglob; cat /sys/devices/platform/coretemp.*/temp1_input /sys/bus/pci/drivers/k10temp/*/temp1_input < /dev/null | ${pkgs.perl}/bin/perl -e 'while (<>) { $n++; $sum += $_; }; print $sum / $n / 1000;'
delft/delft-webserver.nix
+1
-1
@@ -95,7 +95,7 @@ in
95
globalRedirect = "http://nixos.org/";
96
}
97
98
- { hostName = "buildfarm.st.ewi.tudelft.nl";
98
+ { hostName = "wendy.ewi.tudelft.nl";
99
documentRoot = cleanSource ./webroot;
100
enableUserDir = true;
101
extraSubservices = [
delft/machines.nix
+13
-11
@@ -21,7 +21,7 @@ lib: with lib;
21
22
{ # 8-core NixOS build machine.
23
hostName = "kenny";
24
- ipAddress = "192.168.1.19";
24
+ ipAddress = "131.180.119.71";
25
ethernetAddress = "00:22:19:55:cc:0d";
26
systems = [ "x86_64-linux" "i686-linux" ];
27
maxJobs = 8;
@@ -31,7 +31,7 @@ lib: with lib;
31
32
{ # 8-core NixOS build machine.
33
hostName = "stan";
34
- ipAddress = "192.168.1.20";
34
+ ipAddress = "131.180.119.74";
35
ethernetAddress = "00:22:19:55:bf:2e";
36
systems = [ "x86_64-linux" "i686-linux" ];
37
maxJobs = 8;
@@ -41,7 +41,7 @@ lib: with lib;
41
42
{ # 8-core NixOS build machine.
43
hostName = "kyle";
44
- ipAddress = "192.168.1.21";
44
+ ipAddress = "131.180.119.72";
45
ethernetAddress = "00:22:19:55:c1:18";
46
systems = [ "x86_64-linux" "i686-linux" ];
47
maxJobs = 8;
@@ -61,7 +61,7 @@ lib: with lib;
61
62
{ # 64-bit Mac OS X build machine.
63
hostName = "butters";
64
- ipAddress = "192.168.1.23";
64
+ ipAddress = "131.180.119.68";
65
ethernetAddress = "00:24:36:f3:cd:c0";
66
system = "x86_64-darwin";
67
aliases = ["mac64-1"];
@@ -71,42 +71,44 @@ lib: with lib;
71
72
{ # Old Hydra server.
73
hostName = "hydra";
74
- ipAddress = "192.168.1.18";
74
+ ipAddress = "131.180.119.69";
75
ethernetAddress = "00:22:19:55:bf:24";
76
}
77
78
{ # Xen machine.
79
hostName = "mrhankey";
80
ipAddress = "192.168.1.24";
81
- ethernetAddress = "00:1D:09:0E:09:E5";
81
+ ethernetAddress = "00:1d:09:0e:09:e5";
82
}
83
84
{ # Hydra server.
85
hostName = "lucifer";
86
- ipAddress = "192.168.1.25";
87
- ethernetAddress = "84:2B:2B:0B:98:F0";
86
+ ipAddress = "131.180.119.73";
87
+ ethernetAddress = "84:2b:2b:0b:98:f0";
88
}
89
90
- { # 48-core NixOS machine for random hackery.
90
+ { # 48-core NixOS front-end proxy, database server, build machine.
91
hostName = "wendy";
92
- ipAddress = "192.168.1.26";
92
+ ipAddress = "131.180.119.77";
93
ethernetAddress = "f0:4d:a2:40:1b:c0";
94
systems = [ "x86_64-linux" ];
95
}
96
97
{ # 48-core NixOS build machine.
98
hostName = "ike";
99
- ipAddress = "192.168.1.27";
99
+ ipAddress = "131.180.119.70";
100
ethernetAddress = "f0:4d:a2:40:1b:91";
101
systems = [ "x86_64-linux" ];
102
}
103
104
+ /*
105
{ # 48-core NixOS build machine.
106
hostName = "shelley";
107
ipAddress = "192.168.1.28";
108
ethernetAddress = "f0:4d:a2:40:10:6c";
109
systems = [ "x86_64-linux" ];
110
}
111
+ */
112
113
114
/* Xen VMs hosted on mrhankey. Note that 00:16:3e is the prefix for
delft/network.nix
+1
-1
@@ -3,7 +3,7 @@
3
kyle = import ./build-machines-dell-1950.nix;
4
hydra = import ./build-machines-dell-1950.nix;
5
stan = import ./stan.nix;
6
- cartman = import ./cartman.nix;
6
+ #cartman = import ./cartman.nix;
7
lucifer = import ./lucifer.nix;
8
wendy = import ./wendy.nix;
9
ike = import ./build-machines-dell-r815.nix;
delft/stan.nix
+5
-11
@@ -20,20 +20,13 @@
20
networking.firewall.rejectPackets = true;
21
networking.firewall.allowPing = true;
22
23
- networking.bridges.veno1.interfaces = [ "eno1" ];
23
+ networking.bridges.veno1.interfaces = [ config.system.build.mainPhysicalInterface ];
24
25
- virtualisation.libvirtd.enable = true;
26
-
27
- #virtualisation.nova.enableSingleNode = true;
25
+ system.build.mainVirtualInterface = "veno1";
26
29
- virtualisation.nova.extraConfig =
30
- ''
31
- --network_manager=nova.network.manager.FlatDHCPManager
32
- --flat_network_dhcp_start=192.168.80.2
33
- --fixed_range=192.168.80.0/24
34
- --use_ipv6
35
- '';
27
+ virtualisation.libvirtd.enable = true;
28
29
+ /*
30
networking.localCommands =
31
''
32
# Enable IPv6 forwarding. Nova/radvd requires this.
@@ -46,4 +39,5 @@
39
ip -6 addr add 2001:610:685:1:222:19ff:fe55:bf2e/64 dev veno1 || true
40
ip -6 route add default via fe80::204:23ff:fedf:f7bf dev veno1 || true
41
'';
42
+ */
43
}
delft/static-net-config.nix
new
+53
@@ -0,0 +1,53 @@
1
+{ config, pkgs, ... }:
2
+
3
+with pkgs.lib;
4
+
5
+let
6
+
7
+ machines = import ./machines.nix pkgs.lib;
8
+
9
+ machine = findSingle (m: m.hostName == config.deployment.targetHost) {} {} machines;
10
+
11
+ iface = "enx" + replaceChars [":"] [""] machine.ethernetAddress;
12
+
13
+in
14
+
15
+{
16
+ config = mkIf (machine != {}) {
17
+
18
+ # Name network interfaces after their MAC address.
19
+ services.udev.extraRules =
20
+ ''
21
+ SUBSYSTEM!="net", GOTO="my_end"
22
+ IMPORT{builtin}="net_id"
23
+ KERNEL=="vnet*", GOTO="my_end"
24
+ NAME=="", ENV{ID_NET_NAME_MAC}!="", NAME="$env{ID_NET_NAME_MAC}"
25
+ LABEL="my_end"
26
+ '';
27
+
28
+ #networking.useDHCP = false;
29
+
30
+ networking.domain = "ewi.tudelft.nl";
31
+
32
+ networking.defaultGateway = mkDefault "131.180.119.1";
33
+
34
+ #networking.nameservers = mkDefault [ "192.168.1.26" ];
35
+ networking.nameservers = mkDefault [ "8.8.8.8" ];
36
+
37
+ networking.extraHosts =
38
+ ''
39
+ 2001:610:685:1:216:3eff:fe00:1 tweek
40
+ 2001:610:685:1:216:3eff:fe00:5 meerkat
41
+ 2001:610:685:1:216:3eff:fe00:7 beastie
42
+ 2001:610:685:1:216:3eff:fe00:8 demon
43
+ '';
44
+
45
+ system.build.mainPhysicalInterface = iface;
46
+
47
+ networking.interfaces = listToAttrs (singleton (nameValuePair (config.system.build.mainVirtualInterface or config.system.build.mainPhysicalInterface)
48
+ { ipAddress = machine.ipAddress;
49
+ prefixLength = 25;
50
+ }));
51
+
52
+ };
53
+}
delft/sysstat.nix
+2
-15
@@ -3,20 +3,7 @@
3
environment.systemPackages = [ pkgs.sysstat ];
4
5
services.cron.systemCronJobs =
6
- [ "*/1 * * * * root ${pkgs.sysstat}/lib/sa/sa1 -S DISK 1 1"
7
- "53 23 * * * root ${pkgs.sysstat}/lib/sa/sa2 -A"
6
+ [ "*/1 * * * * root mkdir -p /var/log/sa; ${pkgs.sysstat}/lib/sa/sa1 -S DISK 1 1"
7
+ "53 23 * * * root mkdir -p /var/log/sa; ${pkgs.sysstat}/lib/sa/sa2 -A"
8
];
9
-
10
- systemd.services."sysstat-init" =
11
- { description = "Create sysstat directory";
12
- wantedBy = [ "multi-user.target" ];
13
- script =
14
- ''
15
- mkdir -p /var/log/sa
16
- '';
17
- serviceConfig =
18
- { Type = "oneshot";
19
- RemainAfterExit = true;
20
- };
21
- };
9
}
delft/wendy.nix
+18
-90
@@ -2,20 +2,15 @@
2
3
with pkgs.lib;
4
5
-let
6
-
7
- myIP = "130.161.158.181";
8
- machines = import ./machines.nix pkgs.lib;
9
-
10
-in
11
-
5
{
6
require = [ ./build-machines-dell-r815.nix ./delft-webserver.nix ./sysstat.nix ];
7
8
+ /*
9
fileSystems."/backup" =
10
{ device = "130.161.158.5:/dxs/users4/group/buildfarm";
11
fsType = "nfs4";
12
};
13
+ */
14
15
jobs.mturk_webserver_production =
16
{ name = "mturk-webserver-production";
@@ -65,10 +60,12 @@ in
60
'';
61
};
62
63
+ /*
64
services.postgresqlBackup = {
65
enable = true;
66
databases = [ "hydra" "jira" ];
67
};
68
+ */
69
70
services.postgresql = {
71
enable = true;
@@ -89,10 +86,10 @@ in
86
effective_cache_size = 24GB
87
'';
88
authentication = ''
92
- host all all 192.168.1.25/32 md5
93
- host hydra hydra 192.168.1.26/32 md5
94
- host hydra_test hydra 192.168.1.26/32 md5
95
- host zabbix zabbix 192.168.1.26/32 md5
89
+ host all all 131.180.119.77/32 md5
90
+ host hydra hydra 131.180.119.73/32 md5
91
+ host hydra_test hydra 131.180.119.73/32 md5
92
+ host zabbix zabbix 131.180.119.73/32 md5
93
'';
94
};
95
@@ -112,19 +109,13 @@ in
109
'';
110
111
services.cron.systemCronJobs =
115
- [ "15 4 * * * root cp -v /var/backup/postgresql/* /backup/wendy/postgresql/ &> /var/log/backup-db.log"
112
+ [ #"15 4 * * * root cp -v /var/backup/postgresql/* /backup/wendy/postgresql/ &> /var/log/backup-db.log"
113
];
114
118
- services.udev.extraRules =
119
- ''
120
- ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="f0:4d:a2:40:1b:be", NAME="external"
121
- ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="f0:4d:a2:40:1b:c0", NAME="internal"
122
- '';
123
-
115
services.radvd.enable = true;
116
services.radvd.config =
117
''
127
- interface internal {
118
+ interface ${config.system.build.mainPhysicalInterface} {
119
AdvSendAdvert on;
120
prefix 2001:610:685:1::/64 { };
121
RDNSS 2001:610:685:1::1 { };
@@ -132,53 +123,25 @@ in
123
'';
124
125
networking = {
135
- hostName = "wendy";
136
- domain = "buildfarm";
137
-
138
- interfaces.external =
139
- { ipAddress = myIP;
140
- prefixLength = 23;
141
- };
142
-
143
- interfaces.internal =
144
- { ipAddress = (findSingle (m: m.hostName == "wendy") {} {} machines).ipAddress;
145
- prefixLength = 22;
146
- };
147
-
148
- useDHCP = false;
149
-
150
- defaultGateway = "130.161.158.1";
126
152
- nameservers = [ "127.0.0.1" ];
153
-
154
- extraHosts = "192.168.1.26 wendy";
155
-
156
- firewall.allowedTCPPorts = [ 80 443 10051 5999 ];
127
+ firewall.allowedTCPPorts = [ 80 443 10051 5432 5999 ];
128
firewall.allowedUDPPorts = [ 53 67 ];
129
+ /*
130
firewall.extraCommands =
131
''
132
iptables -A nixos-fw -p tcp --dport 5432 -i internal -j nixos-fw-accept
133
'';
162
-
163
- nat.enable = true;
164
- nat.internalIPs = "192.168.1.0/22";
165
- nat.externalInterface = "external";
166
- nat.externalIP = myIP;
134
+ */
135
136
localCommands =
137
''
170
- #${pkgs.iptables}/sbin/iptables -t nat -F PREROUTING
171
-
172
- # lucifer ssh (to give Karl/Armijn access for the BAT project)
173
- #${pkgs.iptables}/sbin/iptables -t nat -A PREROUTING -p tcp -d ${myIP} --dport 2222 -j DNAT --to 192.168.1.26:22
174
-
138
# Cleanup.
139
ip -6 route flush dev sixxs || true
140
ip link set dev sixxs down || true
141
ip tunnel del sixxs || true
142
143
# Set up a SixXS tunnel for IPv6 connectivity.
181
- ip tunnel add sixxs mode sit local ${myIP} remote 192.87.102.107 ttl 64
144
+ ip tunnel add sixxs mode sit local 131.180.119.77 remote 192.87.102.107 ttl 64
145
ip link set dev sixxs mtu 1280 up
146
ip -6 addr add 2001:610:600:88d::2/64 dev sixxs
147
ip -6 route add default via 2001:610:600:88d::1 dev sixxs
@@ -187,49 +150,14 @@ in
150
ip -6 route add 2001:610:685::/48 dev lo || true
151
152
# Create a local network (prefix:1::/64).
190
- ip -6 addr add 2001:610:685:1::1/64 dev internal || true
153
+ ip -6 addr add 2001:610:685:1::1/64 dev ${config.system.build.mainPhysicalInterface} || true
154
155
# Forward traffic to our Nova cloud to "stan".
193
- ip -6 route add 2001:610:685:2::/64 via 2001:610:685:1:222:19ff:fe55:bf2e || true
194
-
195
- # Amazon MTurk experiment.
196
- #${pkgs.iptables}/sbin/iptables -t nat -A PREROUTING -p tcp -d ${myIP} --dport 5998 -j DNAT --to 192.168.1.26:5998
197
- #${pkgs.iptables}/sbin/iptables -t nat -A PREROUTING -p tcp -d ${myIP} --dport 5999 -j DNAT --to 192.168.1.26:5999
156
+ #ip -6 route add 2001:610:685:2::/64 via 2001:610:685:1:222:19ff:fe55:bf2e || true
157
'';
199
- };
158
201
- jobs.dnsmasq =
202
- let
203
-
204
- confFile = pkgs.writeText "dnsmasq.conf"
205
- ''
206
- keep-in-foreground
207
- no-hosts
208
- addn-hosts=${hostsFile}
209
- expand-hosts
210
- domain=buildfarm
211
- interface=internal
212
-
213
- server=130.161.158.4
214
- server=130.161.33.17
215
- server=130.161.180.1
216
- server=8.8.8.8
217
- server=8.8.4.4
218
-
219
- dhcp-range=192.168.1.150,192.168.3.200
220
-
221
- ${flip concatMapStrings machines (m: optionalString (m ? ethernetAddress) ''
222
- dhcp-host=${m.ethernetAddress},${m.ipAddress},${m.hostName}
223
- '')}
224
- '';
225
-
226
- hostsFile = pkgs.writeText "extra-hosts"
227
- (flip concatMapStrings machines (m: "${m.ipAddress} ${m.hostName}\n"));
228
-
229
- in
230
- { startOn = "started network-interfaces";
231
- exec = "${pkgs.dnsmasq}/bin/dnsmasq --conf-file=${confFile}";
232
- };
159
+ dhcpcd.denyInterfaces = [ "sixxs" ];
160
+ };
161
162
# Needed for the Nixpkgs mirror script.
163
environment.pathsToLink = [ "/libexec" ];