Revive the tarball mirroring service
Eelco Dolstra committed
May 9, 2016 at 16:33 UTC
a17ccf87deae4fb86639c8d34ab5938edd68d8c4
3 files changed
+63
-35
nixos-org/network.nix
+18
-1
@@ -32,6 +32,7 @@ in
32
{ inherit region accessKeyId;
33
name = "nixpkgs-tarballs";
34
# All files are readable but not listable.
35
+ # The s3-upload-tarballs user can upload files.
36
policy =
37
''
38
{
@@ -43,6 +44,20 @@ in
44
"Principal": {"AWS": "*"},
45
"Action": ["s3:GetObject"],
46
"Resource": ["${config.arn}/*"]
47
+ },
48
+ {
49
+ "Sid": "AllowUpload",
50
+ "Effect": "Allow",
51
+ "Principal": {"AWS": "arn:aws:iam::080433136561:user/s3-upload-tarballs"},
52
+ "Action": ["s3:PutObject", "s3:PutObjectAcl"],
53
+ "Resource": ["${config.arn}/*"]
54
+ },
55
+ {
56
+ "Sid": "AllowUpload2",
57
+ "Effect": "Allow",
58
+ "Principal": {"AWS": "arn:aws:iam::080433136561:user/s3-upload-tarballs"},
59
+ "Action": ["s3:ListBucket"],
60
+ "Resource": ["${config.arn}"]
61
}
62
]
63
}
@@ -133,8 +148,10 @@ in
148
options = [ "bind" ];
149
};
150
151
+ swapDevices = [ { device = "/tmp/swapfile"; size = 1024; } ];
152
+
153
system.stateVersion = "14.12";
154
138
- imports = [ ./webserver.nix ./hydra-mirror.nix ];
155
+ imports = [ ./webserver.nix ./hydra-mirror.nix ./tarball-mirror.nix ];
156
};
157
}
nixos-org/tarball-mirror.nix
new
+45
@@ -0,0 +1,45 @@
1
+# This module mirrors most tarballs reachable from Nixpkgs's
2
+# release.nix to the content-addressed tarball cache at
3
+# tarballs.nixos.org.
4
+
5
+{ config, lib, pkgs, ... }:
6
+
7
+with lib;
8
+
9
+let
10
+
11
+ nixosRelease = "16.03";
12
+
13
+in
14
+
15
+{
16
+
17
+ users.extraUsers.tarball-mirror =
18
+ { description = "Nixpkgs tarball mirroring user";
19
+ home = "/home/tarball-mirror";
20
+ isNormalUser = true;
21
+ openssh.authorizedKeys.keys = with import ../ssh-keys.nix; [ eelco ];
22
+ };
23
+
24
+ systemd.services.mirror-tarballs =
25
+ { description = "Mirror Nixpkgs Tarballs";
26
+ path = [ config.nix.package pkgs.git pkgs.bash ];
27
+ environment.NIX_REMOTE = "daemon";
28
+ serviceConfig.User = "tarball-mirror";
29
+ serviceConfig.Type = "oneshot";
30
+ serviceConfig.PrivateTmp = true;
31
+ script =
32
+ ''
33
+ cd /home/tarball-mirror/nixpkgs
34
+ git remote update channels
35
+ git checkout channels/nixos-${nixosRelease}
36
+ export AWS_ACCESS_KEY_ID=$(sed 's/aws_access_key_id=\(.*\)/\1/ ; t; d' ~/.aws/credentials)
37
+ export AWS_SECRET_ACCESS_KEY=$(sed 's/aws_secret_access_key=\(.*\)/\1/ ; t; d' ~/.aws/credentials)
38
+ NIX_PATH=nixpkgs=. ./maintainers/scripts/copy-tarballs.pl \
39
+ --expr '(import <nixpkgs/pkgs/top-level/release.nix> { scrubJobs = false; })' \
40
+ --exclude 'registry.npmjs.org|mirror://kde|mirror://xorg|mirror://kernel|mirror://hackage|mirror://gnome|mirror://apache|mirror://mozilla|pypi.python.org'
41
+ '';
42
+ startAt = "05:30";
43
+ };
44
+
45
+}
nixos-org/webserver.nix
-34
@@ -9,8 +9,6 @@ let
9
acmeKeyDir = "/var/lib/acme/nixos.org";
10
acmeWebRoot = "/var/lib/httpd/acme";
11
12
- nixosRelease = "16.03";
13
-
12
nixosVHostConfig =
13
{ hostName = "nixos.org";
14
serverAliases = [ "test.nixos.org" "test2.nixos.org" "ipv6.nixos.org" "localhost" ];
@@ -266,38 +264,6 @@ in
264
[ "ssh-dss AAAAB3NzaC1kc3MAAACBAMrcUf4qQj8XcG1nfG5/6rbfb4a89nV13KcJLBOVWa3Tn4YHeVz1lQDRHvnLK9YKM7MybDXD2wVG5nKuMbJMW5aZPEGrVUM4SQFXtnaNBgmoACrbG978Da/vNjGY89Q7GS/YqA24ASKnc09cRFsTmU0e/9BCbz9zXO4sJ8GaGHz7AAAAFQDZrJCdxTQ8GVvoFjL9Q1s1VHiClwAAAIBK+6r/kP/9VUzfRepEHCVObTIRYIhC9YcIZe2pMyCQSUIAjkGd5hkA8XQecs5/ym5Ddm2j61Kvt2jtGXQVP2F04wIFDuGK4GAfPpYjvLJaXtVxj1Ho4K2W/+WgKG1NEh466myZNsHr3v1MufbxNIS03lg6s8oJI4TmCaWtVHNW+AAAAIEAqh+ablUfEZAr6" ];
265
};
266
269
- users.extraUsers.tarball-mirror =
270
- { description = "Nixpkgs tarball mirroring user";
271
- home = "/home/tarball-mirror";
272
- createHome = true;
273
- useDefaultShell = true;
274
- openssh.authorizedKeys.keys = [ sshKeys.eelco ];
275
- };
276
-
277
- /*
278
- systemd.services.mirror-tarballs =
279
- { description = "Mirror Nixpkgs Tarballs";
280
- path = [ config.nix.package pkgs.curl pkgs.git ];
281
- #environment.DRY_RUN = "1";
282
- environment.NIX_PATH = "nixpkgs=/home/tarball-mirror/nixpkgs";
283
- environment.NIX_REMOTE = "daemon";
284
- environment.CURL_CA_BUNDLE = "/etc/ssl/certs/ca-bundle.crt";
285
- environment.NIX_TARBALLS_CACHE = "/tarballs";
286
- environment.PERL5LIB = "/run/current-system/sw/lib/perl5/site_perl";
287
- serviceConfig.User = "tarball-mirror";
288
- serviceConfig.Type = "oneshot";
289
- script =
290
- ''
291
- export NIX_CURL_FLAGS="--silent --show-error --connect-timeout 30"
292
- cd /home/tarball-mirror/nixpkgs
293
- git remote update channels
294
- git checkout channels/nixos-${nixosRelease}
295
- exec /nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs/maintainers/scripts/copy-tarballs.pl
296
- '';
297
- startAt = "05:30";
298
- };
299
- */
300
-
267
systemd.services.update-channels =
268
{ description = "Update Channels";
269
environment.CURL_CA_BUNDLE = "/etc/ssl/certs/ca-bundle.crt";