tarball-mirror: fix; rotate access token
Martin Weinelt committed
May 16, 2026 at 13:02 UTC
a2beabae05b56073423e241478fe615db596135b
3 files changed
-156
build/secrets/tarball-mirror-aws-credentials.age
Binary files a/build/secrets/tarball-mirror-aws-credentials.age and b/build/secrets/tarball-mirror-aws-credentials.age differ
modules/tarball-mirror.nix
-1
@@ -57,7 +57,6 @@ in
57
cd $dir
58
git remote update origin
59
git checkout -f origin/${branch}
60
- git apply ${./tarball-mirror.patch}
60
# FIXME: use IAM role.
61
export AWS_ACCESS_KEY_ID=$(sed 's/aws_access_key_id=\(.*\)/\1/ ; t; d' ${config.age.secrets.tarball-mirror-aws-credentials.path})
62
export AWS_SECRET_ACCESS_KEY=$(sed 's/aws_secret_access_key=\(.*\)/\1/ ; t; d' ${config.age.secrets.tarball-mirror-aws-credentials.path})
modules/tarball-mirror.patch
deleted
-155
@@ -1,155 +0,0 @@
1
-From 89093ba05e6f9710aa0dcb500f6226f1be80cc86 Mon Sep 17 00:00:00 2001
2
-From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
3
-Date: Wed, 4 Dec 2024 09:39:04 +0100
4
-Subject: [PATCH] copy-tarballs: drop perl bindings
5
-
6
-This hopefully makes it easier to re-write this script in a language
7
-that people understand. Because it's shelling out, it's likely slower
8
-but hopefully still fast enough for our purposes.
9
----
10
- maintainers/scripts/copy-tarballs.pl | 78 +++++++++++++++++++++-------
11
- 1 file changed, 58 insertions(+), 20 deletions(-)
12
-
13
-diff --git a/maintainers/scripts/copy-tarballs.pl b/maintainers/scripts/copy-tarballs.pl
14
-index 30fbac6f002d90..cb117ad2be0762 100755
15
---- a/maintainers/scripts/copy-tarballs.pl
16
-+++ b/maintainers/scripts/copy-tarballs.pl
17
-@@ -1,5 +1,5 @@
18
- #! /usr/bin/env nix-shell
19
--#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix nix.perl-bindings
20
-+#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix
21
-
22
- # This command uploads tarballs to tarballs.nixos.org, the
23
- # content-addressed cache used by fetchurl as a fallback for when
24
-@@ -20,14 +20,51 @@
25
- use File::Slurp;
26
- use JSON;
27
- use Net::Amazon::S3;
28
--use Nix::Store;
29
--
30
--isValidPath("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"); # FIXME: forces Nix::Store initialisation
31
-
32
- sub usage {
33
- die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n";
34
- }
35
-
36
-+sub computeFixedOutputPath {
37
-+ my ($name, $algo, $hash) = @_;
38
-+ my $expr = <<'EXPR';
39
-+{ name, outputHashAlgo, outputHash }:
40
-+builtins.toString (derivation {
41
-+ inherit name outputHashAlgo outputHash;
42
-+ builder = "false";
43
-+ system = "dontcare";
44
-+ outputHashMode = "flat";
45
-+})
46
-+EXPR
47
-+ open(my $fh, "-|",
48
-+ "nix-instantiate",
49
-+ "--eval",
50
-+ "--strict",
51
-+ "-E", $expr,
52
-+ "--argstr", "name", $name,
53
-+ "--argstr", "outputHashAlgo", $algo,
54
-+ "--argstr", "outputHash", $hash) or die "Failed to run nix-instantiate: $!";
55
-+
56
-+ my $storePathJson = <$fh>;
57
-+ chomp $storePathJson;
58
-+ my $storePath = decode_json($storePathJson);
59
-+ close $fh;
60
-+ return $storePath;
61
-+}
62
-+
63
-+sub nixHash {
64
-+ my ($algo, $base16, $path) = @_;
65
-+ open(my $fh, "-|",
66
-+ "nix-hash",
67
-+ "--type", $algo,
68
-+ "--flat",
69
-+ ($base16 ? "--base16" : ()),
70
-+ $path) or die "Failed to run nix-hash: $!";
71
-+ my $hash = <$fh>;
72
-+ chomp $hash;
73
-+ return $hash;
74
-+}
75
-+
76
- my $dryRun = 0;
77
- my $expr;
78
- my @fileNames;
79
-@@ -90,12 +127,12 @@ sub alreadyMirrored {
80
- sub uploadFile {
81
- my ($fn, $name) = @_;
82
-
83
-- my $md5_16 = hashFile("md5", 0, $fn) or die;
84
-- my $sha1_16 = hashFile("sha1", 0, $fn) or die;
85
-- my $sha256_32 = hashFile("sha256", 1, $fn) or die;
86
-- my $sha256_16 = hashFile("sha256", 0, $fn) or die;
87
-- my $sha512_32 = hashFile("sha512", 1, $fn) or die;
88
-- my $sha512_16 = hashFile("sha512", 0, $fn) or die;
89
-+ my $md5_16 = nixHash("md5", 0, $fn) or die;
90
-+ my $sha1_16 = nixHash("sha1", 0, $fn) or die;
91
-+ my $sha256_32 = nixHash("sha256", 1, $fn) or die;
92
-+ my $sha256_16 = nixHash("sha256", 0, $fn) or die;
93
-+ my $sha512_32 = nixHash("sha512", 1, $fn) or die;
94
-+ my $sha512_16 = nixHash("sha512", 0, $fn) or die;
95
-
96
- my $mainKey = "sha512/$sha512_16";
97
-
98
-@@ -130,7 +167,7 @@ sub uploadFile {
99
- my $res = 0;
100
- foreach my $fn (@fileNames) {
101
- eval {
102
-- if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {
103
-+ if (alreadyMirrored("sha512", nixHash("sha512", 0, $fn))) {
104
- print STDERR "$fn is already mirrored\n";
105
- } else {
106
- uploadFile($fn, basename $fn);
107
-@@ -176,7 +213,9 @@ sub uploadFile {
108
-
109
- if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
110
- $algo = $1;
111
-- $hash = `nix hash to-base16 $hash` or die;
112
-+ open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", $hash) or die;
113
-+ $hash = <$fh>;
114
-+ close $fh;
115
- chomp $hash;
116
- }
117
-
118
-@@ -184,11 +223,13 @@ sub uploadFile {
119
-
120
- # Convert non-SRI base-64 to base-16.
121
- if ($hash =~ /^[A-Za-z0-9+\/=]+$/) {
122
-- $hash = `nix hash to-base16 --type '$algo' $hash` or die;
123
-+ open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", "--hash-algo", $algo, $hash) or die;
124
-+ $hash = <$fh>;
125
-+ close $fh;
126
- chomp $hash;
127
- }
128
-
129
-- my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);
130
-+ my $storePath = computeFixedOutputPath($name, $algo, $hash);
131
-
132
- for my $url (@$urls) {
133
- if (defined $ENV{DEBUG}) {
134
-@@ -210,18 +251,15 @@ sub uploadFile {
135
-
136
- print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
137
-
138
-+
139
- if ($dryRun) {
140
- $mirrored++;
141
- last;
142
- }
143
--
144
-- # Substitute the output.
145
-- if (!isValidPath($storePath)) {
146
-- system("nix-store", "-r", $storePath);
147
-- }
148
-+ my $isValidPath = system("nix-store", "-r", $storePath) == 0;
149
-
150
- # Otherwise download the file using nix-prefetch-url.
151
-- if (!isValidPath($storePath)) {
152
-+ if (!$isValidPath) {
153
- $ENV{QUIET} = 1;
154
- $ENV{PRINT_PATH} = 1;
155
- my $fh;