| 1 | #! /usr/bin/env perl |
| 2 | |
| 3 | use strict; |
| 4 | use warnings; |
| 5 | use Data::Dumper; |
| 6 | use Digest::SHA; |
| 7 | use Fcntl qw(:flock); |
| 8 | use File::Basename; |
| 9 | use File::Path; |
| 10 | use File::Slurp; |
| 11 | use File::stat; |
| 12 | use JSON::PP; |
| 13 | use LWP::UserAgent; |
| 14 | use List::MoreUtils qw(uniq); |
| 15 | use Net::Amazon::S3; |
| 16 | use POSIX qw(strftime); |
| 17 | |
| 18 | # Runs the given command, printing the (unescaped) command. |
| 19 | # This command continues on failure. |
| 20 | sub runAllowFailure { |
| 21 | print STDERR " \$ ", join(" ", @_), "\n"; |
| 22 | system(@_); |
| 23 | } |
| 24 | |
| 25 | # Runs the given command, printing the (unescaped) command. |
| 26 | # This command dies on failure. |
| 27 | sub run { |
| 28 | my $context = caller(0); |
| 29 | my $code = runAllowFailure(@_); |
| 30 | unless ($code == 0) { |
| 31 | my $exit = $code >> 8; |
| 32 | my $errno = $code - ($exit << 8); |
| 33 | die "Command failed with code ($exit) errno ($errno).\n"; |
| 34 | } |
| 35 | |
| 36 | return $code; |
| 37 | } |
| 38 | |
| 39 | my $channelName = $ARGV[0]; |
| 40 | my $releaseUrl = $ARGV[1]; |
| 41 | |
| 42 | die "Usage: $0 CHANNEL-NAME RELEASE-URL\n" unless defined $channelName && defined $releaseUrl; |
| 43 | |
| 44 | $channelName =~ /^([a-z]+)-(.*)$/ or die; |
| 45 | my $channelDirRel = $channelName eq "nixpkgs-unstable" ? "nixpkgs" : "$1/$2"; |
| 46 | |
| 47 | |
| 48 | # Configuration. |
| 49 | my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; |
| 50 | my $filesCache = "${TMPDIR}/nixos-files.sqlite"; |
| 51 | my $bucketReleasesName = "nix-releases"; |
| 52 | my $bucketChannelsName = "nix-channels"; |
| 53 | my $dryRun = $ENV{'DRY_RUN'} // 0; |
| 54 | |
| 55 | $ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels"; |
| 56 | |
| 57 | my $bucketReleases; |
| 58 | my $bucketChannels; |
| 59 | |
| 60 | unless ($dryRun) { |
| 61 | # S3 setup. |
| 62 | my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given."; |
| 63 | my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given."; |
| 64 | |
| 65 | my $s3 = Net::Amazon::S3->new( |
| 66 | { aws_access_key_id => $aws_access_key_id, |
| 67 | aws_secret_access_key => $aws_secret_access_key, |
| 68 | retry => 1, |
| 69 | host => "s3-eu-west-1.amazonaws.com", |
| 70 | }); |
| 71 | |
| 72 | $bucketReleases = $s3->bucket($bucketReleasesName) or die; |
| 73 | |
| 74 | my $s3_us = Net::Amazon::S3->new( |
| 75 | { aws_access_key_id => $aws_access_key_id, |
| 76 | aws_secret_access_key => $aws_secret_access_key, |
| 77 | retry => 1, |
| 78 | }); |
| 79 | |
| 80 | $bucketChannels = $s3_us->bucket($bucketChannelsName) or die; |
| 81 | } else { |
| 82 | print STDERR "WARNING: Running in dry-run.\n"; |
| 83 | } |
| 84 | |
| 85 | sub fetch { |
| 86 | my ($url, $type) = @_; |
| 87 | |
| 88 | my $ua = LWP::UserAgent->new; |
| 89 | $ua->default_header('Accept', $type) if defined $type; |
| 90 | |
| 91 | my $response = $ua->get($url); |
| 92 | die "could not download $url: ", $response->status_line, "\n" unless $response->is_success; |
| 93 | |
| 94 | return $response->decoded_content; |
| 95 | } |
| 96 | |
| 97 | my $releaseInfo = decode_json(fetch($releaseUrl, 'application/json')); |
| 98 | |
| 99 | my $releaseId = $releaseInfo->{id} or die; |
| 100 | my $releaseName = $releaseInfo->{nixname} or die; |
| 101 | $releaseName =~ /-([0-9].+)/ or die; |
| 102 | my $releaseVersion = $1; |
| 103 | my $evalId = $releaseInfo->{jobsetevals}->[0] or die; |
| 104 | my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; |
| 105 | my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); |
| 106 | my $releasePrefix = "$channelDirRel/$releaseName"; |
| 107 | |
| 108 | my $rev = $evalInfo->{jobsetevalinputs}->{nixpkgs}->{revision} or die; |
| 109 | |
| 110 | # Get commit date of $rev as unixtime and formatted string |
| 111 | run("git fetch origin $rev >&2"); |
| 112 | my $revUnix = `git show --no-patch --format='%ct' $rev` or die; |
| 113 | my $revDate = strftime("%F %T %Z", localtime($revUnix)); |
| 114 | |
| 115 | print STDERR "\nRelease information:\n"; |
| 116 | print STDERR " - release is: $releaseName (build $releaseId)\n - eval is: $evalId\n - prefix is: $releasePrefix\n - Git commit is: $rev\n - Git commit date is: $revDate\n\n"; |
| 117 | |
| 118 | if ($bucketChannels) { |
| 119 | # Guard against the channel going back in time. |
| 120 | my $curRelease = ""; |
| 121 | |
| 122 | if (defined(my $object = $bucketChannels->get_key($channelName))) { |
| 123 | $curRelease = $object->{'x-amz-website-redirect-location'} // ""; |
| 124 | } |
| 125 | |
| 126 | if (!defined $ENV{'FORCE'}) { |
| 127 | print STDERR "previous release is $curRelease\n"; |
| 128 | $! = 0; # Clear errno to avoid reporting non-fork/exec-related issues |
| 129 | my $d = `NIX_PATH= nix-instantiate --eval -E "builtins.compareVersions (builtins.parseDrvName \\"$curRelease\\").version (builtins.parseDrvName \\"$releaseName\\").version"`; |
| 130 | if ($? != 0) { |
| 131 | warn "Could not execute nix-instantiate: exit $?; errno $!\n"; |
| 132 | exit 1; |
| 133 | } |
| 134 | chomp $d; |
| 135 | if ($d == 1) { |
| 136 | warn("channel would go back in time from $curRelease to $releaseName, bailing out\n"); |
| 137 | exit; |
| 138 | } |
| 139 | exit if $d == 0; |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | if ($bucketReleases && $bucketReleases->head_key("$releasePrefix")) { |
| 144 | print STDERR "release already exists\n"; |
| 145 | } else { |
| 146 | my $tmpDir = "$TMPDIR/release-$channelName/$releaseName"; |
| 147 | File::Path::make_path($tmpDir); |
| 148 | |
| 149 | write_file("$tmpDir/src-url", $evalUrl); |
| 150 | write_file("$tmpDir/git-revision", $rev); |
| 151 | write_file("$tmpDir/binary-cache-url", "https://cache.nixos.org"); |
| 152 | |
| 153 | if (! -e "$tmpDir/store-paths.xz") { |
| 154 | my $storePaths = decode_json(fetch("$evalUrl/store-paths", 'application/json')); |
| 155 | write_file("$tmpDir/store-paths", join("\n", uniq(@{$storePaths})) . "\n"); |
| 156 | } |
| 157 | |
| 158 | sub downloadFile { |
| 159 | my ($jobName, $dstName, $productType, $productPattern) = @_; |
| 160 | |
| 161 | my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); |
| 162 | |
| 163 | my $products = (); |
| 164 | # Key the products by subtype. |
| 165 | foreach my $key (keys $buildInfo->{buildproducts}->%*) { |
| 166 | my $product = $buildInfo->{buildproducts}->{$key}; |
| 167 | my $subType = $product->{subtype}; |
| 168 | |
| 169 | next if defined $productType |
| 170 | && $subType ne $productType; |
| 171 | |
| 172 | next if defined $productPattern |
| 173 | && $product->{path} !~ /$productPattern/; |
| 174 | |
| 175 | if ($products->{$subType}) { |
| 176 | if (defined $productPattern) { |
| 177 | die "Job $jobName has multiple products of subtype $subType that match $productPattern.\nRefine the product regex pattern further to disambiguate."; |
| 178 | } else { |
| 179 | die "Job $jobName has multiple products of the same subtype $subType.\nPass a product regex pattern to disambiguate."; |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | $products->{$subType} = $product; |
| 184 | } |
| 185 | my $size = keys %{$products}; |
| 186 | |
| 187 | if ($size > 1 && !$productType) { |
| 188 | my $types = join(", ", keys %{$products}); |
| 189 | die "Job $jobName has $size build products. Select the right product by subtype [$types] and product match"; |
| 190 | } |
| 191 | |
| 192 | my $product; |
| 193 | if (!$productType) { |
| 194 | # Take the only element |
| 195 | my ($key) = keys %{$products}; |
| 196 | $product = $products->{$key}; |
| 197 | } else { |
| 198 | # Take the selected element |
| 199 | $product = $products->{$productType}; |
| 200 | } |
| 201 | |
| 202 | unless ($product) { |
| 203 | die "No product could be selected for $jobName, with type $productType"; |
| 204 | } |
| 205 | |
| 206 | my $srcFile = $product->{path} or die "job '$jobName' lacks a store path"; |
| 207 | $dstName //= basename($srcFile); |
| 208 | my $dstFile = "$tmpDir/" . $dstName; |
| 209 | |
| 210 | my $sha256_expected = $product->{sha256hash} or die; |
| 211 | |
| 212 | if (! -e $dstFile) { |
| 213 | print STDERR "downloading $srcFile to $dstFile...\n"; |
| 214 | write_file("$dstFile.sha256", "$sha256_expected $dstName"); |
| 215 | runAllowFailure("NIX_REMOTE=s3://nix-cache nix --experimental-features nix-command store cat '$srcFile' > '$dstFile.tmp'") == 0 |
| 216 | or die "unable to fetch $srcFile\n"; |
| 217 | rename("$dstFile.tmp", $dstFile) or die; |
| 218 | } |
| 219 | |
| 220 | if (-e "$dstFile.sha256") { |
| 221 | my $sha256_actual = `nix --experimental-features nix-command hash file --base16 --type sha256 '$dstFile'`; |
| 222 | chomp $sha256_actual; |
| 223 | if ($sha256_expected ne $sha256_actual) { |
| 224 | print STDERR "file $dstFile is corrupt $sha256_expected $sha256_actual\n"; |
| 225 | exit 1; |
| 226 | } |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | if ($channelName =~ /nixos/) { |
| 231 | downloadFile("nixos.channel", "nixexprs.tar.xz", "source-dist", '\.tar\.xz$'); |
| 232 | downloadFile("nixos.channel", "nixexprs.tar.zst", "source-dist", '\.tar\.zst$'); |
| 233 | downloadFile("nixpkgs.tarball", "packages.json.br", "json-br"); |
| 234 | downloadFile("nixos.options", "options.json.br", "json-br"); |
| 235 | |
| 236 | # Minimal installer ISOs were dropped from the small channel |
| 237 | if ($channelName !~ /-small/ || |
| 238 | $channelName =~ /nixos-2([0123]\...|4\.05)-small/) { |
| 239 | downloadFile("nixos.iso_minimal.aarch64-linux"); |
| 240 | downloadFile("nixos.iso_minimal.x86_64-linux"); |
| 241 | } |
| 242 | |
| 243 | # All of these jobs are not present in small channels |
| 244 | if ($channelName !~ /-small/) { |
| 245 | # These jobs were combined into a single job |
| 246 | if ($channelName =~ /nixos-2[01234]/) { |
| 247 | if ($channelName =~ /nixos-2[0123]/) { |
| 248 | downloadFile("nixos.iso_plasma5.aarch64-linux"); |
| 249 | downloadFile("nixos.iso_plasma5.x86_64-linux"); |
| 250 | } else { |
| 251 | downloadFile("nixos.iso_plasma6.aarch64-linux"); |
| 252 | downloadFile("nixos.iso_plasma6.x86_64-linux"); |
| 253 | } |
| 254 | |
| 255 | downloadFile("nixos.iso_gnome.aarch64-linux"); |
| 256 | downloadFile("nixos.iso_gnome.x86_64-linux"); |
| 257 | } else { |
| 258 | downloadFile("nixos.iso_graphical.aarch64-linux"); |
| 259 | downloadFile("nixos.iso_graphical.x86_64-linux"); |
| 260 | } |
| 261 | |
| 262 | if ($channelName =~ /nixos-2[0123]/) { # i686 dropped for > 23.11 |
| 263 | downloadFile("nixos.iso_minimal.i686-linux"); |
| 264 | } |
| 265 | |
| 266 | if ($channelName =~ /nixos-2([0123]\...|4\.05)/) { |
| 267 | downloadFile("nixos.ova.x86_64-linux"); |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | } else { |
| 272 | downloadFile("tarball", "nixexprs.tar.xz", "source-dist", '\.tar\.xz$'); |
| 273 | downloadFile("tarball", "nixexprs.tar.zst", "source-dist", '\.tar\.zst$'); |
| 274 | downloadFile("tarball", "packages.json.br", "json-br"); |
| 275 | } |
| 276 | |
| 277 | # Generate the programs.sqlite database and put it in |
| 278 | # nixexprs.tar.{xz,zst}. Also maintain the debug info repository at |
| 279 | # https://cache.nixos.org/debuginfo. |
| 280 | if ($channelName =~ /nixos/ && -e "$tmpDir/store-paths") { |
| 281 | # Unpack nixpkgs from tarball. |
| 282 | File::Path::make_path("$tmpDir/unpack"); |
| 283 | run("tar", |
| 284 | "--extract", |
| 285 | "--file", "$tmpDir/nixexprs.tar.zst", |
| 286 | "--zstd", |
| 287 | "--directory", "$tmpDir/unpack" |
| 288 | ); |
| 289 | my $exprDir = glob("$tmpDir/unpack/*"); |
| 290 | |
| 291 | # Create artifacts. |
| 292 | run("nix-channel-index", |
| 293 | "--output", "$exprDir/programs.sqlite", |
| 294 | "--debug-output", "$exprDir/debug.sqlite", |
| 295 | "--nixpkgs", "$exprDir/nixpkgs", |
| 296 | "--platform", "aarch64-linux", |
| 297 | "--platform", "x86_64-linux" |
| 298 | ); |
| 299 | |
| 300 | run("index-debuginfo", "$exprDir/debug.sqlite", "s3://nix-cache"); |
| 301 | |
| 302 | # Remove the downloaded tarballs and intermediate artifacts before repacking. |
| 303 | run("rm", |
| 304 | "--force", |
| 305 | "$tmpDir/nixexprs.tar.xz", |
| 306 | "$tmpDir/nixexprs.tar.xz.sha256", |
| 307 | "$tmpDir/nixexprs.tar.zst", |
| 308 | "$tmpDir/nixexprs.tar.zst.sha256", |
| 309 | "$exprDir/debug.sqlite" |
| 310 | ); |
| 311 | |
| 312 | # Repack tarballs with the generated artifacts. |
| 313 | run("tar", |
| 314 | "--create", |
| 315 | "--file=$tmpDir/nixexprs.tar.xz", |
| 316 | "--xz", |
| 317 | "--format=gnu", |
| 318 | "--sort=name", |
| 319 | "--owner=0", |
| 320 | "--group=0", |
| 321 | "--mtime=\@315532800", # matches SOURCE_DATE_EPOCH from stdenv |
| 322 | "--numeric-owner", |
| 323 | "--directory=$tmpDir/unpack", |
| 324 | basename($exprDir) |
| 325 | ); |
| 326 | run("tar", |
| 327 | "--create", |
| 328 | "--file=$tmpDir/nixexprs.tar.zst", |
| 329 | "--use-compress-program=zstd -19 -T0", |
| 330 | "--format=gnu", |
| 331 | "--sort=name", |
| 332 | "--owner=0", |
| 333 | "--group=0", |
| 334 | "--mtime=\@315532800", # matches SOURCE_DATE_EPOCH from stdenv |
| 335 | "--numeric-owner", |
| 336 | "--directory=$tmpDir/unpack", |
| 337 | basename($exprDir) |
| 338 | ); |
| 339 | |
| 340 | # Clean up. |
| 341 | run("rm", |
| 342 | "--recursive", |
| 343 | "--force", |
| 344 | "$tmpDir/unpack" |
| 345 | ); |
| 346 | } |
| 347 | |
| 348 | if (-e "$tmpDir/store-paths") { |
| 349 | run("xz", "$tmpDir/store-paths"); |
| 350 | } |
| 351 | |
| 352 | my $now = strftime("%F %T %Z", localtime); |
| 353 | my $title = "$channelName release $releaseName"; |
| 354 | my $githubLink = "https://github.com/NixOS/nixpkgs/commits/$rev"; |
| 355 | |
| 356 | my $html = "<html><head>"; |
| 357 | $html .= "<title>$title</title></head>"; |
| 358 | $html .= "<body><h1>$title</h1>"; |
| 359 | $html .= "<p>Released on $now from <a href='$githubLink'>Git commit <tt>$rev</tt></a> from $revDate "; |
| 360 | $html .= "via <a href='$evalUrl'>Hydra evaluation $evalId</a>.</p>"; |
| 361 | $html .= "<table><thead><tr><th>File name</th><th>Size</th><th>SHA-256 hash</th></tr></thead><tbody>"; |
| 362 | |
| 363 | if ($bucketReleases) { |
| 364 | # Upload the release to S3. |
| 365 | for my $fn (sort glob("$tmpDir/*")) { |
| 366 | my $basename = basename $fn; |
| 367 | my $key = "$releasePrefix/" . $basename; |
| 368 | |
| 369 | unless (defined $bucketReleases->head_key($key)) { |
| 370 | print STDERR "mirroring $fn to s3://$bucketReleasesName/$key...\n"; |
| 371 | |
| 372 | # Default headers |
| 373 | my $configuration = (); |
| 374 | $configuration->{content_type} = "application/octet-stream"; |
| 375 | |
| 376 | if ($fn =~ /.sha256|src-url|binary-cache-url|git-revision/) { |
| 377 | # Text files |
| 378 | $configuration->{content_type} = "text/plain"; |
| 379 | } elsif ($fn =~ /.json.br$/) { |
| 380 | # JSON encoded as brotli |
| 381 | $configuration->{content_type} = "application/json"; |
| 382 | $configuration->{content_encoding} = "br"; |
| 383 | } |
| 384 | |
| 385 | $bucketReleases->add_key_filename( |
| 386 | $key, $fn, $configuration |
| 387 | ) or die $bucketReleases->err . ": " . $bucketReleases->errstr; |
| 388 | } |
| 389 | |
| 390 | next if $basename =~ /.sha256$/; |
| 391 | |
| 392 | my $size = stat($fn)->size; |
| 393 | my $sha256 = Digest::SHA::sha256_hex(read_file($fn)); |
| 394 | $html .= "<tr>"; |
| 395 | $html .= "<td><a href='/$key'>$basename</a></td>"; |
| 396 | $html .= "<td align='right'>$size</td>"; |
| 397 | $html .= "<td><tt>$sha256</tt></td>"; |
| 398 | $html .= "</tr>"; |
| 399 | } |
| 400 | |
| 401 | $html .= "</tbody></table></body></html>"; |
| 402 | |
| 403 | $bucketReleases->add_key($releasePrefix, $html, |
| 404 | { content_type => "text/html" }) |
| 405 | or die $bucketReleases->err . ": " . $bucketReleases->errstr; |
| 406 | } |
| 407 | |
| 408 | File::Path::remove_tree($tmpDir); |
| 409 | } |
| 410 | |
| 411 | if ($dryRun) { |
| 412 | print STDERR "WARNING: dry-run finished...\n"; |
| 413 | exit(0); |
| 414 | } |
| 415 | |
| 416 | # Update the nixos-* branch in the nixpkgs repo. |
| 417 | run("git remote update origin >&2"); |
| 418 | run("git push origin $rev:refs/heads/$channelName >&2"); |
| 419 | |
| 420 | # maxage=600: Serve from cache for 5 minutes. |
| 421 | # stale-while-revaliadate=1800: Serve from cache while updating in the background for 30 minutes. |
| 422 | # https://web.dev/stale-while-revalidate/ |
| 423 | # https://developer.fastly.com/learning/concepts/cache-freshness/ |
| 424 | my $cache_control = "maxage=600,stale-while-revalidate=1800,public"; |
| 425 | |
| 426 | sub redirect { |
| 427 | my ($from, $to) = @_; |
| 428 | $to = "https://releases.nixos.org/" . $to; |
| 429 | print STDERR "redirect $from -> $to\n"; |
| 430 | $bucketChannels->add_key($from, "", { "x-amz-website-redirect-location" => $to, "cache-control" => $cache_control }) |
| 431 | or die $bucketChannels->err . ": " . $bucketChannels->errstr; |
| 432 | } |
| 433 | |
| 434 | # Update channels on channels.nixos.org. |
| 435 | redirect($channelName, $releasePrefix); |
| 436 | redirect("$channelName/nixexprs.tar.xz", "$releasePrefix/nixexprs.tar.xz?rev=$rev&lastModified=$revUnix"); |
| 437 | redirect("$channelName/nixexprs.tar.zst", "$releasePrefix/nixexprs.tar.zst?rev=$rev&lastModified=$revUnix"); |
| 438 | redirect("$channelName/git-revision", "$releasePrefix/git-revision"); |
| 439 | redirect("$channelName/packages.json.br", "$releasePrefix/packages.json.br"); |
| 440 | redirect("$channelName/store-paths.xz", "$releasePrefix/store-paths.xz"); |
| 441 | |
| 442 | # Create redirects relevant only to NixOS channels. |
| 443 | # FIXME: create only redirects to files that exist. |
| 444 | if ($channelName =~ /nixos/) { |
| 445 | # Options listing |
| 446 | redirect("$channelName/options.json.br", "$releasePrefix/options.json.br"); |
| 447 | |
| 448 | # Redirects for latest images. |
| 449 | for my $arch ("x86_64-linux", "i686-linux", "aarch64-linux") { |
| 450 | # i686 dropped for > 23.11 |
| 451 | next if $arch eq "i686-linux" && $channelName !~ /nixos-2[0123]/; |
| 452 | |
| 453 | for my $artifact ("nixos-graphical", |
| 454 | "nixos-plasma5", |
| 455 | "nixos-plasma6", |
| 456 | "nixos-gnome", |
| 457 | "nixos-minimal", |
| 458 | ) |
| 459 | { |
| 460 | redirect("$channelName/latest-$artifact-$arch.iso", "$releasePrefix/$artifact-$releaseVersion-$arch.iso"); |
| 461 | redirect("$channelName/latest-$artifact-$arch.iso.sha256", "$releasePrefix/$artifact-$releaseVersion-$arch.iso.sha256"); |
| 462 | } |
| 463 | |
| 464 | redirect("$channelName/latest-nixos-$arch.ova", "$releasePrefix/nixos-$releaseVersion-$arch.ova"); |
| 465 | redirect("$channelName/latest-nixos-$arch.ova.sha256", "$releasePrefix/nixos-$releaseVersion-$arch.ova.sha256"); |
| 466 | } |
| 467 | } |