@cryptotaxi247 / infra-1 / commits / 0bf99023

introduce staging binary cache to migration

This introduces two buckets. One will be the old cache that we want to move to glacier eventually. The other bucket is the new cache that we want to use in the future.

Jörg Thalheim committed Oct 15, 2024 at 14:39 UTC 0bf990234d5eca926a3219f40019dbd3ade06ed6
10 files changed +660
terraform/cache-bucket/main.tf new
+132
@@ -0,0 +1,132 @@
1 +variable "bucket_name" {
2 + type = string
3 +}
4 +
5 +resource "aws_s3_bucket" "cache" {
6 + provider = aws
7 + bucket = var.bucket_name
8 +
9 + lifecycle_rule {
10 + enabled = true
11 +
12 + transition {
13 + days = 365
14 + storage_class = "STANDARD_IA"
15 + }
16 + }
17 +
18 + cors_rule {
19 + allowed_headers = ["Authorization"]
20 + allowed_methods = ["GET"]
21 + allowed_origins = ["*"]
22 + max_age_seconds = 3000
23 + }
24 +}
25 +
26 +resource "aws_s3_bucket_public_access_block" "cache" {
27 + bucket = aws_s3_bucket.cache.bucket
28 +
29 + block_public_acls = false
30 + block_public_policy = false
31 +}
32 +
33 +resource "aws_s3_bucket_object" "cache-nix-cache-info" {
34 + provider = aws
35 + depends_on = [aws_s3_bucket_public_access_block.cache]
36 +
37 + bucket = aws_s3_bucket.cache.bucket
38 + content_type = "text/x-nix-cache-info"
39 + etag = filemd5("${path.module}/../cache-staging/nix-cache-info")
40 + key = "nix-cache-info"
41 + source = "${path.module}/../cache-staging/nix-cache-info"
42 +}
43 +
44 +resource "aws_s3_bucket_object" "cache-index-html" {
45 + provider = aws
46 + depends_on = [aws_s3_bucket_public_access_block.cache]
47 +
48 + bucket = aws_s3_bucket.cache.bucket
49 + content_type = "text/html"
50 + etag = filemd5("${path.module}/../cache-staging/index.html")
51 + key = "index.html"
52 + source = "${path.module}/../cache-staging/index.html"
53 +}
54 +
55 +resource "aws_s3_bucket_policy" "cache" {
56 + provider = aws
57 + bucket = aws_s3_bucket.cache.id
58 + depends_on = [aws_s3_bucket_public_access_block.cache]
59 +
60 + # imported from existing
61 + policy = <<EOF
62 +{
63 + "Version": "2008-10-17",
64 + "Statement": [
65 + {
66 + "Sid": "AllowPublicRead",
67 + "Effect": "Allow",
68 + "Principal": {
69 + "AWS": "*"
70 + },
71 + "Action": "s3:GetObject",
72 + "Resource": "arn:aws:s3:::${var.bucket_name}/*"
73 + },
74 + {
75 + "Sid": "AllowUploadDebuginfoWrite",
76 + "Effect": "Allow",
77 + "Principal": {
78 + "AWS": "arn:aws:iam::080433136561:user/s3-upload-releases"
79 + },
80 + "Action": [
81 + "s3:PutObject",
82 + "s3:PutObjectAcl"
83 + ],
84 + "Resource": "arn:aws:s3:::${var.bucket_name}/debuginfo/*"
85 + },
86 + {
87 + "Sid": "AllowUploadDebuginfoRead",
88 + "Effect": "Allow",
89 + "Principal": {
90 + "AWS": "arn:aws:iam::080433136561:user/s3-upload-releases"
91 + },
92 + "Action": "s3:GetObject",
93 + "Resource": "arn:aws:s3:::${var.bucket_name}/*"
94 + },
95 + {
96 + "Sid": "AllowUploadDebuginfoRead2",
97 + "Effect": "Allow",
98 + "Principal": {
99 + "AWS": "arn:aws:iam::080433136561:user/s3-upload-releases"
100 + },
101 + "Action": [
102 + "s3:ListBucket",
103 + "s3:GetBucketLocation"
104 + ],
105 + "Resource": "arn:aws:s3:::${var.bucket_name}"
106 + }
107 + ]
108 +}
109 +EOF
110 +}
111 +
112 +resource "aws_s3_bucket_request_payment_configuration" "cache" {
113 + provider = aws
114 + bucket = aws_s3_bucket.cache.id
115 + payer = "Requester"
116 +}
117 +
118 +output "bucket" {
119 + value = aws_s3_bucket.cache.bucket
120 +}
121 +
122 +output "bucket_domain_name" {
123 + value = aws_s3_bucket.cache.bucket_domain_name
124 +}
125 +
126 +output "bucket_regional_domain_name" {
127 + value = aws_s3_bucket.cache.bucket_regional_domain_name
128 +}
129 +
130 +output "region" {
131 + value = aws_s3_bucket.cache.region
132 +}
terraform/cache-bucket/providers.tf new
+7
@@ -0,0 +1,7 @@
1 +terraform {
2 + required_providers {
3 + aws = {
4 + source = "registry.terraform.io/hashicorp/aws"
5 + }
6 + }
7 +}
terraform/cache-staging.tf new
+333
@@ -0,0 +1,333 @@
1 +locals {
2 + cache_staging_domain = "cache-staging.nixos.org"
3 +}
4 +
5 +# This is the old bucket we want to archive.
6 +module "cache-staging-202010" {
7 + source = "./cache-bucket"
8 + bucket_name = "nix-cache-staging"
9 + providers = {
10 + aws = aws.us
11 + }
12 +}
13 +
14 +# This is the new bucket we want to use in future.
15 +module "cache-staging-202410" {
16 + source = "./cache-bucket"
17 + bucket_name = "nix-cache-staging-202410"
18 + providers = {
19 + # move the new bucket to EU
20 + aws = aws
21 + }
22 +}
23 +
24 +# The fastly configuration below will first try the new bucket and than the old bucket.
25 +# As demonstation we have two files in the buckets:
26 +# $ curl https://cache-staging.nixos.org/new-cache │
27 +# new
28 +# $ curl https://cache-staging.nixos.org/old-cache
29 +# old
30 +
31 +resource "aws_s3_object" "old-cache-test-file" {
32 + provider = aws.us
33 + depends_on = [module.cache-staging-202010]
34 +
35 + bucket = module.cache-staging-202010.bucket
36 + content_type = "text/plain"
37 + etag = filemd5("${path.module}/cache-staging/old-cache-test-file")
38 + key = "old-cache"
39 + source = "${path.module}/cache-staging/old-cache-test-file"
40 +}
41 +resource "aws_s3_object" "new-cache-test-file" {
42 + provider = aws
43 + depends_on = [module.cache-staging-202410]
44 +
45 + bucket = module.cache-staging-202410.bucket
46 + content_type = "text/plain"
47 + etag = filemd5("${path.module}/cache-staging/new-cache-test-file")
48 + key = "new-cache"
49 + source = "${path.module}/cache-staging/new-cache-test-file"
50 +}
51 +
52 +resource "fastly_service_vcl" "cache-staging" {
53 + name = local.cache_staging_domain
54 + default_ttl = 86400
55 +
56 + backend {
57 + address = module.cache-staging-202010.bucket_regional_domain_name
58 + auto_loadbalance = false
59 + between_bytes_timeout = 10000
60 + connect_timeout = 5000
61 + error_threshold = 0
62 + first_byte_timeout = 15000
63 + max_conn = 200
64 + name = "old_bucket"
65 + port = 443
66 + # For the old bucket we want to use Ashburn as our bucket is in us-east-1
67 + shield = "iad-va-us"
68 + ssl_cert_hostname = module.cache-staging-202010.bucket_regional_domain_name
69 + ssl_check_cert = true
70 + use_ssl = true
71 + weight = 100
72 + }
73 +
74 + backend {
75 + address = module.cache-staging-202410.bucket_regional_domain_name
76 + auto_loadbalance = false
77 + between_bytes_timeout = 10000
78 + connect_timeout = 5000
79 + error_threshold = 0
80 + first_byte_timeout = 15000
81 + max_conn = 200
82 + name = "new_bucket"
83 + port = 443
84 + # The new bucket is in EU (eu-west-1)
85 + shield = "dub-dublin-ie"
86 + ssl_cert_hostname = module.cache-staging-202410.bucket_regional_domain_name
87 + ssl_check_cert = true
88 + use_ssl = true
89 +
90 + # newer bucket has higher priority
91 + weight = 200
92 + }
93 +
94 + # Temporarily disabled due to nix-index bugs: see https://github.com/nix-community/nix-index/issues/249
95 + #request_setting {
96 + # name = "Redirect HTTP to HTTPS"
97 + # force_ssl = true
98 + #}
99 +
100 + condition {
101 + name = "is-404"
102 + priority = 0
103 + statement = "beresp.status == 404"
104 + type = "CACHE"
105 + }
106 +
107 + condition {
108 + name = "Match /"
109 + priority = 10
110 + statement = "req.url ~ \"^/$\""
111 + type = "REQUEST"
112 + }
113 +
114 + condition {
115 + name = "Restarts > 0"
116 + type = "REQUEST"
117 + priority = 20
118 + statement = "req.restarts > 0"
119 + }
120 +
121 + domain {
122 + name = "cache-staging.nixos.org"
123 + }
124 +
125 + header {
126 + name = "Landing page"
127 + request_condition = "Match /"
128 + ignore_if_set = false
129 + priority = 10
130 + type = "request"
131 +
132 + action = "set"
133 + destination = "url"
134 + source = "\"/index.html\""
135 +
136 + }
137 +
138 + header {
139 + name = "Use old bucket"
140 + request_condition = "Restarts > 0"
141 + ignore_if_set = false
142 + priority = 20
143 + type = "request"
144 +
145 + action = "set"
146 + destination = "backend"
147 + source = "F_old_bucket"
148 + }
149 +
150 + # Clean headers for caching
151 + header {
152 + destination = "http.x-amz-request-id"
153 + type = "cache"
154 + action = "delete"
155 + name = "remove x-amz-request-id"
156 + }
157 + header {
158 + destination = "http.x-amz-version-id"
159 + type = "cache"
160 + action = "delete"
161 + name = "remove x-amz-version-id"
162 + }
163 + header {
164 + destination = "http.x-amz-id-2"
165 + type = "cache"
166 + action = "delete"
167 + name = "remove x-amz-id-2"
168 + }
169 +
170 + # Enable Streaming Miss.
171 + # https://docs.fastly.com/en/guides/streaming-miss
172 + # https://github.com/NixOS/nixos-org-configurations/issues/212#issuecomment-1187568233
173 + header {
174 + priority = 20
175 + destination = "do_stream"
176 + type = "cache"
177 + action = "set"
178 + name = "Enabling Streaming Miss"
179 + source = "true"
180 + }
181 +
182 + # Allow CORS GET requests.
183 + header {
184 + destination = "http.access-control-allow-origin"
185 + type = "response"
186 + action = "set"
187 + name = "CORS Allow"
188 + source = "\"*\""
189 + }
190 +
191 + response_object {
192 + name = "404-page"
193 + cache_condition = "is-404"
194 + content = "404"
195 + content_type = "text/plain"
196 + response = "Not Found"
197 + status = 404
198 + }
199 +
200 + snippet {
201 + name = "Variables for aws s3 auth"
202 + type = "miss"
203 + priority = 90
204 + content = <<-EOT
205 +declare local var.awsAccessKey STRING;
206 +declare local var.awsSecretKey STRING;
207 +declare local var.awsS3Bucket STRING;
208 +declare local var.awsRegion STRING;
209 +declare local var.awsS3Host STRING;
210 +
211 +declare local var.canonicalHeaders STRING;
212 +declare local var.signedHeaders STRING;
213 +declare local var.canonicalRequest STRING;
214 +declare local var.canonicalQuery STRING;
215 +declare local var.stringToSign STRING;
216 +declare local var.dateStamp STRING;
217 +declare local var.signature STRING;
218 +declare local var.scope STRING;
219 +EOT
220 + }
221 +
222 + # Authenticate Fastly<->S3 requests. See Fastly documentation:
223 + # https://docs.fastly.com/en/guides/amazon-s3#using-an-amazon-s3-private-bucket
224 + snippet {
225 + name = "Authenticate S3 requests for new bucket"
226 + type = "miss"
227 + priority = 100
228 + content = templatefile("${path.module}/cache-staging/s3-authn.vcl", {
229 + backend_name = "F_new_bucket"
230 + aws_region = module.cache-staging-202410.region
231 + bucket = module.cache-staging-202410.bucket
232 + backend_domain = module.cache-staging-202410.bucket_domain_name
233 + access_key = local.cache-iam.key
234 + secret_key = local.cache-iam.secret
235 + })
236 + }
237 +
238 + snippet {
239 + name = "Authenticate S3 requests for old bucket"
240 + type = "miss"
241 + priority = 100
242 + content = templatefile("${path.module}/cache-staging/s3-authn.vcl", {
243 + backend_name = "F_old_bucket"
244 + aws_region = module.cache-staging-202010.region
245 + bucket = module.cache-staging-202010.bucket
246 + backend_domain = module.cache-staging-202010.bucket_domain_name
247 + access_key = local.cache-iam.key
248 + secret_key = local.cache-iam.secret
249 + })
250 + }
251 +
252 + snippet {
253 + content = "set req.url = querystring.remove(req.url);"
254 + name = "Remove all query strings"
255 + priority = 50
256 + type = "recv"
257 + }
258 +
259 +
260 + # Work around the 2GB size limit for large files
261 + #
262 + # See https://docs.fastly.com/en/guides/segmented-caching
263 + snippet {
264 + content = <<-EOT
265 + if (req.url.path ~ "^/nar/") {
266 + set req.enable_segmented_caching = true;
267 + }
268 + EOT
269 + name = "Enable segment caching for NAR files"
270 + priority = 60
271 + type = "recv"
272 + }
273 +
274 + snippet {
275 + name = "Fallback to old bucket on 403 or return 404"
276 + type = "fetch"
277 + priority = 90
278 + content = <<-EOT
279 + if (beresp.status == 403) {
280 + if (req.backend == F_new_bucket) {
281 + restart;
282 + } else {
283 + set beresp.status = 404;
284 + }
285 + }
286 + EOT
287 + }
288 +
289 + # We will switch to this snipped once we retire the old bucket instead of the fallback above
290 + #snippet {
291 + # name = "Return 404 on 403"
292 + # type = "fetch"
293 + # priority = 90
294 + # content = <<-EOT
295 + # if (beresp.status == 403) {
296 + # set beresp.status = 404;
297 + # }
298 + # EOT
299 + #}
300 +
301 + # Add a snippet to set a custom header based on the backend used
302 + snippet {
303 + name = "Set-Backend-Header"
304 + type = "deliver"
305 + priority = 70
306 + content = <<-EOT
307 + if (req.backend == F_old_bucket) {
308 + set resp.http.X-Bucket = "${module.cache-staging-202010.bucket}";
309 + } else if (req.backend == F_new_bucket) {
310 + set resp.http.X-Bucket = "${module.cache-staging-202410.bucket}";
311 + }
312 + EOT
313 + }
314 +
315 + logging_s3 {
316 + name = "${local.cache_staging_domain}-to-s3"
317 + bucket_name = local.fastlylogs["bucket_name"]
318 + compression_codec = "zstd"
319 + domain = local.fastlylogs["s3_domain"]
320 + format = local.fastlylogs["format"]
321 + format_version = 2
322 + path = "${local.cache_staging_domain}/"
323 + period = local.fastlylogs["period"]
324 + message_type = "blank"
325 + s3_iam_role = local.fastlylogs["iam_role_arn"]
326 + }
327 +}
328 +
329 +resource "fastly_tls_subscription" "cache-staging" {
330 + domains = [for domain in fastly_service_vcl.cache-staging.domain : domain.name]
331 + configuration_id = local.fastly_tls12_sni_configuration_id
332 + certificate_authority = "globalsign"
333 +}
terraform/cache-staging/diagnostic.sh new
+54
@@ -0,0 +1,54 @@
1 +#!/usr/bin/env nix-shell
2 +#!nix-shell -i bash -p bind.dnsutils -p mtr -p curl
3 +# shellcheck shell=bash
4 +# impure: needs ping
5 +#
6 +# Run this script if you are having issues with cache.nixos.org and paste the
7 +# output URL in a new issue in the same repo.
8 +#
9 +
10 +domain=${1:-cache-staging.nixos.org}
11 +
12 +run() {
13 + echo "> $*"
14 + "$@" |& sed -e "s/^/ /"
15 + printf "Exit: %s\n\n\n" "$?"
16 +}
17 +
18 +curl_w="
19 +time_namelookup: %{time_namelookup}
20 +time_connect: %{time_connect}
21 +time_appconnect: %{time_appconnect}
22 +time_pretransfer: %{time_pretransfer}
23 +time_redirect: %{time_redirect}
24 +time_starttransfer: %{time_starttransfer}
25 +time_total: %{time_total}
26 +"
27 +
28 +curl_test() {
29 + curl -w "$curl_w" -v -o /dev/null "$@"
30 +}
31 +
32 +ix() {
33 + url=$(cat | curl -F 'f:1=<-' ix.io 2>/dev/null)
34 + echo "Pasted at: $url"
35 +}
36 +
37 +(
38 + echo "domain=$domain"
39 + run dig -t A "$domain"
40 + run ping -c1 "$domain"
41 + run ping -4 -c1 "$domain"
42 + run ping -6 -c1 "$domain"
43 + run mtr -c 20 -w -r "$domain"
44 + run curl_test -4 "http://$domain/"
45 + run curl_test -6 "http://$domain/"
46 + run curl_test -4 "https://$domain/"
47 + run curl_test -6 "https://$domain/"
48 + run curl -I -4 "https://$domain/"
49 + run curl -I -4 "https://$domain/"
50 + run curl -I -4 "https://$domain/"
51 + run curl -I -6 "https://$domain/"
52 + run curl -I -6 "https://$domain/"
53 + run curl -I -6 "https://$domain/"
54 +) | tee /dev/stderr | ix
terraform/cache-staging/index.html new
+60
@@ -0,0 +1,60 @@
1 +<!DOCTYPE html>
2 +<html lang="en">
3 + <head>
4 + <title>cache-staging.nixos.org is up</title>
5 + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
6 + <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
7 + <meta name="viewport" content="width=device-width, initial-scale=1.0" />
8 + <link rel="stylesheet" href="https://nixos.org/bootstrap/css/bootstrap.min.css" />
9 + <link rel="stylesheet" href="https://nixos.org/bootstrap/css/bootstrap-responsive.min.css" />
10 + <style>
11 + body {
12 + padding-top: 0;
13 + margin-top: 4em;
14 + margin-bottom: 4em;
15 + }
16 + body > div {
17 + max-width: 800px;
18 + }
19 + p {
20 + text-align: center;
21 + }
22 + .cache {
23 + font-style: italic;
24 + }
25 + </style>
26 + </head>
27 + <body>
28 + <div class="container jumbotron">
29 + <div class="jumbotron">
30 + <p class="lead">
31 + <a href="https://nixos.org/nixos">
32 + <img src="https://nixos.org/logo/nixos-hires.png" width="500px" alt="logo" />
33 + </a>
34 + </p>
35 +
36 + <p class="lead">
37 + <code>https://cache.nixos.org/</code> provides
38 + prebuilt binaries for Nixpkgs and NixOS. It is
39 + used automatically by the Nix package manager to
40 + speed up builds.
41 + </p>
42 + </p>
43 + </div>
44 + <hr />
45 + <div class="help">
46 + <p>
47 + If you are having trouble, please reach out through one of the
48 + <a href="https://nixos.org/nixos/support.html">support channels</a>
49 + with the results of
50 + <a href="https://github.com/NixOS/nixos-org-configurations/blob/master/terraform/cache/diagnostic.sh">this diagnostics script</a>
51 + which will help us figure out where the issue lies.
52 + </p>
53 + <p>
54 + For questions, or support, <a href="https://nixos.org/nixos/support.html">
55 + the support page</a> from the NixOS website describes how to get in touch.
56 + </p>
57 + </div>
58 + </div>
59 + </body>
60 +</html>
terraform/cache-staging/new-cache-test-file new
+1
@@ -0,0 +1 @@
1 +new
terraform/cache-staging/nix-cache-info new
+3
@@ -0,0 +1,3 @@
1 +StoreDir: /nix/store
2 +WantMassQuery: 1
3 +Priority: 40
terraform/cache-staging/old-cache-test-file new
+1
@@ -0,0 +1 @@
1 +old
terraform/cache-staging/s3-authn.vcl new
+64
@@ -0,0 +1,64 @@
1 +# VCL snippet to authenticate Fastly<->S3 requests.
2 +#
3 +# https://docs.fastly.com/en/guides/amazon-s3#using-an-amazon-s3-private-bucket
4 +
5 +if (req.method == "GET" && !req.backend.is_shield && req.backend == ${backend_name}) {
6 + set var.awsAccessKey = "${access_key}";
7 + set var.awsSecretKey = "${secret_key}";
8 + set var.awsS3Bucket = "${bucket}";
9 + set var.awsRegion = "${aws_region}"; # Change this value to your own data
10 + set var.awsS3Host = var.awsS3Bucket ".s3." var.awsRegion ".amazonaws.com";
11 +
12 + set bereq.http.x-amz-content-sha256 = digest.hash_sha256("");
13 + set bereq.http.x-amz-date = strftime({"%Y%m%dT%H%M%SZ"}, now);
14 + set bereq.http.x-amz-request-payer = "requester";
15 + set bereq.http.host = var.awsS3Host;
16 +
17 + set bereq.url = querystring.remove(bereq.url);
18 + set bereq.url = regsuball(urlencode(urldecode(bereq.url.path)), {"%2F"}, "/");
19 + set var.dateStamp = strftime({"%Y%m%d"}, now);
20 + set var.canonicalHeaders = ""
21 + "host:" bereq.http.host LF
22 + "x-amz-content-sha256:" bereq.http.x-amz-content-sha256 LF
23 + "x-amz-date:" bereq.http.x-amz-date LF
24 + "x-amz-request-payer:" bereq.http.x-amz-request-payer LF
25 + ;
26 + set var.canonicalQuery = "";
27 + set var.signedHeaders = "host;x-amz-content-sha256;x-amz-date;x-amz-request-payer";
28 + set var.canonicalRequest = ""
29 + "GET" LF
30 + bereq.url.path LF
31 + var.canonicalQuery LF
32 + var.canonicalHeaders LF
33 + var.signedHeaders LF
34 + digest.hash_sha256("")
35 + ;
36 +
37 + set var.scope = var.dateStamp "/" var.awsRegion "/s3/aws4_request";
38 +
39 + set var.stringToSign = ""
40 + "AWS4-HMAC-SHA256" LF
41 + bereq.http.x-amz-date LF
42 + var.scope LF
43 + regsub(digest.hash_sha256(var.canonicalRequest),"^0x", "")
44 + ;
45 +
46 + set var.signature = digest.awsv4_hmac(
47 + var.awsSecretKey,
48 + var.dateStamp,
49 + var.awsRegion,
50 + "s3",
51 + var.stringToSign
52 + );
53 +
54 + set bereq.http.Authorization = "AWS4-HMAC-SHA256 "
55 + "Credential=${access_key}/" var.scope ", "
56 + "SignedHeaders=" var.signedHeaders ", "
57 + "Signature=" + regsub(var.signature,"^0x", "")
58 + ;
59 +
60 + unset bereq.http.Accept;
61 + unset bereq.http.Accept-Language;
62 + unset bereq.http.User-Agent;
63 + unset bereq.http.Fastly-Client-IP;
64 +}
terraform/dns.tf
+5
@@ -131,6 +131,11 @@ locals {
131 type = "CNAME"
132 value = "dualstack.v2.shared.global.fastly.net"
133 },
134 + {
135 + hostname = "cache-staging.nixos.org"
136 + type = "CNAME"
137 + value = "dualstack.v2.shared.global.fastly.net"
138 + },
139 {
140 hostname = "channels.nixos.org"
141 type = "CNAME"