Capture some logging to experiment with looking at the cache for 'popularity contest' type research
Graham Christensen committed
Jun 22, 2022 at 21:09 UTC
208f1374298458a28066556d28bd1b2a42facbbe
5 files changed
+97
terraform/cache.tf
+13
@@ -224,6 +224,19 @@ resource "fastly_service_v1" "cache" {
224
priority = 100
225
type = "fetch"
226
}
227
+
228
+ s3logging {
229
+ name = "${local.cache_domain}-to-s3"
230
+ bucket_name = module.fastlylogs.bucket_name
231
+ compression_codec = "zstd"
232
+ domain = module.fastlylogs.s3_domain
233
+ format = module.fastlylogs.format
234
+ format_version = 2
235
+ path = "${local.cache_domain}/"
236
+ period = module.fastlylogs.period
237
+ message_type = "blank"
238
+ s3_iam_role = module.fastlylogs.iam_role_arn
239
+ }
240
}
241
242
resource "fastly_tls_subscription" "cache" {
terraform/channels.tf
+13
@@ -196,6 +196,19 @@ resource "fastly_service_v1" "channels" {
196
priority = 100
197
type = "fetch"
198
}
199
+
200
+ s3logging {
201
+ name = "${local.channels_domain}-to-s3"
202
+ bucket_name = module.fastlylogs.bucket_name
203
+ compression_codec = "zstd"
204
+ domain = module.fastlylogs.s3_domain
205
+ format = module.fastlylogs.format
206
+ format_version = 2
207
+ path = "${local.channels_domain}/"
208
+ period = module.fastlylogs.period
209
+ message_type = "blank"
210
+ s3_iam_role = module.fastlylogs.iam_role_arn
211
+ }
212
}
213
214
resource "fastly_tls_subscription" "channels" {
terraform/fastlylog/outputs.tf
new
+45
@@ -0,0 +1,45 @@
1
+output "bucket_name" {
2
+ value = aws_s3_bucket.logs.bucket
3
+}
4
+
5
+output "s3_domain" {
6
+ # I don't kno whow to get this from the S3 bucket resource :/.
7
+ value = "s3.eu-west-1.amazonaws.com"
8
+}
9
+
10
+output "iam_role_arn" {
11
+ value = aws_iam_role.fastly_log_forwarder.arn
12
+}
13
+
14
+
15
+
16
+output "period" {
17
+ # Frequency that Fastly servers will push a file to S3, in seconds
18
+ value = 3600
19
+}
20
+
21
+output "format" {
22
+ value = "{${join(",", [for k, v in {
23
+ asn = "%%{client.as.number}V",
24
+ elapsed_usec = "%%{json.escape(time.elapsed.usec)}V",
25
+ fastly_is_edge = "%%{if(fastly.ff.visits_this_service == 0, \"true\", \"false\")}V",
26
+ fastly_server = "\"%%{json.escape(server.identity)}V\"",
27
+ geo_country = "\"%%{json.escape(client.geo.country_name)}V\"",
28
+ geo_region = "\"%%{json.escape(client.geo.region.utf8)}V\"",
29
+ geo_speed = "\"%%{json.escape(client.geo.conn_speed)}V\"",
30
+ host = "\"%%{json.escape(if(req.http.Fastly-Orig-Host, req.http.Fastly-Orig-Host, req.http.Host))}V\"",
31
+ request_method = "\"%%{json.escape(req.method)}V\"",
32
+ request_protocol = "\"%%{json.escape(req.proto)}V\"",
33
+ request_referer = "\"%%{json.escape(req.http.referer)}V\"",
34
+ request_size = "%%{json.escape(req.bytes_read)}V",
35
+ request_user_agent = "\"%%{json.escape(req.http.User-Agent)}V\"",
36
+ response_body_size = "%%{resp.body_bytes_written}V",
37
+ response_reason = "%%{if(resp.response, \"%22\"+json.escape(resp.response)+\"%22\", \"null\")}V",
38
+ response_state = "\"%%{json.escape(fastly_info.state)}V\"",
39
+ response_status = "\"%%{resp.status}V\"",
40
+ timestamp = "\"%%{strftime(\\{\"%Y-%m-%dT%H:%M:%S%z\"\\}, time.start)}V\"",
41
+ tls_client_cipher = "\"%%{json.escape(if(tls.client.cipher, tls.client.cipher, \"null\"))}V\"",
42
+ tls_client_protocol = "\"%%{json.escape(if(tls.client.protocol, tls.client.protocol, \"null\"))}V\"",
43
+ url = "\"%%{json.escape(req.url)}V\"",
44
+ } : "\"${k}\": ${v}"])}}"
45
+}
\ No newline at end of file
terraform/nixpkgs-tarballs.tf
+13
@@ -256,6 +256,19 @@ resource "fastly_service_v1" "nixpkgs-tarballs" {
256
priority = 100
257
type = "fetch"
258
}
259
+
260
+ s3logging {
261
+ name = "${local.tarballs_domain}-to-s3"
262
+ bucket_name = module.fastlylogs.bucket_name
263
+ compression_codec = "zstd"
264
+ domain = module.fastlylogs.s3_domain
265
+ format = module.fastlylogs.format
266
+ format_version = 2
267
+ path = "${local.tarballs_domain}/"
268
+ period = module.fastlylogs.period
269
+ message_type = "blank"
270
+ s3_iam_role = module.fastlylogs.iam_role_arn
271
+ }
272
}
273
274
resource "fastly_tls_subscription" "nixpkgs-tarballs" {
terraform/releases.tf
+13
@@ -201,6 +201,19 @@ resource "fastly_service_v1" "releases" {
201
priority = 100
202
type = "fetch"
203
}
204
+
205
+ s3logging {
206
+ name = "${local.releases_domain}-to-s3"
207
+ bucket_name = module.fastlylogs.bucket_name
208
+ compression_codec = "zstd"
209
+ domain = module.fastlylogs.s3_domain
210
+ format = module.fastlylogs.format
211
+ format_version = 2
212
+ path = "${local.releases_domain}/"
213
+ period = module.fastlylogs.period
214
+ message_type = "blank"
215
+ s3_iam_role = module.fastlylogs.iam_role_arn
216
+ }
217
}
218
219
resource "fastly_tls_subscription" "releases" {