Initial
Eelco Dolstra committed
Sep 27, 2022 at 14:33 UTC
73e401238ca2f55b43f4173c465122911de0a2f0
4 files changed
+207
cron.sh
new
+18
@@ -0,0 +1,18 @@
1
+#! /bin/sh -e
2
+
3
+export AWS_PROFILE=nixos-org
4
+
5
+now=$(date +%s)
6
+#now=$((now - 86400))
7
+prev_week=$((($now / 86400 / 7)))
8
+
9
+from_date_incl=$(date +%F --date="@$(($prev_week * 86400 * 7 - 2 * 86400))")
10
+to_date_incl=$(date +%F --date="@$(($prev_week * 86400 * 7 + 5 * 86400))")
11
+
12
+echo "Ingesting [$from_date_incl, $to_date_incl)."
13
+
14
+#exit 0
15
+
16
+./ingest-raw-logs.sh "$from_date_incl" "$to_date_incl"
17
+
18
+./run-queries.sh
ingest-raw-logs.sh
new
+71
@@ -0,0 +1,71 @@
1
+#! /bin/sh -e
2
+
3
+region=eu-west-1
4
+
5
+from_date_incl="$1"
6
+to_date_excl="$2"
7
+
8
+[[ -n $from_date_incl ]]
9
+[[ -n $to_date_excl ]]
10
+
11
+run_query() {
12
+ local name="$1"
13
+ local query="$2"
14
+
15
+ res=$(aws athena start-query-execution \
16
+ --region $region \
17
+ --result-configuration OutputLocation=s3://nixos-athena/ingestion/$name/ \
18
+ --query-string "$query")
19
+
20
+ execution_id="$(printf "%s" "$res" | jq -r -e .QueryExecutionId)"
21
+ [[ -n $execution_id ]]
22
+
23
+ echo "Started query $name as $execution_id."
24
+
25
+ printf "Waiting..."
26
+ while true; do
27
+ res="$(aws athena get-query-execution --region $region --query-execution-id $execution_id)"
28
+ status="$(printf %s "$res" | jq -r -e .QueryExecution.Status.State)"
29
+ if [[ $status = RUNNING || $status = QUEUED ]]; then
30
+ printf "."
31
+ sleep 1
32
+ continue
33
+ fi
34
+ if [[ $status = SUCCEEDED ]]; then
35
+ printf " done.\n"
36
+ break
37
+ fi
38
+ printf "\nFailed: %s (%s)\n" "$status" "$res"
39
+ exit 1
40
+ done
41
+}
42
+
43
+run_query fill-urls \
44
+ "
45
+ insert into urls
46
+ with requests2 as (select *, date_format(date_parse(timestamp, '%Y-%m-%dT%T+0000'), '%Y-%m-%d') as day from requests)
47
+ select url, count(*) as nr, sum(response_body_size) as total_bytes, sum(elapsed_usec) as total_elapsed, host, day
48
+ from requests2
49
+ where (response_status >= '200' and response_status <= '399') and (day >= '$from_date_incl' and day < '$to_date_excl')
50
+ group by host, day, url;
51
+ "
52
+
53
+run_query fill-nix-cache-info \
54
+ "
55
+ insert into nix_cache_info
56
+ with requests2 as (select *, date_format(date_parse(timestamp, '%Y-%m-%dT%T+0000'), '%Y-%m-%d') as day from requests)
57
+ select count(*) as nr, asn, geo_country, geo_region, request_user_agent, day
58
+ from requests2
59
+ where host = 'cache.nixos.org' and url = '/nix-cache-info' and (day >= '$from_date_incl' and day < '$to_date_excl')
60
+ group by day, asn, geo_country, geo_region, request_user_agent;
61
+ "
62
+
63
+run_query fill-clients \
64
+ "
65
+ insert into clients
66
+ with requests2 as (select *, date_format(date_parse(timestamp, '%Y-%m-%dT%T+0000'), '%Y-%m-%d') as day from requests)
67
+ select asn, geo_country, geo_region, count(*) as nr, sum(response_body_size) as total_bytes, sum(elapsed_usec) as total_elapsed, host, day
68
+ from requests2
69
+ where (day >= '$from_date_incl' and day < '$to_date_excl')
70
+ group by host, day, asn, geo_country, geo_region;
71
+ "
run-queries.sh
new
+111
@@ -0,0 +1,111 @@
1
+#! /bin/sh -e
2
+
3
+region=eu-west-1
4
+
5
+report_date="$(date +%Y-%m-%d)"
6
+
7
+run_query() {
8
+ local name="$1"
9
+ local query="$2"
10
+
11
+ res=$(aws athena start-query-execution \
12
+ --region $region \
13
+ --result-configuration OutputLocation=s3://nixos-metrics/$report_date/$name/ \
14
+ --query-string "$query")
15
+
16
+ execution_id="$(printf "%s" "$res" | jq -r -e .QueryExecutionId)"
17
+ [[ -n $execution_id ]]
18
+
19
+ echo "Started query $name as $execution_id."
20
+
21
+ redirect=latest/$name.csv
22
+ aws s3api put-object \
23
+ --bucket nixos-metrics \
24
+ --key $redirect \
25
+ --website-redirect-location /$report_date/$name/$execution_id.csv > /dev/null
26
+
27
+ echo "Created redirect http://nixos-metrics.s3-website-eu-west-1.amazonaws.com/$redirect."
28
+}
29
+
30
+if true; then
31
+
32
+run_query traffic-per-day \
33
+ "
34
+ select day, host, sum(nr) as nr_requests, sum(total_bytes) as total_bytes
35
+ from urls
36
+ group by day, host
37
+ order by day, host
38
+ "
39
+
40
+run_query traffic-per-country \
41
+ "
42
+ select geo_country, sum(nr) as nr_requests, sum(total_bytes) as total_bytes
43
+ from clients
44
+ group by geo_country
45
+ order by total_bytes desc
46
+ "
47
+
48
+run_query cache-info-requests-per-day \
49
+ "
50
+ select day, sum(nr) as cache_info_requests
51
+ from nix_cache_info
52
+ group by day
53
+ order by day
54
+ "
55
+
56
+run_query cache-info-requests-per-day-not-hosted \
57
+ "
58
+ select day, sum(nr) as cache_info_requests
59
+ from nix_cache_info
60
+ where asn not in (select asn_nr from hosting_asns)
61
+ group by day
62
+ order by day
63
+ "
64
+
65
+run_query cache-info-requests-per-day-per-ua \
66
+ "
67
+ with tmp as
68
+ (select *, regexp_replace(regexp_replace(request_user_agent, '.* Nix', 'Nix'), 'pre[^ ]*', 'pre*') as cleaned_ua from nix_cache_info)
69
+ select day, cleaned_ua, sum(nr) as cache_info_requests
70
+ from tmp
71
+ group by day, cleaned_ua
72
+ order by day, cache_info_requests desc
73
+ "
74
+
75
+run_query flake-registry-requests-per-day \
76
+ "
77
+ select day, sum(nr) as total_requests
78
+ from urls
79
+ where host = 'channels.nixos.org' and url like '%/flake-registry.json'
80
+ group by day
81
+ order by day
82
+ "
83
+
84
+run_query top-store-paths \
85
+ "
86
+ select path, sum(nr) as total_requests
87
+ from urls
88
+ join all_paths on regexp_replace(regexp_replace(url, '.narinfo', ''), '/', '') = regexp_replace(regexp_replace(path, '/nix/store/', ''), '-.*', '')
89
+ where
90
+ host = 'cache.nixos.org'
91
+ and url like '%.narinfo'
92
+ group by path
93
+ having sum(nr) > 100
94
+ order by total_requests desc
95
+ "
96
+
97
+run_query narinfo-queries-per-release \
98
+ "
99
+ with tmp as
100
+ (select distinct path, regexp_replace(regexp_replace(regexp_replace(regexp_replace(release_name, 'pre.*', 'pre'), 'alpha.*', ''), 'beta.*', 'beta'), '\.[0-9]+\.[0-9a-f][0-9a-f][0-9a-f][0-9a-f]+$', '') as release from release_paths)
101
+ select release, sum(nr) as total_requests
102
+ from urls
103
+ join tmp on regexp_replace(regexp_replace(url, '.narinfo', ''), '/', '') = regexp_replace(regexp_replace(path, '/nix/store/', ''), '-.*', '')
104
+ where
105
+ host = 'cache.nixos.org'
106
+ and url like '%.narinfo'
107
+ group by release
108
+ order by total_requests desc
109
+ "
110
+
111
+fi
update-asn-list.sh
new
+7
@@ -0,0 +1,7 @@
1
+#! /bin/sh -e
2
+
3
+curl --fail https://ftp.ripe.net/ripe/asnames/asn.txt > /tmp/asn.txt
4
+
5
+sed -e 's/^\([0-9]\+\) \(.\+\), \([A-Z][A-Z]\)$/\1\t\2\t\3/; t; d' < /tmp/asn.txt > /tmp/asn.tsv
6
+
7
+aws s3 cp /tmp/asn.tsv s3://nixos-athena/all-asns/list.tsv