Implement PackageCloud cleanup (#13236)
* add script for netdata packagecloud cleanup * add script for netdata packagecloud cleanup * remove old package-cloud cleanup code * add workflow for packagecloud cleanup * Debug msg * allow individual matrix jobs to complete independently to each other * perform cleanup without dry-run * remove debug msg * remove cleanup line from the slack notification
maneamarius committed
Jun 28, 2022 at 14:22 UTC
2b5999c39337e1fc0f6c25a22e48f2d23fc6e9f8
4 files changed
+226
-98
.github/scripts/netdata-pkgcloud-cleanup.py
new
+190
@@ -0,0 +1,190 @@
1
+#!/bin/env python3
2
+
3
+import requests
4
+from requests.auth import HTTPBasicAuth
5
+from datetime import date, datetime, timedelta
6
+import os
7
+import sys
8
+import argparse
9
+from pprint import pprint
10
+from datetime import datetime
11
+from dateutil import parser
12
+
13
+
14
+class PackageCloud:
15
+ NUM_PACKAGE_MINOR_TO_KEEP = 5
16
+ NUM_RETENTION_DAYS = 30
17
+ # number of pages to process. Use '0' to process all
18
+ MAX_PAGES = 0
19
+
20
+ def __init__(self, repo_type, dry_run=True, auth_token=None):
21
+ self.headers = {
22
+ "Accept" : "application/json",
23
+ "Content-Type" : "application/json",
24
+ }
25
+ self.dry_run = dry_run
26
+ self.repo_type = repo_type
27
+ if repo_type == "stable":
28
+ repo = "netdata/netdata"
29
+ elif repo_type == "devel":
30
+ repo = "netdata/netdata-devel"
31
+ elif repo_type == "edge":
32
+ repo = "netdata/netdata-edge"
33
+ else:
34
+ print(f"ERROR: unknown repo type '{repo_type}'!\nAccepted values are: stable,devel,edge")
35
+ sys.exit(1)
36
+ self.base_url = f"https://packagecloud.io/api/v1/repos/{repo}"
37
+ self.auth = HTTPBasicAuth(username=auth_token, password='') if auth_token else None
38
+
39
+ def get_all_packages(self):
40
+ page = 1
41
+ all_pkg_list = []
42
+ while True:
43
+ url = f"{self.base_url}/packages.json?page={page}"
44
+ if page > self.MAX_PAGES and self.MAX_PAGES != 0:
45
+ break
46
+ else:
47
+ pkg_list = requests.get(url, auth=self.auth, headers=self.headers).json()
48
+ if len(pkg_list) == 0:
49
+ break
50
+ else:
51
+ print(f"Processing page: {page}")
52
+ for element in pkg_list:
53
+ self.is_pkg_older_than_days(element, 30)
54
+ if element['name'] != 'netdata-repo' and element['name'] != 'netdata-repo-edge':
55
+ all_pkg_list.append(element)
56
+ page += 1
57
+ return all_pkg_list
58
+
59
+ def delete_package(self, destroy_url):
60
+ if self.dry_run:
61
+ print(f" - DRY_RUN mode. Not deleting package '{destroy_url}'.")
62
+ else:
63
+ print(f" - Deleting package: {destroy_url}")
64
+ url = f"https://packagecloud.io{destroy_url}"
65
+ response = requests.delete(url, auth=self.auth, headers=self.headers).json()
66
+ response = None
67
+ if not response:
68
+ print(f" Package deleted successfully.")
69
+ else:
70
+ print(f" Failed deleting package!")
71
+
72
+ def get_destroy_url(self, pkg_url):
73
+ url = f"https://packagecloud.io{pkg_url}"
74
+ response = requests.get(url, auth=self.auth, headers=self.headers)
75
+ response.raise_for_status()
76
+ return response.json()['destroy_url']
77
+
78
+ def get_packages_for_distro(self, distro, all_pkg_list):
79
+ distro_pkg_list = [ pkg for pkg in all_pkg_list if pkg['distro_version'] == distro ]
80
+ return distro_pkg_list
81
+
82
+ def get_packages_for_arch(self, arch, all_pkg_list):
83
+ arch_pkg_list = [ pkg for pkg in all_pkg_list if pkg['package_url'].split('/')[11] == arch ]
84
+ return arch_pkg_list
85
+
86
+ def get_arches(self, pkg_list):
87
+ arches = list(set([pkg['package_url'].split('/')[11] for pkg in pkg_list ]))
88
+ return arches
89
+
90
+ def get_pkg_list(self, pkg_name, pkg_list):
91
+ filtered_list = [ pkg for pkg in pkg_list if pkg['name'] == pkg_name ]
92
+ return filtered_list
93
+
94
+ def get_minor_versions(self, all_versions):
95
+ minor_versions = ['.'.join(version.split('.')[:-1]) for version in all_versions ]
96
+ minor_versions = list(set(minor_versions))
97
+ minor_versions.sort()
98
+ return minor_versions
99
+
100
+ def is_pkg_older_than_days(self, pkg, num_days):
101
+ pkg_create_date = datetime.strptime(pkg['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ')
102
+ time_difference = datetime.now() - pkg_create_date
103
+ return time_difference.days > num_days
104
+
105
+ def cleanup_repo(self):
106
+ if self.repo_type == 'stable':
107
+ self.cleanup_stable_repo()
108
+ else:
109
+ self.cleanup_edge_repo()
110
+
111
+ def cleanup_edge_repo(self):
112
+ all_pkg_list = self.get_all_packages()
113
+ pkgs_to_delete = []
114
+ pkgs_to_keep = []
115
+ for package in all_pkg_list:
116
+ if self.is_pkg_older_than_days(package, self.NUM_RETENTION_DAYS):
117
+ pkgs_to_delete.append(package)
118
+ else:
119
+ pkgs_to_keep.append(package)
120
+ print(f"Keeping the following packages (newer than {self.NUM_RETENTION_DAYS} days):")
121
+ for pkg in pkgs_to_keep:
122
+ print(f" > pkg: {pkg['package_html_url']} / created_at: {pkg['created_at']}")
123
+ print(f"Deleting the following packages (older than {self.NUM_RETENTION_DAYS} days):")
124
+ for pkg in pkgs_to_delete:
125
+ print(f" > pkg: {pkg['package_html_url']} / created_at: {pkg['created_at']}")
126
+ self.delete_package(pkg['destroy_url'])
127
+
128
+ def cleanup_stable_repo(self):
129
+ all_pkg_list = self.get_all_packages()
130
+ all_distros = list(set([ pkg['distro_version'] for pkg in all_pkg_list ]))
131
+ all_distros = sorted(all_distros)
132
+ print(f"<> Distributions list: {all_distros}")
133
+
134
+ for distro in all_distros:
135
+ print(f">> Processing distro: {distro}")
136
+ pkg_list_distro = self.get_packages_for_distro(distro, all_pkg_list)
137
+ arches = self.get_arches(pkg_list_distro)
138
+ print(f" <> Arch list: {arches}")
139
+ for arch in arches:
140
+ print(f" >> Processing arch: {distro} -> {arch}")
141
+ pkg_list_arch = self.get_packages_for_arch(arch, pkg_list_distro)
142
+ pkg_names = [pkg['name'] for pkg in pkg_list_arch]
143
+ pkg_names = list(set(pkg_names))
144
+ print(f" <> Package names: {pkg_names}")
145
+ for pkg_name in pkg_names:
146
+ print(f" >> Processing package: {distro} -> {arch} -> {pkg_name}")
147
+ pkg_list = self.get_pkg_list(pkg_name, pkg_list_arch)
148
+ pkg_versions = [pkg['version'] for pkg in pkg_list]
149
+ pkg_minor_versions = self.get_minor_versions(pkg_versions)
150
+ pkg_minor_to_keep = pkg_minor_versions[-self.NUM_PACKAGE_MINOR_TO_KEEP:]
151
+ print(f" <> Minor Package Versions to Keep: {pkg_minor_to_keep}")
152
+ pkg_minor_to_delete = list(set(pkg_minor_versions) - set(pkg_minor_to_keep))
153
+ print(f" <> Minor Package Versions to Delete: {pkg_minor_to_delete}")
154
+ urls_to_keep = [pkg['package_url'] for pkg in pkg_list if '.'.join(pkg['version'].split('.')[:-1]) in pkg_minor_to_keep]
155
+ urls_to_delete = [pkg['package_url'] for pkg in pkg_list if '.'.join(pkg['version'].split('.')[:-1]) in pkg_minor_to_delete]
156
+ for pkg_url in urls_to_delete:
157
+ destroy_url = self.get_destroy_url(pkg_url)
158
+ self.delete_package(destroy_url)
159
+
160
+
161
+def configure():
162
+ parser = argparse.ArgumentParser()
163
+ parser.add_argument('--repo-type', '-r', required=True,
164
+ help='Repository type against to perform cleanup')
165
+ parser.add_argument('--dry-run', '-d', action='store_true',
166
+ help='Dry-run Mode')
167
+ args = parser.parse_args()
168
+ try:
169
+ token = os.environ['PKGCLOUD_TOKEN']
170
+ except Exception as e:
171
+ print(f"FATAL: 'PKGCLOUD_TOKEN' environment variable is not set!", file=sys.stderr)
172
+ sys.exit(1)
173
+ repo_type = args.repo_type
174
+ dry_run = args.dry_run
175
+ conf = {
176
+ 'repo_type': args.repo_type,
177
+ 'dry_run': args.dry_run,
178
+ 'token': token
179
+ }
180
+ return conf
181
+
182
+
183
+def main():
184
+ config = configure()
185
+ pkg_cloud = PackageCloud(config['repo_type'], config['dry_run'], config['token'])
186
+ pkg_cloud.cleanup_repo()
187
+
188
+
189
+if __name__ == "__main__":
190
+ main()
.github/scripts/old_package_purging.sh
deleted
-88
@@ -1,88 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Script to handle package cloud retention policy
4
-# Our open source subscription is limited,
5
-# so we use this script to control the number of packages maintained historically
6
-#
7
-# Dependencies:
8
-# - PACKAGE_CLOUD_RETENTION_DAYS
9
-# This is to indicate for how many days back we want to maintain the various RPM and DEB packages on package cloud
10
-#
11
-# Copyright : SPDX-License-Identifier: GPL-3.0-or-later
12
-#
13
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
14
-#
15
-set -e
16
-
17
-delete_files_for_version() {
18
- local v="$1"
19
-
20
- # Delete the selected filenames in version
21
- FILES_IN_VERSION=$(jq --sort-keys --arg v "${v}" '.[] | select ( .version | contains($v))' "${PKG_LIST_FILE}" | grep filename | cut -d':' -f 2)
22
-
23
- # Iterate through the files and delete them
24
- for pkg in ${FILES_IN_VERSION/\\n/}; do
25
- pkg=${pkg/,/}
26
- pkg=${pkg/\"/}
27
- pkg=${pkg/\"/}
28
- echo "Attempting yank on ${pkg}.."
29
- .github/scripts/package_cloud_wrapper.sh yank "${REPO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}"
30
- done
31
-}
32
-
33
-# If we are not in netdata git repo, at the top level directory, fail
34
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
35
-CWD=$(git rev-parse --show-cdup)
36
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
37
- echo "Run as .github/scripts/$(basename "$0") from top level directory of netdata git repository"
38
- echo "Old packages yanking cancelled"
39
- exit 1
40
-fi
41
-
42
-if [ -z "${REPO}" ]; then
43
- echo "No REPO variable found"
44
- exit 1
45
-fi
46
-
47
-if [ -z ${PKG_CLOUD_TOKEN} ]; then
48
- echo "No PKG_CLOUD_TOKEN variable found"
49
- exit 1
50
-fi
51
-
52
-if [ -z ${PACKAGE_CLOUD_RETENTION_DAYS} ]; then
53
- echo "No PACKAGE_CLOUD_RETENTION_DAYS variable found"
54
- exit 1
55
-fi
56
-
57
-TMP_DIR="$(mktemp -d /tmp/netdata-old-package-yanking-XXXXXX)"
58
-PKG_LIST_FILE="${TMP_DIR}/complete_package_list.json"
59
-DATE_EPOCH="1970-01-01"
60
-DATE_UNTIL_TO_DELETE=$(date --date="${PACKAGE_CLOUD_RETENTION_DAYS} day ago" +%Y-%m-%d)
61
-
62
-
63
-echo "Created temp directory: ${TMP_DIR}"
64
-echo "We will be purging contents up until ${DATE_UNTIL_TO_DELETE}"
65
-
66
-echo "Calling package could to retrieve all available packages on ${REPO}"
67
-curl -sS "https://${PKG_CLOUD_TOKEN}:@packagecloud.io/api/v1/repos/${REPO}/packages.json" > "${PKG_LIST_FILE}"
68
-
69
-# Get versions within the desired duration
70
-#
71
-VERSIONS_TO_PURGE=$(jq --arg s "${DATE_EPOCH}" --arg e "${DATE_UNTIL_TO_DELETE}" '
72
-[($s, $e) | strptime("%Y-%m-%d")[0:3]] as $r
73
- | map(select(
74
- (.created_at[:19] | strptime("%Y-%m-%dT%H:%M:%S")[0:3]) as $d
75
- | $d >= $r[0] and $d <= $r[1]
76
-))' "${PKG_LIST_FILE}" | grep '"version":' | sort -u | sed -e 's/ //g' | cut -d':' -f2)
77
-
78
-echo "We will be deleting the following versions: ${VERSIONS_TO_PURGE}"
79
-for v in ${VERSIONS_TO_PURGE/\n//}; do
80
- v=${v/\"/}
81
- v=${v/\"/}
82
- v=${v/,/}
83
- echo "Remove all files for version $v"
84
- delete_files_for_version "${v}"
85
-done
86
-
87
-# Done, clean up
88
-[ -d "${TMP_DIR}" ] && rm -rf "${TMP_DIR}"
.github/workflows/packagecloud.yml
new
+36
@@ -0,0 +1,36 @@
1
+---
2
+# Runs PackageCloud cleanup every day at 9pm
3
+name: PackageCloud Cleanup
4
+on:
5
+ schedule:
6
+ - cron: '0 21 * * *'
7
+ workflow_dispatch: null
8
+
9
+jobs:
10
+ cleanup:
11
+ name: PackageCloud Cleanup
12
+ runs-on: ubuntu-latest
13
+ if: always()
14
+ strategy:
15
+ fail-fast: false
16
+ matrix:
17
+ repos:
18
+ - stable
19
+ - edge
20
+ - devel
21
+ steps:
22
+ - name: Checkout
23
+ uses: actions/checkout@v3
24
+ id: checkout
25
+ with:
26
+ submodules: recursive
27
+ - name: Prepare environment
28
+ id: prepare
29
+ run: |
30
+ pip3 install requests python-dateutil
31
+ - name: Run PackageCloud Cleanup
32
+ id: cleanup
33
+ env:
34
+ PKGCLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
35
+ run: |
36
+ python3 .github/scripts/netdata-pkgcloud-cleanup.py -r ${{ matrix.repos }}
.github/workflows/packaging.yml
-10
@@ -227,15 +227,6 @@ jobs:
227
"$(basename "${pkgfile}")" || true
228
.github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} "${pkgfile}"
229
done
230
- - name: Clean
231
- id: cleanup
232
- if: github.event_name == 'workflow_dispatch'
233
- shell: bash
234
- env:
235
- REPO: ${{ needs.version-check.outputs.repo }}
236
- PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
237
- PACKAGE_CLOUD_RETENTION_DAYS: ${{ needs.version-check.outputs.retention }}
238
- run: .github/scripts/old_package_purging.sh
230
- name: Failure Notification
231
uses: rtCamp/action-slack-notify@v2
232
env:
@@ -252,7 +243,6 @@ jobs:
243
Build: ${{ steps.build.outcome }}
244
Test: ${{ steps.test.outcome }}
245
Publish: ${{ steps.upload.outcome }}
255
- Cleanup: ${{ steps.cleanup.outcome }}
246
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
247
if: >-
248
${{