| 1 | #!/bin/env python3 |
| 2 | |
| 3 | import requests |
| 4 | from requests.auth import HTTPBasicAuth |
| 5 | from datetime import date, datetime, timedelta |
| 6 | import os |
| 7 | import sys |
| 8 | import argparse |
| 9 | from pprint import pprint |
| 10 | from datetime import datetime |
| 11 | from dateutil import parser |
| 12 | |
| 13 | |
| 14 | class PackageCloud: |
| 15 | NUM_PACKAGE_MINOR_TO_KEEP = 5 |
| 16 | NUM_RETENTION_DAYS = 30 |
| 17 | # number of pages to process. Use '0' to process all |
| 18 | MAX_PAGES = 0 |
| 19 | |
| 20 | def __init__(self, repo_type, dry_run=True, auth_token=None): |
| 21 | self.headers = { |
| 22 | "Accept" : "application/json", |
| 23 | "Content-Type" : "application/json", |
| 24 | } |
| 25 | self.dry_run = dry_run |
| 26 | self.repo_type = repo_type |
| 27 | if repo_type == "stable": |
| 28 | repo = "netdata/netdata" |
| 29 | elif repo_type == "devel": |
| 30 | repo = "netdata/netdata-devel" |
| 31 | elif repo_type == "edge": |
| 32 | repo = "netdata/netdata-edge" |
| 33 | else: |
| 34 | print(f"ERROR: unknown repo type '{repo_type}'!\nAccepted values are: stable,devel,edge") |
| 35 | sys.exit(1) |
| 36 | self.base_url = f"https://packagecloud.io/api/v1/repos/{repo}" |
| 37 | self.auth = HTTPBasicAuth(username=auth_token, password='') if auth_token else None |
| 38 | |
| 39 | def get_all_packages(self): |
| 40 | page = 1 |
| 41 | all_pkg_list = [] |
| 42 | while True: |
| 43 | url = f"{self.base_url}/packages.json?page={page}" |
| 44 | if page > self.MAX_PAGES and self.MAX_PAGES != 0: |
| 45 | break |
| 46 | else: |
| 47 | pkg_list = requests.get(url, auth=self.auth, headers=self.headers).json() |
| 48 | if len(pkg_list) == 0: |
| 49 | break |
| 50 | else: |
| 51 | print(f"Processing page: {page}") |
| 52 | for element in pkg_list: |
| 53 | self.is_pkg_older_than_days(element, 30) |
| 54 | if element['name'] != 'netdata-repo' and element['name'] != 'netdata-repo-edge': |
| 55 | all_pkg_list.append(element) |
| 56 | page += 1 |
| 57 | return all_pkg_list |
| 58 | |
| 59 | def delete_package(self, destroy_url): |
| 60 | if self.dry_run: |
| 61 | print(f" - DRY_RUN mode. Not deleting package '{destroy_url}'.") |
| 62 | else: |
| 63 | print(f" - Deleting package: {destroy_url}") |
| 64 | url = f"https://packagecloud.io{destroy_url}" |
| 65 | response = requests.delete(url, auth=self.auth, headers=self.headers).json() |
| 66 | response = None |
| 67 | if not response: |
| 68 | print(f" Package deleted successfully.") |
| 69 | else: |
| 70 | print(f" Failed deleting package!") |
| 71 | |
| 72 | def get_destroy_url(self, pkg_url): |
| 73 | url = f"https://packagecloud.io{pkg_url}" |
| 74 | response = requests.get(url, auth=self.auth, headers=self.headers) |
| 75 | response.raise_for_status() |
| 76 | return response.json()['destroy_url'] |
| 77 | |
| 78 | def get_packages_for_distro(self, distro, all_pkg_list): |
| 79 | distro_pkg_list = [ pkg for pkg in all_pkg_list if pkg['distro_version'] == distro ] |
| 80 | return distro_pkg_list |
| 81 | |
| 82 | def get_packages_for_arch(self, arch, all_pkg_list): |
| 83 | arch_pkg_list = [ pkg for pkg in all_pkg_list if pkg['package_url'].split('/')[11] == arch ] |
| 84 | return arch_pkg_list |
| 85 | |
| 86 | def get_arches(self, pkg_list): |
| 87 | arches = list(set([pkg['package_url'].split('/')[11] for pkg in pkg_list ])) |
| 88 | return arches |
| 89 | |
| 90 | def get_pkg_list(self, pkg_name, pkg_list): |
| 91 | filtered_list = [ pkg for pkg in pkg_list if pkg['name'] == pkg_name ] |
| 92 | return filtered_list |
| 93 | |
| 94 | def get_minor_versions(self, all_versions): |
| 95 | minor_versions = ['.'.join(version.split('.')[:-1]) for version in all_versions ] |
| 96 | minor_versions = list(set(minor_versions)) |
| 97 | minor_versions.sort() |
| 98 | return minor_versions |
| 99 | |
| 100 | def is_pkg_older_than_days(self, pkg, num_days): |
| 101 | pkg_create_date = datetime.strptime(pkg['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ') |
| 102 | time_difference = datetime.now() - pkg_create_date |
| 103 | return time_difference.days > num_days |
| 104 | |
| 105 | def cleanup_repo(self): |
| 106 | if self.repo_type == 'stable': |
| 107 | self.cleanup_stable_repo() |
| 108 | else: |
| 109 | self.cleanup_edge_repo() |
| 110 | |
| 111 | def cleanup_edge_repo(self): |
| 112 | all_pkg_list = self.get_all_packages() |
| 113 | pkgs_to_delete = [] |
| 114 | pkgs_to_keep = [] |
| 115 | for package in all_pkg_list: |
| 116 | if self.is_pkg_older_than_days(package, self.NUM_RETENTION_DAYS): |
| 117 | pkgs_to_delete.append(package) |
| 118 | else: |
| 119 | pkgs_to_keep.append(package) |
| 120 | print(f"Keeping the following packages (newer than {self.NUM_RETENTION_DAYS} days):") |
| 121 | for pkg in pkgs_to_keep: |
| 122 | print(f" > pkg: {pkg['package_html_url']} / created_at: {pkg['created_at']}") |
| 123 | print(f"Deleting the following packages (older than {self.NUM_RETENTION_DAYS} days):") |
| 124 | for pkg in pkgs_to_delete: |
| 125 | print(f" > pkg: {pkg['package_html_url']} / created_at: {pkg['created_at']}") |
| 126 | self.delete_package(pkg['destroy_url']) |
| 127 | |
| 128 | def cleanup_stable_repo(self): |
| 129 | all_pkg_list = self.get_all_packages() |
| 130 | all_distros = list(set([ pkg['distro_version'] for pkg in all_pkg_list ])) |
| 131 | all_distros = sorted(all_distros) |
| 132 | print(f"<> Distributions list: {all_distros}") |
| 133 | |
| 134 | for distro in all_distros: |
| 135 | print(f">> Processing distro: {distro}") |
| 136 | pkg_list_distro = self.get_packages_for_distro(distro, all_pkg_list) |
| 137 | arches = self.get_arches(pkg_list_distro) |
| 138 | print(f" <> Arch list: {arches}") |
| 139 | for arch in arches: |
| 140 | print(f" >> Processing arch: {distro} -> {arch}") |
| 141 | pkg_list_arch = self.get_packages_for_arch(arch, pkg_list_distro) |
| 142 | pkg_names = [pkg['name'] for pkg in pkg_list_arch] |
| 143 | pkg_names = list(set(pkg_names)) |
| 144 | print(f" <> Package names: {pkg_names}") |
| 145 | for pkg_name in pkg_names: |
| 146 | print(f" >> Processing package: {distro} -> {arch} -> {pkg_name}") |
| 147 | pkg_list = self.get_pkg_list(pkg_name, pkg_list_arch) |
| 148 | pkg_versions = [pkg['version'] for pkg in pkg_list] |
| 149 | pkg_minor_versions = self.get_minor_versions(pkg_versions) |
| 150 | pkg_minor_to_keep = pkg_minor_versions[-self.NUM_PACKAGE_MINOR_TO_KEEP:] |
| 151 | print(f" <> Minor Package Versions to Keep: {pkg_minor_to_keep}") |
| 152 | pkg_minor_to_delete = list(set(pkg_minor_versions) - set(pkg_minor_to_keep)) |
| 153 | print(f" <> Minor Package Versions to Delete: {pkg_minor_to_delete}") |
| 154 | urls_to_keep = [pkg['package_url'] for pkg in pkg_list if '.'.join(pkg['version'].split('.')[:-1]) in pkg_minor_to_keep] |
| 155 | urls_to_delete = [pkg['package_url'] for pkg in pkg_list if '.'.join(pkg['version'].split('.')[:-1]) in pkg_minor_to_delete] |
| 156 | for pkg_url in urls_to_delete: |
| 157 | destroy_url = self.get_destroy_url(pkg_url) |
| 158 | self.delete_package(destroy_url) |
| 159 | |
| 160 | |
| 161 | def configure(): |
| 162 | parser = argparse.ArgumentParser() |
| 163 | parser.add_argument('--repo-type', '-r', required=True, |
| 164 | help='Repository type against to perform cleanup') |
| 165 | parser.add_argument('--dry-run', '-d', action='store_true', |
| 166 | help='Dry-run Mode') |
| 167 | args = parser.parse_args() |
| 168 | try: |
| 169 | token = os.environ['PKGCLOUD_TOKEN'] |
| 170 | except Exception as e: |
| 171 | print(f"FATAL: 'PKGCLOUD_TOKEN' environment variable is not set!", file=sys.stderr) |
| 172 | sys.exit(1) |
| 173 | repo_type = args.repo_type |
| 174 | dry_run = args.dry_run |
| 175 | conf = { |
| 176 | 'repo_type': args.repo_type, |
| 177 | 'dry_run': args.dry_run, |
| 178 | 'token': token |
| 179 | } |
| 180 | return conf |
| 181 | |
| 182 | |
| 183 | def main(): |
| 184 | config = configure() |
| 185 | pkg_cloud = PackageCloud(config['repo_type'], config['dry_run'], config['token']) |
| 186 | pkg_cloud.cleanup_repo() |
| 187 | |
| 188 | |
| 189 | if __name__ == "__main__": |
| 190 | main() |