netdata/packaging: Add purging logic for package cloud repositories (#6732)
* netdata/packaging: Add purging logic for package cloud repositories 1) Introduce scriptlet that is executing the cleanup on a designated repository, with a designated retention period 2) Add .travis.yml configuration to execute purging logic with the right config on the nightly operations * netdata/packaging: uncomment the temp dir clean up * netdata/packaging: fix codacy * netdata/packaging: [ci skip] Purge only -edge and -devel packages
Paul Emm. Katsoulakis committed
Aug 29, 2019 at 17:35 UTC
a2bd5e1bc93bd3c77c0069cf1c4c18de0405843f
2 files changed
+99
-1
.travis.yml
+6
-1
@@ -29,6 +29,7 @@ install:
29
- export DEPLOY_REPO="netdata" # Default production packaging repository
30
- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi;
31
- export PACKAGING_USER="$(echo ${TRAVIS_REPO_SLUG} | cut -d'/' -f1)"
32
+ - export PACKAGE_CLOUD_RETENTION_DAYS=30
33
34
35
@@ -574,7 +575,11 @@ jobs:
575
git:
576
depth: false
577
577
-
578
+ - name: Clean up package cloud nightly repository from old versions
579
+ before_script: post_message "TRAVIS_MESSAGE" "Starting package cloud repository clean up" "${NOTIF_CHANNEL}"
580
+ script:
581
+ - DEPLOY_REPO="netdata-edge" .travis/package_management/old_package_purging.sh
582
+ - DEPLOY_REPO="netdata-devel" .travis/package_management/old_package_purging.sh
583
584
# This is the nightly execution step
585
#
.travis/package_management/old_package_purging.sh
new
+93
@@ -0,0 +1,93 @@
1
+#!/usr/bin/env bash
2
+#
3
+# Script to handle package cloud retention policy
4
+# Our open source subscription is limited,
5
+# so we use this sript to control the number of packages maintained historically
6
+#
7
+# Dependencies:
8
+# - PACKAGE_CLOUD_RETENTION_DAYS
9
+# This is to indicate for how many days back we want to maintain the various RPM and DEB packages on package cloud
10
+#
11
+# Copyright : SPDX-License-Identifier: GPL-3.0-or-later
12
+#
13
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
14
+#
15
+set -e
16
+
17
+delete_files_for_version() {
18
+ local v="$1"
19
+
20
+ # Delete the selected filenames in version
21
+ FILES_IN_VERSION=$(jq --sort-keys --arg v "${v}" '.[] | select ( .version | contains($v))' "${PKG_LIST_FILE}" | grep filename | cut -d':' -f 2)
22
+
23
+ # Iterate through the files and delete them
24
+ for pkg in ${FILES_IN_VERSION/\\n/}; do
25
+ pkg=${pkg/,/}
26
+ pkg=${pkg/\"/}
27
+ pkg=${pkg/\"/}
28
+ echo "Attempting yank on ${pkg}.."
29
+ .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}"
30
+ done
31
+}
32
+
33
+# If we are not in netdata git repo, at the top level directory, fail
34
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
35
+CWD=$(git rev-parse --show-cdup)
36
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
37
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
38
+ echo "Old packages yanking cancelled"
39
+ exit 1
40
+fi
41
+
42
+if [ -z "${PACKAGING_USER}" ]; then
43
+ echo "No PACKAGING_USER variable found"
44
+ exit 1
45
+fi
46
+
47
+if [ -z "${DEPLOY_REPO}" ]; then
48
+ echo "No DEPLOY_REPO variable found"
49
+ exit 1
50
+fi
51
+
52
+if [ -z ${PKG_CLOUD_TOKEN} ]; then
53
+ echo "No PKG_CLOUD_TOKEN variable found"
54
+ exit 1
55
+fi
56
+
57
+if [ -z ${PACKAGE_CLOUD_RETENTION_DAYS} ]; then
58
+ echo "No PACKAGE_CLOUD_RETENTION_DAYS variable found"
59
+ exit 1
60
+fi
61
+
62
+TMP_DIR="$(mktemp -d /tmp/netdata-old-package-yanking-XXXXXX)"
63
+PKG_LIST_FILE="${TMP_DIR}/complete_package_list.json"
64
+DATE_EPOCH="1970-01-01"
65
+DATE_UNTIL_TO_DELETE=$(date --date="${PACKAGE_CLOUD_RETENTION_DAYS} day ago" +%Y-%m-%d)
66
+
67
+
68
+echo "Created temp directory: ${TMP_DIR}"
69
+echo "We will be purging contents up until ${DATE_UNTIL_TO_DELETE}"
70
+
71
+echo "Calling package could to retrieve all available packages on ${PACKAGING_USER}/${DEPLOY_REPO}"
72
+curl -sS "https://${PKG_CLOUD_TOKEN}:@packagecloud.io/api/v1/repos/${PACKAGING_USER}/${DEPLOY_REPO}/packages.json" > "${PKG_LIST_FILE}"
73
+
74
+# Get versions within the desired duration
75
+#
76
+VERSIONS_TO_PURGE=$(jq --arg s "${DATE_EPOCH}" --arg e "${DATE_UNTIL_TO_DELETE}" '
77
+[($s, $e) | strptime("%Y-%m-%d")[0:3]] as $r
78
+ | map(select(
79
+ (.created_at[:19] | strptime("%Y-%m-%dT%H:%M:%S")[0:3]) as $d
80
+ | $d >= $r[0] and $d <= $r[1]
81
+))' "${PKG_LIST_FILE}" | grep '"version":' | sort -u | sed -e 's/ //g' | cut -d':' -f2)
82
+
83
+echo "We will be deleting the following versions: ${VERSIONS_TO_PURGE}"
84
+for v in ${VERSIONS_TO_PURGE/\n//}; do
85
+ v=${v/\"/}
86
+ v=${v/\"/}
87
+ v=${v/,/}
88
+ echo "Remove all files for version $v"
89
+ delete_files_for_version "${v}"
90
+done
91
+
92
+# Done, clean up
93
+[ -d "${TMP_DIR}" ] && rm -rf "${TMP_DIR}"