Add a scheduled CI job to flag supported platforms going EOL upstream. (#14581)
* Add a scheduled CI job to flag supported platforms going EOL upstream. By default, it runs at 03:00 UTC every Monday and checks the upstream EOL date for each platform we support that needs such checking. If the platform will be EOL upstream within the next 30 days, an issue is opened flagging the platform for removal from CI and our support document and auto-assigned to the agent SRE team members. The workflow can also be manually triggered (mostly intended for testing). Data about upstream EOL dates is retrieved from https://endoflife.date via their new public API. Happily, our own definition of what constitutes EOL for our purposes matches up 1:1 with how they categorize platforms as EOL. * Fix logic issue in issue creation. * Explicitly enable error handling for issue checks.
Austin S. Hemmelgarn committed
Feb 24, 2023 at 10:00 UTC
a20b8c4ab47c84f8e9f1473e78472ff4bedd57f1
4 files changed
+188
-2
.github/data/distros.yml
+15
-2
@@ -1,6 +1,6 @@
1
# This defines the full set of distros we run CI on.
2
---
3
-platform_map: # map packaging architectures to docker platforms
3
+platform_map: # map packaging architectures to docker platforms
4
aarch64: linux/arm64/v8
5
amd64: linux/amd64
6
arm64: linux/arm64/v8
@@ -8,7 +8,7 @@ platform_map: # map packaging architectures to docker platforms
8
armhfp: linux/arm/v7
9
i386: linux/i386
10
x86_64: linux/amd64
11
-arch_order: # sort order for per-architecture jobs in CI
11
+arch_order: # sort order for per-architecture jobs in CI
12
- amd64
13
- x86_64
14
- i386
@@ -20,6 +20,7 @@ include:
20
- &alpine
21
distro: alpine
22
version: edge
23
+ eol_check: false
24
env_prep: |
25
apk add -U bash
26
jsonc_removal: |
@@ -28,15 +29,20 @@ include:
29
ebpf-core: true
30
- <<: *alpine
31
version: "3.17"
32
+ eol_check: true
33
- <<: *alpine
34
version: "3.16"
35
+ eol_check: true
36
- <<: *alpine
37
version: "3.15"
38
+ eol_check: true
39
- <<: *alpine
40
version: "3.14"
41
+ eol_check: true
42
43
- distro: archlinux
44
version: latest
45
+ eol_check: false
46
env_prep: |
47
pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi
48
test:
@@ -47,6 +53,7 @@ include:
53
version: "9"
54
jsonc_removal: |
55
dnf remove -y json-c-devel
56
+ eol_check: true
57
packages: &alma_packages
58
type: rpm
59
repo_distro: el/9
@@ -69,6 +76,7 @@ include:
76
77
- distro: centos
78
version: "7"
79
+ eol_check: false
80
packages:
81
type: rpm
82
repo_distro: el/7
@@ -84,6 +92,7 @@ include:
92
distro: debian
93
version: "12"
94
base_image: debian:bookworm
95
+ eol_check: true
96
env_prep: |
97
apt-get update
98
jsonc_removal: |
@@ -118,6 +127,7 @@ include:
127
- &fedora
128
distro: fedora
129
version: "37"
130
+ eol_check: true
131
jsonc_removal: |
132
dnf remove -y json-c-devel
133
packages: &fedora_packages
@@ -139,6 +149,7 @@ include:
149
- &opensuse
150
distro: opensuse
151
version: "15.4"
152
+ eol_check: true
153
base_image: opensuse/leap:15.4
154
jsonc_removal: |
155
zypper rm -y libjson-c-devel
@@ -154,6 +165,7 @@ include:
165
- &oracle
166
distro: oraclelinux
167
version: "8"
168
+ eol_check: true
169
jsonc_removal: |
170
dnf remove -y json-c-devel
171
packages: &oracle_packages
@@ -173,6 +185,7 @@ include:
185
- &ubuntu
186
distro: ubuntu
187
version: "22.10"
188
+ eol_check: true
189
env_prep: |
190
rm -f /etc/apt/apt.conf.d/docker && apt-get update
191
jsonc_removal: |
.github/scripts/gen-matrix-eol-check.py
new
+23
@@ -0,0 +1,23 @@
1
+#!/usr/bin/env python3
2
+'''Generate the build matrix for the EOL check jobs.'''
3
+
4
+import json
5
+
6
+from ruamel.yaml import YAML
7
+
8
+yaml = YAML(typ='safe')
9
+entries = list()
10
+
11
+with open('.github/data/distros.yml') as f:
12
+ data = yaml.load(f)
13
+
14
+for item in data['include']:
15
+ if 'eol_check' in item and item['eol_check']:
16
+ entries.append({
17
+ 'distro': item['distro'],
18
+ 'release': item['version'],
19
+ })
20
+
21
+entries.sort(key=lambda k: (k['distro'], k['release']))
22
+matrix = json.dumps({'include': entries}, sort_keys=True)
23
+print(matrix)
.github/scripts/platform-impending-eol.py
new
+41
@@ -0,0 +1,41 @@
1
+#!/usr/bin/env python3
2
+'''Check if a given distro is going to be EOL soon.
3
+
4
+ This queries the public API of https://endoflife.date to fetch EOL dates.
5
+
6
+ ‘soon’ is defined by LEAD_DAYS, currently 30 days.'''
7
+
8
+import datetime
9
+import json
10
+import sys
11
+import urllib.request
12
+
13
+URL_BASE = 'https://endoflife.date/api'
14
+NOW = datetime.date.today()
15
+LEAD_DAYS = datetime.timedelta(days=30)
16
+
17
+DISTRO = sys.argv[1]
18
+RELEASE = sys.argv[2]
19
+
20
+
21
+with urllib.request.urlopen(f'{ URL_BASE }/{ DISTRO }/{ RELEASE }.json') as response:
22
+ match response.status:
23
+ case 200:
24
+ data = json.load(response)
25
+ case 404:
26
+ sys.exit(f'No data available for { DISTRO } { RELEASE }.')
27
+ case _:
28
+ sys.exit(
29
+ f'Failed to retrieve data for { DISTRO } { RELEASE } ' +
30
+ f'(status: { response.status }).'
31
+ )
32
+
33
+eol = datetime.date.fromisoformat(data['eol'])
34
+
35
+offset = abs(eol - NOW)
36
+
37
+if offset <= LEAD_DAYS:
38
+ print(data['eol'])
39
+ sys.exit(2)
40
+else:
41
+ sys.exit(0)
.github/workflows/platform-eol-check.yml
new
+109
@@ -0,0 +1,109 @@
1
+---
2
+# Auto-generate issues for EOL of platforms that are approaching their EOL date.
3
+# Uses https://endoflife.date and their new API to check for EOL dates.
4
+#
5
+# Issues are created when the EOL date is within the next 30 days.
6
+name: Check Platform EOL
7
+on: # Run weekly and whenever manually triggered
8
+ schedule:
9
+ - cron: '0 3 * * 1'
10
+ workflow_dispatch: null
11
+concurrency: # Simple single-instance concurrency.
12
+ group: eol-check-${{ github.repository }}
13
+ cancel-in-progress: true
14
+jobs:
15
+ # Prepare the build matrix.
16
+ # This uses output from .github/scripts/gen-matrix-eol-check.py
17
+ matrix:
18
+ name: Prepare Build Matrix
19
+ runs-on: ubuntu-latest
20
+ outputs:
21
+ matrix: ${{ steps.set-matrix.outputs.matrix }}
22
+ steps:
23
+ - name: Checkout
24
+ id: checkout
25
+ uses: actions/checkout@v3
26
+ - name: Prepare tools
27
+ id: prepare
28
+ run: |
29
+ sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
30
+ - name: Read build matrix
31
+ id: set-matrix
32
+ run: |
33
+ matrix="$(.github/scripts/gen-matrix-eol-check.py)"
34
+ echo "Generated matrix: ${matrix}"
35
+ echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
36
+
37
+ eol-check:
38
+ name: EOL Check
39
+ runs-on: ubuntu-latest
40
+ needs:
41
+ - matrix
42
+ strategy:
43
+ matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
44
+ fail-fast: false # We want to check everything, so don’t bail on the first failure.
45
+ max-parallel: 2 # Cap of two jobs at a time to limit impact on other CI.
46
+ steps:
47
+ - name: Checkout
48
+ id: checkout
49
+ uses: actions/checkout@v3
50
+ # Actually check the EOL date for the platform.
51
+ - name: Check EOL Date
52
+ id: check
53
+ run: |
54
+ d="$(.github/scripts/platform-impending-eol.py ${{ matrix.distro }} ${{ matrix.release }})"
55
+ case $? in
56
+ 0) echo "pending=false" >> "${GITHUB_OUTPUT}" ;;
57
+ 2)
58
+ echo "pending=true" >> "${GITHUB_OUTPUT}"
59
+ echo "date=${d}" >> "${GITHUB_OUTPUT}"
60
+ ;;
61
+ *)
62
+ echo "::error::Failed to check EOL date for ${{ matrix.distro }} ${{ matrix.release }}"
63
+ exit 1
64
+ ;;
65
+ esac
66
+ # Figure out the issue title.
67
+ # This is it’s own step so we only have to set it in one place.
68
+ - name: Determine Issue Title
69
+ id: title
70
+ if: steps.check.outputs.pending == 'true'
71
+ run: |
72
+ echo "title=[Platform EOL]: ${{ matrix.full_name }} will be EOL soon." >> "${GITHUB_OUTPUT}"
73
+ # Check if there is an existing issue in the repo for the platform EOL.
74
+ # The actual command line to make the check is unfortunately complicated because
75
+ # GitHub think that it’s sensible to exit with a status of 0 if there no results
76
+ # for a search.
77
+ - name: Check for Existing Issue
78
+ id: existing
79
+ if: steps.check.outputs.pending == 'true'
80
+ env:
81
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
82
+ run: |
83
+ set -e
84
+ count=$(gh issue list -R netdata/netdata -s all -S '${{ steps.title.outputs.title }} in:title' --json 'id' -q '. | length')
85
+ if [ "${count}" -ge 1 ]; then
86
+ echo 'exists=true' >> "${GITHUB_OUTPUT}"
87
+ else
88
+ echo 'exists=false' >> "${GITHUB_OUTPUT}"
89
+ fi
90
+ # If the platform is near EOL and there is no existing issue, create one.
91
+ - name: Create EOL Issue
92
+ id: create-issue
93
+ if: steps.check.outputs.pending == 'true' && steps.existing.outputs.exists == 'false'
94
+ uses: imjohnbo/issue-bot@v3
95
+ with:
96
+ assignees: Ferroin, tkatsoulas
97
+ labels: area/packaging, needs triage
98
+ title: ${{ steps.title.outputs.title }}
99
+ body: |
100
+ Based on information from https://endoflife.date/${{ matrix.distro }},
101
+ upstream support for ${{ matrix.full_name }} will be ending
102
+ on ${{ steps.check.outputs.date }}. A PR should be created
103
+ to remove this platform from our platform support document,
104
+ CI, and packaging code.
105
+
106
+ - [ ]: Remove platform from `packaging/PLATFORM_SUPPORT.md`
107
+ - [ ]: Remove platform from `.github/data/distros.yml`
108
+ - [ ]: Remove platform package builder from helper-images repo (if applicable).
109
+ - [ ]: Verify any other platform support code that needs to be cleaned up.