Introduce agent release metadata pipelines (#16366)
* For stable releases we updated when an new version published or deleted in netdata/netdata repo * You can manually update to the latest (per channel) major versions. * We update the newly published version during CI build process (if an release occurred) for nightly releases. --------- Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud> Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud>
Tasos Katsoulas committed
Nov 16, 2023 at 17:13 UTC
4141e137ec7fd9a5a68af825c73e4a7fa3000442
8 files changed
+339
.github/scripts/check_latest_versions.py
new
+33
@@ -0,0 +1,33 @@
1
+import sys
2
+import os
3
+import modules.version_manipulation as ndvm
4
+import modules.github_actions as cigh
5
+
6
+
7
+def main(command_line_args):
8
+ """
9
+ Inputs: Single version or multiple versions
10
+ Outputs:
11
+ Create files with the versions that needed update under temp_dir/staging-new-releases
12
+ Setting the GitHub outputs, 'versions_needs_update' to 'true'
13
+ """
14
+ versions = [str(arg) for arg in command_line_args]
15
+ # Create a temp output folder for the release that need update
16
+ staging = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'staging-new-releases')
17
+ os.makedirs(staging, exist_ok=True)
18
+ for version in versions:
19
+ temp_value = ndvm.compare_version_with_remote(version)
20
+ if temp_value:
21
+ path, filename = ndvm.get_release_path_and_filename(version)
22
+ release_path = os.path.join(staging, path)
23
+ os.makedirs(release_path, exist_ok=True)
24
+ file_release_path = os.path.join(release_path, filename)
25
+ with open(file_release_path, "w") as file:
26
+ print("Creating local copy of the release version update at: ", file_release_path)
27
+ file.write(version)
28
+ if cigh.run_as_github_action():
29
+ cigh.update_github_output("versions_needs_update", "true")
30
+
31
+
32
+if __name__ == "__main__":
33
+ main(sys.argv[1:])
.github/scripts/check_latest_versions_per_channel.py
new
+9
@@ -0,0 +1,9 @@
1
+import check_latest_versions
2
+import modules.version_manipulation as ndvm
3
+import sys
4
+
5
+if __name__ == "__main__":
6
+ channel = sys.argv[1]
7
+ sorted_agents_by_major = ndvm.sort_and_grouby_major_agents_of_channel(channel)
8
+ latest_per_major = [values[0] for values in sorted_agents_by_major.values()]
9
+ check_latest_versions.main(latest_per_major)
.github/scripts/modules/github_actions.py
new
+27
@@ -0,0 +1,27 @@
1
+import os
2
+
3
+
4
+def update_github_env(key, value):
5
+ try:
6
+ env_file = os.getenv('GITHUB_ENV')
7
+ print(env_file)
8
+ with open(env_file, "a") as file:
9
+ file.write(f"{key}={value}")
10
+ print(f"Updated GITHUB_ENV with {key}={value}")
11
+ except Exception as e:
12
+ print(f"Error updating GITHUB_ENV. Error: {e}")
13
+
14
+
15
+def update_github_output(key, value):
16
+ try:
17
+ env_file = os.getenv('GITHUB_OUTPUT')
18
+ print(env_file)
19
+ with open(env_file, "a") as file:
20
+ file.write(f"{key}={value}")
21
+ print(f"Updated GITHUB_OUTPUT with {key}={value}")
22
+ except Exception as e:
23
+ print(f"Error updating GITHUB_OUTPUT. Error: {e}")
24
+
25
+
26
+def run_as_github_action():
27
+ return os.environ.get('GITHUB_ACTIONS') == 'true'
.github/scripts/modules/requirements.txt
new
+1
@@ -0,0 +1 @@
1
+PyGithub==2.1.1
.github/scripts/modules/version_manipulation.py
new
+141
@@ -0,0 +1,141 @@
1
+import os
2
+import re
3
+import requests
4
+from itertools import groupby
5
+from github import Github
6
+from github.GithubException import GithubException
7
+
8
+repos_URL = {
9
+ "stable": "netdata/netdata",
10
+ "nightly": "netdata/netdata-nightlies"
11
+}
12
+
13
+GH_TOKEN = os.getenv("GH_TOKEN")
14
+if GH_TOKEN is None or GH_TOKEN != "":
15
+ print("Token is not defined or empty, continuing with limitation on requests per sec towards Github API")
16
+
17
+
18
+def identify_channel(_version):
19
+ nightly_pattern = r'v(\d+)\.(\d+)\.(\d+)-(\d+)-nightly'
20
+ stable_pattern = r'v(\d+)\.(\d+)\.(\d+)'
21
+ if re.match(nightly_pattern, _version):
22
+ _channel = "nightly"
23
+ _pattern = nightly_pattern
24
+ elif re.match(stable_pattern, _version):
25
+ _channel = "stable"
26
+ _pattern = stable_pattern
27
+ else:
28
+ print("Invalid version format.")
29
+ return None
30
+ return _channel, _pattern
31
+
32
+
33
+def padded_version(item):
34
+ key_value = '10000'
35
+ for value in item[1:]:
36
+ key_value += f'{value:05}'
37
+ return int(key_value)
38
+
39
+
40
+def extract_version(title):
41
+ if identify_channel(title):
42
+ _, _pattern = identify_channel(title)
43
+ try:
44
+ match = re.match(_pattern, title)
45
+ if match:
46
+ return tuple(map(int, match.groups()))
47
+ except Exception as e:
48
+ print(f"Unexpected error: {e}")
49
+ return None
50
+
51
+
52
+def get_release_path_and_filename(_version):
53
+ nightly_pattern = r'v(\d+)\.(\d+)\.(\d+)-(\d+)-nightly'
54
+ stable_pattern = r'v(\d+)\.(\d+)\.(\d+)'
55
+ if match := re.match(nightly_pattern, _version):
56
+ msb = match.group(1)
57
+ _path = "nightly"
58
+ _filename = f"v{msb}"
59
+ elif match := re.match(stable_pattern, _version):
60
+ msb = match.group(1)
61
+ _path = "stable"
62
+ _filename = f"v{msb}"
63
+ else:
64
+ print("Invalid version format.")
65
+ exit(1)
66
+ return (_path, _filename)
67
+
68
+
69
+def compare_version_with_remote(version):
70
+ """
71
+ If the version = fun (version) you need to update the version in the
72
+ remote. If the version remote doesn't exist, returns the version
73
+ :param channel: any version of the agent
74
+ :return: the greater from version and version remote.
75
+ """
76
+
77
+ prefix = "https://packages.netdata.cloud/releases"
78
+ path, filename = get_release_path_and_filename(version)
79
+
80
+ remote_url = f"{prefix}/{path}/{filename}"
81
+ response = requests.get(remote_url)
82
+
83
+ if response.status_code == 200:
84
+ version_remote = response.text.rstrip()
85
+
86
+ version_components = extract_version(version)
87
+ remote_version_components = extract_version(version_remote)
88
+
89
+ absolute_version = padded_version(version_components)
90
+ absolute_remote_version = padded_version(remote_version_components)
91
+
92
+ if absolute_version > absolute_remote_version:
93
+ print(f"Version in the remote: {version_remote}, is older than the current: {version}, I need to update")
94
+ return (version)
95
+ else:
96
+ print(f"Version in the remote: {version_remote}, is newer than the current: {version}, no action needed")
97
+ return (None)
98
+ else:
99
+ # Remote version not found
100
+ print(f"Version in the remote not found, updating the predefined latest path with the version: {version}")
101
+ return (version)
102
+
103
+
104
+def sort_and_grouby_major_agents_of_channel(channel):
105
+ """
106
+ Fetches the GH API and read either netdata/netdata or netdata/netdata-nightlies repo. It fetches all of their
107
+ releases implements a grouping by their major release number.
108
+ Every k,v in this dictionary is in the form; "vX": [descending ordered list of Agents in this major release].
109
+ :param channel: "nightly" or "stable"
110
+ :return: None or dict() with the Agents grouped by major version # (vX)
111
+ """
112
+ try:
113
+ G = Github(GH_TOKEN)
114
+ repo = G.get_repo(repos_URL[channel])
115
+ releases = repo.get_releases()
116
+ except GithubException as e:
117
+ print(f"GitHub API request failed: {e}")
118
+ return None
119
+
120
+ except Exception as e:
121
+ print(f"An unexpected error occurred: {e}")
122
+ return None
123
+
124
+ extracted_titles = [extract_version(item.title) for item in releases if
125
+ extract_version(item.title) is not None]
126
+ # Necessary sorting for implement the group by
127
+ extracted_titles.sort(key=lambda x: x[0])
128
+ # Group titles by major version
129
+ grouped_by_major = {major: list(group) for major, group in groupby(extracted_titles, key=lambda x: x[0])}
130
+ sorted_grouped_by_major = {}
131
+ for key, values in grouped_by_major.items():
132
+ sorted_values = sorted(values, key=padded_version, reverse=True)
133
+ sorted_grouped_by_major[key] = sorted_values
134
+ # Transform them in the correct form
135
+ if channel == "stable":
136
+ result_dict = {f"v{key}": [f"v{a}.{b}.{c}" for a, b, c in values] for key, values in
137
+ sorted_grouped_by_major.items()}
138
+ else:
139
+ result_dict = {f"v{key}": [f"v{a}.{b}.{c}-{d}-nightly" for a, b, c, d in values] for key, values in
140
+ sorted_grouped_by_major.items()}
141
+ return result_dict
.github/scripts/upload-new-version-tags.sh
new
+18
@@ -0,0 +1,18 @@
1
+#!/bin/bash
2
+
3
+set -e
4
+
5
+host="packages.netdata.cloud"
6
+user="netdatabot"
7
+
8
+prefix="/var/www/html/releases"
9
+staging="${TMPDIR:-/tmp}/staging-new-releases"
10
+
11
+mkdir -p "${staging}"
12
+
13
+for source_dir in "${staging}"/*; do
14
+ if [ -d "${source_dir}" ]; then
15
+ base_name=$(basename "${source_dir}")
16
+ scp -r "${source_dir}"/* "${user}@${host}:${prefix}/${base_name}"
17
+ fi
18
+done
.github/workflows/build.yml
+37
@@ -865,6 +865,37 @@ jobs:
865
makeLatest: true
866
tag: ${{ steps.version.outputs.version }}
867
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
868
+ - name: Checkout netdata main Repo # Checkout back to netdata/netdata repo to the update latest packaged versions
869
+ id: checkout-netdata
870
+ uses: actions/checkout@v4
871
+ with:
872
+ token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
873
+ - name: Init python environment for publish release metadata
874
+ uses: actions/setup-python@v4
875
+ id: init-python
876
+ with:
877
+ python-version: "3.12"
878
+ - name: Setup python environment
879
+ id: setup-python
880
+ run: |
881
+ pip install -r .github/scripts/modules/requirements.txt
882
+ - name: Check if the version is latest and published
883
+ id: check-latest-version
884
+ run: |
885
+ python .github/scripts/check_latest_versions.py ${{ steps.version.outputs.version }}
886
+ - name: SSH setup
887
+ id: ssh-setup
888
+ if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
889
+ uses: shimataro/ssh-key-action@v2
890
+ with:
891
+ key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
892
+ name: id_ecdsa
893
+ known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
894
+ - name: Sync newer releases
895
+ id: sync-releases
896
+ if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
897
+ run: |
898
+ .github/scripts/upload-new-version-tags.sh
899
- name: Failure Notification
900
uses: rtCamp/action-slack-notify@v2
901
env:
@@ -880,6 +911,12 @@ jobs:
911
Fetch artifacts: ${{ steps.fetch.outcome }}
912
Prepare version info: ${{ steps.version.outcome }}
913
Create release: ${{ steps.create-release.outcome }}
914
+ Checkout back netdata/netdata: ${{ steps.checkout-netdata.outcome }}
915
+ Init python environment: ${{ steps.init-python.outcome }}
916
+ Setup python environment: ${{ steps.setup-python.outcome }}
917
+ Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }}
918
+ Setup ssh: ${{ steps.ssh-setup.outcome }}
919
+ Sync with the releases: ${{ steps.sync-releases.outcome }}
920
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
921
if: >-
922
${{
.github/workflows/monitor-releases.yml
new
+73
@@ -0,0 +1,73 @@
1
+---
2
+name: Monitor-releases
3
+
4
+on:
5
+ release:
6
+ types: [released, deleted]
7
+ workflow_dispatch:
8
+ inputs:
9
+ channel:
10
+ description: 'Specify the release channel'
11
+ required: true
12
+ default: 'stable'
13
+
14
+
15
+concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type.
16
+ group: monitor-{{ github.event.inputs.channel }}-releases-${{ github.ref }}-${{ github.event_name }}
17
+ cancel-in-progress: true
18
+
19
+jobs:
20
+ update-stable-agents-metadata:
21
+ name: update-stable-agents-metadata
22
+ if: ${{ github.ref == 'refs/heads/master' }}
23
+ runs-on: ubuntu-latest
24
+ steps:
25
+ - name: Checkout
26
+ id: checkout
27
+ uses: actions/checkout@v4
28
+ with:
29
+ token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
30
+ - name: Init python environment
31
+ uses: actions/setup-python@v4
32
+ id: init-python
33
+ with:
34
+ python-version: "3.12"
35
+ - name: Setup python environment
36
+ id: setup-python
37
+ run: |
38
+ pip install -r .github/scripts/modules/requirements.txt
39
+ - name: Check for newer versions
40
+ id: check-newer-releases
41
+ run: |
42
+ python .github/scripts/check_latest_versions_per_channel.py "${{ github.event.inputs.channel }}"
43
+ - name: SSH setup
44
+ id: ssh-setup
45
+ if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true'
46
+ uses: shimataro/ssh-key-action@v2
47
+ with:
48
+ key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
49
+ name: id_ecdsa
50
+ known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
51
+ - name: Sync newer releases
52
+ id: sync-releases
53
+ if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true'
54
+ run: |
55
+ .github/scripts/upload-new-version-tags.sh
56
+ - name: Failure Notification
57
+ uses: rtCamp/action-slack-notify@v2
58
+ env:
59
+ SLACK_COLOR: 'danger'
60
+ SLACK_FOOTER: ''
61
+ SLACK_ICON_EMOJI: ':github-actions:'
62
+ SLACK_TITLE: 'Failed to prepare changelog:'
63
+ SLACK_USERNAME: 'GitHub Actions'
64
+ SLACK_MESSAGE: |-
65
+ ${{ github.repository }}: Failed to update stable Agent's metadata.
66
+ Checkout: ${{ steps.checkout.outcome }}
67
+ Init python: ${{ steps.init-python.outcome }}
68
+ Setup python: ${{ steps.setup-python.outcome }}
69
+ Check for newer stable releaes: ${{ steps.check-newer-releases.outcome }}
70
+ Setup ssh: ${{ steps.ssh-setup.outcome }}
71
+ Syncing newer release to packages.netdata.cloud : ${{ steps.sync-releases.outcome }}
72
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
73
+ if: failure()