Add Docker tags for the last few nightly builds. (#19734)
This adds new Docker tags for our nightly builds to track prior nightly builds relative to the current one. The new tags take the form `edge-x` where the `x` is a number that represents how many builds previous to the current build the tag tracks. For example, `edge-1` is the build just prior to the current nightly version, while `edge-3` is three builds prior to the current nightly build. These tags are rotated every time a nightly build is published. This approach was chosen to avoid the significant complexity that would be required to properly clean up old nightly build tags if we were tagging by actual nightly version instead of tagging relative to the current build. The current configuration will track the 4 most recent nightly builds. The special tag `edge-0` is a new alias for the current nightly build. This is only needed to allow the tag update logic to work correctly, as the workflow is intentionally configured to publish the new nightly images before rotating the tags for the old ones (so that if rotation fails it does not prevent us from publishing the current nightly).
Austin S. Hemmelgarn committed
Apr 7, 2025 at 07:43 UTC
4627b4f163f676fff7221c939212907d61412271
3 files changed
+100
-23
.github/scripts/gen-docker-tags.py
+52
-21
@@ -1,36 +1,67 @@
1
#!/usr/bin/env python3
2
3
+from __future__ import annotations
4
+
5
+import os
6
import sys
7
5
-github_event = sys.argv[1]
6
-version = sys.argv[2]
8
+from typing import Final
9
+
10
+github_event: Final = sys.argv[1]
11
+version: Final = sys.argv[2]
12
+
13
+REPO: Final = 'netdata/netdata'
14
+REPOS: Final = {
15
+ 'docker': REPO,
16
+ 'quay': f'quay.io/{REPO}',
17
+ 'ghcr': f'ghcr.io/{REPO}',
18
+}
19
8
-REPO = 'netdata/netdata'
20
+REPO: Final = 'netdata/netdata'
21
+QUAY_REPO: Final = f'quay.io/{REPO}'
22
+GHCR_REPO: Final = f'ghcr.io/{REPO}'
23
+NIGHTLY_TAG: Final = 'edge'
24
10
-REPOS = (
11
- REPO,
12
- f'quay.io/{REPO}',
13
- f'ghcr.io/{REPO}',
14
-)
25
+tags = {
26
+ k: [] for k in REPOS.keys()
27
+}
28
+
29
+nightly = False
30
31
match version:
32
case '':
18
- tags = (f'{REPO}:test',)
33
+ for h in REPOS.keys():
34
+ tags[h] = [f'{REPOS[h]}:test']
35
case 'nightly':
20
- tags = tuple([
21
- f'{r}:{t}' for r in REPOS for t in ('edge', 'latest')
22
- ])
36
+ for h in REPOS.keys():
37
+ tags[h] = [f'{REPOS[h]}:{t}' for t in (NIGHTLY_TAG, 'latest')]
38
+
39
+ nightly = True
40
case _:
41
v = f'v{version}'.split('.')
42
26
- tags = tuple([
27
- f'{r}:{t}' for r in REPOS for t in (
28
- v[0],
29
- '.'.join(v[0:2]),
30
- '.'.join(v[0:3]),
31
- )
32
- ])
43
+ versions: Final = (
44
+ v[0],
45
+ '.'.join(v[0:2]),
46
+ '.'.join(v[0:3]),
47
+ 'stable',
48
+ )
49
+
50
+ for h in REPOS.keys():
51
+ tags[h] = [f'{REPOS[h]}:{t}' for t in versions]
52
+
53
+all_tags = [x for y in tags.values() for x in y]
54
+
55
+
56
+def write_output(name: str, value: str) -> None:
57
+ with open(os.getenv('GITHUB_OUTPUT'), 'a') as f:
58
+ f.write(f'{name}={value}')
59
+
60
34
- tags = tags + tuple([f'{r}:stable' for r in REPOS])
61
+write_output('tags', ','.join(all_tags))
62
+write_output('nightly', '1' if nightly else '0')
63
+write_output('nightly_tag', NIGHTLY_TAG)
64
36
-print(','.join(tags))
65
+for h in REPOS.keys():
66
+ write_output(f'{h}_tags', ','.join(tags[h]))
67
+ write_output(f'{h}_repo', REPOS[h])
.github/scripts/rotate-docker-tags.sh
new
+22
@@ -0,0 +1,22 @@
1
+#!/usr/bin/env sh
2
+
3
+set -e
4
+
5
+repo="${1}"
6
+tag="${2}"
7
+count="${3}"
8
+
9
+retag_image() {
10
+ new_tag="${1}"
11
+ old_tag="${2}"
12
+
13
+ if docker manifest inspect "${old_tag}" > /dev/null; then
14
+ docker buildx imagetools create --tag "${new_tag}" "${old_tag}"
15
+ fi
16
+}
17
+
18
+for i in $(seq "${count}" -1 1); do
19
+ retag_image "${repo}:${tag}-${i}" "${repo}:${tag}-$((i - 1))"
20
+done
21
+
22
+retag_image "${repo}:${tag}-0" "${repo}:${tag}"
.github/workflows/docker.yml
+26
-2
@@ -22,6 +22,7 @@ on:
22
required: true
23
env:
24
DISABLE_TELEMETRY: 1
25
+ NIGHTLY_COUNT: 4
26
concurrency:
27
group: docker-${{ github.ref }}-${{ github.event_name }}
28
cancel-in-progress: true
@@ -260,6 +261,14 @@ jobs:
261
if: github.event_name == 'workflow_dispatch'
262
outputs:
263
tags: ${{ steps.tag.outputs.tags }}
264
+ nightly: ${{ steps.tag.outputs.nightly }}
265
+ nightly_tag: ${{ steps.tag.outputs.nightly_tag }}
266
+ docker_tags: ${{ steps.tag.outputs.docker_tags }}
267
+ quay_tags: ${{ steps.tag.outputs.quay_tags }}
268
+ ghcr_tags: ${{ steps.tag.outputs.ghcr_tags }}
269
+ docker_repo: ${{ steps.tag.outputs.docker_repo }}
270
+ quay_repo: ${{ steps.tag.outputs.quay_repo }}
271
+ ghcr_repo: ${{ steps.tag.outputs.ghcr_repo }}
272
steps:
273
- name: Checkout
274
id: checkout
@@ -268,9 +277,9 @@ jobs:
277
id: tag
278
run: |
279
if [ ${{ github.event_name }} = 'workflow_dispatch' ]; then
271
- echo "tags=$(.github/scripts/gen-docker-tags.py ${{ github.event_name }} ${{ github.event.inputs.version }})" >> "${GITHUB_OUTPUT}"
280
+ .github/scripts/gen-docker-tags.py ${{ github.event_name }} ${{ github.event.inputs.version }}
281
else
273
- echo "tags=$(.github/scripts/gen-docker-tags.py ${{ github.event_name }} '')" >> "${GITHUB_OUTPUT}"
282
+ .github/scripts/gen-docker-tags.py ${{ github.event_name }} ''
283
fi
284
285
build-images-docker-hub:
@@ -403,6 +412,10 @@ jobs:
412
id: manifest
413
if: github.repository == 'netdata/netdata'
414
run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests '' "${{ needs.gen-tags.outputs.tags }}")
415
+ - name: Rotate Old Image Tags
416
+ id: rotate
417
+ if: github.repository == 'netdata/netdata' && needs.gen-tags.outputs.nightly == '1'
418
+ run: .github/scripts/rotate-docker-tags.sh ${{ needs.gen-tags.outputs.docker_repo }} ${{ needs.gen-tags.outputs.nightly_tag }} ${NIGHTLY_COUNT}
419
- name: Failure Notification
420
uses: rtCamp/action-slack-notify@v2
421
env:
@@ -418,6 +431,7 @@ jobs:
431
Setup buildx: ${{ steps.prepare.outcome }}
432
Login to registry: ${{ steps.login.outcome }}
433
Create and push manifest: ${{ steps.manifest.outcome }}
434
+ Rotate old image tags: ${{ steps.rotate.outcome }}
435
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
436
if: >-
437
${{
@@ -557,6 +571,10 @@ jobs:
571
id: manifest
572
if: github.repository == 'netdata/netdata'
573
run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'quay.io' "${{ needs.gen-tags.outputs.tags }}")
574
+ - name: Rotate Old Image Tags
575
+ id: rotate
576
+ if: github.repository == 'netdata/netdata' && needs.gen-tags.outputs.nightly == '1'
577
+ run: .github/scripts/rotate-docker-tags.sh ${{ needs.gen-tags.outputs.quay_repo }} ${{ needs.gen-tags.outputs.nightly_tag }} ${NIGHTLY_COUNT}
578
- name: Failure Notification
579
uses: rtCamp/action-slack-notify@v2
580
env:
@@ -572,6 +590,7 @@ jobs:
590
Setup buildx: ${{ steps.prepare.outcome }}
591
Login to registry: ${{ steps.login.outcome }}
592
Create and push manifest: ${{ steps.manifest.outcome }}
593
+ Rotate old image tags: ${{ steps.rotate.outcome }}
594
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
595
if: >-
596
${{
@@ -711,6 +730,10 @@ jobs:
730
id: manifest
731
if: github.repository == 'netdata/netdata'
732
run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'ghcr.io' "${{ needs.gen-tags.outputs.tags }}")
733
+ - name: Rotate Old Image Tags
734
+ id: rotate
735
+ if: github.repository == 'netdata/netdata' && needs.gen-tags.outputs.nightly == '1'
736
+ run: .github/scripts/rotate-docker-tags.sh ${{ needs.gen-tags.outputs.ghcr_repo }} ${{ needs.gen-tags.outputs.nightly_tag }} ${NIGHTLY_COUNT}
737
- name: Failure Notification
738
uses: rtCamp/action-slack-notify@v2
739
env:
@@ -726,6 +749,7 @@ jobs:
749
Setup buildx: ${{ steps.prepare.outcome }}
750
Login to registry: ${{ steps.login.outcome }}
751
Create and push manifest: ${{ steps.manifest.outcome }}
752
+ Rotate old image tags: ${{ steps.rotate.outcome }}
753
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
754
if: >-
755
${{