Switch to using GitHub Actions for publishing Docker images. (#10365)
* Switch to using GitHub Actions for publishing Docker images. This simplifies handling of multiarch images and publishing to multiple registries, and unifies testing with the code actually being used to build and publish the images. This also removes a handful of scripts that are no longer needed due to this change, and switches our Dockerfile to not needing an architecture to be specified in a build argument, instead relying on proper multiarch docker image support. * Fix YAML syntax. * Add separate tagging for nightly and stable builds. * Correct YAML syntax errors.
Austin S. Hemmelgarn committed
Jan 15, 2021 at 09:45 UTC
7ffe03690b02ec9961837f09f3c225172fb31fad
6 files changed
+45
-338
.github/workflows/docker.yml
+30
-20
@@ -13,37 +13,47 @@ on:
13
- '.github/workflows/docker.yml'
14
- 'netdata-installer.sh'
15
- 'packaging/**'
16
+ workflow_dispatch:
17
+ inputs:
18
+ version:
19
+ name: Version Tag
20
+ default: latest
21
+ required: true
22
jobs:
23
docker-build:
24
name: Docker Build
19
- strategy:
20
- matrix:
21
- arch:
22
- - linux/amd64
23
- - linux/i386
24
- - linux/arm/v7
25
- - linux/arm64
26
- include:
27
- - arch: linux/amd64
28
- base: amd64
29
- - arch: linux/i386
30
- base: i386
31
- - arch: linux/arm/v7
32
- base: armhf
33
- - arch: linux/arm64
34
- base: aarch64
25
runs-on: ubuntu-latest
26
steps:
27
- name: Checkout
28
uses: actions/checkout@v2
29
+ - name: Determine if we should push changes and which tags to use
30
+ if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly'
31
+ run: |
32
+ echo "publish=true" >> $GITHUB_ENV
33
+ echo "tags=netdata/netdata:latest,netdata/netdata:stable,netdata/netdata:${{ github.event.inputs.version }}" >> $GITHUB_ENV
34
+ - name: Determine if we should push changes and which tags to use
35
+ if: github.event_name == 'workflow_dispatch' && github.event.inputs.version == 'nightly'
36
+ run: |
37
+ echo "publish=true" >> $GITHUB_ENV
38
+ echo "tags=netdata/netdata:latest,netdata/netdata:edge" >> $GITHUB_ENV
39
+ - name: Determine if we should push changes and which tags to use
40
+ if: github.event_name != 'workflow_dispatch'
41
+ run: |
42
+ echo "publish=false" >> $GITHUB_ENV
43
+ echo "tags=netdata/netdata:test" >> $GITHUB_ENV
44
- name: Setup QEMU
45
uses: docker/setup-qemu-action@v1
46
- name: Setup Buildx
47
uses: docker/setup-buildx-action@v1
48
+ - name: Docker Hub Login
49
+ if: github.event_name == 'workflow_dispatch'
50
+ uses: docker/login-action@v1
51
+ with:
52
+ username: ${{ secrets.DOCKER_HUB_USERNAME }}
53
+ password: ${{ secrets.DOCKER_HUB_PASSWORD }}
54
- name: Docker Build
55
uses: docker/build-push-action@v2
56
with:
46
- platforms: ${{ matrix.arch }}
47
- push: false
48
- build-args: |
49
- ARCH=${{ matrix.base }}
57
+ platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64
58
+ push: ${{ env.publish }}
59
+ tags: ${{ env.tags }}
.travis.yml
+11
-75
@@ -103,14 +103,6 @@ jobs:
103
env: CFLAGS='-O1 -Wall -Wextra -Wformat-signedness -fstack-protector-all -fno-common -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1'
104
after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)"
105
106
- - name: Docker container build process (alpine installation)
107
- script:
108
- - "sudo echo '{\"experimental\": true}' > /etc/docker/daemon.json && sudo systemctl restart docker"
109
- - packaging/docker/build.sh
110
- env:
111
- - ARCH=amd64
112
- after_failure: post_message "TRAVIS_MESSAGE" "Docker build process failed"
113
-
106
- name: Build/Install for ubuntu 20.04 (not containerized)
107
script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME
108
after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 18.04"
@@ -353,48 +345,7 @@ jobs:
345
346
# We only publish if a TAG has been set during packaging
347
- stage: Publish for release
356
- _template: &RELEASE_TEMPLATE
357
- env:
358
- - RELEASE_CHANNEL: stable
359
- git:
360
- depth: false
361
- script:
362
- - echo "GIT Branch:" && git branch
363
- - echo "Last commit:" && git log -1
364
- - echo "GIT Describe:" && git describe
365
- - echo "packaging/version:" && cat packaging/version
366
- - "sudo echo '{\"experimental\": true}' > /etc/docker/daemon.json && sudo systemctl restart docker"
367
- - packaging/docker/check_login.sh
368
- && echo "Switching to latest master branch, to pick up tagging if any" && git checkout master && git pull
369
- && tick packaging/docker/build.sh
370
- && packaging/docker/publish.sh
371
- after_failure: post_message "TRAVIS_MESSAGE" "<!here> Docker image publishing failed"
372
-
373
- name: Build & Publish docker image for i386
374
- <<: *RELEASE_TEMPLATE
375
- env:
376
- - ALLOW_SOFT_FAILURE_HERE=true
377
- - ARCH=i386
378
-
379
- - name: Build & Publish docker image for amd64
380
- <<: *RELEASE_TEMPLATE
381
- env:
382
- - ALLOW_SOFT_FAILURE_HERE=true
383
- - ARCH=amd64
384
-
385
- - name: Build & Publish docker image for armhf
386
- <<: *RELEASE_TEMPLATE
387
- env:
388
- - ALLOW_SOFT_FAILURE_HERE=true
389
- - ARCH=armhf
390
-
391
- - name: Build & Publish docker image for aarch64
392
- <<: *RELEASE_TEMPLATE
393
- env:
394
- - ALLOW_SOFT_FAILURE_HERE=true
395
- - ARCH=aarch64
396
-
397
- - name: Create release draft
348
+ name: Create release draft
349
git:
350
depth: false
351
env:
@@ -409,6 +360,10 @@ jobs:
360
- .travis/draft_release.sh
361
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed"
362
363
+ - name: Trigger Docker image build and publish
364
+ script: >-
365
+ curl -X POST -H 'Accept: application/vnd.github.v3+json' 'https://api.github.com/repos/netdata/netdata/actions/workflows/docker.yml' -d '{"ref": "master", "inputs": {"version": "${build_version}"}}'
366
+ after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during release" "${NOTIF_CHANNEL}"
367
368
- stage: Trigger deb and rpm package build (release)
369
name: Trigger deb and rpm package build
@@ -452,31 +407,7 @@ jobs:
407
&& packaging/docker/publish.sh
408
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed"
409
455
- name: Build & Publish docker image for i386
456
- <<: *NIGHTLY_TEMPLATE
457
- env:
458
- - ALLOW_SOFT_FAILURE_HERE=true
459
- - ARCH=i386
460
-
461
- - name: Build & Publish docker image for amd64
462
- <<: *NIGHTLY_TEMPLATE
463
- env:
464
- - ALLOW_SOFT_FAILURE_HERE=true
465
- - ARCH=amd64
466
-
467
- - name: Build & Publish docker image for armhf
468
- <<: *NIGHTLY_TEMPLATE
469
- env:
470
- - ALLOW_SOFT_FAILURE_HERE=true
471
- - ARCH=armhf
472
-
473
- - name: Build & Publish docker image for aarch64
474
- <<: *NIGHTLY_TEMPLATE
475
- env:
476
- - ALLOW_SOFT_FAILURE_HERE=true
477
- - ARCH=aarch64
478
-
479
- - name: Create nightly release artifacts, publish to GCS
410
+ name: Create nightly release artifacts, publish to GCS
411
script:
412
- echo "GIT Branch:" && git branch
413
- echo "Last commit:" && git log -1
@@ -526,6 +457,11 @@ jobs:
457
condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata"
458
after_deploy: rm -f .travis/gcs-credentials.json
459
460
+ - name: Trigger Docker image build and publish
461
+ script: >-
462
+ curl -X POST -H 'Accept: application/vnd.github.v3+json' 'https://api.github.com/repos/netdata/netdata/actions/workflows/docker.yml' -d '{"ref": "master", "inputs": {"version": "nightly"}}'
463
+ after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during nightly release" "${NOTIF_CHANNEL}"
464
+
465
- stage: Trigger deb and rpm package build (nightly release)
466
name: Trigger deb and rpm package build
467
script: .travis/trigger_package_generation.sh "[Build latest]"
packaging/docker/Dockerfile
+4
-6
@@ -1,11 +1,9 @@
1
# SPDX-License-Identifier: GPL-3.0-or-later
2
# author : paulfantom
3
4
-# Cross-arch building is achieved by specifying ARCH as a build parameter with `--build-arg` option.
5
-# It is automated in `build.sh` script
6
-ARG ARCH=amd64
4
# This image contains preinstalled dependecies
8
-FROM netdata/builder:${ARCH} as builder
5
+# hadolint ignore=DL3007
6
+FROM netdata/builder:latest as builder
7
8
# One of 'nightly' or 'stable'
9
ARG RELEASE_CHANNEL=nightly
@@ -54,9 +52,9 @@ RUN mkdir -p /app/usr/sbin/ \
52
chmod +x /app/usr/sbin/run.sh
53
54
#####################################################################
57
-ARG ARCH
55
# This image contains preinstalled dependecies
59
-FROM netdata/base:${ARCH}
56
+# hadolint ignore=DL3007
57
+FROM netdata/base:latest as base
58
59
# Configure system
60
ARG NETDATA_UID=201
packaging/docker/build.sh
deleted
-80
@@ -1,80 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-#
4
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
5
-#
6
-# Author : Pawel Krupa (paulfantom)
7
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
8
-
9
-set -e
10
-
11
-if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then
12
- echo "This mechanism currently can only run on BASH version 4 and above"
13
- exit 1
14
-fi
15
-
16
-VERSION="$1"
17
-
18
-if [ -z "${ARCH}" ]; then
19
- echo "ARCH not set, build cannot proceed"
20
- exit 1
21
-fi
22
-
23
-if [ "${RELEASE_CHANNEL}" != "nightly" ] && [ "${RELEASE_CHANNEL}" != "stable" ]; then
24
- echo "RELEASE_CHANNEL must be set to either 'nightly' or 'stable' - build cannot proceed"
25
- exit 1
26
-fi
27
-
28
-if [ -z ${REPOSITORY} ]; then
29
- REPOSITORY="${TRAVIS_REPO_SLUG}"
30
- if [ -z ${REPOSITORY} ]; then
31
- echo "REPOSITORY not set, build cannot proceed"
32
- exit 1
33
- else
34
- echo "REPOSITORY was not detected, attempted to use TRAVIS_REPO_SLUG setting: ${TRAVIS_REPO_SLUG}"
35
- fi
36
-fi
37
-
38
-# Ensure there is a version, the most appropriate one
39
-if [ "${VERSION}" == "" ]; then
40
- VERSION=$(git tag --points-at)
41
- if [ "${VERSION}" == "" ]; then
42
- VERSION="latest"
43
- fi
44
-fi
45
-
46
-# If we are not in netdata git repo, at the top level directory, fail
47
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
48
-CWD=$(git rev-parse --show-cdup)
49
-if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
50
- echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
51
- echo "Docker build process aborted"
52
- exit 1
53
-fi
54
-
55
-case "${ARCH}" in
56
- amd64) DOCKER_PLATFORM="linux/amd64" ;;
57
- i386) DOCKER_PLATFORM="linux/i386" ;;
58
- armhf) DOCKER_PLATFORM="linux/arm/v7" ;;
59
- aarch64) DOCKER_PLATFORM="linux/arm64" ;;
60
-esac
61
-
62
-echo "Docker image build in progress.."
63
-echo "Version : ${VERSION}"
64
-echo "Repository : ${REPOSITORY}"
65
-echo "Architecture: ${ARCH}"
66
-
67
-docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
68
-
69
-# Build images using multi-arch Dockerfile.
70
-TAG="${REPOSITORY,,}:${VERSION}-${ARCH}"
71
-echo "Building tag ${TAG}.."
72
-docker build --no-cache \
73
- --build-arg ARCH="${ARCH}" \
74
- --build-arg RELEASE_CHANNEL="${RELEASE_CHANNEL}" \
75
- --platform "${DOCKER_PLATFORM}" \
76
- --tag "${TAG}" \
77
- --file packaging/docker/Dockerfile .
78
-echo "..Done!"
79
-
80
-echo "Docker build process completed!"
packaging/docker/check_login.sh
deleted
-41
@@ -1,41 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# This is a credential checker script, to help get early input on docker credentials status
4
-# If these are wrong, then build/publish has no point running
5
-#
6
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
7
-#
8
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
9
-
10
-set -e
11
-
12
-if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then
13
- echo "This mechanism currently can only run on BASH version 4 and above"
14
- exit 1
15
-fi
16
-
17
-DOCKER_CMD="docker "
18
-
19
-# There is no reason to continue if we cannot log in to docker hub
20
-if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PWD+x} ]; then
21
- echo "No docker hub username or password found, aborting without publishing"
22
- exit 1
23
-fi
24
-
25
-# If we are not in netdata git repo, at the top level directory, fail
26
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
27
-CWD=$(git rev-parse --show-cdup)
28
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
29
- echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
30
- echo "Docker build process aborted"
31
- exit 1
32
-fi
33
-
34
-# Login to docker hub to allow futher operations
35
-echo "Attempting to login to docker"
36
-echo "$DOCKER_PWD" | $DOCKER_CMD login -u "$DOCKER_USERNAME" --password-stdin
37
-
38
-echo "Docker login successful!"
39
-$DOCKER_CMD logout
40
-
41
-echo "Docker login validation completed"
packaging/docker/publish.sh
deleted
-116
@@ -1,116 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Cross-arch docker publish helper script
4
-# Needs docker in version >18.02 due to usage of manifests
5
-#
6
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
7
-#
8
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
9
-
10
-set -e
11
-
12
-if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then
13
- echo "This mechanism currently can only run on BASH version 4 and above"
14
- exit 1
15
-fi
16
-
17
-WORKDIR="$(mktemp -d)" # Temporary folder, removed after script is done
18
-VERSION="$1"
19
-
20
-if [ -z "${ARCH}" ]; then
21
- echo "ARCH not set, build cannot proceed"
22
- exit 1
23
-fi
24
-
25
-DOCKER_CMD="docker --config ${WORKDIR}"
26
-GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"}
27
-GIT_USER=${GIT_USER:-"netdatabot"}
28
-
29
-if [ -z ${REPOSITORY} ]; then
30
- REPOSITORY="${TRAVIS_REPO_SLUG}"
31
- if [ -z ${REPOSITORY} ]; then
32
- echo "REPOSITORY not set, publish cannot proceed"
33
- exit 1
34
- else
35
- echo "REPOSITORY was not detected, attempted to use TRAVIS_REPO_SLUG setting: ${TRAVIS_REPO_SLUG}"
36
- fi
37
-fi
38
-
39
-# Ensure there is a version, the most appropriate one
40
-if [ "${VERSION}" == "" ]; then
41
- VERSION=$(git tag --points-at)
42
- if [ "${VERSION}" == "" ]; then
43
- VERSION="latest"
44
- fi
45
-fi
46
-MANIFEST_LIST="${REPOSITORY}:${VERSION}"
47
-
48
-# There is no reason to continue if we cannot log in to docker hub
49
-if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PWD+x} ]; then
50
- echo "No docker hub username or password found, aborting without publishing"
51
- exit 1
52
-fi
53
-
54
-# If we are not in netdata git repo, at the top level directory, fail
55
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
56
-CWD=$(git rev-parse --show-cdup)
57
-if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
58
- echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
59
- echo "Docker build process aborted"
60
- exit 1
61
-fi
62
-
63
-echo "Docker image publishing in progress.."
64
-echo "Version : ${VERSION}"
65
-echo "Repository : ${REPOSITORY}"
66
-echo "Architecture : ${ARCH}"
67
-echo "Manifest list: ${MANIFEST_LIST}"
68
-
69
-# Create temporary docker CLI config with experimental features enabled (manifests v2 need it)
70
-echo '{"experimental":"enabled"}' > "${WORKDIR}"/config.json
71
-
72
-# Login to docker hub to allow futher operations
73
-echo "$DOCKER_PWD" | $DOCKER_CMD login -u "$DOCKER_USERNAME" --password-stdin
74
-
75
-# Push images to registry
76
-TAG="${MANIFEST_LIST}-${ARCH}"
77
-echo "Publishing image ${TAG}.."
78
-$DOCKER_CMD push "${TAG}"
79
-
80
-published() {
81
- curl -s "https://registry.hub.docker.com/v2/repositories/${REPOSITORY}/tags" | jq -e -r '.results[] | select(.name == "'"${VERSION}-${ARCH}"'")' > /dev/null
82
-}
83
-retry 5 published
84
-
85
-echo "Image ${TAG} published succesfully!"
86
-
87
-# Recreate docker manifest list
88
-echo "Getting tag list for version '${VERSION}'.."
89
-TAGS=($(curl -s https://registry.hub.docker.com/v2/repositories/${REPOSITORY}/tags/ | jq -r '.results[]["name"]' | grep "^${VERSION}-"))
90
-
91
-echo "Creating manifest list.."
92
-$DOCKER_CMD manifest create --amend "${MANIFEST_LIST}" "${TAGS[@]/#/${REPOSITORY}:}"
93
-
94
-# Annotate manifest with CPU architecture information
95
-declare -A ARCH_MAP
96
-ARCH_MAP=(["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64")
97
-
98
-echo "Executing manifest annotate.."
99
-for TAG in "${TAGS[@]}"; do
100
- ARCH="${TAG#${VERSION}-}"
101
- echo "Annotating manifest for $ARCH, with TAG: ${REPOSITORY}:${TAG} (Manifest list: ${MANIFEST_LIST})"
102
- $DOCKER_CMD manifest annotate "${MANIFEST_LIST}" "${REPOSITORY}:${TAG}" --os linux --arch "${ARCH_MAP[$ARCH]}"
103
-done
104
-
105
-# Push manifest to docker hub
106
-echo "Pushing manifest list to docker.."
107
-$DOCKER_CMD manifest push -p "${MANIFEST_LIST}"
108
-
109
-# Show current manifest (debugging purpose only)
110
-echo "Evaluating manifest list entry"
111
-$DOCKER_CMD manifest inspect "${MANIFEST_LIST}"
112
-
113
-# Cleanup
114
-rm -r "${WORKDIR}"
115
-
116
-echo "Docker publishing process completed!"