Split docker build script into build and publish - attempt small cleanups along the way (#5591)
* Refactor Travis CI docker image packaging process (preparatory step) 1) Split build and publish on two scripts 2) Update travis code to simply call both scripts wherever only build was called Note: This is just an initial step to verify the implementation split. There will be follow up commits to rework the implementation details * Cleanup build process, first round * Publish docker images: Slight refactor on input preparation and printed outputs * Just like magic numbers are evil, manually settings the same path across the code is also dangerous - declare a WORKDIR and replace * Use the single archs array for the loops, reference the desired ARCHs in one place * Use MANIFEST_LIST for clarity * Declare one docker command to use everywhere, to avoid confusion or misses * From the way its used, seems like a temporary storage facility, make sure its cleaned up after we are done * Add a bit more verbosity, plus a typo * Attempt to run both scripts on the same step * Do not call publish script within build stage, publish never ran there (to confirm with @paulfantom the details, couldnt track the workflow to justify) * Cleanup comments as per feedback * packaging/docker/publishing: indentation * netdata/packaging/docker: Do not use gitignore to know whether you run on the top level directory of the repo 1) Remove check for .gitignore as its unstable 2) Use show top level and cdup to make sure you are under netdata and at the top level directory This does not provide any functional change, it is only a more stable way of verifying we are running from TLD * netdata/packaging/docker: Leave a comment for the next iteration * netdata/packaging/docker: missed renamed of variable, sowwy * netdata/packaging/docker: Use mktemp for temporary directory creation. Cleaner, safer * netdata/packaging/travis: Make sure you get rid of the json file after completion Just being cautious here, dont let any leftovers behind regardless the fact that this is an ephemeral host the process is running on * netdata/packaging: make sure we dont break if the file is already removed
Paul Katsoulakis committed
Mar 23, 2019 at 14:26 UTC
a6192373597493189519937a85a9147aa16f350d
6 files changed
+148
-72
.travis.yml
+6
-6
@@ -32,11 +32,11 @@ jobs:
32
33
- stage: build
34
# TODO: enable when travis OSX become stable. Probably after 12.01.2019
35
-# name: OSX
36
-# install: brew install fakeroot ossp-uuid
37
-# script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it
38
-# os: osx
39
-# - name: ubuntu 14.04 (not containerized)
35
+# name: OSX
36
+# install: brew install fakeroot ossp-uuid
37
+# script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it
38
+# os: osx
39
+# - name: ubuntu 14.04 (not containerized)
40
name: ubuntu 14.04 (not containerized)
41
install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot
42
script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME
@@ -80,7 +80,7 @@ jobs:
80
bucket: "netdata-nightlies"
81
skip_cleanup: true
82
local_dir: "artifacts"
83
-
83
+ after_deploy: rm -f .travis/gcs-credentials.json
84
notifications:
85
webhooks: https://app.fossa.io/hooks/travisci
86
slack:
.travis/nightlies.sh
+1
@@ -37,6 +37,7 @@ fi
37
38
echo "--- BUILD & PUBLISH DOCKER IMAGES ---"
39
packaging/docker/build.sh || BAD_THING_HAPPENED=1
40
+packaging/docker/publish.sh || BAD_THING_HAPPENED=1
41
42
echo "--- BUILD ARTIFACTS ---"
43
.travis/create_artifacts.sh || BAD_THING_HAPPENED=1
.travis/releaser.sh
+1
@@ -65,6 +65,7 @@ fi
65
66
echo "---- CREATING TAGGED DOCKER CONTAINERS ----"
67
./packaging/docker/build.sh
68
+./packaging/docker/publish.sh
69
70
echo "---- CREATING RELEASE ARTIFACTS -----"
71
# Artifacts are stored in `artifacts/` directory
packaging/docker/build-test.sh
+7
-3
@@ -1,7 +1,11 @@
1
#!/bin/bash
2
-# SPDX-License-Identifier: GPL-3.0-or-later
3
-# Author : Chris Akritidis (cakrit)
4
-# Cross-arch docker build helper script
2
+# Docker build wrapper, for testing manually the docker build process
3
+# TODO: This script should consume build.sh after setting up required parameters
4
+#
5
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
+#
7
+# Author : Chris Akritidis (chris@netdata.cloud)
8
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
9
10
printhelp() {
11
echo "Usage: packaging/docker/build-test.sh -r <REPOSITORY> -v <VERSION> -u <DOCKER_USERNAME> -p <DOCKER_PASSWORD> [-s]
packaging/docker/build.sh
+39
-63
@@ -1,82 +1,58 @@
1
#!/bin/bash
2
-# SPDX-License-Identifier: GPL-3.0-or-later
3
-# Author : Pawel Krupa (paulfantom)
2
# Cross-arch docker build helper script
5
-# Needs docker in version >18.02 due to usage of manifests
3
+#
4
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
5
+#
6
+# Author : Pawel Krupa (paulfantom)
7
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
8
9
set -e
10
9
-if [ ! -f .gitignore ]; then
10
- echo "Run as ./packaging/docker/$(basename "$0") from top level directory of git repository"
11
- exit 1
12
-fi
11
+VERSION="$1"
12
+REPOSITORY="${REPOSITORY:-netdata}"
13
+declare -A ARCH_MAP
14
+ARCH_MAP=(["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64")
15
+DEVEL_ARCHS=(amd64)
16
+ARCHS="${!ARCH_MAP[@]}"
17
14
-if [ "$1" == "" ]; then
15
- VERSION=$(git tag --points-at)
16
-else
17
- VERSION="$1"
18
+# When development mode is set, build on DEVEL_ARCHS
19
+if [ ! -z ${DEVEL+x} ]; then
20
+ declare -a ARCHS=(${DEVEL_ARCHS[@]})
21
fi
22
+
23
+# Ensure there is a version, the most appropriate one
24
if [ "${VERSION}" == "" ]; then
20
- VERSION="latest"
25
+ VERSION=$(git tag --points-at)
26
+ if [ "${VERSION}" == "" ]; then
27
+ VERSION="latest"
28
+ fi
29
fi
30
23
-declare -A ARCH_MAP
24
-ARCH_MAP=( ["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64")
25
-if [ -z ${DEVEL+x} ]; then
26
- declare -a ARCHITECTURES=(i386 armhf aarch64 amd64)
27
-else
28
- declare -a ARCHITECTURES=(amd64)
29
- unset DOCKER_PASSWORD
30
- unset DOCKER_USERNAME
31
+# If we are not in netdata git repo, at the top level directory, fail
32
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
33
+CWD=$(git rev-parse --show-cdup)
34
+if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
35
+ echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
36
+ echo "Docker build process aborted"
37
+ exit 1
38
fi
39
33
-REPOSITORY="${REPOSITORY:-netdata}"
34
-echo "Building ${VERSION} of ${REPOSITORY} container"
40
+echo "Docker image build in progress.."
41
+echo "Version : ${VERSION}"
42
+echo "Repository : ${REPOSITORY}"
43
+echo "Architectures : ${ARCHS}"
44
45
docker run --rm --privileged multiarch/qemu-user-static:register --reset
46
47
# Build images using multi-arch Dockerfile.
39
-for ARCH in "${ARCHITECTURES[@]}"; do
48
+for ARCH in "${ARCHS[@]}"; do
49
+ TAG="${REPOSITORY}:${VERSION}-${ARCH}"
50
+ echo "Building tag ${TAG}.."
51
eval docker build \
41
- --build-arg ARCH="${ARCH}" \
42
- --tag "${REPOSITORY}:${VERSION}-${ARCH}" \
43
- --file packaging/docker/Dockerfile ./
52
+ --build-arg ARCH="${ARCH}" \
53
+ --tag "${TAG}" \
54
+ --file packaging/docker/Dockerfile ./
55
+ echo "..Done!"
56
done
57
46
-# There is no reason to continue if we cannot log in to docker hub
47
-if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then
48
- echo "No docker hub username or password specified. Exiting without pushing images to registry"
49
- exit 0
50
-fi
51
-
52
-# Create temporary docker CLI config with experimental features enabled (manifests v2 need it)
53
-mkdir -p /tmp/docker
54
-echo '{"experimental":"enabled"}' > /tmp/docker/config.json
55
-
56
-# Login to docker hub to allow futher operations
57
-echo "$DOCKER_PASSWORD" | docker --config /tmp/docker login -u "$DOCKER_USERNAME" --password-stdin
58
-
59
-# Push images to registry
60
-for ARCH in amd64 i386 armhf aarch64; do
61
- docker --config /tmp/docker push "${REPOSITORY}:${VERSION}-${ARCH}" &
62
-done
63
-wait
64
-
65
-# Recreate docker manifest
66
-docker --config /tmp/docker manifest create --amend \
67
- "${REPOSITORY}:${VERSION}" \
68
- "${REPOSITORY}:${VERSION}-i386" \
69
- "${REPOSITORY}:${VERSION}-armhf" \
70
- "${REPOSITORY}:${VERSION}-aarch64" \
71
- "${REPOSITORY}:${VERSION}-amd64"
72
-
73
-# Annotate manifest with CPU architecture information
74
-for ARCH in i386 armhf aarch64 amd64; do
75
- docker --config /tmp/docker manifest annotate "${REPOSITORY}:${VERSION}" "${REPOSITORY}:${VERSION}-${ARCH}" --os linux --arch "${ARCH_MAP[$ARCH]}"
76
-done
77
-
78
-# Push manifest to docker hub
79
-docker --config /tmp/docker manifest push -p "${REPOSITORY}:${VERSION}"
80
-
81
-# Show current manifest (debugging purpose only)
82
-docker --config /tmp/docker manifest inspect "${REPOSITORY}:${VERSION}"
58
+echo "Docker build process completed!"
packaging/docker/publish.sh
new
+94
@@ -0,0 +1,94 @@
1
+#!/bin/bash
2
+# Cross-arch docker publish helper script
3
+# Needs docker in version >18.02 due to usage of manifests
4
+#
5
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
+#
7
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
8
+
9
+set -e
10
+
11
+WORKDIR="$(mktemp -d)" # Temporary folder, removed after script is done
12
+VERSION="$1"
13
+REPOSITORY="${REPOSITORY:-netdata}"
14
+MANIFEST_LIST="${REPOSITORY}:${VERSION}"
15
+declare -A ARCH_MAP
16
+ARCH_MAP=(["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64")
17
+DEVEL_ARCHS=(amd64)
18
+ARCHS="${!ARCH_MAP[@]}"
19
+DOCKER_CMD="docker --config ${WORKDIR}"
20
+
21
+# When development mode is set, build on DEVEL_ARCHS
22
+if [ ! -z ${DEVEL+x} ]; then
23
+ declare -a ARCHS=(${DEVEL_ARCHS[@]})
24
+fi
25
+
26
+# Ensure there is a version, the most appropriate one
27
+if [ "${VERSION}" == "" ]; then
28
+ VERSION=$(git tag --points-at)
29
+ if [ "${VERSION}" == "" ]; then
30
+ VERSION="latest"
31
+ fi
32
+fi
33
+
34
+# There is no reason to continue if we cannot log in to docker hub
35
+if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then
36
+ echo "No docker hub username or password found, aborting without publishing"
37
+ exit 1
38
+fi
39
+
40
+# If we are not in netdata git repo, at the top level directory, fail
41
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
42
+CWD=$(git rev-parse --show-cdup)
43
+if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
44
+ echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
45
+ echo "Docker build process aborted"
46
+ exit 1
47
+fi
48
+
49
+echo "Docker image publishing in progress.."
50
+echo "Version : ${VERSION}"
51
+echo "Repository : ${REPOSITORY}"
52
+echo "Architectures : ${ARCHS}"
53
+echo "Manifest list : ${MANIFEST_LIST}"
54
+
55
+# Create temporary docker CLI config with experimental features enabled (manifests v2 need it)
56
+echo '{"experimental":"enabled"}' > "${WORKDIR}"/config.json
57
+
58
+# Login to docker hub to allow futher operations
59
+echo "$DOCKER_PASSWORD" | $DOCKER_CMD login -u "$DOCKER_USERNAME" --password-stdin
60
+
61
+# Push images to registry
62
+for ARCH in ${ARCHS[@]}; do
63
+ TAG="${MANIFEST_LIST}-${ARCH}"
64
+ echo "Publishing image ${TAG}.."
65
+ $DOCKER_CMD push "${TAG}" &
66
+ echo "Image ${TAG} published succesfully!"
67
+done
68
+
69
+echo "Waiting for images publishing to complete"
70
+wait
71
+
72
+# Recreate docker manifest list
73
+$DOCKER_CMD manifest create --amend "${MANIFEST_LIST}" \
74
+ "${MANIFEST_LIST}-i386" \
75
+ "${MANIFEST_LIST}-armhf" \
76
+ "${MANIFEST_LIST}-aarch64" \
77
+ "${MANIFEST_LIST}-amd64"
78
+
79
+# Annotate manifest with CPU architecture information
80
+for ARCH in ${ARCHS[@]}; do
81
+ TAG="${MANIFEST_LIST}-${ARCH}"
82
+ $DOCKER_CMD manifest annotate "${MANIFEST_LIST}" "${TAG}" --os linux --arch "${ARCH_MAP[$ARCH]}"
83
+done
84
+
85
+# Push manifest to docker hub
86
+$DOCKER_CMD manifest push -p "${MANIFEST_LIST}"
87
+
88
+# Show current manifest (debugging purpose only)
89
+$DOCKER_CMD manifest inspect "${MANIFEST_LIST}"
90
+
91
+# Cleanup
92
+rm -r "${WORKDIR}"
93
+
94
+echo "Docker publishing process completed!"