netdata/packaging/ci: Create manual nightly deployment tool (#5899)
* netdata/packaging/ci: Create manual nightly deployment tool 1. Alter docker tools to shebang with /usr/bin/env, safer to detect the right BASH (>4.0) 2. Generate script packaging/manual_nightly_deployment.sh, that runs only from TLD of netdata GIT and provides the means to deploy to docker and GCS the latest nightly build of netdata. Only option passed to the script is whether you want to deploy in docker, gcs or both. Note: for GCS deployment the user has to define which bucket to update. Note2: this tool has a hard dependency on gsutil to be properly setup on the development environment * netdata/packaging/ci: Do not change global config for mail and user Since we are now starting to use these scripts externally, we need to make sure we dont mess developer config. It seems that we are touching global config, in an attempt to modify a single commit message. Alter this and instead use the --author option for the single commit that we care about instead of altering the universe Cheers
Paul Emm. Katsoulakis committed
May 3, 2019 at 15:24 UTC
7f3c5f6c0123434c5bc500512c4ee139d7342dbd
7 files changed
+131
-15
.travis/create_artifacts.sh
-4
@@ -14,10 +14,6 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
14
fi;
15
16
echo "--- Initialize git configuration ---"
17
-export GIT_MAIL="bot@netdata.cloud"
18
-export GIT_USER="netdatabot"
19
-git config user.email "${GIT_MAIL}"
20
-git config user.name "${GIT_USER}"
17
git checkout master
18
git pull
19
.travis/draft_release.sh
-4
@@ -23,10 +23,6 @@ if [ ! -f .gitignore ]; then
23
fi
24
25
echo "--- Initialize git configuration ---"
26
-export GIT_MAIL="bot@netdata.cloud"
27
-export GIT_USER="netdatabot"
28
-git config user.email "${GIT_MAIL}"
29
-git config user.name "${GIT_USER}"
26
git checkout master
27
git pull
28
.travis/generate_changelog_and_tag_release.sh
+1
-3
@@ -48,8 +48,6 @@ fi
48
echo "--- Initialize git configuration ---"
49
export GIT_MAIL="bot@netdata.cloud"
50
export GIT_USER="netdatabot"
51
-git config user.email "${GIT_MAIL}"
52
-git config user.name "${GIT_USER}"
51
git checkout master
52
git pull
53
@@ -62,7 +60,7 @@ echo "---- GENERATE CHANGELOG -----"
60
git add CHANGELOG.md
61
62
echo "---- COMMIT AND PUSH CHANGES ----"
65
-git commit -m "[ci skip] release $GIT_TAG"
63
+git commit -m "[ci skip] release $GIT_TAG" --author "${GIT_USER} <${GIT_MAIL}>"
64
git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER"
65
git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')"
66
git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" --tags
.travis/generate_changelog_for_nightlies.sh
+1
-3
@@ -38,8 +38,6 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
38
exit 0
39
fi
40
41
-git config user.email "${GIT_MAIL}"
42
-git config user.name "${GIT_USER}"
41
git checkout master
42
git pull
43
@@ -58,7 +56,7 @@ echo "Changelog created! Adding packaging/version(${NEW_VERSION}) and CHANGELOG.
56
echo "${NEW_VERSION}" > packaging/version
57
git add packaging/version && echo "1) Added packaging/version to repository" || FAIL=1
58
git add CHANGELOG.md && echo "2) Added changelog file to repository" || FAIL=1
61
-git commit -m '[ci skip] create nightly packages and update changelog' && echo "3) Committed changes to repository" || FAIL=1
59
+git commit -m '[ci skip] create nightly packages and update changelog' --author "${GIT_USER} <${GIT_MAIL}>" && echo "3) Committed changes to repository" || FAIL=1
60
git push "https://${GITHUB_TOKEN}:@${PUSH_URL}" && echo "4) Pushed changes to remote ${PUSH_URL}" || FAIL=1
61
62
# In case of a failure, wrap it up and bail out cleanly
packaging/docker/build.sh
+1
-1
@@ -1,5 +1,5 @@
1
#!/usr/bin/env bash
2
-# Cross-arch docker build helper script
2
+#
3
#
4
# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
5
#
packaging/docker/publish.sh
+1
@@ -1,4 +1,5 @@
1
#!/usr/bin/env bash
2
+#
3
# Cross-arch docker publish helper script
4
# Needs docker in version >18.02 due to usage of manifests
5
#
packaging/manual_nightly_deployment.sh
new
+127
@@ -0,0 +1,127 @@
1
+#!/usr/bin/env bash
2
+#
3
+# This tool allows netdata team to manually deploy nightlies
4
+# It emulates the nightly operations required for a new version to be published for our users
5
+#
6
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
7
+#
8
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
9
+#
10
+set -e
11
+
12
+# If we are not in netdata git repo, at the top level directory, fail
13
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
14
+CWD=$(git rev-parse --show-cdup || echo "")
15
+if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
16
+ echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository"
17
+ echo "Changelog generation process aborted"
18
+ exit 1
19
+fi
20
+
21
+if [ $# -lt 1 ] || [ $# -gt 2 ]; then
22
+ echo "Run as ./$(basename "$0") [docker|gcs]|all] from the top level directory of netdata GIT repository"
23
+ exit 1
24
+fi
25
+
26
+GSUTIL_BINARY=$(command -v gsutil 2> /dev/null)
27
+if [ -z "${GSUTIL_BINARY}" ]; then
28
+ echo "No gsutil utility available, you need gsutil deployed to manually deploy to GCS"
29
+ exit 1
30
+fi;
31
+
32
+# Function declarations
33
+publish_docker() {
34
+
35
+ # Ensure REPOSITORY present
36
+ if [ -z "${REPOSITORY}" ]; then
37
+ echo "Please provide the repository to deploy the containers:"
38
+ read -r REPOSITORY
39
+ export REPOSITORY
40
+ else
41
+ echo "Docker publishing to ${REPOSITORY}"
42
+ fi
43
+
44
+ # Ensure DOCKER_USERNAME present
45
+ if [ -z "${DOCKER_USERNAME}" ]; then
46
+ echo "For repository ${REPOSITORY}, Please provide the docker USERNAME to use:"
47
+ read -r DOCKER_USERNAME
48
+ export DOCKER_USERNAME
49
+ else
50
+ echo "Using docker username ${DOCKER_USERNAME}"
51
+ fi
52
+
53
+ # Ensure DOCKER_PASS present
54
+ if [ -z "${DOCKER_PASS}" ]; then
55
+ echo "Username ${DOCKER_USERNAME} received, now give me the password:"
56
+ read -r -s DOCKER_PASS
57
+ export DOCKER_PASS
58
+ else
59
+ echo "Docker password has already been set to env, using that"
60
+ fi
61
+
62
+ echo "Building Docker images.."
63
+ packaging/docker/build.sh
64
+
65
+ echo "Publishing Docker images.."
66
+ packaging/docker/publish.sh
67
+}
68
+
69
+publish_nightly_binaries() {
70
+ echo "Publishing nightly binaries to GCS"
71
+
72
+ echo "Please select the bucket to sync, from the ones available to you:"
73
+ bucket_list=$(${GSUTIL_BINARY} list | tr '\n' ' ')
74
+ declare -A buckets
75
+ idx=0
76
+ for abucket in ${bucket_list}; do
77
+ echo "${idx}. ${abucket}"
78
+ buckets["${idx}"]=${abucket}
79
+ ((idx=idx+1))
80
+ done
81
+ read -p"Selection>" -r -n 1 selected_bucket
82
+
83
+ echo "Ok!"
84
+ echo "Syncing artifacts directory contents with GCS bucket: ${buckets[${selected_bucket}]}"
85
+ if [ -d artifacts ]; then
86
+ ${GSUTIL_BINARY} -m rsync -r artifacts "${buckets["${selected_bucket}"]}"
87
+ echo "GCS Sync complete!"
88
+ else
89
+ echo "Directory artifacts does not exist, nothing to do on GCS"
90
+ fi
91
+}
92
+
93
+prepare_and_publish_gcs() {
94
+ # Prepare the artifacts directory
95
+ echo "Preparing artifacts directory contents"
96
+ .travis/create_artifacts.sh
97
+
98
+ # Publish it to GCS
99
+ publish_nightly_binaries
100
+
101
+ # Clean up
102
+ echo "Cleaning up repository"
103
+ make clean || echo "Nothing to clean"
104
+ make distclean || echo "Nothing to distclean"
105
+ rm -rf artifacts
106
+}
107
+
108
+# Mandatory variable declarations
109
+export TRAVIS_REPO_SLUG="netdata/netdata"
110
+
111
+echo "Manual nightly deployment procedure started"
112
+case "$1" in
113
+ "docker")
114
+ publish_docker
115
+ ;;
116
+ "gcs")
117
+ prepare_and_publish_gcs
118
+ ;;
119
+ "all")
120
+ publish_docker
121
+ prepare_and_publish_gcs
122
+ ;;
123
+ *)
124
+ echo "ERROR: Invalid request parameter $1. Valid values are: docker, gcs, all"
125
+ ;;
126
+esac
127
+echo "Manual nightly deployment completed!"