netdata/ci: nits and fixes around package release workflow (#6914)
* netdata/packaging: bring el/6 first, because this appears on all cases. Otherwise travis craps out the steps * netdata/ci: Attempt to sanitise workflow for master branch commits 1) Rename Packaging for release, now named: Support activities on main branch 2) Adjust instruction messages to reflect more realistically the activities 3) Put labeler first on the stage, since this will always run 4) Make changelog and tag for release conditionally run, so that it does not confuse people * netdata/ci: Sanitise workflow for release more (explained below) We need to clean up tagging logic more and in order to do this we need to bring tagger logic to travis.yml Then we have the complete control of tagging and triggering for changelog generation in travis. To mitigate this, we need to: 1) make tagger script a lirbary. Keep the tagging logic in a method and remove the things needed when it was an executable (checks, executions etc) 2) make travis handle GIT_TAG checking and setting at the beginning. We need this outside of any conditionals so that we have GIT_TAG result available inside the steps 3) COMMIT_TAG_POINTS_AT can now become GIT_TAG, since this is what we actually wanted. We just named differently before to avoid any weird conflicts 4) make changelog generation for release to NOT source tagger any more. Also make it fail in case GIT_TAG is not there, we wont run the script if git tag not in place so that should be error if it happens 5) Rename .travis/generate_changelog_for_release.sh -> .travis/create_changelog_for_release.sh 6) Rename .travis/generate_changelog_and_tag_release.sh -> .travis/generate_changelog_for_release.sh * netdata/ci: reinstate filename here. we basically do the git tag operation here still * netdata/ci: simplify more the changelog generation process 1) Rename create_changelog_for_release.sh -> create_changelog.sh 2) Add comments and the standard path checks in create_changelog.sh 3) Do some minor changes to help diff with nightlies changelog (no functional change though, primarily messages and move flags lower) 4) Inside generate_changelog_for_nightlies.sh, use create_changelog.sh and remove code related to create changelog process 5) in nightlies.sh, use standard shebang adn fix some messages Note: might follow up with more changes, this is just a first batch * netdata/packaging: reinstate accidentally removed OPTS from other iterationns (different engineer)
Paul Emm. Katsoulakis committed
Sep 23, 2019 at 19:23 UTC
ae4bb0286e615da649cd1f7c2011931cc35acf29
6 files changed
+75
-102
.travis.yml
+18
-14
@@ -31,6 +31,10 @@ install:
31
- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi;
32
- export PACKAGE_CLOUD_RETENTION_DAYS=30
33
- if [ ! "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export DEPLOY_REPO="netdata-devel"; fi;
34
+ # These are release-related artifacts and have to be evaluated before we start doing conditional checks inside stages
35
+ - source ".travis/tagger.sh"
36
+ - export GIT_TAG="$(git tag --points-at)"
37
+ - if [[ -z "${GIT_TAG}" ]]; then echo "Running set tag for release" && set_tag_for_release; fi;
38
39
40
@@ -58,7 +62,7 @@ stages:
62
if: branch = master AND type = cron
63
64
# Scheduled releases
61
- - name: Packaging for release
65
+ - name: Support activities on main branch
66
if: branch = master AND type != pull_request AND type != cron
67
68
- name: Publish for release
@@ -240,22 +244,22 @@ jobs:
244
245
246
243
- - stage: Packaging for release
247
+ - stage: Support activities on main branch
248
+ name: Run labeler on github issues
249
+ script: .travis/labeler.sh # labeler should be replaced with GitHub Actions when they hit GA
250
245
- name: Generate changelog and TAG the release (only on special commit msg)
246
- before_script: post_message "TRAVIS_MESSAGE" "Packaging step for release initiated" "${NOTIF_CHANNEL}"
251
+ - name: Generate changelog for release (only on special and tagged commit msg)
252
+ before_script: post_message "TRAVIS_MESSAGE" "Support activities on main branch initiated" "${NOTIF_CHANNEL}"
253
script:
254
- echo "GIT Branch:" && git branch
255
- echo "Last commit:" && git log -1
256
- echo "GIT Describe:" && git describe
257
- echo "packaging/version:" && cat packaging/version
258
- .travis/generate_changelog_and_tag_release.sh
253
- after_failure: post_message "TRAVIS_MESSAGE" "<!here> Packaging for release failed"
259
+ after_failure: post_message "TRAVIS_MESSAGE" "<!here> Changelog generation and tag of release, failed"
260
git:
261
depth: false
256
-
257
- - name: Run labeler on github issues
258
- script: .travis/labeler.sh # labeler should be replaced with GitHub Actions when they hit GA
262
+ if: commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ AND tag !~ /(-rc)/ OR (env(GIT_TAG) IS present AND NOT env(GIT_TAG) IS blank)
263
264
265
@@ -411,19 +415,19 @@ jobs:
415
- if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi;
416
- if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi;
417
414
- name: "Build & Publish RPM package for Enterprise Linux 7"
418
+ name: "Build & Publish RPM package for Enterprise Linux 6"
419
<<: *RPM_TEMPLATE
416
- if: commit_message =~ /\[Package (amd64|arm64) RPM( Enterprise Linux)?\]/
420
+ if: commit_message =~ /\[Package (amd64|arm64|i386) RPM( Enterprise Linux)?\]/
421
env:
418
- - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7"
422
+ - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="6" BUILD_STRING="el/6"
423
- PACKAGE_TYPE="rpm" REPO_TOOL="yum"
424
- ALLOW_SOFT_FAILURE_HERE=true
425
422
- - name: "Build & Publish RPM package for Enterprise Linux 6"
426
+ - name: "Build & Publish RPM package for Enterprise Linux 7"
427
<<: *RPM_TEMPLATE
424
- if: commit_message =~ /\[Package (amd64|arm64|i386) RPM( Enterprise Linux)?\]/
428
+ if: commit_message =~ /\[Package (amd64|arm64) RPM( Enterprise Linux)?\]/
429
env:
426
- - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="6" BUILD_STRING="el/6"
430
+ - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7"
431
- PACKAGE_TYPE="rpm" REPO_TOOL="yum"
432
- ALLOW_SOFT_FAILURE_HERE=true
433
.travis/create_changelog.sh
renamed
+17
-9
@@ -1,16 +1,24 @@
1
-#!/bin/bash
2
-
1
+#!/usr/bin/env bash
2
+#
3
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
4
+#
5
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
6
set -e
7
5
-if [ ! -f .gitignore ]; then
6
- echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
7
- exit 1
8
+# If we are not in netdata git repo, at the top level directory, fail
9
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
10
+CWD=$(git rev-parse --show-cdup || echo "")
11
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
12
+ echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository"
13
+ echo "Changelog creation aborted"
14
+ exit 1
15
fi
16
17
ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}')
18
PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}')
19
GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"}
20
GIT_USER=${GIT_USER:-"netdatabot"}
21
+
22
if [ -z ${GIT_TAG+x} ]; then
23
OPTS=""
24
else
@@ -18,7 +26,7 @@ else
26
fi
27
28
if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
21
- echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do"
29
+ echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do here"
30
exit 0
31
fi
32
@@ -31,8 +39,8 @@ docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest
39
--project "${PROJECT}" \
40
--token "${GITHUB_TOKEN}" \
41
--since-tag "v1.10.0" \
34
- --no-issues \
42
--unreleased-label "**Next release**" \
43
+ --no-issues \
44
--exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \
37
- --max-issues 500 \
38
- --bug-labels IGNOREBUGS
45
+ --max-issues 500 \
46
+ --bug-labels IGNOREBUGS ${OPTS}
.travis/generate_changelog_and_tag_release.sh
+6
-9
@@ -30,14 +30,11 @@ if [ ! -f .gitignore ]; then
30
exit 1
31
fi
32
33
-echo "--- Executing Tagging facility to determine TAG ---"
34
-source .travis/tagger.sh
35
-
36
-echo "--- Changelog generator and tagger script starting ---"
37
-# If tagger script hasn't produced a TAG, there is nothing to do so bail out happy
33
+echo "--- Changelog generator script starting ---"
34
+# If we dont have a produced TAG there is nothing to do, so bail out happy
35
if [ -z "${GIT_TAG}" ]; then
39
- echo "GIT_TAG is empty, nothing to do for now (Value: $GIT_TAG)"
40
- exit 0
36
+ echo "GIT_TAG is empty, that is not suppose to happen (Value: $GIT_TAG)"
37
+ exit 1
38
fi
39
40
if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
@@ -55,8 +52,8 @@ echo "---- UPDATE VERSION FILE ----"
52
echo "$GIT_TAG" >packaging/version
53
git add packaging/version
54
58
-echo "---- GENERATE CHANGELOG -----"
59
-./.travis/generate_changelog_for_release.sh
55
+echo "---- Create CHANGELOG -----"
56
+./.travis/create_changelog.sh
57
git add CHANGELOG.md
58
59
echo "---- COMMIT AND PUSH CHANGES ----"
.travis/generate_changelog_for_nightlies.sh
+4
-25
@@ -1,6 +1,6 @@
1
-#!/bin/bash
1
+#!/usr/bin/env bash
2
#
3
-# Changelog generation scriptlet.
3
+# Changelog generation scriptlet, for nightlies
4
#
5
# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
#
@@ -20,39 +20,18 @@ fi
20
LAST_TAG="$1"
21
COMMITS_SINCE_RELEASE="$2"
22
NEW_VERSION="${LAST_TAG}-$((COMMITS_SINCE_RELEASE + 1))-nightly"
23
-ORG=$(echo "$TRAVIS_REPO_SLUG" | cut -d '/' -f1)
24
-PROJECT=$(echo "$TRAVIS_REPO_SLUG" | cut -d '/' -f 2)
23
GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"}
24
GIT_USER=${GIT_USER:-"netdatabot"}
25
PUSH_URL=$(git config --get remote.origin.url | sed -e 's/^https:\/\///')
26
FAIL=0
29
-if [ -z ${GIT_TAG+x} ]; then
30
- OPTS=""
31
-else
32
- OPTS="--future-release ${GIT_TAG}"
33
-fi
34
-echo "We got $COMMITS_SINCE_RELEASE changes since $LAST_TAG, re-generating changelog"
27
28
if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
29
echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do here"
30
exit 0
31
fi
32
41
-git checkout master
42
-git pull
43
-
44
-echo "Running project markmandel for github changelog generation"
45
-#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \
46
-docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \
47
- --user "${ORG}" \
48
- --project "${PROJECT}" \
49
- --token "${GITHUB_TOKEN}" \
50
- --since-tag "v1.10.0" \
51
- --unreleased-label "**Next release**" \
52
- --no-issues \
53
- --exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \
54
- --max-issues 500 \
55
- --bug-labels IGNOREBUGS
33
+echo "Running changelog creation mechanism"
34
+.travis/create_changelog.sh
35
36
echo "Changelog created! Adding packaging/version(${NEW_VERSION}) and CHANGELOG.md to the repository"
37
echo "${NEW_VERSION}" > packaging/version
.travis/nightlies.sh
+3
-2
@@ -1,4 +1,4 @@
1
-#!/bin/bash
1
+#!/usr/bin/env bash
2
#
3
# This is the nightly changelog generation script
4
# It is responsible for two major activities:
@@ -29,7 +29,7 @@ PREVIOUS_NIGHTLY_COUNT="$(rev <packaging/version | cut -d- -f 2 | rev)"
29
30
# If no commits since release, just stop
31
if [ "${COMMITS_SINCE_RELEASE}" == "${PREVIOUS_NIGHTLY_COUNT}" ]; then
32
- echo "No changes since last nighthly release"
32
+ echo "No changes since last nighthly release, nothing else to do"
33
exit 0
34
fi
35
@@ -39,6 +39,7 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
39
fi
40
41
echo "--- Running Changelog generation ---"
42
+echo "We got $COMMITS_SINCE_RELEASE changes since $LAST_TAG, re-generating changelog"
43
NIGHTLIES_CHANGELOG_FAILED=0
44
.travis/generate_changelog_for_nightlies.sh "${LAST_TAG}" "${COMMITS_SINCE_RELEASE}" || NIGHTLIES_CHANGELOG_FAILED=1
45
.travis/tagger.sh
+27
-43
@@ -1,6 +1,4 @@
1
-#!/bin/bash
2
-#
3
-# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
1
+# #BASH library
2
#
3
# Tags are generated by searching for a keyword in last commit message. Keywords are:
4
# - [patch] or [fix] to bump patch number
@@ -12,8 +10,6 @@
10
# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo
11
# - git-semver python package (pip install git-semver)
12
#
15
-# Note: Exported variables needed by .travis/draft_release.sh
16
-#
13
# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
14
#
15
# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
@@ -21,24 +17,11 @@
17
# Author : Pawel Krupa (paulfantom)
18
# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
19
24
-set -e
25
-
26
-
27
-# If we are not in netdata git repo, at the top level directory, fail
28
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
29
-CWD=$(git rev-parse --show-cdup || echo "")
30
-if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
31
- echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository"
32
- echo "Changelog generation process aborted"
33
- exit 1
34
-fi
35
-
36
-
20
# Figure out what will be new release candidate tag based only on previous ones.
21
# This assumes that RELEASES are in format of "v0.1.2" and prereleases (RCs) are using "v0.1.2-rc0"
22
function set_tag_release_candidate() {
23
LAST_TAG=$(git semver)
41
- echo "Last tag found is: ${LAST_TAG}"
24
+ echo "${0}: Last tag found is: ${LAST_TAG}"
25
26
if [[ $LAST_TAG =~ -rc* ]]; then
27
VERSION=$(echo "$LAST_TAG" | cut -d'-' -f 1)
@@ -47,31 +30,32 @@ function set_tag_release_candidate() {
30
else
31
VERSION="$(git semver --next-minor)"
32
RC=0
50
- echo "Warning: Will set version to ${VERSION} (Last tag: ${LAST_TAG}) while tagged for release candidate generation"
33
+ echo "${0}: Warning: Will set version to ${VERSION} (Last tag: ${LAST_TAG}) while tagged for release candidate generation"
34
fi
35
+
36
GIT_TAG="v${VERSION}-rc${RC}"
37
+ echo "${0}: Generated a new tag, set to: (${GIT_TAG})"
38
}
39
55
-echo "Determining TAG"
56
-
57
-# Check if current commit is tagged or not
58
-GIT_TAG=$(git tag --points-at)
59
-
60
-if [ -z "${GIT_TAG}" ]; then
61
- git semver
62
- # Figure out next tag based on commit message
63
- echo "Last commit message: ${TRAVIS_COMMIT_MESSAGE}"
64
- case "${TRAVIS_COMMIT_MESSAGE}" in
65
- *"[netdata patch release]"*) GIT_TAG="v$(git semver --next-patch)" ;;
66
- *"[netdata minor release]"*) GIT_TAG="v$(git semver --next-minor)" ;;
67
- *"[netdata major release]"*) GIT_TAG="v$(git semver --next-major)" ;;
68
- *"[netdata release candidate]"*) set_tag_release_candidate ;;
69
- *)
70
- echo "Keyword not detected. Exiting..."
71
- exit 0
72
- ;;
73
- esac
74
-fi
75
-
76
-echo "Setting up GIT_TAG to ${GIT_TAG}"
77
-export GIT_TAG
40
+function set_tag_for_release() {
41
+ echo "${0}: Checking for tag existence"
42
+ if [ -z "${GIT_TAG}" ]; then
43
+ echo "${0}: No tag was found, generating a new tag"
44
+ git semver
45
+
46
+ echo "${0}: Last commit message: ${TRAVIS_COMMIT_MESSAGE}"
47
+
48
+ # Figure out next tag based on commit message
49
+ case "${TRAVIS_COMMIT_MESSAGE}" in
50
+ *"[netdata patch release]"*) GIT_TAG="v$(git semver --next-patch)" ;;
51
+ *"[netdata minor release]"*) GIT_TAG="v$(git semver --next-minor)" ;;
52
+ *"[netdata major release]"*) GIT_TAG="v$(git semver --next-major)" ;;
53
+ *"[netdata release candidate]"*) set_tag_release_candidate ;;
54
+ *)
55
+ echo "${0}: Keyword not detected. Nothing to set for GIT_TAG"
56
+ ;;
57
+ esac
58
+ else
59
+ echo "${0}: We seem to already have a GIT_TAG set to (${GIT_TAG})"
60
+ fi
61
+}