Migrated release build process from Travis CI to GitHub Actions. (#11055)
* Initial version of release build workflow. * Consolidate static and dist build CI. * Typo fixes. * Ensure dependencies are present for dist tarball build. * Further typo fixes. * Convert Travis CI to trigger newrelease build workflow. * Fix actionlint errors. * Fix distfile name handling. * Fix handling of dist directory.
Austin S. Hemmelgarn committed
Oct 15, 2021 at 11:57 UTC
a3643623bda776dd0123980eb214d3247975179f
6 files changed
+269
-121
.github/scripts/run_install_with_dist_file.sh
+1
-1
@@ -18,7 +18,7 @@ shift
18
19
printf >&2 "Opening dist archive %s ... " "${distfile}"
20
tar -xovf "${distfile}"
21
-distdir="$(echo "${distfile}" | cut -d. -f1,2,3)"
21
+distdir="$(echo "${distfile}" | rev | cut -d. -f3- | rev)"
22
cp -a packaging/installer/install-required-packages.sh "${distdir}/install-required-packages.sh"
23
if [ ! -d "${distdir}" ]; then
24
printf >&2 "ERROR: %s is not a directory" "${distdir}"
.github/workflows/build-and-install.yml
-22
@@ -11,28 +11,6 @@ concurrency:
11
group: builder-${{ github.ref }}
12
cancel-in-progress: true
13
jobs:
14
- static-build:
15
- name: Static Build
16
- runs-on: ubuntu-latest
17
- strategy:
18
- matrix:
19
- arch:
20
- - x86_64
21
- - armv7l
22
- - aarch64
23
- steps:
24
- - name: Git clone repository
25
- uses: actions/checkout@v2
26
- with:
27
- fetch-depth: 0
28
- submodules: recursive
29
- - name: Build
30
- run: .github/scripts/build-static.sh ${{ matrix.arch }}
31
- - name: Save Packages
32
- uses: actions/upload-artifact@v2
33
- with:
34
- name: ${{ matrix.arch }}-static-builds
35
- path: ${{ github.workspace }}/artifacts/*
14
source-build:
15
name: Build & Install
16
strategy:
.github/workflows/checks.yml
-37
@@ -53,43 +53,6 @@ jobs:
53
- name: Build
54
run: |
55
docker build -f .github/dockerfiles/Dockerfile.clang .
56
- dist-checks:
57
- name: Dist
58
- runs-on: ubuntu-latest
59
- steps:
60
- - name: Checkout
61
- uses: actions/checkout@v2
62
- with:
63
- submodules: recursive
64
- - name: Prepare environment
65
- run: |
66
- ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
67
- sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev \
68
- libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler \
69
- libnetfilter-acct-dev
70
- - name: Configure
71
- run: |
72
- autoreconf -ivf
73
- ./configure \
74
- --with-zlib \
75
- --with-math \
76
- --with-user=netdata \
77
- CFLAGS=-O2
78
- - name: Make dist
79
- run: |
80
- make dist
81
- - name: Verify & Set distfile
82
- run: |
83
- ls -lah netdata-*.tar.gz
84
- echo "DISTFILE=$(ls netdata-*.tar.gz)" >> $GITHUB_ENV
85
- - name: Save Packages
86
- uses: actions/upload-artifact@v2
87
- with:
88
- name: dist-archive
89
- path: ${{ github.workspace }}/${{ env.DISTFILE }}
90
- - name: Run run_install_with_dist_file.sh
91
- run: |
92
- ./.github/scripts/run_install_with_dist_file.sh "${DISTFILE}"
56
gitignore-check:
57
name: .gitignore
58
runs-on: ubuntu-latest
.github/workflows/release-build.yml
new
+240
@@ -0,0 +1,240 @@
1
+---
2
+# Ci code for building release artifacts.
3
+name: Release Build
4
+on:
5
+ push: # Master branch checks only validate the build and generate artifacts for testing.
6
+ branches:
7
+ - master
8
+ pull_request: null # PR checks only validate the build and generate artifacts for testing.
9
+ workflow_dispatch:
10
+ inputs:
11
+ type:
12
+ description: Build Type
13
+ default: nightly
14
+ required: true
15
+ version:
16
+ description: Version Tag
17
+ default: nightly
18
+ required: true
19
+concurrency:
20
+ group: release-${{ github.ref }}-${{ github.event_name }}
21
+ cancel-in-progress: true
22
+jobs:
23
+ build-dist: # Build the distribution tarball and store it as an artifact.
24
+ name: Build Distribution Tarball
25
+ runs-on: ubuntu-latest
26
+ steps:
27
+ - name: Checkout
28
+ uses: actions/checkout@v2
29
+ with:
30
+ fetch-depth: 0
31
+ submodules: recursive
32
+ - name: Mark Stable
33
+ if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly'
34
+ run: |
35
+ sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh
36
+ - name: Build
37
+ run: |
38
+ mkdir -p artifacts
39
+ ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
40
+ autoreconf -ivf
41
+ ./configure --prefix=/usr \
42
+ --sysconfdir=/etc \
43
+ --localstatedir=/var \
44
+ --libexecdir=/usr/libexec \
45
+ --with-zlib \
46
+ --with-math \
47
+ --with-user=netdata
48
+ make dist
49
+ echo "DISTFILE=$(find . -name 'netdata-*.tar.gz')" >> "$GITHUB_ENV"
50
+ cp netdata-*.tar.gz artifacts/
51
+ - name: Test
52
+ run: |
53
+ .github/scripts/run_install_with_dist_file.sh "${DISTFILE}"
54
+ - name: Store
55
+ uses: actions/upload-artifact@v2
56
+ with:
57
+ name: dist-tarball
58
+ path: artifacts/*.tar.gz
59
+ retention-days: 30
60
+ - name: Failure Notification
61
+ uses: rtCamp/action-slack-notify@v2
62
+ env:
63
+ SLACK_COLOR: 'danger'
64
+ SLACK_FOOTER: ''
65
+ SLACK_ICON_EMOJI: ':github-actions:'
66
+ SLACK_TITLE: 'Distribution tarball build failed:'
67
+ SLACK_USERNAME: 'GitHub Actions'
68
+ SLACK_MESSAGE: "Distribution tarball build failed."
69
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
70
+ if: >-
71
+ ${{
72
+ failure()
73
+ && startsWith(github.ref, 'refs/heads/master')
74
+ && github.event_name != 'pull_request'
75
+ }}
76
+
77
+ build-static: # Build the static binary archives, and store them as artifacts.
78
+ name: Build Static
79
+ runs-on: ubuntu-latest
80
+ strategy:
81
+ matrix:
82
+ arch:
83
+ - 'x86_64'
84
+ - 'armv7l'
85
+ - 'aarch64'
86
+ steps:
87
+ - name: Checkout
88
+ uses: actions/checkout@v2
89
+ with:
90
+ fetch-depth: 0
91
+ submodules: recursive
92
+ - name: Mark Stable
93
+ if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly'
94
+ run: |
95
+ sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh packaging/makeself/install-or-update.sh
96
+ - name: Build
97
+ run: .github/scripts/build-static.sh ${{ matrix.arch }}
98
+ - name: Store
99
+ uses: actions/upload-artifact@v2
100
+ with:
101
+ name: static-archive
102
+ path: artifacts/*.gz.run
103
+ retention-days: 30
104
+ - name: Failure Notification
105
+ uses: rtCamp/action-slack-notify@v2
106
+ env:
107
+ SLACK_COLOR: 'danger'
108
+ SLACK_FOOTER: ''
109
+ SLACK_ICON_EMOJI: ':github-actions:'
110
+ SLACK_TITLE: 'Static build for ${{ matrix.arch }} failed:'
111
+ SLACK_USERNAME: 'GitHub Actions'
112
+ SLACK_MESSAGE: "Static build for ${{ matrix.arch }} failed."
113
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
114
+ if: >-
115
+ ${{
116
+ failure()
117
+ && startsWith(github.ref, 'refs/heads/master')
118
+ && github.event_name != 'pull_request'
119
+ }}
120
+
121
+ prepare-upload: # Consolidate the artifacts for uploading or releasing.
122
+ name: Prepare Artifacts
123
+ if: github.event_name == 'workflow_dispatch'
124
+ runs-on: ubuntu-latest
125
+ needs:
126
+ - build-dist
127
+ - build-static
128
+ steps:
129
+ - name: Prepare Environment
130
+ run: mkdir -p artifacts
131
+ - name: Retrieve Artifacts
132
+ uses: actions/download-artifact@v2
133
+ - name: Prepare Artifacts
134
+ working-directory: ./artifacts/
135
+ run: |
136
+ mv ../dist-tarball/* artifacts
137
+ mv ../static-archive/* artifcts
138
+ cp ../packaging/version artifacts/latest-version.txt
139
+ sha256sum -b ./* > "sha256sums.txt"
140
+ - name: Store Artifacts
141
+ uses: action/upload-artifact@v2
142
+ with:
143
+ name: final-artifacts
144
+ path: artifacts/*
145
+ retention-days: 30
146
+ - name: Failure Notification
147
+ uses: rtCamp/action-slack-notify@v2
148
+ env:
149
+ SLACK_COLOR: 'danger'
150
+ SLACK_FOOTER: ''
151
+ SLACK_ICON_EMOJI: ':github-actions:'
152
+ SLACK_TITLE: 'Failed to prepare release artifacts for upload:'
153
+ SLACK_USERNAME: 'GitHub Actions'
154
+ SLACK_MESSAGE: "Failed to prepare release artifacts for upload."
155
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
156
+ if: >-
157
+ ${{
158
+ failure()
159
+ && startsWith(github.ref, 'refs/heads/master')
160
+ && github.event_name != 'pull_request'
161
+ }}
162
+
163
+ upload-nightly: # Upload the nightly build artifacts to GCS.
164
+ name: Upload Nightly Artifacts
165
+ runs-on: ubuntu-latest
166
+ if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly'
167
+ needs:
168
+ - prepare-upload
169
+ steps:
170
+ - name: Retrieve Artifacts
171
+ uses: actions/download-artifact@v2
172
+ with:
173
+ name: final-artifacts
174
+ - name: Setup Gcloud
175
+ uses: google-github-actions/setup-gcloud@v0.2.1
176
+ with:
177
+ project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }}
178
+ service_account_key: ${{ secrets.GCP_STORAGE_SERVICE_ACCOUNT_KEY }}
179
+ export_default_credentials: true
180
+ - name: Upload Artifacts
181
+ uses: google-github-actions/upload-cloud-storage@v0.2.1
182
+ with:
183
+ destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }}
184
+ gzip: false
185
+ path: ./
186
+ - name: Failure Notification
187
+ uses: rtCamp/action-slack-notify@v2
188
+ env:
189
+ SLACK_COLOR: 'danger'
190
+ SLACK_FOOTER: ''
191
+ SLACK_ICON_EMOJI: ':github-actions:'
192
+ SLACK_TITLE: 'Failed to upload nightly release artifacts:'
193
+ SLACK_USERNAME: 'GitHub Actions'
194
+ SLACK_MESSAGE: "Failed to upload nightly release artifacts."
195
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
196
+ if: >-
197
+ ${{
198
+ failure()
199
+ && startsWith(github.ref, 'refs/heads/master')
200
+ && github.event_name != 'pull_request'
201
+ }}
202
+
203
+ upload-release: # Create the draft release and upload the build artifacts.
204
+ name: Create Release Draft
205
+ runs-on: ubuntu-latest
206
+ if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release'
207
+ needs:
208
+ - prepare-upload
209
+ steps:
210
+ - name: Checkout
211
+ uses: actions/checkout@v2
212
+ - name: Retrieve Artifacts
213
+ uses: actions/download-artifact@v2
214
+ with:
215
+ name: final-artifacts
216
+ - name: Create Release
217
+ uses: ncipollo/release-action@v1
218
+ with:
219
+ allowUpdates: false
220
+ artifactErrorsFailBuild: true
221
+ artifacts: 'sha256sums.txt,netdata-*.tar.gz,netdata-*.gz.run'
222
+ draft: true
223
+ tag: ${{ github.event.inputs.version }}
224
+ token: ${{ secrets.NETDATABOT_TOKEN }}
225
+ - name: Failure Notification
226
+ uses: rtCamp/action-slack-notify@v2
227
+ env:
228
+ SLACK_COLOR: 'danger'
229
+ SLACK_FOOTER: ''
230
+ SLACK_ICON_EMOJI: ':github-actions:'
231
+ SLACK_TITLE: 'Failed to draft release:'
232
+ SLACK_USERNAME: 'GitHub Actions'
233
+ SLACK_MESSAGE: "Failed to draft release."
234
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
235
+ if: >-
236
+ ${{
237
+ failure()
238
+ && startsWith(github.ref, 'refs/heads/master')
239
+ && github.event_name == 'wworkflow_dispatch'
240
+ }}
.travis.yml
+8
-61
@@ -104,20 +104,11 @@ jobs:
104
105
# We only publish if a TAG has been set during packaging
106
- stage: Publish for release
107
- name: Create release draft
108
- git:
109
- depth: false
110
- env:
111
- - RELEASE_CHANNEL=stable
112
- before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" "${NOTIF_CHANNEL}"
107
+ name: Trigger release build and draft release creation
108
script:
114
- - echo "GIT Branch:" && git branch
115
- - echo "Last commit:" && git log -1
116
- - echo "GIT Describe:" && git describe
117
- - echo "packaging/version:" && cat packaging/version
118
- - echo "Generating release artifacts" && .travis/create_artifacts.sh # Could/should be a common storage to put this and share between jobs
119
- - .travis/draft_release.sh
120
- after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed"
109
+ - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version | sed 's/^v//')"
110
+ - .travis/trigger_release_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "release"
111
+ after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger release artifact build during nightly release" "${NOTIF_CHANNEL}"
112
113
- name: Trigger Docker image build and publish
114
script:
@@ -149,55 +140,11 @@ jobs:
140
# This is the nightly execution step
141
#
142
- stage: Nightly release
152
- name: Create nightly release artifacts, publish to GCS
143
+ name: Trigger nightly artifact build and upload
144
script:
154
- - echo "GIT Branch:" && git branch
155
- - echo "Last commit:" && git log -1
156
- - echo "GIT Describe:" && git describe
157
- - echo "packaging/version:" && cat packaging/version
158
- - .travis/create_artifacts.sh
159
- after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly artifacts generation failed"
160
- git:
161
- depth: false
162
- before_deploy:
163
- echo "Preparing creds under ${TRAVIS_REPO_SLUG}";
164
- if [ "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
165
- openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d;
166
- else
167
- echo "Beta deployment stage in progress";
168
- openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d;
169
- fi;
170
- deploy:
171
- # Beta storage, used for testing purposes
172
- - provider: gcs
173
- edge:
174
- branch: gcs-ng
175
- project_id: netdata-storage
176
- credentials: .travis/gcs-credentials.json
177
- bucket: "netdata-dev-nightlies"
178
- skip_cleanup: true
179
- local_dir: "artifacts"
180
- on:
181
- # Only deploy on netdata/netdata, master branch, when artifacts directory is created
182
- repo: ${TRAVIS_REPO_SLUG}
183
- branch: master
184
- condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} != "netdata/netdata"
185
-
186
- # Production storage
187
- - provider: gcs
188
- edge:
189
- branch: gcs-ng
190
- project_id: netdata-storage
191
- credentials: .travis/gcs-credentials.json
192
- bucket: "netdata-nightlies"
193
- skip_cleanup: true
194
- local_dir: "artifacts"
195
- on:
196
- # Only deploy on netdata/netdata, master branch, when artifacts directory is created
197
- repo: netdata/netdata
198
- branch: master
199
- condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata"
200
- after_deploy: rm -f .travis/gcs-credentials.json
145
+ - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version | sed 's/^v//')"
146
+ - .travis/trigger_release_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "nightly"
147
+ after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger release artifact build during nightly release" "${NOTIF_CHANNEL}"
148
149
- name: Trigger Docker image build and publish
150
script: .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "nightly"
.travis/trigger_artifact_build.sh
new
+20
@@ -0,0 +1,20 @@
1
+#!/bin/sh
2
+
3
+token="${1}"
4
+version="${2}"
5
+type="${3}"
6
+
7
+resp="$(curl -X POST \
8
+ -H 'Accept: application/vnd.github.v3+json' \
9
+ -H "Authorization: Bearer ${token}" \
10
+ "https://api.github.com/repos/netdata/netdata/actions/workflows/release-build.yml/dispatches" \
11
+ -d "{\"ref\": \"master\", \"inputs\": {\"version\": \"${version}\", \"type\": \"${type}\"}}")"
12
+
13
+if [ -z "${resp}" ]; then
14
+ echo "Successfully triggered release artifact build."
15
+ exit 0
16
+else
17
+ echo "Failed to trigger release artifact build. Output:"
18
+ echo "${resp}"
19
+ exit 1
20
+fi