Skip Go code in CI if it hasn’t changed. (#17077)
* Skip building Go components for Docker CI if they have not changed. * Properly handle Go code in general checks PR. * Skip Go code in build checks if it hasn’t changed. * Fix linting issues. * Fix propagation of installer flags. * Fix propagation of environment variables through static build process. * Fix handling of extra install options in static builds. * Skip starting the agent in updater checks. * Fix actionlint warning.
Austin S. Hemmelgarn committed
Apr 10, 2024 at 09:38 UTC
5ce422daf086189fc8a97d77c71d5308e355025a
10 files changed
+135
-73
.github/scripts/build-static.sh
+1
-1
@@ -22,7 +22,7 @@ prepare_build() {
22
build_static() {
23
progress "Building static ${BUILDARCH}"
24
(
25
- USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
25
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}" USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
26
) >&2
27
}
28
.github/scripts/run-updater-check.sh
+1
-1
@@ -4,7 +4,7 @@ echo ">>> Installing CI support packages..."
4
/netdata/.github/scripts/ci-support-pkgs.sh
5
mkdir -p /etc/cron.daily # Needed to make auto-update checking work correctly on some platforms.
6
echo ">>> Installing Netdata..."
7
-/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1
7
+/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --dont-start-it --disable-telemetry "${EXTRA_INSTALL_FLAGS:+--local-build-options "${EXTRA_INSTALL_FLAGS}"}" || exit 1
8
echo "::group::>>> Pre-Update Environment File Contents"
9
cat /etc/netdata/.environment
10
echo "::endgroup::"
.github/workflows/build.yml
+46
-17
@@ -25,6 +25,7 @@ jobs:
25
runs-on: ubuntu-latest
26
outputs:
27
run: ${{ steps.check-run.outputs.run }}
28
+ skip-go: ${{ steps.check-go.outputs.skip-go }}
29
steps:
30
- name: Checkout
31
id: checkout
@@ -32,8 +33,8 @@ jobs:
33
with:
34
fetch-depth: 0
35
submodules: recursive
35
- - name: Check files
36
- id: check-files
36
+ - name: Check source files
37
+ id: check-source-files
38
uses: tj-actions/changed-files@v44
39
with:
40
since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
@@ -44,6 +45,19 @@ jobs:
45
**/*.hh
46
**/*.in
47
**/*.patch
48
+ src/aclk/aclk-schemas/
49
+ src/ml/dlib/
50
+ src/fluent-bit/
51
+ src/web/server/h2o/libh2o/
52
+ files_ignore: |
53
+ netdata.spec.in
54
+ **/*.md
55
+ - name: Check build files
56
+ id: check-build-files
57
+ uses: tj-actions/changed-files@v43
58
+ with:
59
+ since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
60
+ files: |
61
**/*.cmake
62
CMakeLists.txt
63
netdata-installer.sh
@@ -59,29 +73,39 @@ jobs:
73
packaging/*.sh
74
packaging/*.version
75
packaging/*.checksums
62
- src/aclk/aclk-schemas/
63
- src/ml/dlib/
64
- src/fluent-bit/
65
- src/web/server/h2o/libh2o/
76
files_ignore: |
67
- netdata.spec.in
77
**/*.md
78
- name: List all changed files in pattern
79
continue-on-error: true
80
env:
72
- ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
81
+ CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
82
+ CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
83
run: |
74
- for file in ${ALL_CHANGED_FILES}; do
84
+ for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
85
echo "$file was changed"
86
done
87
- name: Check Run
88
id: check-run
89
run: |
80
- if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
90
+ if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
91
echo 'run=true' >> "${GITHUB_OUTPUT}"
92
else
93
echo 'run=false' >> "${GITHUB_OUTPUT}"
94
fi
95
+ - name: Check Go
96
+ id: check-go
97
+ env:
98
+ OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
99
+ run: |
100
+ if [ '${{ github.event_name }}' == 'pull_request' ]; then
101
+ if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
102
+ echo 'skip-go=' >> "${GITHUB_OUTPUT}"
103
+ else
104
+ echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
105
+ fi
106
+ else
107
+ echo 'skip-go=' >> "${GITHUB_OUTPUT}"
108
+ fi
109
110
build-dist: # Build the distribution tarball and store it as an artifact.
111
name: Build Distribution Tarball
@@ -204,7 +228,9 @@ jobs:
228
key: ${{ steps.cache-key.outputs.key }}
229
- name: Build
230
if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs.
207
- run: .github/scripts/build-static.sh ${{ matrix.arch }}
231
+ run: |
232
+ export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
233
+ .github/scripts/build-static.sh ${{ matrix.arch }}
234
- name: Build
235
if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
236
id: build
@@ -212,7 +238,9 @@ jobs:
238
with:
239
timeout_minutes: 180
240
max_attempts: 3
215
- command: .github/scripts/build-static.sh ${{ matrix.arch }}
241
+ command: |
242
+ export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
243
+ .github/scripts/build-static.sh ${{ matrix.arch }}
244
- name: Store
245
id: store
246
if: needs.file-check.outputs.run == 'true'
@@ -432,19 +460,19 @@ jobs:
460
if: needs.file-check.outputs.run == 'true'
461
run: |
462
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
435
- /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build'
463
+ /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
464
- name: netdata-installer on ${{ matrix.distro }}, require cloud
465
id: build-cloud
466
if: needs.file-check.outputs.run == 'true'
467
run: |
468
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
441
- /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build'
469
+ /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
470
- name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
471
id: build-no-jsonc
472
if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true'
473
run: |
474
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
447
- /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build'
475
+ /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
476
- name: Failure Notification
477
uses: rtCamp/action-slack-notify@v2
478
env:
@@ -545,8 +573,9 @@ jobs:
573
id: updater-check
574
if: needs.file-check.outputs.run == 'true'
575
run: |
548
- docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata test:${{ matrix.artifact_key }} \
549
- /netdata/.github/scripts/run-updater-check.sh
576
+ docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \
577
+ -e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \
578
+ test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh
579
- name: Failure Notification
580
uses: rtCamp/action-slack-notify@v2
581
env:
.github/workflows/checks.yml
+37
-16
@@ -16,6 +16,7 @@ jobs:
16
runs-on: ubuntu-latest
17
outputs:
18
run: ${{ steps.check-run.outputs.run }}
19
+ skip-go: ${{ steps.check-go.outputs.skip-go }}
20
steps:
21
- name: Checkout
22
id: checkout
@@ -23,8 +24,8 @@ jobs:
24
with:
25
fetch-depth: 0
26
submodules: recursive
26
- - name: Check files
27
- id: check-files
27
+ - name: Check source files
28
+ id: check-source-files
29
uses: tj-actions/changed-files@v44
30
with:
31
since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
@@ -35,41 +36,60 @@ jobs:
36
**/*.hh
37
**/*.in
38
**/*.patch
39
+ src/aclk/aclk-schemas/
40
+ src/ml/dlib/
41
+ src/fluent-bit/
42
+ src/web/server/h2o/libh2o/
43
+ files_ignore: |
44
+ netdata.spec.in
45
+ **/*.md
46
+ - name: Check build files
47
+ id: check-build-files
48
+ uses: tj-actions/changed-files@v43
49
+ with:
50
+ since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
51
+ files: |
52
**/*.cmake
53
CMakeLists.txt
54
.gitignore
55
.github/data/distros.yml
56
.github/workflows/build.yml
43
- .github/scripts/build-static.sh
44
- .github/scripts/get-static-cache-key.sh
45
- .github/scripts/gen-matrix-build.py
46
- .github/scripts/run-updater-check.sh
57
packaging/cmake/
58
packaging/*.version
59
packaging/*.checksums
50
- src/aclk/aclk-schemas/
51
- src/ml/dlib/
52
- src/fluent-bit/
53
- src/web/server/h2o/libh2o/
60
files_ignore: |
55
- netdata.spec.in
61
**/*.md
62
- name: List all changed files in pattern
63
continue-on-error: true
64
env:
60
- ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
65
+ CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
66
+ CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
67
run: |
62
- for file in ${ALL_CHANGED_FILES}; do
68
+ for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
69
echo "$file was changed"
70
done
71
- name: Check Run
72
id: check-run
73
run: |
68
- if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
74
+ if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
75
echo 'run=true' >> "${GITHUB_OUTPUT}"
76
else
77
echo 'run=false' >> "${GITHUB_OUTPUT}"
78
fi
79
+ - name: Check Go
80
+ id: check-go
81
+ env:
82
+ OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
83
+ run: |
84
+ if [ '${{ github.event_name }}' == 'pull_request' ]; then
85
+ if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
86
+ echo 'skip-go=' >> "${GITHUB_OUTPUT}"
87
+ else
88
+ echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
89
+ fi
90
+ else
91
+ echo 'skip-go=' >> "${GITHUB_OUTPUT}"
92
+ fi
93
94
libressl-checks:
95
name: LibreSSL
@@ -94,7 +114,8 @@ jobs:
114
./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata;
115
apk del openssl openssl-dev;
116
apk add libressl libressl-dev protobuf-dev;
97
- ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build;'
117
+ ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build --disable-go;'
118
+
119
clang-checks:
120
name: Clang
121
needs:
@@ -134,7 +155,7 @@ jobs:
155
run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
156
- name: Build netdata
157
if: needs.file-check.outputs.run == 'true'
137
- run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build
158
+ run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build ${{ needs.file-check.outputs.skip-go }}
159
- name: Check that repo is clean
160
if: needs.file-check.outputs.run == 'true'
161
run: |
.github/workflows/docker.yml
+39
-29
@@ -31,6 +31,7 @@ jobs:
31
runs-on: ubuntu-latest
32
outputs:
33
run: ${{ steps.check-run.outputs.run }}
34
+ skip-go: ${{ steps.check-go.outputs.skip-go }}
35
steps:
36
- name: Checkout
37
id: checkout
@@ -39,8 +40,8 @@ jobs:
40
with:
41
fetch-depth: 0
42
submodules: recursive
42
- - name: Check files
43
- id: check-files
43
+ - name: Check source files
44
+ id: check-source-files
45
if: github.event_name != 'workflow_dispatch'
46
uses: tj-actions/changed-files@v44
47
with:
@@ -52,7 +53,20 @@ jobs:
53
**/*.hh
54
**/*.in
55
**/*.patch
55
- **/*.cmake
56
+ src/aclk/aclk-schemas/
57
+ src/ml/dlib/
58
+ src/fluent-bit/
59
+ src/web/server/h2o/libh2o/
60
+ files_ignore: |
61
+ netdata.spec.in
62
+ **/*.md
63
+ - name: Check build system files
64
+ id: check-build-files
65
+ if: github.event_name != 'workflow_dispatch'
66
+ uses: tj-actions/changed-files@v42
67
+ with:
68
+ since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
69
+ files: |
70
.dockerignore
71
CMakeLists.txt
72
netdata-installer.sh
@@ -66,30 +80,40 @@ jobs:
80
packaging/runtime-check.sh
81
packaging/*.version
82
packaging/*.checksums
69
- src/aclk/aclk-schemas/
70
- src/ml/dlib/
71
- src/fluent-bit/
72
- src/web/server/h2o/libh2o/
83
files_ignore: |
74
- netdata.spec.in
84
**/*.md
85
- name: List all changed files in pattern
86
continue-on-error: true
87
if: github.event_name != 'workflow_dispatch'
88
env:
80
- ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
89
+ CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
90
+ CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
91
run: |
82
- for file in ${ALL_CHANGED_FILES}; do
92
+ for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
93
echo "$file was changed"
94
done
95
- name: Check Run
96
id: check-run
97
run: |
88
- if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
98
+ if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
99
echo 'run=true' >> "${GITHUB_OUTPUT}"
100
else
101
echo 'run=false' >> "${GITHUB_OUTPUT}"
102
fi
103
+ - name: Check Go
104
+ id: check-go
105
+ env:
106
+ OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
107
+ run: |
108
+ if [ '${{ github.event_name }}' == 'pull_request' ]; then
109
+ if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
110
+ echo 'skip-go=' >> "${GITHUB_OUTPUT}"
111
+ else
112
+ echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
113
+ fi
114
+ else
115
+ echo 'skip-go=' >> "${GITHUB_OUTPUT}"
116
+ fi
117
118
build-images:
119
name: Build Docker Images
@@ -143,7 +167,9 @@ jobs:
167
tags: netdata/netdata:test
168
load: true
169
cache-to: type=local,dest=/tmp/build-cache,mode=max
146
- build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
170
+ build-args: |
171
+ OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
172
+ EXTRA_INSTALL_OPTS=${{ needs.file-check.outputs.skip-go }}
173
- name: Test Image
174
id: test
175
if: needs.file-check.outputs.run == 'true' && matrix.platform == 'linux/amd64'
@@ -257,24 +283,8 @@ jobs:
283
with:
284
platforms: ${{ matrix.platform }}
285
cache-from: type=local,src=/tmp/build-cache
260
- build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
286
outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true
262
- - name: Export Digest
263
- id: export-digest
264
- if: github.repository == 'netdata/netdata'
265
- run: |
266
- mkdir -p /tmp/digests
267
- digest="${{ steps.build.outputs.digest }}"
268
- touch "/tmp/digests/${digest#sha256:}"
269
- - name: Upload digest
270
- id: upload-digest
271
- if: github.repository == 'netdata/netdata'
272
- uses: actions/upload-artifact@v4
273
- with:
274
- name: docker-digests-${{ steps.artifact-name.outputs.platform }}
275
- path: /tmp/digests/*
276
- if-no-files-found: error
277
- retention-days: 1
287
+ build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
288
- name: Failure Notification
289
uses: rtCamp/action-slack-notify@v2
290
env:
netdata-installer.sh
+1
-1
@@ -211,7 +211,7 @@ USAGE: ${PROGRAM} [options]
211
--disable-dbengine Explicitly disable DB engine support.
212
--enable-plugin-go Enable the Go plugin. Default: Enabled when possible.
213
--disable-plugin-go Disable the Go plugin.
214
- --disable-go Equivalent to --disable-go-plugin
214
+ --disable-go Disable all Go components.
215
--enable-plugin-nfacct Enable nfacct plugin. Default: enable it when libmnl and libnetfilter_acct are available.
216
--disable-plugin-nfacct Explicitly disable the nfacct plugin.
217
--enable-plugin-xenstat Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available.
packaging/makeself/build-static.sh
+6
-5
@@ -1,4 +1,4 @@
1
-#!/usr/bin/env bash
1
+#!/bin/bash
2
3
# SPDX-License-Identifier: GPL-3.0-or-later
4
@@ -54,10 +54,11 @@ fi
54
# Run the build script inside the container
55
if [ -t 1 ]; then
56
run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
57
- --platform "${platform}" "${DOCKER_IMAGE_NAME}" \
58
- /bin/sh /netdata/packaging/makeself/build.sh "${@}"
57
+ --platform "${platform}" ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \
58
+ "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}"
59
else
60
run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
61
- -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" "${DOCKER_IMAGE_NAME}" \
62
- /bin/sh /netdata/packaging/makeself/build.sh "${@}"
61
+ -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" \
62
+ ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \
63
+ "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}"
64
fi
packaging/makeself/build.sh
+1
-1
@@ -1,4 +1,4 @@
1
-#!/usr/bin/env sh
1
+#!/bin/bash
2
# SPDX-License-Identifier: GPL-3.0-or-later
3
4
# -----------------------------------------------------------------------------
packaging/makeself/jobs/70-netdata-git.install.sh
+2
-1
@@ -37,7 +37,8 @@ run ./netdata-installer.sh \
37
--dont-scrub-cflags-even-though-it-may-break-things \
38
--one-time-build \
39
--disable-logsmanagement \
40
- --enable-lto
40
+ --enable-lto \
41
+ ${EXTRA_INSTALL_FLAGS:+${EXTRA_INSTALL_FLAGS}} \
42
43
# shellcheck disable=SC2015
44
[ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Finishing netdata install" || true
packaging/makeself/run-all-jobs.sh
+1
-1
@@ -1,4 +1,4 @@
1
-#!/usr/bin/env bash
1
+#!/bin/bash
2
# SPDX-License-Identifier: GPL-3.0-or-later
3
4
set -e