master
yml 491 lines 19.2 KB
Raw
1 ---
2 # Handles building of binary packages for the agent.
3 name: Packages
4 on:
5 pull_request:
6 types:
7 - opened
8 - reopened
9 - labeled
10 - synchronize
11 push:
12 branches:
13 - master
14 workflow_dispatch:
15 inputs:
16 type:
17 description: Package build type
18 default: devel
19 required: true
20 version:
21 description: Package version
22 required: false
23 env:
24 DISABLE_TELEMETRY: 1
25 REPO_PREFIX: netdata/netdata
26 concurrency:
27 group: packages-${{ github.ref }}-${{ github.event_name }}
28 cancel-in-progress: true
29 jobs:
30 file-check: # Check what files changed if we’re being run in a PR or on a push.
31 name: Check Modified Files
32 runs-on: ubuntu-latest
33 outputs:
34 run: ${{ steps.check-run.outputs.run }}
35 steps:
36 - name: Checkout
37 id: checkout
38 uses: actions/checkout@v6
39 with:
40 fetch-depth: 0
41 submodules: recursive
42 - name: Check files
43 id: check-files
44 uses: step-security/changed-files@v45
45 with:
46 since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
47 files: |
48 **/*.c
49 **/*.cc
50 **/*.h
51 **/*.hh
52 **/*.in
53 **/*.patch
54 **/*.cmake
55 netdata.spec.in
56 CMakeLists.txt
57 .github/data/distros.yml
58 .github/workflows/packaging.yml
59 .github/scripts/gen-matrix-packaging.py
60 .github/scripts/prepare-topology-ip-intel-stock.sh
61 .github/scripts/pkg-test.sh
62 packaging/cmake/
63 packaging/*.sh
64 packaging/*.version
65 packaging/*.checksums
66 src/go/tools/topology-ip-intel-downloader/**
67 src/crates/
68 src/aclk/aclk-schemas/
69 src/ml/dlib/
70 files_ignore: |
71 src/go/otel-collector/release-config.yaml.in
72 **/*.md
73 packaging/repoconfig/
74 - name: List all changed files in pattern
75 continue-on-error: true
76 env:
77 ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
78 run: |
79 for file in ${ALL_CHANGED_FILES}; do
80 echo "$file was changed"
81 done
82 - name: Check Run
83 id: check-run
84 run: |
85 if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
86 echo 'run=true' >> "${GITHUB_OUTPUT}"
87 else
88 echo 'run=false' >> "${GITHUB_OUTPUT}"
89 fi
90
91 matrix:
92 name: Prepare Build Matrix
93 runs-on: ubuntu-latest
94 outputs:
95 matrix: ${{ steps.set-matrix.outputs.matrix }}
96 steps:
97 - name: Checkout
98 id: checkout
99 uses: actions/checkout@v6
100 - name: Prepare tools
101 id: prepare
102 run: |
103 sudo apt-get update || true
104 sudo apt-get install -y python3-ruamel.yaml
105 - name: Read build matrix
106 id: set-matrix
107 run: |
108 if [ "${{ github.event_name }}" = "pull_request" ] && \
109 [ "${{ !contains(github.event.pull_request.labels.*.name, 'run-ci/packaging') }}" = "true" ]; then
110 matrix="$(.github/scripts/gen-matrix-packaging.py 1)"
111 else
112 matrix="$(.github/scripts/gen-matrix-packaging.py 0)"
113 fi
114 echo "Generated matrix: ${matrix}"
115 echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
116 - name: Failure Notification
117 uses: rtCamp/action-slack-notify@v2
118 env:
119 SLACK_COLOR: 'danger'
120 SLACK_ICON_EMOJI: ':github-actions:'
121 SLACK_TITLE: 'Package Build matrix generation failed:'
122 SLACK_USERNAME: 'GitHub Actions'
123 SLACK_MESSAGE: |-
124 ${{ github.repository }}: Failed to generate build matrix for package build.
125 Checkout: ${{ steps.checkout.outcome }}
126 Prepare Tools: ${{ steps.prepare.outcome }}
127 Read Build Matrix: ${{ steps.set-matrix.outcome }}
128 SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
129 if: >-
130 ${{
131 failure()
132 && github.event_name != 'pull_request'
133 && startsWith(github.ref, 'refs/heads/master')
134 && github.repository == 'netdata/netdata'
135 }}
136
137 version-check:
138 name: Version check
139 runs-on: ubuntu-latest
140 outputs:
141 repo: ${{ steps.check-version.outputs.repo }}
142 version: ${{ steps.check-version.outputs.version }}
143 retention: ${{ steps.check-version.outputs.retention }}
144 steps:
145 - name: Checkout
146 id: checkout
147 uses: actions/checkout@v6
148 - name: Check Version
149 id: check-version
150 run: |
151 if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
152 case "${{ github.event.inputs.type }}" in
153 "release")
154 echo "repo=${REPO_PREFIX}" >> "${GITHUB_OUTPUT}"
155 echo "version=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
156 echo "retention=365" >> "${GITHUB_OUTPUT}"
157 ;;
158 "nightly")
159 echo "repo=${REPO_PREFIX}-edge" >> "${GITHUB_OUTPUT}"
160 echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
161 echo "retention=30" >> "${GITHUB_OUTPUT}"
162 ;;
163 *)
164 echo "repo=${REPO_PREFIX}-devel" >> "${GITHUB_OUTPUT}"
165 echo "version=0.${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
166 echo "retention=30" >> "${GITHUB_OUTPUT}"
167 ;;
168 esac
169 else
170 echo "version=$(cut -d'-' -f 1 packaging/version | tr -d 'v')" >> "${GITHUB_OUTPUT}"
171 echo "retention=0" >> "${GITHUB_OUTPUT}"
172 fi
173 - name: Failure Notification
174 uses: rtCamp/action-slack-notify@v2
175 env:
176 SLACK_COLOR: 'danger'
177 SLACK_ICON_EMOJI: ':github-actions:'
178 SLACK_TITLE: 'Package Build version check failed:'
179 SLACK_USERNAME: 'GitHub Actions'
180 SLACK_MESSAGE: |-
181 ${{ github.repository }}: Failed to generate version information for package build.
182 Checkout: ${{ steps.checkout.outcome }}
183 Check Version: ${{ steps.check-version.outcome }}
184 SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
185 if: >-
186 ${{
187 failure()
188 && github.event_name != 'pull_request'
189 && startsWith(github.ref, 'refs/heads/master')
190 && github.repository == 'netdata/netdata'
191 }}
192
193 prepare-ibm-mq-libs:
194 name: Prepare IBM MQ client libraries for build
195 runs-on: ubuntu-latest
196 needs:
197 - file-check
198 steps:
199 - name: Skip Check
200 id: skip
201 if: needs.file-check.outputs.run != 'true'
202 run: echo "SKIPPED"
203 - name: Checkout
204 id: checkout
205 if: needs.file-check.outputs.run == 'true'
206 uses: actions/checkout@v6
207 with:
208 fetch-depth: 1
209 submodules: false
210 - name: Fetch IBM MQ libs
211 id: fetch-ibm-mq
212 if: needs.file-check.outputs.run == 'true'
213 run: .github/scripts/prep-ibm-mq-libs.sh
214 - name: Upload IBM MQ libs artifact
215 id: upload-ibm-mq
216 if: needs.file-check.outputs.run == 'true'
217 uses: actions/upload-artifact@v7.0.0
218 with:
219 name: ibm-mq-libs
220 path: ${{ github.workspace }}/tmp/ibm_mq
221 include-hidden-files: true
222 - name: Failure Notification
223 uses: rtCamp/action-slack-notify@v2
224 env:
225 SLACK_COLOR: 'danger'
226 SLACK_ICON_EMOJI: ':github-actions:'
227 SLACK_TITLE: 'Packaging IBM MQ libs fetch failed:'
228 SLACK_USERNAME: 'GitHub Actions'
229 SLACK_MESSAGE: |-
230 ${{ github.repository }}: Failed to fetch IBM MQ libs for package builds.
231 Checkout: ${{ steps.checkout.outcome }}
232 Fetch IBM MQ libs: ${{ steps.fetch-ibm-mq.outcome }}
233 Upload IBM MQ libs: ${{ steps.upload-ibm-mq.outcome }}
234 SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
235 if: >-
236 ${{
237 failure()
238 && github.event_name != 'pull_request'
239 && startsWith(github.ref, 'refs/heads/master')
240 && github.repository == 'netdata/netdata'
241 && needs.file-check.outputs.run == 'true'
242 }}
243
244 prepare-topology-ip-intel-stock:
245 name: Prepare topology-ip-intel stock payload
246 runs-on: ubuntu-latest
247 needs:
248 - file-check
249 steps:
250 - name: Skip Check
251 id: skip
252 if: needs.file-check.outputs.run != 'true'
253 run: echo "SKIPPED"
254 - name: Checkout
255 id: checkout
256 if: needs.file-check.outputs.run == 'true'
257 uses: actions/checkout@v6
258 with:
259 fetch-depth: 0
260 submodules: recursive
261 - name: Setup Go
262 id: setup-go
263 if: needs.file-check.outputs.run == 'true'
264 uses: actions/setup-go@v6
265 with:
266 go-version-file: src/go/go.mod
267 - name: Prepare synthetic stock payload
268 id: prepare-synthetic
269 if: needs.file-check.outputs.run == 'true' && github.event_name == 'pull_request'
270 run: |
271 .github/scripts/prepare-topology-ip-intel-stock.sh \
272 --mode synthetic \
273 --output-dir "${GITHUB_WORKSPACE}/.topology-ip-intel-stock"
274 - name: Prepare release stock payload
275 id: prepare-release
276 if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request'
277 uses: nick-invision/retry@v4
278 with:
279 max_attempts: 3
280 retry_wait_seconds: 30
281 timeout_seconds: 1800
282 command: >
283 .github/scripts/prepare-topology-ip-intel-stock.sh
284 --mode release
285 --output-dir "${GITHUB_WORKSPACE}/.topology-ip-intel-stock"
286 - name: Upload topology-ip-intel stock payload
287 id: upload-stock
288 if: needs.file-check.outputs.run == 'true'
289 uses: actions/upload-artifact@v7.0.0
290 with:
291 name: topology-ip-intel-stock
292 path: ${{ github.workspace }}/.topology-ip-intel-stock/
293 include-hidden-files: true
294 - name: Failure Notification
295 uses: rtCamp/action-slack-notify@v2
296 env:
297 SLACK_COLOR: 'danger'
298 SLACK_ICON_EMOJI: ':github-actions:'
299 SLACK_TITLE: 'Topology stock payload preparation failed:'
300 SLACK_USERNAME: 'GitHub Actions'
301 SLACK_MESSAGE: |-
302 ${{ github.repository }}: Failed to prepare topology-ip-intel stock payload.
303 Checkout: ${{ steps.checkout.outcome }}
304 Setup Go: ${{ steps.setup-go.outcome }}
305 Prepare synthetic payload: ${{ steps.prepare-synthetic.outcome }}
306 Prepare release payload: ${{ steps.prepare-release.outcome }}
307 Upload stock payload: ${{ steps.upload-stock.outcome }}
308 SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
309 if: >-
310 ${{
311 failure()
312 && github.event_name != 'pull_request'
313 && startsWith(github.ref, 'refs/heads/master')
314 && github.repository == 'netdata/netdata'
315 && needs.file-check.outputs.run == 'true'
316 }}
317
318 build:
319 name: Build
320 runs-on: ${{ matrix.runner }}
321 env:
322 DOCKER_CLI_EXPERIMENTAL: enabled
323 needs:
324 - matrix
325 - version-check
326 - file-check
327 - prepare-ibm-mq-libs
328 - prepare-topology-ip-intel-stock
329 strategy:
330 matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
331 # We intentiaonally disable the fail-fast behavior so that a
332 # build failure for one version doesn't prevent us from publishing
333 # successfully built and tested packages for another version.
334 fail-fast: false
335 max-parallel: 16
336 steps:
337 - name: Skip Check
338 id: skip
339 if: needs.file-check.outputs.run != 'true'
340 run: echo "SKIPPED"
341 - name: Checkout
342 id: checkout
343 if: needs.file-check.outputs.run == 'true'
344 uses: actions/checkout@v6
345 with:
346 fetch-depth: 0 # We need full history for versioning
347 submodules: recursive
348 - name: Set Sentry telemetry env vars
349 id: set-telemetry-env-vars
350 run: |
351 if [ "${{ github.repository }}" = 'netdata/netdata' ] && \
352 [ "${{ matrix.bundle_sentry }}" = 'true' ] && \
353 [ "${{ github.event_name }}" = 'workflow_dispatch' ]; then
354 echo "RELEASE_PIPELINE=Production" >> "${GITHUB_ENV}"
355 echo "UPLOAD_SENTRY=true" >> "${GITHUB_ENV}"
356 else
357 echo "RELEASE_PIPELINE=Unknown" >> "${GITHUB_ENV}"
358 echo "UPLOAD_SENTRY=false" >> "${GITHUB_ENV}"
359 fi
360 - name: Setup QEMU
361 id: qemu
362 if: matrix.qemu && needs.file-check.outputs.run == 'true'
363 run: |
364 sudo apt-get update
365 sudo apt-get upgrade -y
366 sudo apt-get install -y qemu-user-static
367 - name: Fetch images
368 id: fetch-images
369 if: needs.file-check.outputs.run == 'true'
370 uses: nick-invision/retry@v4
371 with:
372 max_attempts: 3
373 retry_wait_seconds: 30
374 timeout_seconds: 900
375 command: |
376 docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}
377 docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }}
378 - name: Download IBM MQ libs
379 id: fetch-ibm-mq
380 if: (matrix.arch == 'amd64' || matrix.arch == 'x86_64') && needs.file-check.outputs.run == 'true'
381 uses: actions/download-artifact@v8
382 with:
383 name: ibm-mq-libs
384 path: ${{ github.workspace }}/tmp/ibm_mq
385 - name: Download topology-ip-intel stock payload
386 id: fetch-topology-stock
387 if: needs.file-check.outputs.run == 'true'
388 uses: actions/download-artifact@v8
389 with:
390 name: topology-ip-intel-stock
391 path: ${{ github.workspace }}/.topology-ip-intel-stock
392 - name: Build Packages
393 id: build
394 if: needs.file-check.outputs.run == 'true'
395 shell: bash
396 run: |
397 docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e VERSION=${{ needs.version-check.outputs.version }} \
398 -e ENABLE_SENTRY=${{ matrix.bundle_sentry }} -e RELEASE_PIPELINE=${{ env.RELEASE_PIPELINE }} \
399 -e BUILD_DESTINATION=${{ matrix.distro }}${{ matrix.version }}_${{ matrix.arch }} -e UPLOAD_SENTRY=${{ env.UPLOAD_SENTRY }} \
400 -e SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_CLI_TOKEN }} -e NETDATA_SENTRY_DSN=${{ secrets.SENTRY_DSN }} \
401 -e NETDATA_TOPOLOGY_IP_INTEL_STOCK_DIR=/netdata/.topology-ip-intel-stock \
402 -e GOOS=$(echo ${{ matrix.platform }} | cut -f 1 -d '/') -e GOARCH=$(echo ${{ matrix.platform }} | cut -f 2 -d '/') \
403 --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }}
404 - name: Save Packages
405 id: artifacts
406 if: needs.file-check.outputs.run == 'true'
407 continue-on-error: true
408 uses: actions/upload-artifact@v7.0.1
409 with:
410 name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages
411 path: ${{ github.workspace }}/artifacts/*${{ matrix.format }}
412 compression-level: 0
413 - name: Test Packages
414 id: test
415 if: needs.file-check.outputs.run == 'true'
416 shell: bash
417 run: |
418 docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e DISTRO=${{ matrix.distro }} \
419 -e VERSION=${{ needs.version-check.outputs.version }} -e DISTRO_VERSION=${{ matrix.version }} \
420 --platform=${{ matrix.platform }} -v "$PWD":/netdata ${{ matrix.base_image }} \
421 /netdata/.github/scripts/pkg-test.sh
422 - name: Import GPG Keys
423 id: import-keys
424 if: needs.file-check.outputs.run == 'true' && matrix.format == 'deb' && github.event_name != 'pull_request'
425 uses: crazy-max/ghaction-import-gpg@v7
426 with:
427 gpg_private_key: ${{ secrets.NETDATABOT_PACKAGE_SIGNING_KEY }}
428 - name: Sign DEB Packages
429 id: sign-deb
430 if: needs.file-check.outputs.run == 'true' && matrix.format == 'deb' && github.event_name != 'pull_request'
431 shell: bash
432 run: .github/scripts/deb-sign.sh artifacts ${{ steps.import-keys.outputs.fingerprint }}
433 - name: SSH setup
434 id: ssh-setup
435 if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true'
436 uses: shimataro/ssh-key-action@v2
437 with:
438 key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
439 name: id_ecdsa
440 known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
441 - name: Upload to packages.netdata.cloud
442 id: package-upload
443 continue-on-error: true
444 if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true'
445 run: |
446 .github/scripts/package-upload.sh \
447 packages.netdata.cloud \
448 ${{ matrix.repo_distro }} \
449 ${{ matrix.arch }} \
450 ${{ matrix.format }} \
451 ${{ needs.version-check.outputs.repo }}
452 - name: Upload to packages2.netdata.cloud
453 id: package2-upload
454 if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true'
455 run: |
456 .github/scripts/package-upload.sh \
457 packages2.netdata.cloud \
458 ${{ matrix.repo_distro }} \
459 ${{ matrix.arch }} \
460 ${{ matrix.format }} \
461 ${{ needs.version-check.outputs.repo }}
462 - name: Failure Notification
463 uses: rtCamp/action-slack-notify@v2
464 env:
465 SLACK_COLOR: 'danger'
466 SLACK_ICON_EMOJI: ':github-actions:'
467 SLACK_TITLE: 'Package Build failed:'
468 SLACK_USERNAME: 'GitHub Actions'
469 SLACK_MESSAGE: |-
470 ${{ github.repository }}: ${{ matrix.repo_distro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed.
471 Checkout: ${{ steps.checkout.outcome }}
472 Setup QEMU: ${{ steps.qemu.outcome }}
473 Fetch images: ${{ steps.fetch-images.outcome }}
474 Fetch IBM MQ libs: ${{ steps.fetch-ibm-mq.outcome }}
475 Fetch topology stock payload: ${{ steps.fetch-topology-stock.outcome }}
476 Build: ${{ steps.build.outcome }}
477 Test: ${{ steps.test.outcome }}
478 Import GPG Keys: ${{ steps.import-keys.outcome }}
479 Sign DEB Packages: ${{ steps.sign-deb.outcome }}
480 Import SSH Key: ${{ steps.ssh-setup.outcome }}
481 Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }}
482 Publish to packages2.netdata.cloud: ${{ steps.package2-upload.outcome }}
483 SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
484 if: >-
485 ${{
486 failure()
487 && github.event_name != 'pull_request'
488 && startsWith(github.ref, 'refs/heads/master')
489 && github.repository == 'netdata/netdata'
490 && needs.file-check.outputs.run == 'true'
491 }}