Assorted CI fixes for IBM MQ library handlign in packaging workflow. (#22491)
* Use more robust error reporting for IBM MQ download failures in CI. * Use a single shared download of IBM MQ libs per packaging run. This should eliminate the issues we’ve had with single platforms failing to build due to download failures. * Address review issues. * Correctly fail after too many failed retries. * Better handle max tries and retry delay. * Address review comments.
Austin S. Hemmelgarn committed
Jun 1, 2026 at 09:59 UTC
785ec827b6ffcb4aec66215f7e2395cc89e1a897
2 files changed
+119
-14
.github/scripts/prep-ibm-mq-libs.sh
+62
-12
@@ -9,13 +9,74 @@ cache_dir="artifacts/cache"
9
tmp_dir="tmp"
10
hash_path="${tmp_dir}/hash"
11
dl_path="${tmp_dir}/${file}"
12
+dl_log="${tmp_dir}/dl.log"
13
cache_path="${cache_dir}/${file}"
14
extract_path="${tmp_dir}/ibm_mq"
15
+retry_delay=90
16
+max_tries=5
17
need_to_fetch=1
18
19
rm -rf "${extract_path}"
20
mkdir -p "${tmp_dir}" "${cache_dir}" "${extract_path}"
21
22
+fetch_url() {
23
+ local success=0
24
+ local ret
25
+ local status
26
+
27
+ for i in $(seq "${max_tries}") ; do
28
+ echo "Download attempt ${i}"
29
+ set +e
30
+ rm -f "${dl_log}"
31
+ curl --output "${dl_path}" \
32
+ --fail \
33
+ --location \
34
+ --max-time 300 \
35
+ --connect-timeout 90 \
36
+ --remote-time \
37
+ --write-out "%{http_code}" \
38
+ "${url}" > "${dl_log}"
39
+ ret="$?"
40
+ set -e
41
+
42
+ case "${ret}" in
43
+ 0)
44
+ echo "Download successful."
45
+ success=1
46
+ break
47
+ ;;
48
+ 22|78)
49
+ status="$(tail -n 1 "${dl_log}")"
50
+ case "${status}" in
51
+ 403|404)
52
+ echo "Download failed, got ${status} from remote server."
53
+ exit 1
54
+ ;;
55
+ *) echo "Download failed, got ${status} from remote server, retrying in ${retry_delay} seconds." ;;
56
+ esac
57
+ ;;
58
+ 28) echo "Operation timed out, retrying in ${retry_delay} seconds." ;;
59
+ 18|52) echo "Incorrect amount of data returned, retrying in ${retry_delay} seconds." ;;
60
+ 56|92|95) echo "Network communications error, retrying in ${retry_delay} seconds." ;;
61
+ 5|6|7) echo "Failed to connect to remote server, retrying in ${retry_delay} seconds." ;;
62
+ 35|60|83) echo "TLS error connecting to remote server, retrying in ${retry_delay} seconds." ;;
63
+ *)
64
+ echo "Unknown error (exit code ${ret}) downloading remote file."
65
+ exit 1
66
+ ;;
67
+ esac
68
+
69
+ if [ "${i}" -ne "${max_tries}" ]; then
70
+ sleep "${retry_delay}"
71
+ fi
72
+ done
73
+
74
+ if [ "${success}" -ne 1 ]; then
75
+ echo "Exhausted all retry attempts."
76
+ exit 1
77
+ fi
78
+}
79
+
80
hash_valid() {
81
local path="${1}"
82
echo "${sha256} ${path}" > "${hash_path}"
@@ -36,19 +97,8 @@ fi
97
98
if [ "${need_to_fetch}" -eq 1 ]; then
99
rm -f "${dl_path}"
39
- curl --output "${dl_path}" \
40
- --fail \
41
- --location \
42
- --max-time 300 \
43
- --connect-timeout 90 \
44
- --retry 5 \
45
- --retry-all-errors \
46
- --retry-delay 90 \
47
- --remote-time \
48
- "${url}"
49
-
100
+ fetch_url
101
hash_valid "${dl_path}"
51
-
102
cp "${dl_path}" "${cache_path}"
103
fi
104
.github/workflows/packaging.yml
+57
-2
@@ -190,6 +190,57 @@ jobs:
190
&& github.repository == 'netdata/netdata'
191
}}
192
193
+ prepare-ibm-mq-libs:
194
+ name: Prepare IBM MQ client libraries for build
195
+ runs-on: ubuntu-latest
196
+ needs:
197
+ - file-check
198
+ steps:
199
+ - name: Skip Check
200
+ id: skip
201
+ if: needs.file-check.outputs.run != 'true'
202
+ run: echo "SKIPPED"
203
+ - name: Checkout
204
+ id: checkout
205
+ if: needs.file-check.outputs.run == 'true'
206
+ uses: actions/checkout@v6
207
+ with:
208
+ fetch-depth: 1
209
+ submodules: false
210
+ - name: Fetch IBM MQ libs
211
+ id: fetch-ibm-mq
212
+ if: needs.file-check.outputs.run == 'true'
213
+ run: .github/scripts/prep-ibm-mq-libs.sh
214
+ - name: Upload IBM MQ libs artifact
215
+ id: upload-ibm-mq
216
+ if: needs.file-check.outputs.run == 'true'
217
+ uses: actions/upload-artifact@v7.0.0
218
+ with:
219
+ name: ibm-mq-libs
220
+ path: ${{ github.workspace }}/tmp/ibm_mq
221
+ include-hidden-files: true
222
+ - name: Failure Notification
223
+ uses: rtCamp/action-slack-notify@v2
224
+ env:
225
+ SLACK_COLOR: 'danger'
226
+ SLACK_ICON_EMOJI: ':github-actions:'
227
+ SLACK_TITLE: 'Packaging IBM MQ libs fetch failed:'
228
+ SLACK_USERNAME: 'GitHub Actions'
229
+ SLACK_MESSAGE: |-
230
+ ${{ github.repository }}: Failed to fetch IBM MQ libs for package builds.
231
+ Checkout: ${{ steps.checkout.outcome }}
232
+ Fetch IBM MQ libs: ${{ steps.fetch-ibm-mq.outcome }}
233
+ Upload IBM MQ libs: ${{ steps.upload-ibm-mq.outcome }}
234
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
235
+ if: >-
236
+ ${{
237
+ failure()
238
+ && github.event_name != 'pull_request'
239
+ && startsWith(github.ref, 'refs/heads/master')
240
+ && github.repository == 'netdata/netdata'
241
+ && needs.file-check.outputs.run == 'true'
242
+ }}
243
+
244
prepare-topology-ip-intel-stock:
245
name: Prepare topology-ip-intel stock payload
246
runs-on: ubuntu-latest
@@ -273,6 +324,7 @@ jobs:
324
- matrix
325
- version-check
326
- file-check
327
+ - prepare-ibm-mq-libs
328
- prepare-topology-ip-intel-stock
329
strategy:
330
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
@@ -323,10 +375,13 @@ jobs:
375
command: |
376
docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}
377
docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }}
326
- - name: Fetch IBM MQ libs
378
+ - name: Download IBM MQ libs
379
id: fetch-ibm-mq
380
if: (matrix.arch == 'amd64' || matrix.arch == 'x86_64') && needs.file-check.outputs.run == 'true'
329
- run: .github/scripts/prep-ibm-mq-libs.sh
381
+ uses: actions/download-artifact@v8
382
+ with:
383
+ name: ibm-mq-libs
384
+ path: ${{ github.workspace }}/tmp/ibm_mq
385
- name: Download topology-ip-intel stock payload
386
id: fetch-topology-stock
387
if: needs.file-check.outputs.run == 'true'