Handle fetching IBM MQ libraries in CI package build jobs outside of CMake (#21862)
* Support FETCHCONTENT_SOURCE_DIR for IBM MQ libs. This allows us to handle fetching them as a separate step in CI, letting us better handle the errors resulting from IBM’s questioanble infrastructure. * Pre-fetch IBM MQ libraries in package build CI jobs. This lets us better handle infrastructure issues when trying to fetch the libraries. * Fix duplicate CONFIGURE_COMMAND. * Address review comments from copilot. * Fix additional bugs. * Don’t use loop for file handling.
Austin S. Hemmelgarn committed
Mar 6, 2026 at 07:01 UTC
15ffa4faeb78cee19bece455f9ceaf1193a74faa
8 files changed
+131
-5
.github/scripts/prep-ibm-mq-libs.sh
new
+55
@@ -0,0 +1,55 @@
1
+#!/bin/bash
2
+
3
+set -euo pipefail
4
+
5
+url="$(cat packaging/vendor/ibm_mq.url)"
6
+sha256="$(cat packaging/vendor/ibm_mq.sha256)"
7
+file="${url##*/}"
8
+cache_dir="artifacts/cache"
9
+tmp_dir="tmp"
10
+hash_path="${tmp_dir}/hash"
11
+dl_path="${tmp_dir}/${file}"
12
+cache_path="${cache_dir}/${file}"
13
+extract_path="${tmp_dir}/ibm_mq"
14
+need_to_fetch=1
15
+
16
+rm -rf "${extract_path}"
17
+mkdir -p "${tmp_dir}" "${cache_dir}" "${extract_path}"
18
+
19
+hash_valid() {
20
+ local path="${1}"
21
+ echo "${sha256} ${path}" > "${hash_path}"
22
+ if sha256sum -c "${hash_path}"; then
23
+ return 0
24
+ else
25
+ return 1
26
+ fi
27
+}
28
+
29
+if [ -f "${cache_path}" ]; then
30
+ if ! hash_valid "${cache_path}" ; then
31
+ rm -f "${cache_path}"
32
+ else
33
+ need_to_fetch=0
34
+ fi
35
+fi
36
+
37
+if [ "${need_to_fetch}" -eq 1 ]; then
38
+ rm -f "${dl_path}"
39
+ curl --output "${dl_path}" \
40
+ --fail \
41
+ --location \
42
+ --max-time 300 \
43
+ --connect-timeout 90 \
44
+ --retry 5 \
45
+ --retry-all-errors \
46
+ --retry-delay 90 \
47
+ --remote-time \
48
+ "${url}"
49
+
50
+ hash_valid "${dl_path}"
51
+
52
+ cp "${dl_path}" "${cache_path}"
53
+fi
54
+
55
+tar -xvzf "${cache_path}" -C "${extract_path}"
.github/workflows/packaging.yml
+6
-1
@@ -246,6 +246,10 @@ jobs:
246
command: |
247
docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}
248
docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }}
249
+ - name: Fetch IBM MQ libs
250
+ id: fetch-ibm-mq
251
+ if: matrix.arch == 'x86_64' && needs.file-check.outputs.run == 'true'
252
+ run: .github/scripts/prep-ibm-mq-libs.sh
253
- name: Build Packages
254
id: build
255
if: needs.file-check.outputs.run == 'true'
@@ -264,7 +268,7 @@ jobs:
268
uses: actions/upload-artifact@v7.0.0
269
with:
270
name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages
267
- path: ${{ github.workspace }}/artifacts/*
271
+ path: ${{ github.workspace }}/artifacts/*${{ matrix.format }}
272
- name: Test Packages
273
id: test
274
if: needs.file-check.outputs.run == 'true'
@@ -326,6 +330,7 @@ jobs:
330
Checkout: ${{ steps.checkout.outcome }}
331
Setup QEMU: ${{ steps.qemu.outcome }}
332
Fetch images: ${{ steps.fetch-images.outcome }}
333
+ Fetch IBM MQ libs: ${{ steps.fetch-ibm-mq.outcome }}
334
Build: ${{ steps.build.outcome }}
335
Test: ${{ steps.test.outcome }}
336
Import GPG Keys: ${{ steps.import-keys.outcome }}
netdata.spec.in
+1
@@ -448,6 +448,7 @@ advanced correlations and fast root cause analysis, native horizontal scalabilit
448
-DENABLE_PLUGIN_SCRIPTS=On \
449
%if %{_have_ibm_plugin}
450
-DENABLE_PLUGIN_IBM=On \
451
+ $([ -d "%{_topdir}/SOURCES/%{name}-%{version}/tmp/ibm_mq" ] && echo -DFETCHCONTENT_SOURCE_DIR_IBM_MQ=%{_topdir}/SOURCES/%{name}-%{version}/tmp/ibm_mq) \
452
%else
453
-DENABLE_PLUGIN_IBM=Off \
454
%endif
packaging/build-package.sh
+2
@@ -58,6 +58,8 @@ add_cmake_option ENABLE_LIBBACKTRACE On
58
59
add_cmake_option BUILD_FOR_PACKAGING On
60
61
+[ -d "${SOURCE_DIR}/tmp/ibm_mq" ] && add_cmake_option FETCHCONTENT_SOURCE_DIR_IBM_MQ "${SOURCE_DIR}/tmp/ibm_mq"
62
+
63
case "${PKG_TYPE}" in
64
DEB)
65
case "$(dpkg-architecture -q DEB_TARGET_ARCH)" in
packaging/cmake/Modules/NetdataIBMPlugin.cmake
+22
-4
@@ -2,6 +2,7 @@
2
# Build configuration for ibm.d.plugin which requires CGO
3
4
include(FetchContent)
5
+include(NetdataUtil)
6
7
# add_ibm_plugin_target: Add target for building ibm.d.plugin with CGO
8
#
@@ -12,6 +13,8 @@ macro(add_ibm_plugin_target)
13
set(IBM_MQ_BUILD_DIR "${CMAKE_BINARY_DIR}/${IBM_MQ_DIR_NAME}")
14
set(IBM_MQ_INSTALL_SUFFIX "lib/netdata/${IBM_MQ_DIR_NAME}")
15
16
+ get_vendored_url_and_hash(ibm_mq IBM_MQ)
17
+
18
FetchContent_Declare(
19
ibm_mq
20
SOURCE_DIR "${IBM_MQ_BUILD_DIR}"
@@ -19,12 +22,27 @@ macro(add_ibm_plugin_target)
22
BUILD_COMMAND "${CMAKE_COMMAND} -E true"
23
INSTALL_COMMAND "${CMAKE_COMMAND} -E true"
24
DOWNLOAD_NO_PROGRESS True
22
- URL https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/9.4.3.1-IBM-MQC-Redist-LinuxX64.tar.gz
23
- URL_HASH SHA256=9df98ab0ab69954fa080dc7765c9a3eb7cd6c683e5d596ae10c4fbb32eec2ad8
25
+ STEP_TARGETS build
26
+ URL ${IBM_MQ_URL}
27
+ URL_HASH SHA256=${IBM_MQ_HASH}
28
)
29
26
- message(STATUS "Fetching IBM MQ client libraries for build")
27
- FetchContent_MakeAvailable(ibm_mq)
30
+ if(DEFINED FETCHCONTENT_SOURCE_DIR_IBM_MQ)
31
+ message(STATUS "Copying IBM MQ client library sources from ${FETCHCONTENT_SOURCE_DIR_IBM_MQ} to ${IBM_MQ_BUILD_DIR}")
32
+ file(REMOVE_RECURSE "${IBM_MQ_BUILD_DIR}")
33
+ execute_process(
34
+ COMMAND ${CMAKE_COMMAND} -E copy_directory ${FETCHCONTENT_SOURCE_DIR_IBM_MQ} ${IBM_MQ_BUILD_DIR}
35
+ RESULT_VARIABLE IBM_MQ_COPY_STATUS
36
+ )
37
+
38
+ if(IBM_MQ_COPY_STATUS)
39
+ message(FATAL_ERROR "Failed to prepare IBM MQ client library directory")
40
+ endif()
41
+ FetchContent_MakeAvailable(ibm_mq)
42
+ else()
43
+ message(STATUS "Fetching IBM MQ client libraries for build")
44
+ FetchContent_MakeAvailable(ibm_mq)
45
+ endif()
46
47
# Find all Go source files - resolve symlinks for consistent dependency tracking
48
get_filename_component(RESOLVED_SOURCE_DIR "${CMAKE_SOURCE_DIR}" REALPATH)
packaging/cmake/Modules/NetdataUtil.cmake
+43
@@ -241,3 +241,46 @@ function(precompile_python dir component)
241
COMPONENT ${component}
242
)
243
endfunction()
244
+
245
+# Load the URL and hash for a vendored component
246
+function(get_vendored_url_and_hash component prefix)
247
+ set(url_file "${CMAKE_SOURCE_DIR}/packaging/vendor/${component}.url")
248
+ set(hash_file "${CMAKE_SOURCE_DIR}/packaging/vendor/${component}.sha256")
249
+
250
+ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.29)
251
+ if(NOT IS_READABLE ${url_file})
252
+ message(FATAL_ERROR "Unable to read data from ${url_file}")
253
+ endif()
254
+ else()
255
+ if(NOT EXISTS ${url_file})
256
+ message(FATAL_ERROR "Unable to read data from ${url_file}")
257
+ endif()
258
+ endif()
259
+
260
+ file(READ "${url_file}" "url")
261
+ string(STRIP "${url}" "url")
262
+
263
+ if("${url}" STREQUAL "")
264
+ message(FATAL_ERROR "${url_file} is empty")
265
+ endif()
266
+
267
+ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.29)
268
+ if(NOT IS_READABLE ${hash_file})
269
+ message(FATAL_ERROR "Unable to read data from ${hash_file}")
270
+ endif()
271
+ else()
272
+ if(NOT EXISTS ${hash_file})
273
+ message(FATAL_ERROR "Unable to read data from ${hash_file}")
274
+ endif()
275
+ endif()
276
+
277
+ file(READ "${hash_file}" "hash")
278
+ string(STRIP "${hash}" "hash")
279
+
280
+ if("${hash}" STREQUAL "")
281
+ message(FATAL_ERROR "${hash_file} is empty")
282
+ endif()
283
+
284
+ set("${prefix}_URL" "${url}" PARENT_SCOPE)
285
+ set("${prefix}_HASH" "${hash}" PARENT_SCOPE)
286
+endfunction()
packaging/vendor/ibm_mq.sha256
new
+1
@@ -0,0 +1 @@
1
+9df98ab0ab69954fa080dc7765c9a3eb7cd6c683e5d596ae10c4fbb32eec2ad8
packaging/vendor/ibm_mq.url
new
+1
@@ -0,0 +1 @@
1
+https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/9.4.3.1-IBM-MQC-Redist-LinuxX64.tar.gz