Added eBPF collector support to DEB and RPM packages. (#9628)
* Add DEB/RPM package build tests to Travis. * Add working support for bundling eBPF in binary packages. * Show stdout and stderr from commands run in LXC. * Add proper bundling code for libbpf. * Use AC_CHECK_FILE for libbpf.a external dep. This way it gets properly logged in both configure output and the configure log.
Austin S. Hemmelgarn committed
Jul 30, 2020 at 07:09 UTC
8a105bad8188b7581c92811c069676d3110f130b
10 files changed
+99
-8
.travis.yml
+40
-1
@@ -165,6 +165,46 @@ jobs:
165
script: 'for i in $(seq 0 4); do printf "[XXX: Run #%s]\n" "$i";docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:31" tests/updater_checks.sh && break; done'
166
after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 31"
167
168
+ - name: DEB package test
169
+ git:
170
+ depth: false
171
+ before_install:
172
+ - sudo apt-get install -y wget lxc python3-lxc python-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl
173
+ before_script:
174
+ - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
175
+ script:
176
+ - echo "GIT Branch:" && git branch
177
+ - echo "Last commit:" && git log -1
178
+ - echo "GIT Describe:" && git describe
179
+ - echo "packaging/version:" && cat packaging/version
180
+ - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
181
+ - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
182
+ - sudo chown -R root:travis "/var/lib/lxc"
183
+ - sudo chmod -R 750 "/var/lib/lxc"
184
+ - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
185
+ env:
186
+ - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster"
187
+ - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
188
+
189
+ - name: RPM package test
190
+ git:
191
+ depth: false
192
+ before_install:
193
+ - sudo apt-get install -y wget lxc lxc-templates python3-lxc python-lxc
194
+ before_script:
195
+ - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
196
+ script:
197
+ - echo "GIT Branch:" && git branch
198
+ - echo "Last commit:" && git log -1
199
+ - echo "GIT Describe:" && git describe
200
+ - echo "packaging/version:" && cat packaging/version
201
+ - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
202
+ - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
203
+ - sudo chmod -R 755 "/var/lib/lxc"
204
+ - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
205
+ env:
206
+ - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="31" BUILD_STRING="fedora/31"
207
+ - PACKAGE_TYPE="rpm" REPO_TOOL="dnf"
208
209
- stage: Support activities on main branch
210
name: Generate changelog for release (only on special and tagged commit msg)
@@ -181,7 +221,6 @@ jobs:
221
depth: false
222
if: commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ AND tag !~ /(-rc)/ OR (env(GIT_TAG) IS present AND NOT env(GIT_TAG) IS blank)
223
184
-
224
# ###### Packaging workflow section ######
225
# References:
226
# https://us.images.linuxcontainers.org
.travis/package_management/common.py
+1
-1
@@ -51,7 +51,7 @@ def replace_tag(tag_name, spec, new_tag_content):
51
52
def run_command(container, command):
53
print("Running command: %s" % command)
54
- command_result = container.attach_wait(lxc.attach_run_command, command)
54
+ command_result = container.attach_wait(lxc.attach_run_command, command, stdout=sys.stdout.buffer, stderr=sys.stdout.buffer)
55
56
if command_result != 0:
57
raise Exception("Command failed with exit code %d" % command_result)
.travis/package_management/functions.sh
+2
-2
@@ -24,8 +24,8 @@ function detect_arch_from_commit {
24
;;
25
26
*)
27
- echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. No BUILD_ARCH can be provided"
28
- exit 1
27
+ echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. Assuming amd64"
28
+ export BUILD_ARCH="amd64"
29
;;
30
esac
31
Makefile.am
+2
@@ -72,6 +72,8 @@ dist_noinst_DATA = \
72
packaging/mosquitto.version \
73
packaging/mosquitto.checksums \
74
packaging/bundle-dashboard.sh \
75
+ packaging/bundle-ebpf.sh \
76
+ packaging/bundle-libbpf.sh \
77
packaging/bundle-mosquitto.sh \
78
packaging/check-kernel-config.sh \
79
packaging/libbpf.checksums \
configure.ac
+7
-1
@@ -960,11 +960,17 @@ AC_CHECK_TYPE(
960
[#include <linux/bpf.h>]
961
)
962
963
+AC_CHECK_FILE(
964
+ externaldeps/libbpf/libbpf.a,
965
+ [have_libbpf=yes],
966
+ [have_libbpf=no]
967
+)
968
+
969
AC_MSG_CHECKING([if ebpf.plugin should be enabled])
970
if test "${build_target}" = "linux" -a \
971
"${have_libelf}" = "yes" -a \
972
"${have_bpf}" = "yes" -a \
967
- -f externaldeps/libbpf/libbpf.a; then
973
+ "${have_libbpf}" = "yes"; then
974
OPTIONAL_BPF_CFLAGS="${LIBELF_CFLAGS} -I externaldeps/libbpf/include"
975
OPTIONAL_BPF_LIBS="externaldeps/libbpf/libbpf.a ${LIBELF_LIBS}"
976
AC_DEFINE([HAVE_LIBBPF], [1], [libbpf usability])
contrib/debian/rules
+6
@@ -36,6 +36,7 @@ override_dh_installinit:
36
override_dh_auto_configure:
37
packaging/bundle-mosquitto.sh .
38
packaging/bundle-lws.sh .
39
+ packaging/bundle-libbpf.sh .
40
autoreconf -ivf
41
dh_auto_configure -- --prefix=/usr --sysconfdir=/etc --localstatedir=/var --libdir=/usr/lib \
42
--libexecdir=/usr/libexec --with-user=netdata --with-math --with-zlib --with-webdir=/var/lib/netdata/www
@@ -76,6 +77,8 @@ override_dh_install:
77
ln -s "/usr/share/netdata/www/$$D" "$(TOP)/var/lib/netdata/www/$$D"; \
78
done
79
80
+ packaging/bundle-ebpf.sh . ${TOP}/usr/libexec/netdata/plugins.d
81
+
82
# Install go
83
#
84
debian/install_go.sh $$(cat ${CURDIR}/packaging/go.d.version) $(TOP)/usr/lib/netdata $(TOP)/usr/libexec/netdata
@@ -93,6 +96,9 @@ override_dh_installdocs:
96
--target $(TOP)/usr/share/doc/netdata/ \
97
{} \;
98
99
+override_dh_shlibdeps:
100
+ dh_shlibdeps -X libnetdata_ebpf
101
+
102
override_dh_fixperms:
103
dh_fixperms
104
netdata.spec.in
+2
@@ -239,6 +239,7 @@ happened, on your systems and applications.
239
%setup -q -n %{name}-%{version}
240
export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-mosquitto.sh ${RPM_BUILD_DIR}/%{name}-%{version}
241
export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-lws.sh ${RPM_BUILD_DIR}/%{name}-%{version}
242
+export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-libbpf.sh ${RPM_BUILD_DIR}/%{name}-%{version}
243
244
%build
245
# Conf step
@@ -412,6 +413,7 @@ install_go
413
install -m 0640 -p go.d.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/go.d.plugin"
414
415
${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-dashboard.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_datadir}/%{name}/web
416
+${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-ebpf.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d
417
418
%pre
419
packaging/bundle-dashboard.sh
+3
-3
@@ -6,9 +6,9 @@ WEBDIR="${2}"
6
DASHBOARD_TARBALL="dashboard.tar.gz"
7
DASHBOARD_VERSION="$(cat "${SRCDIR}/packaging/dashboard.version")"
8
9
-mkdir -p "${SRCDIR}/tmp"
9
+mkdir -p "${SRCDIR}/tmp/dashboard"
10
curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/dashboard/releases/download/${DASHBOARD_VERSION}/${DASHBOARD_TARBALL}" > "${DASHBOARD_TARBALL}" || exit 1
11
sha256sum -c "${SRCDIR}/packaging/dashboard.checksums" || exit 1
12
-tar -xzf "${DASHBOARD_TARBALL}" -C "${SRCDIR}/tmp" || exit 1
12
+tar -xzf "${DASHBOARD_TARBALL}" -C "${SRCDIR}/tmp/dashboard" || exit 1
13
# shellcheck disable=SC2046
14
-cp -a $(find "${SRCDIR}/tmp/build" -mindepth 1 -maxdepth 1) "${WEBDIR}"
14
+cp -a $(find "${SRCDIR}/tmp/dashboard/build" -mindepth 1 -maxdepth 1) "${WEBDIR}"
packaging/bundle-ebpf.sh
new
+14
@@ -0,0 +1,14 @@
1
+#!/bin/sh
2
+
3
+SRCDIR="${1}"
4
+PLUGINDIR="${2}"
5
+
6
+EBPF_VERSION="$(cat "${SRCDIR}/packaging/ebpf.version")"
7
+EBPF_TARBALL="netdata-kernel-collector-glibc-${EBPF_VERSION}.tar.xz"
8
+
9
+mkdir -p "${SRCDIR}/tmp/ebpf"
10
+curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/kernel-collector/releases/download/${EBPF_VERSION}/${EBPF_TARBALL}" > "${EBPF_TARBALL}" || exit 1
11
+grep "${EBPF_TARBALL}" "${SRCDIR}/packaging/ebpf.checksums" | sha256sum -c - || exit 1
12
+tar -xaf "${EBPF_TARBALL}" -C "${SRCDIR}/tmp/ebpf" || exit 1
13
+# shellcheck disable=SC2046
14
+cp -a $(find "${SRCDIR}/tmp/ebpf" -mindepth 1 -maxdepth 1) "${PLUGINDIR}"
packaging/bundle-libbpf.sh
new
+22
@@ -0,0 +1,22 @@
1
+#!/bin/sh
2
+
3
+LIBBPF_TARBALL="v$(cat "${1}/packaging/libbpf.version").tar.gz"
4
+LIBBPF_BUILD_PATH="${1}/externaldeps/libbpf/libbpf-$(cat "${1}/packaging/libbpf.version")"
5
+
6
+if [ "$(uname -m)" = x86_64 ]; then
7
+ lib_subdir="lib64"
8
+else
9
+ lib_subdir="lib"
10
+fi
11
+
12
+mkdir -p "${1}/externaldeps/libbpf" || exit 1
13
+curl -sSL --connect-timeout 10 --retry 3 "https://github.com/libbpf/libbpf/archive/${LIBBPF_TARBALL}" > "${LIBBPF_TARBALL}" || exit 1
14
+sha256sum -c "${1}/packaging/libbpf.checksums" || exit 1
15
+tar -xzf "${LIBBPF_TARBALL}" -C "${1}/externaldeps/libbpf" || exit 1
16
+LPWD="${PWD}"
17
+cd "${LIBBPF_BUILD_PATH}" || exit 1
18
+patch -p1 < "${1}/libnetdata/ebpf/libbpf.c.diff" || exit 1
19
+cd "${LPWD}" || exit 1
20
+make -C "${LIBBPF_BUILD_PATH}/src" BUILD_STATIC_ONLY=1 OBJDIR=build/ DESTDIR=../ install || exit 1
21
+cp -a "${LIBBPF_BUILD_PATH}/usr/${lib_subdir}/libbpf.a" "${1}/externaldeps/libbpf" || exit 1
22
+cp -a "${LIBBPF_BUILD_PATH}/usr/include" "${1}/externaldeps/libbpf" || exit 1