Added support for bundling protobuf as part of the install. (#11374)
* Add support for bundling protobu as part of the install. * Fix typo. * Fix `make dist`. * Fix handling of protobuf usage. * Add explicit check for ACLK-NG in builds. * only protos in dist from aclk-schemas Co-authored-by: Timotej Šiškovič <timotej@netdata.cloud>
Austin S. Hemmelgarn committed
Aug 16, 2021 at 13:04 UTC
ade4b8798173142fd7cf70be949343217dbe72b0
7 files changed
+118
-2
.github/workflows/build-and-install.yml
+3
@@ -124,6 +124,9 @@ jobs:
124
- name: netdata-installer on ${{ matrix.distro }}, require cloud
125
run: |
126
docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
127
+ - name: netdata-installer on ${{ matrix.distro }}, require cloud, require ACLK-NG
128
+ run: |
129
+ docker run -w /netdata -e NETDATA_CONFIGURE_OPTIONS='--with-aclk-ng' test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
130
- name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
131
if: matrix.rmjsonc != ''
132
run: |
Makefile.am
+18
-1
@@ -57,7 +57,6 @@ dist_noinst_DATA = \
57
contrib \
58
docs \
59
mqtt_websockets \
60
- aclk/aclk-schemas \
60
netdata.cppcheck \
61
netdata.spec \
62
package.json \
@@ -84,6 +83,8 @@ dist_noinst_DATA = \
83
packaging/libwebsockets.version \
84
packaging/mosquitto.checksums \
85
packaging/mosquitto.version \
86
+ packaging/protobuf.checksums \
87
+ packaging/protobuf.version \
88
packaging/version \
89
$(NULL)
90
@@ -599,6 +600,22 @@ ACLK_NG_FILES = \
600
aclk/schema-wrappers/schema_wrapper_utils.h \
601
$(NULL)
602
603
+ACLK_NG_PROTO_DEFINITIONS = \
604
+ aclk/aclk-schemas/proto/aclk/v1/lib.proto \
605
+ aclk/aclk-schemas/proto/agent/v1/connection.proto \
606
+ aclk/aclk-schemas/proto/alarm/v1/config.proto \
607
+ aclk/aclk-schemas/proto/alarm/v1/stream.proto \
608
+ aclk/aclk-schemas/proto/chart/v1/config.proto \
609
+ aclk/aclk-schemas/proto/chart/v1/dimension.proto \
610
+ aclk/aclk-schemas/proto/chart/v1/instance.proto \
611
+ aclk/aclk-schemas/proto/chart/v1/stream.proto \
612
+ aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto \
613
+ aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto \
614
+ aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto \
615
+ $(NULL)
616
+
617
+dist_noinst_DATA += $(ACLK_NG_PROTO_DEFINITIONS)
618
+
619
ACLK_NG_PROTO_BUILT_FILES = aclk/aclk-schemas/proto/agent/v1/connection.pb.cc \
620
aclk/aclk-schemas/proto/agent/v1/connection.pb.h \
621
aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.pb.cc \
aclk/schema-wrappers/schema_wrapper_utils.h
+1
@@ -3,6 +3,7 @@
3
#ifndef SCHEMA_WRAPPER_UTILS_H
4
#define SCHEMA_WRAPPER_UTILS_H
5
6
+#include <sys/time.h>
7
#include <google/protobuf/timestamp.pb.h>
8
9
#if GOOGLE_PROTOBUF_VERSION < 3001000
netdata-installer.sh
+79
-1
@@ -236,6 +236,7 @@ USAGE: ${PROGRAM} [options]
236
--disable-lto
237
--disable-x86-sse Disable SSE instructions. By default SSE optimizations are enabled.
238
--use-system-lws Use a system copy of libwebsockets instead of bundling our own (default is to use the bundled copy).
239
+ --use-system-protobuf Use a system copy of libprotobuf instead of bundling our own (default is to use the bundled copy).
240
--zlib-is-really-here or
241
--libs-are-really-here If you get errors about missing zlib or libuuid but you know it is available, you might
242
have a broken pkg-config. Use this option to proceed without checking pkg-config.
@@ -278,6 +279,7 @@ while [ -n "${1}" ]; do
279
"--zlib-is-really-here") LIBS_ARE_HERE=1 ;;
280
"--libs-are-really-here") LIBS_ARE_HERE=1 ;;
281
"--use-system-lws") USE_SYSTEM_LWS=1 ;;
282
+ "--use-system-protobuf") USE_SYSTEM_PROTOBUF=1 ;;
283
"--dont-scrub-cflags-even-though-it-may-break-things") DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS=1 ;;
284
"--dont-start-it") DONOTSTART=1 ;;
285
"--dont-wait") DONOTWAIT=1 ;;
@@ -310,7 +312,10 @@ while [ -n "${1}" ]; do
312
"--enable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-kinesis/} --enable-backend-kinesis" ;;
313
"--disable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-kinesis/} --disable-backend-kinesis" ;;
314
"--enable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-prometheus-remote-write/} --enable-backend-prometheus-remote-write" ;;
313
- "--disable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-prometheus-remote-write/} --disable-backend-prometheus-remote-write" ;;
315
+ "--disable-backend-prometheus-remote-write")
316
+ NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-prometheus-remote-write/} --disable-backend-prometheus-remote-write"
317
+ NETDATA_DISABLE_PROMETHEUS=1
318
+ ;;
319
"--enable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-mongodb/} --enable-backend-mongodb" ;;
320
"--disable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-mongodb/} --disable-backend-mongodb" ;;
321
"--enable-lto") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-lto/} --enable-lto" ;;
@@ -725,6 +730,79 @@ bundle_libwebsockets
730
731
# -----------------------------------------------------------------------------
732
733
+build_protobuf() {
734
+ local env_cmd=''
735
+
736
+ if [ -z "${DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS}" ]; then
737
+ env_cmd="env CFLAGS=-fPIC CXXFLAGS= LDFLAGS="
738
+ fi
739
+
740
+ pushd "${1}" > /dev/null || return 1
741
+ run ${env_cmd} ./configure --disable-shared \
742
+ --without-zlib \
743
+ --disable-dependency-tracking \
744
+ --with-pic || return 1
745
+ run ${env_cmd} ${make} -j$(find_processors) || return 1
746
+ popd > /dev/null || return 1
747
+}
748
+
749
+copy_protobuf() {
750
+ target_dir="${PWD}/externaldeps/protobuf"
751
+
752
+ run mkdir -p "${target_dir}" || return 1
753
+ run cp -a "${1}/src" "${target_dir}" || return 1
754
+}
755
+
756
+bundle_protobuf() {
757
+ if [ -n "${NETDATA_DISABLE_CLOUD}" ] && [ -n "${NETDATA_DISABLE_PROMETHEUS}" ]; then
758
+ echo "Skipping protobuf"
759
+ return 0
760
+ fi
761
+
762
+ if [ -n "${USE_SYSTEM_PROTOBUF}" ]; then
763
+ echo "Skipping protobuf"
764
+ defer_error "You have requested use of a system copy of protobuf. This should work, but it is not recommended as it's very likely to break if you upgrade the currently installed version of protobuf."
765
+ return 0
766
+ fi
767
+
768
+ PROTOBUF_PACKAGE_VERSION="$(cat packaging/protobuf.version)"
769
+
770
+ tmp="$(mktemp -d -t netdata-protobuf-XXXXXX)"
771
+ PROTOBUF_PACKAGE_BASENAME="protobuf-cpp-${PROTOBUF_PACKAGE_VERSION}.tar.gz"
772
+
773
+ if fetch_and_verify "protobuf" \
774
+ "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_PACKAGE_VERSION}/${PROTOBUF_PACKAGE_BASENAME}" \
775
+ "${PROTOBUF_PACKAGE_BASENAME}" \
776
+ "${tmp}" \
777
+ "${NETDATA_LOCAL_TARBALL_VERRIDE_PROTOBUF}"; then
778
+ if run tar -xf "${tmp}/${PROTOBUF_PACKAGE_BASENAME}" -C "${tmp}" &&
779
+ build_protobuf "${tmp}/protobuf-${PROTOBUF_PACKAGE_VERSION}" &&
780
+ copy_protobuf "${tmp}/protobuf-${PROTOBUF_PACKAGE_VERSION}" &&
781
+ rm -rf "${tmp}"; then
782
+ run_ok "protobuf built and prepared."
783
+ NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS} --with-bundled-protobuf"
784
+ else
785
+ run_failed "Failed to build protobuf."
786
+ if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then
787
+ exit 1
788
+ else
789
+ defer_error_highlighted "Failed to build protobuf. You may not be able to connect this node to Netdata Cloud."
790
+ fi
791
+ fi
792
+ else
793
+ run_failed "Unable to fetch sources for protobuf."
794
+ if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then
795
+ exit 1
796
+ else
797
+ defer_error_highlighted "Unable to fetch sources for protobuf. You may not be able to connect this node to Netdata Cloud."
798
+ fi
799
+ fi
800
+}
801
+
802
+bundle_protobuf
803
+
804
+# -----------------------------------------------------------------------------
805
+
806
build_judy() {
807
local env_cmd=''
808
local libtoolize="libtoolize"
packaging/installer/install-required-packages.sh
+15
@@ -721,6 +721,20 @@ declare -A pkg_gcc=(
721
['default']="gcc"
722
)
723
724
+# g++, required for building protobuf
725
+# All three cases of this not being required are systems that implicitly
726
+# include g++ when installing gcc.
727
+declare -A pkg_gxx=(
728
+ ['alpine']="g++"
729
+ ['arch']="NOTREQUIRED"
730
+ ['clearlinux']="c-basic"
731
+ ['debian']="g++"
732
+ ['gentoo']="NOTREQUIRED"
733
+ ['macos']="NOTREQUIRED"
734
+ ['ubuntu']="g++"
735
+ ['default']="gcc-c++"
736
+)
737
+
738
declare -A pkg_gdb=(
739
['gentoo']="sys-devel/gdb"
740
['macos']="NOTREQUIRED"
@@ -1291,6 +1305,7 @@ packages() {
1305
1306
require_cmd gcc ||
1307
require_cmd gcc-multilib || suitable_package gcc
1308
+ require_cmd g++ || suitable_package gxx
1309
1310
require_cmd make || suitable_package make
1311
require_cmd autoconf || suitable_package autoconf
packaging/protobuf.checksums
new
+1
@@ -0,0 +1 @@
1
+51cec99f108b83422b7af1170afd7aeb2dd77d2bcbb7b6bad1f92509e9ccf8cb protobuf-cpp-3.17.3.tar.gz
packaging/protobuf.version
new
+1
@@ -0,0 +1 @@
1
+3.17.3