Added libwebsockets bundling code to netdata-installer.sh (#8144)
* Generalize handling of fetch and verification of archives. This generalizes the logic used for fetching and verifying source archives for external dependencies that we potentially bundle as part of our regular build. It also slightly improves the error handling for the bundling of libmosquitto. * Add code to bundle libwebsockets during install. This gets used when building on a system without libwebsockets3. Due to how the libwebsockets build system works, this adds cmake as an installer dependency on systems that do not have libwebsockets3. * Add libwebsockets-dev to install-required-packages.sh * Fix error caused by resolving merge conflict incorrectly. * Unconditionally bunlde libwebsockets if cloud is enabled on install. As discussed with the product team, we're taking this approach to ensure SOCKS5 proxy support.
Austin S. Hemmelgarn committed
Feb 25, 2020 at 21:51 UTC
b500359f0c6c8dfcc2dbbbdc48ca18f38df41ba5
6 files changed
+126
-26
netdata-installer.sh
+65
-22
@@ -258,7 +258,7 @@ while [ -n "${1}" ]; do
258
"--disable-go") NETDATA_DISABLE_GO=1 ;;
259
"--enable-ebpf") NETDATA_ENABLE_EBPF=1 ;;
260
"--disable-cloud")
261
- NETDATA_DISABLE_LIBMOSQUITTO=1
261
+ NETDATA_DISABLE_CLOUD=1
262
NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-aclk/} --disable-aclk"
263
;;
264
"--install")
@@ -435,10 +435,6 @@ trap build_error EXIT
435
436
# -----------------------------------------------------------------------------
437
438
-fetch_libmosquitto() {
439
- download_tarball "${1}" "${2}" "libmosquitto" "cloud"
440
-}
441
-
438
build_libmosquitto() {
439
run make -C "${1}/lib"
440
}
@@ -453,7 +449,7 @@ copy_libmosquitto() {
449
}
450
451
bundle_libmosquitto() {
456
- if [ -n "${NETDATA_DISABLE_LIBMOSQUITTO}" ]; then
452
+ if [ -n "${NETDATA_DISABLE_CLOUD}" ]; then
453
return 0
454
fi
455
@@ -470,32 +466,79 @@ bundle_libmosquitto() {
466
tmp="$(mktemp -d -t netdata-mosquitto-XXXXXX)"
467
MOSQUITTO_PACKAGE_BASENAME="${MOSQUITTO_PACKAGE_VERSION}.tar.gz"
468
473
- if [ -z "${NETDATA_LOCAL_TARBALL_OVERRIDE_MOSQUITTO}" ]; then
474
- fetch_libmosquitto "https://github.com/netdata/mosquitto/archive/${MOSQUITTO_PACKAGE_BASENAME}" "${tmp}/${MOSQUITTO_PACKAGE_BASENAME}"
469
+ if fetch_and_verify "mosquitto" \
470
+ "https://github.com/netdata/mosquitto/archive/${MOSQUITTO_PACKAGE_BASENAME}" \
471
+ "${MOSQUITTO_PACKAGE_BASENAME}" \
472
+ "${tmp}" \
473
+ "${NETDATA_LOCAL_TARBALL_OVERRIDE_MOSQUITTO}"
474
+ then
475
+ if run tar -xf "${tmp}/${MOSQUITTO_PACKAGE_BASENAME}" -C "${tmp}" && \
476
+ build_libmosquitto "${tmp}/mosquitto-${MOSQUITTO_PACKAGE_VERSION}" && \
477
+ copy_libmosquitto "${tmp}/mosquitto-${MOSQUITTO_PACKAGE_VERSION}" && \
478
+ rm -rf "${tmp}"
479
+ then
480
+ run_ok "libmosquitto built and prepared."
481
+ else
482
+ run_failed "Failed to build libmosquitto. The install process will continue, but you will not be able to connect this node to Netdata Cloud."
483
+ fi
484
else
476
- progress "Using provided mosquitto tarball ${NETDATA_LOCAL_TARBALL_OVERRIDE_MOSQUITTO}"
477
- run cp "${NETDATA_LOCAL_TARBALL_OVERRIDE_MOSQUITTO}" "${tmp}/${MOSQUITTO_PACKAGE_BASENAME}"
485
+ run_failed "Unable to fetch sources for libmosquitto. The install process will continue, but you will not be able to connect this node to Netdata Cloud."
486
fi
487
+}
488
480
- if [ ! -f "${tmp}/${MOSQUITTO_PACKAGE_BASENAME}" ] || [ ! -s "${tmp}/${MOSQUITTO_PACKAGE_BASENAME}" ]; then
481
- run_failed "unable to find a usable libmosquitto source archive, Netdata Cloud will not be available"
482
- return 0
483
- fi
489
+bundle_libmosquitto
490
485
- grep "${MOSQUITTO_PACKAGE_BASENAME}\$" "${INSTALLER_DIR}/packaging/mosquitto.checksums" > "${tmp}/sha256sums.txt" 2> /dev/null
491
+# -----------------------------------------------------------------------------
492
487
- # Checksum validation
488
- if ! (cd "${tmp}" && safe_sha256sum -c "sha256sums.txt"); then
489
- run_failed "mosquitto files checksum validation failed."
493
+build_libwebsockets() {
494
+ pushd "${1}" > /dev/null || exit 1
495
+ cmake -D LWS_WITH_SOCKS5:bool=ON .
496
+ make
497
+ popd > /dev/null || exit 1
498
+}
499
+
500
+copy_libwebsockets() {
501
+ target_dir="${PWD}/externaldeps/libwebsockets"
502
+
503
+ run mkdir -p "${target_dir}" || return 1
504
+
505
+ run cp "${1}/lib/libwebsockets.a" "${target_dir}/libwebsockets.a" || return 1
506
+ run cp -r "${1}/include" "${target_dir}" || return 1
507
+}
508
+
509
+bundle_libwebsockets() {
510
+ if [ -n "${DISABLE_CLOUD}" ] ; then
511
return 0
512
fi
513
493
- run tar -xf "${tmp}/${MOSQUITTO_PACKAGE_BASENAME}" -C "${tmp}"
494
-
495
- build_libmosquitto "${tmp}/mosquitto-${MOSQUITTO_PACKAGE_VERSION}" && copy_libmosquitto "${tmp}/mosquitto-${MOSQUITTO_PACKAGE_VERSION}" && rm "${tmp}"
514
+ progress "Prepare libwebsockets"
515
+
516
+ LIBWEBSOCKETS_PACKAGE_VERSION="$(cat packaging/libwebsockets.version)"
517
+
518
+ tmp="$(mktemp -d -t netdata-libwebsockets-XXXXXX)"
519
+ LIBWEBSOCKETS_PACKAGE_BASENAME="v${LIBWEBSOCKETS_PACKAGE_VERSION}.tar.gz"
520
+
521
+ if fetch_and_verify "libwebsockets" \
522
+ "https://github.com/warmcat/libwebsockets/archive/${LIBWEBSOCKETS_PACKAGE_BASENAME}" \
523
+ "${LIBWEBSOCKETS_PACKAGE_BASENAME}" \
524
+ "${tmp}" \
525
+ "${NETDATA_LOCAL_TARBALL_OVERRIDE_LIBWEBSOCKETS}"
526
+ then
527
+ if run tar -xf "${tmp}/${LIBWEBSOCKETS_PACKAGE_BASENAME}" -C "${tmp}" && \
528
+ build_libwebsockets "${tmp}/libwebsockets-${LIBWEBSOCKETS_PACKAGE_VERSION}" && \
529
+ copy_libwebsockets "${tmp}/libwebsockets-${LIBWEBSOCKETS_PACKAGE_VERSION}" && \
530
+ rm -rf "${tmp}"
531
+ then
532
+ run_ok "libwebsockets built and prepared."
533
+ else
534
+ run_failed "Failed to build libwebsockets. The install process will continue, but you may not be able to connect this node to Netdata Cloud."
535
+ fi
536
+ else
537
+ run_failed "Unable to fetch sources for libwebsockets. The install process will continue, but you may not be able to connect this node to Netdata Cloud."
538
+ fi
539
}
540
498
-bundle_libmosquitto
541
+bundle_libwebsockets
542
543
# -----------------------------------------------------------------------------
544
echo >&2
packaging/installer/functions.sh
+31
@@ -103,6 +103,37 @@ get() {
103
fi
104
}
105
106
+# -----------------------------------------------------------------------------
107
+# external component handling
108
+
109
+fetch_and_verify() {
110
+ local component=${1}
111
+ local url=${2}
112
+ local base_name=${3}
113
+ local tmp=${4}
114
+ local override=${5}
115
+
116
+ if [ -z "${override}" ]; then
117
+ get "${url}" > "${tmp}/${base_name}"
118
+ else
119
+ progress "Using provided ${component} archive ${override}"
120
+ run cp "${override}" "${tmp}/${base_name}"
121
+ fi
122
+
123
+ if [ ! -f "${tmp}/${base_name}" ] || [ ! -s "${tmp}/${base_name}" ]; then
124
+ run_failed "Unable to find usable archive for ${component}"
125
+ return 1
126
+ fi
127
+
128
+ grep "${base_name}\$" "${INSTALLER_DIR}/packaging/${component}.checksums" > "${tmp}/sha256sums.txt" 2> /dev/null
129
+
130
+ # Checksum validation
131
+ if ! (cd "${tmp}" && safe_sha256sum -c "sha256sums.txt"); then
132
+ run_failed "${component} files checksum validation failed."
133
+ return 1
134
+ fi
135
+}
136
+
137
# -----------------------------------------------------------------------------
138
139
netdata_banner() {
packaging/installer/install-required-packages.sh
+14
@@ -591,6 +591,12 @@ declare -A pkg_automake=(
591
['default']="automake"
592
)
593
594
+# Required to build libwebsockets and libmosquitto on some systems.
595
+declare -A pkg_cmake=(
596
+ ['clearlinux']="c-basic"
597
+ ['default']="cmake"
598
+)
599
+
600
declare -A pkg_bridge_utils=(
601
['gentoo']="net-misc/bridge-utils"
602
['clearlinux']="network-basic"
@@ -703,6 +709,12 @@ declare -A pkg_libmnl_dev=(
709
['default']=""
710
)
711
712
+declare -A pkg_libwebsockets_dev=(
713
+ ['debian']="libwebsockets-dev"
714
+ ['ubuntu']="libwebsockets-dev"
715
+ ['default']="libwebsockets-devel"
716
+)
717
+
718
declare -A pkg_lm_sensors=(
719
['alpine']="lm_sensors"
720
['arch']="lm_sensors"
@@ -1104,6 +1116,7 @@ packages() {
1116
require_cmd autogen || suitable_package autogen
1117
require_cmd automake || suitable_package automake
1118
require_cmd pkg-config || suitable_package pkg-config
1119
+ require_cmd cmake || suitable_package cmake
1120
1121
# -------------------------------------------------------------------------
1122
# debugging tools for development
@@ -1157,6 +1170,7 @@ packages() {
1170
suitable_package libz-dev
1171
suitable_package libuuid-dev
1172
suitable_package libmnl-dev
1173
+ suitable_package libwebsockets-dev
1174
fi
1175
1176
# -------------------------------------------------------------------------
packaging/installer/methods/manual.md
+14
-4
@@ -61,16 +61,16 @@ This is how to do it by hand:
61
62
```sh
63
# Debian / Ubuntu
64
-apt-get install zlib1g-dev uuid-dev libuv1-dev liblz4-dev libjudy-dev libssl-dev libmnl-dev gcc make git autoconf autoconf-archive autogen automake pkg-config curl python
64
+apt-get install zlib1g-dev uuid-dev libuv1-dev liblz4-dev libjudy-dev libssl-dev libmnl-dev gcc make git autoconf autoconf-archive autogen automake pkg-config curl python cmake libwebsockets-dev
65
66
# Fedora
67
-dnf install zlib-devel libuuid-devel libuv-devel lz4-devel Judy-devel openssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python
67
+dnf install zlib-devel libuuid-devel libuv-devel lz4-devel Judy-devel openssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python cmake libwebsockets-devel
68
69
# CentOS / Red Hat Enterprise Linux
70
-yum install autoconf automake curl gcc git libmnl-devel libuuid-devel openssl-devel libuv-devel lz4-devel Judy-devel make nc pkgconfig python zlib-devel
70
+yum install autoconf automake curl gcc git libmnl-devel libuuid-devel openssl-devel libuv-devel lz4-devel Judy-devel make nc pkgconfig python zlib-devel cmake libwebsockets-devel
71
72
# openSUSE
73
-zypper install zlib-devel libuuid-devel libuv-devel liblz4-devel judy-devel libopenssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python
73
+zypper install zlib-devel libuuid-devel libuv-devel liblz4-devel judy-devel libopenssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python cmake libwebsockets-devel
74
```
75
76
Once Netdata is compiled, to run it the following packages are required (already installed using the above commands):
@@ -115,6 +115,16 @@ Netdata DB engine can be enabled when these are installed (they are optional):
115
116
*Netdata will greatly benefit if you have the above packages installed, but it will still work without them.*
117
118
+Netdata Cloud support may require the following packages to be installed:
119
+
120
+| package | description
121
+|:--------:| -----------------------
122
+| `libwebsockets-devel` | Version 3 or higher is at needed build time for websockets support for Netdata Cloud
123
+| `libwebsockets` | Version 3 or higher is needed at runtime for websockets support for Netdata Cloud
124
+| `cmake` | Needed at build time if you aren't using your distribution's version of libwebsockets or are building on a platform other than Linux
125
+
126
+*Netdata will greatly benefit if you have the above packages installed, but it will still work without them.*
127
+
128
### CentOS / RHEL 6.x
129
130
On CentOS / RHEL 6.x, many of the dependencies for Netdata are only
packaging/libwebsockets.checksums
new
+1
@@ -0,0 +1 @@
1
+166d6e17cab64bfc10c2a71799c298284540a1fa63f6ea3de5caccb34502243c v3.2.2.tar.gz
packaging/libwebsockets.version
new
+1
@@ -0,0 +1 @@
1
+3.2.2