Fix build for the AWS Kinesis exporting connector (#9823)
Co-authored-by: James Mills <prologic@shortcircuit.net.au>
Vladimir Kobal committed
Sep 21, 2020 at 10:53 UTC
a8c4f9f2a9e1f833ac346bd492cb447e97f4856e
5 files changed
+173
-53
.github/dockerfiles/Dockerfile.build_test
-1
@@ -7,6 +7,5 @@ ENV PRE=${PRE}
7
8
COPY . /netdata
9
10
-RUN chmod +x /netdata/rmjsonc.sh
10
RUN /bin/sh /netdata/prep-cmd.sh
11
RUN /netdata/packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
.github/workflows/build-and-install.yml
+81
-1
@@ -108,7 +108,7 @@ jobs:
108
RMJSONC: ${{ matrix.rmjsonc }}
109
run: |
110
echo $PRE > ./prep-cmd.sh
111
- echo $RMJSONC > ./rmjsonc.sh
111
+ echo $RMJSONC > ./rmjsonc.sh && chmod +x ./rmjsonc.sh
112
docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
113
- name: Regular build on ${{ matrix.distro }}
114
run: |
@@ -124,3 +124,83 @@ jobs:
124
run: |
125
docker run -w /netdata test \
126
/bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
127
+ aws-kinesis-build:
128
+ name: With AWS Kinesis SDK
129
+ strategy:
130
+ fail-fast: false
131
+ matrix:
132
+ distro:
133
+ # XXX: CentOS 8.x is broken (libjudy)
134
+ # - 'centos:8'
135
+ - 'debian:buster'
136
+ - 'fedora:32'
137
+ - 'ubuntu:20.04'
138
+ include:
139
+ #- distro: 'centos:8'
140
+ # pre: >-
141
+ # yum -y update &&
142
+ # yum -y groupinstall 'Development Tools' &&
143
+ # yum -y install libcurl-devel openssl-devel libuuid-devel
144
+ # build_kinesis: >-
145
+ # git clone https://github.com/aws/aws-sdk-cpp.git &&
146
+ # cmake -DCMAKE_INSTALL_PREFIX=/usr
147
+ # -DBUILD_ONLY=kinesis
148
+ # ./aws-sdk-cpp &&
149
+ # make &&
150
+ # make install
151
+ - distro: 'debian:buster'
152
+ pre: >-
153
+ apt-get update &&
154
+ apt-get install -y build-essential &&
155
+ apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
156
+ build_kinesis: >-
157
+ git clone https://github.com/aws/aws-sdk-cpp.git &&
158
+ cmake -DCMAKE_INSTALL_PREFIX=/usr
159
+ -DBUILD_ONLY=kinesis
160
+ ./aws-sdk-cpp &&
161
+ make &&
162
+ make install
163
+ - distro: 'fedora:32'
164
+ pre: >-
165
+ dnf -y update &&
166
+ dnf -y groupinstall 'Development Tools' &&
167
+ dnf -y install libcurl-devel openssl-devel libuuid-devel
168
+ build_kinesis: >-
169
+ git clone https://github.com/aws/aws-sdk-cpp.git &&
170
+ cmake -DCMAKE_INSTALL_PREFIX=/usr
171
+ -DBUILD_ONLY=kinesis
172
+ ./aws-sdk-cpp &&
173
+ make &&
174
+ make install
175
+ - distro: 'ubuntu:20.04'
176
+ pre: >-
177
+ apt-get update &&
178
+ apt-get install -y build-essential &&
179
+ apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
180
+ build_kinesis: >-
181
+ git clone https://github.com/aws/aws-sdk-cpp.git &&
182
+ cmake -DCMAKE_INSTALL_PREFIX=/usr
183
+ -DBUILD_ONLY=kinesis
184
+ ./aws-sdk-cpp &&
185
+ make &&
186
+ make install
187
+ runs-on: ubuntu-latest
188
+ steps:
189
+ - name: Git clone repository
190
+ uses: actions/checkout@v2
191
+ - name: install-required-packages.sh on ${{ matrix.distro }}
192
+ env:
193
+ PRE: ${{ matrix.pre }}
194
+ BUILD_KINESIS: ${{ matrix.build_kinesis }}
195
+ run: |
196
+ echo $PRE > ./prep-cmd.sh
197
+ echo $BUILD_KINESIS > ./build-kinesis.sh && chmod +x ./build-kinesis.sh
198
+ docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
199
+ - name: Build on ${{ matrix.distro }}
200
+ env:
201
+ RUNCMD: >-
202
+ ./build-kinesis.sh &&
203
+ ./netdata-installer.sh --dont-wait --dont-start-it --enable-backend-kinesis
204
+ run: |
205
+ docker run -w /netdata test \
206
+ /bin/sh -c "$RUNCMD"
CMakeLists.txt
+35
-8
@@ -237,27 +237,51 @@ pkg_check_modules(CURL libcurl)
237
# ${CURL_CFLAGS_OTHER}
238
# ${CURL_INCLUDE_DIRS}
239
240
+# -----------------------------------------------------------------------------
241
+# Detect libaws-c-common
242
+
243
+find_library(HAVE_AWS_CHECKSUMS aws-checksums)
244
+# later we use:
245
+# ${HAVE_AWS_CHECKSUMS}
246
+
247
+# -----------------------------------------------------------------------------
248
+# Detect libaws-c-common
249
+
250
+find_library(HAVE_AWS_COMMON aws-c-common)
251
+# later we use:
252
+# ${HAVE_AWS_COMMON}
253
+
254
+# -----------------------------------------------------------------------------
255
+# Detect libaws-c-common
256
+
257
+find_library(HAVE_AWS_EVENT_STREAM aws-c-event-stream)
258
+# later we use:
259
+# ${HAVE_AWS_EVENT_STREAM}
260
261
# -----------------------------------------------------------------------------
262
# Detect libaws-cpp-sdk-core
263
244
-find_library(HAVE_AWS aws-cpp-sdk-core)
264
+pkg_check_modules(AWS_CORE aws-cpp-sdk-core)
265
# later we use:
246
-# ${HAVE_AWS}
266
+# ${AWS_CORE_LIBRARIES}
267
+# ${AWS_CORE_CFLAGS_OTHER}
268
+# ${AWS_CORE_INCLUDE_DIRS}
269
270
# -----------------------------------------------------------------------------
271
# Detect libaws-cpp-sdk-kinesis
272
251
-find_library(HAVE_KINESIS aws-cpp-sdk-kinesis)
273
+pkg_check_modules(KINESIS aws-cpp-sdk-kinesis)
274
# later we use:
253
-# ${HAVE_KINESIS}
275
+# ${KINESIS_LIBRARIES}
276
+# ${KINESIS_CFLAGS_OTHER}
277
+# ${KINESIS_INCLUDE_DIRS}
278
279
# -----------------------------------------------------------------------------
280
# Detect libgrpc
281
282
pkg_check_modules(GRPC grpc)
283
# later we use:
260
-# ${GRPCF_LIBRARIES}
284
+# ${GRPC_LIBRARIES}
285
# ${GRPC_CFLAGS_OTHER}
286
# ${GRPC_INCLUDE_DIRS}
287
@@ -844,7 +868,8 @@ add_definitions(
868
# -----------------------------------------------------------------------------
869
# kinesis backend
870
847
-IF(HAVE_KINESIS AND HAVE_AWS AND CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
871
+IF(KINESIS_LIBRARIES AND AWS_CORE_LIBRARIES AND HAVE_AWS_EVENT_STREAM AND HAVE_AWS_COMMON AND HAVE_AWS_CHECKSUMS AND
872
+ CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
873
SET(ENABLE_BACKEND_KINESIS True)
874
ELSE()
875
SET(ENABLE_BACKEND_KINESIS False)
@@ -853,8 +878,10 @@ ENDIF()
878
IF(ENABLE_BACKEND_KINESIS)
879
message(STATUS "kinesis backend: enabled")
880
list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES} ${KINESIS_EXPORTING_FILES})
856
- list(APPEND NETDATA_COMMON_LIBRARIES aws-cpp-sdk-kinesis aws-cpp-sdk-core ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
857
- list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
881
+ list(APPEND NETDATA_COMMON_LIBRARIES ${KINESIS_LIBRARIES} ${AWS_CORE_LIBRARIES}
882
+ ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
883
+ list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${KINESIS_INCLUDE_DIRS} ${AWS_CORE_INCLUDE_DIRS}
884
+ ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
885
list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER})
886
ELSE()
887
message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)")
configure.ac
+49
-38
@@ -1074,46 +1074,45 @@ PKG_CHECK_MODULES(
1074
[have_libcurl=no]
1075
)
1076
1077
-AC_CHECK_LIB(
1077
+PKG_CHECK_MODULES(
1078
+ [AWS_CPP_SDK_CORE],
1079
[aws-cpp-sdk-core],
1079
- [cJSON_free],
1080
- [have_libaws_cpp_sdk_core=yes],
1081
- [have_libaws_cpp_sdk_core=no],
1082
- [${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}]
1080
+ [AC_CHECK_LIB(
1081
+ [aws-cpp-sdk-core],
1082
+ [cJSON_free],
1083
+ [have_libaws_cpp_sdk_core=yes],
1084
+ [have_libaws_cpp_sdk_core=no]
1085
+ )],
1086
+ [have_libaws_cpp_sdk_core=no]
1087
)
1088
1085
-AC_MSG_CHECKING([for Aws::Kinesis::Model::PutRecordRequest in -laws-cpp-sdk-kinesis])
1086
-
1087
-if test "${have_libaws_cpp_sdk_core}" = "yes" -a "${have_libcrypto}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcurl}" = "yes"; then
1088
- AC_LANG_SAVE
1089
- AC_LANG_CPLUSPLUS
1090
- save_LIBS="${LIBS}"
1091
- LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
1092
- save_CXXFLAGS="${CXXFLAGS}"
1093
- CXXFLAGS="${CXXFLAGS} -std=c++11"
1089
+PKG_CHECK_MODULES(
1090
+ [AWS_CPP_SDK_KINESIS],
1091
+ [aws-cpp-sdk-kinesis],
1092
+ [have_libaws_cpp_sdk_kinesis=yes],
1093
+ [have_libaws_cpp_sdk_kinesis=no]
1094
+)
1095
1095
- AC_TRY_LINK(
1096
- [
1097
- #include <aws/core/Aws.h>
1098
- #include <aws/core/client/ClientConfiguration.h>
1099
- #include <aws/core/auth/AWSCredentials.h>
1100
- #include <aws/core/utils/Outcome.h>
1101
- #include <aws/kinesis/KinesisClient.h>
1102
- #include <aws/kinesis/model/PutRecordRequest.h>
1103
- ],
1104
- [Aws::Kinesis::Model::PutRecordRequest request;],
1105
- [have_libaws_cpp_sdk_kinesis=yes],
1106
- [have_libaws_cpp_sdk_kinesis=no]
1107
- )
1096
+AC_CHECK_LIB(
1097
+ [aws-checksums],
1098
+ [aws_checksums_crc32],
1099
+ [have_libaws_checksums=yes],
1100
+ [have_libaws_checksums=no]
1101
+)
1102
1109
- LIBS="${save_LIBS}"
1110
- CXXFLAGS="${save_CXXFLAGS}"
1111
- AC_LANG_RESTORE
1112
-else
1113
- have_libaws_cpp_sdk_kinesis=no
1114
-fi
1103
+AC_CHECK_LIB(
1104
+ [aws-c-common],
1105
+ [aws_default_allocator],
1106
+ [have_libaws_c_common=yes],
1107
+ [have_libaws_c_common=no]
1108
+)
1109
1116
-AC_MSG_RESULT([${have_libaws_cpp_sdk_kinesis}])
1110
+AC_CHECK_LIB(
1111
+ [aws-c-event-stream],
1112
+ [aws_event_stream_library_init],
1113
+ [have_libaws_c_event_stream=yes],
1114
+ [have_libaws_c_event_stream=no]
1115
+)
1116
1117
test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \
1118
AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK])
@@ -1130,14 +1129,26 @@ test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \
1129
test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \
1130
AC_MSG_ERROR([libcrypto required but not found])
1131
1132
+test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_checksums}" != "yes" \
1133
+ -a "${have_libaws_c_common}" != "yes" \
1134
+ -a "${have_libaws_c_event_stream}" != "yes" && \
1135
+ AC_MSG_ERROR([AWS SKD third party dependencies required but not found])
1136
+
1137
AC_MSG_CHECKING([if kinesis backend should be enabled])
1134
-if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" = "yes" \
1135
- -a "${have_libcurl}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcrypto}" = "yes"; then
1138
+if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" \
1139
+ -a "${have_libaws_cpp_sdk_core}" = "yes" \
1140
+ -a "${have_libaws_checksums}" = "yes" \
1141
+ -a "${have_libaws_c_common}" = "yes" \
1142
+ -a "${have_libaws_c_event_stream}" = "yes" \
1143
+ -a "${have_libcurl}" = "yes" \
1144
+ -a "${have_libssl}" = "yes" \
1145
+ -a "${have_libcrypto}" = "yes"; then
1146
enable_backend_kinesis="yes"
1147
AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])
1148
OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"
1139
- CXX11FLAG="-std=c++11"
1140
- OPTIONAL_KINESIS_LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
1149
+ CXX11FLAG="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}"
1150
+ OPTIONAL_KINESIS_LIBS="${AWS_CPP_SDK_KINESIS_LIBS} ${AWS_CPP_SDK_CORE_LIBS} \
1151
+ ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
1152
else
1153
enable_backend_kinesis="no"
1154
fi
exporting/aws_kinesis/README.md
+8
-5
@@ -10,20 +10,23 @@ sidebar_label: AWS Kinesis Data Streams
10
## Prerequisites
11
12
To use AWS Kinesis for metric collecting and processing, you should first
13
-[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++. Netdata
14
-works with the SDK version 1.7.121. Other versions might work correctly as well, but they were not tested with Netdata.
13
+[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++.
14
`libcrypto`, `libssl`, and `libcurl` are also required to compile Netdata with Kinesis support enabled. Next, Netdata
15
should be re-installed from the source. The installer will detect that the required libraries are now available.
16
18
-If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the
19
-building process could take a very long time. Note that the default installation path for the libraries is
17
+If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY=kinesis`. Otherwise, the
18
+build process could take a very long time. Note, that the default installation path for the libraries is
19
`/usr/local/lib64`. Many Linux distributions don't include this path as the default one for a library search, so it is
20
advisable to use the following options to `cmake` while building the AWS SDK:
21
22
```sh
24
-cmake -DCMAKE_INSTALL_LIBDIR=/usr/lib -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
23
+sudo cmake -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
24
```
25
26
+The `-DCMAKE_INSTALL_PREFIX=/usr` option also ensures that
27
+[third party dependencies](https://github.com/aws/aws-sdk-cpp#third-party-dependencies) are installed in your system
28
+during the SDK build process.
29
+
30
## Configuration
31
32
To enable data sending to the Kinesis service, run `./edit-config exporting.conf` in the Netdata configuration directory