Add AWS Kinesis backend (#5914)
* Add Kinesis backend * Separate config file * Send data in chunks * Fix minor issues * Add error handling * Use existing JSON functions * Do not retry on send failure * Implement building with autotools * Implement building with CMake * Fix CMake variables * Fix build when C++ compiler is not available * Add checks for C++11 * Don't reinitialize API * Don't reinitialize client * Minor cleanup * Fix Codacy warning * Separate sending records and receiving results * Add documentation * Make connection timeout configurable * Fix operation metrics * Fix typo * Change parameter names for credentials * Allow using the default SDK credentials configuration
Vladimir Kobal committed
May 13, 2019 at 15:45 UTC
198c7fa3e34c71bc075bcfb067da80242e74bddf
15 files changed
+722
-101
CMakeLists.txt
+77
-2
@@ -3,8 +3,8 @@
3
# This file is only used for development (netdata in Clion)
4
# It can build netdata, but you are on your own...
5
6
-cmake_minimum_required(VERSION 3.0.2)
7
-project(netdata C)
6
+cmake_minimum_required(VERSION 3.1.0)
7
+project(netdata C CXX)
8
9
find_package(Threads REQUIRED)
10
find_package(PkgConfig REQUIRED)
@@ -157,6 +157,50 @@ IF(MACOS)
157
ENDIF(MACOS)
158
159
160
+# -----------------------------------------------------------------------------
161
+# Detect libcrypto
162
+
163
+pkg_check_modules(CRYPTO libcrypto)
164
+# later we use:
165
+# ${CRYPTO_LIBRARIES}
166
+# ${CRYPTO_CFLAGS_OTHER}
167
+# ${CRYPTO_INCLUDE_DIRS}
168
+
169
+
170
+# -----------------------------------------------------------------------------
171
+# Detect libssl
172
+
173
+pkg_check_modules(SSL libssl)
174
+# later we use:
175
+# ${SSL_LIBRARIES}
176
+# ${SSL_CFLAGS_OTHER}
177
+# ${SSL_INCLUDE_DIRS}
178
+
179
+
180
+# -----------------------------------------------------------------------------
181
+# Detect libcurl
182
+
183
+pkg_check_modules(CURL libcurl)
184
+# later we use:
185
+# ${CURL_LIBRARIES}
186
+# ${CURL_CFLAGS_OTHER}
187
+# ${CURL_INCLUDE_DIRS}
188
+
189
+
190
+# -----------------------------------------------------------------------------
191
+# Detect libaws-cpp-sdk-core
192
+
193
+find_library(HAVE_AWS aws-cpp-sdk-core)
194
+# later we use:
195
+# ${HAVE_AWS}
196
+
197
+# -----------------------------------------------------------------------------
198
+# Detect libaws-cpp-sdk-kinesis
199
+
200
+find_library(HAVE_KINESIS aws-cpp-sdk-kinesis)
201
+# later we use:
202
+# ${HAVE_KINESIS}
203
+
204
# -----------------------------------------------------------------------------
205
# netdata files
206
@@ -440,6 +484,13 @@ set(BACKENDS_PLUGIN_FILES
484
backends/prometheus/backend_prometheus.h
485
)
486
487
+set(KINESIS_BACKEND_FILES
488
+ backends/aws_kinesis/aws_kinesis.c
489
+ backends/aws_kinesis/aws_kinesis.h
490
+ backends/aws_kinesis/aws_kinesis_put_record.cc
491
+ backends/aws_kinesis/aws_kinesis_put_record.h
492
+ )
493
+
494
set(DAEMON_FILES
495
daemon/common.c
496
daemon/common.h
@@ -485,6 +536,25 @@ add_definitions(
536
-DVARLIB_DIR="/var/lib/netdata"
537
)
538
539
+# -----------------------------------------------------------------------------
540
+# kinesis backend
541
+
542
+IF(HAVE_KINESIS AND HAVE_AWS AND CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
543
+ SET(ENABLE_BACKEND_KINESIS True)
544
+ELSE()
545
+ SET(ENABLE_BACKEND_KINESIS False)
546
+ENDIF()
547
+
548
+IF(ENABLE_BACKEND_KINESIS)
549
+ message(STATUS "kinesis backend: enabled")
550
+ list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES})
551
+ list(APPEND NETDATA_COMMON_LIBRARIES aws-cpp-sdk-kinesis aws-cpp-sdk-core ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
552
+ list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
553
+ list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER})
554
+ELSE()
555
+ message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)")
556
+ENDIF()
557
+
558
# -----------------------------------------------------------------------------
559
# netdata
560
@@ -522,6 +592,11 @@ ELSEIF(MACOS)
592
593
ENDIF()
594
595
+IF(ENABLE_BACKEND_KINESIS)
596
+ set_property(TARGET netdata PROPERTY CXX_STANDARD 11)
597
+ set_property(TARGET netdata PROPERTY CMAKE_CXX_STANDARD_REQUIRED ON)
598
+ENDIF()
599
+
600
IF(IPMI_LIBRARIES)
601
SET(ENABLE_PLUGIN_FREEIPMI True)
602
ELSE()
Makefile.am
+18
-1
@@ -412,6 +412,13 @@ BACKENDS_PLUGIN_FILES = \
412
backends/prometheus/backend_prometheus.h \
413
$(NULL)
414
415
+KINESIS_BACKEND_FILES = \
416
+ backends/aws_kinesis/aws_kinesis.c \
417
+ backends/aws_kinesis/aws_kinesis.h \
418
+ backends/aws_kinesis/aws_kinesis_put_record.cc \
419
+ backends/aws_kinesis/aws_kinesis_put_record.h \
420
+ $(NULL)
421
+
422
DAEMON_FILES = \
423
daemon/common.c \
424
daemon/common.h \
@@ -471,13 +478,18 @@ NETDATA_COMMON_LIBS = \
478
$(OPTIONAL_ZLIB_LIBS) \
479
$(OPTIONAL_UUID_LIBS) \
480
$(NULL)
474
-
481
+# TODO: Find more graceful way to add libs for AWS Kinesis
482
483
sbin_PROGRAMS += netdata
484
netdata_SOURCES = $(NETDATA_FILES)
485
netdata_LDADD = \
486
$(NETDATA_COMMON_LIBS) \
487
$(NULL)
488
+if ENABLE_BACKEND_KINESIS
489
+ netdata_LINK = $(CXXLD) $(CXXFLAGS) -o $@
490
+else
491
+ netdata_LINK = $(CCLD) $(CFLAGS) -o $@
492
+endif
493
494
if ENABLE_PLUGIN_APPS
495
plugins_PROGRAMS += apps.plugin
@@ -531,3 +543,8 @@ if ENABLE_PLUGIN_XENSTAT
543
$(OPTIONAL_XENSTAT_LIBS) \
544
$(NULL)
545
endif
546
+
547
+if ENABLE_BACKEND_KINESIS
548
+ netdata_SOURCES += $(KINESIS_BACKEND_FILES)
549
+ netdata_LDADD += $(OPTIONAL_KINESIS_LIBS)
550
+endif
backends/Makefile.am
+1
@@ -8,6 +8,7 @@ SUBDIRS = \
8
json \
9
opentsdb \
10
prometheus \
11
+ aws_kinesis \
12
$(NULL)
13
14
dist_noinst_DATA = \
backends/README.md
+20
-14
@@ -32,24 +32,28 @@ X seconds (though, it can send them per second if you need it to).
32
33
- **prometheus** is described at [prometheus page](prometheus/) since it pulls data from netdata.
34
35
+ - **AWS Kinesis Data Streams**
36
+
37
+ metrics are sent to the service in `JSON` format.
38
+
39
2. Only one backend may be active at a time.
40
41
3. Netdata can filter metrics (at the chart level), to send only a subset of the collected metrics.
42
43
4. Netdata supports three modes of operation for all backends:
44
41
- - `as-collected` sends to backends the metrics as they are collected, in the units they are collected.
42
- So, counters are sent as counters and gauges are sent as gauges, much like all data collectors do.
45
+ - `as-collected` sends to backends the metrics as they are collected, in the units they are collected.
46
+ So, counters are sent as counters and gauges are sent as gauges, much like all data collectors do.
47
For example, to calculate CPU utilization in this format, you need to know how to convert kernel ticks to percentage.
48
45
- - `average` sends to backends normalized metrics from the netdata database.
46
- In this mode, all metrics are sent as gauges, in the units netdata uses. This abstracts data collection
47
- and simplifies visualization, but you will not be able to copy and paste queries from other sources to convert units.
48
- For example, CPU utilization percentage is calculated by netdata, so netdata will convert ticks to percentage and
49
+ - `average` sends to backends normalized metrics from the netdata database.
50
+ In this mode, all metrics are sent as gauges, in the units netdata uses. This abstracts data collection
51
+ and simplifies visualization, but you will not be able to copy and paste queries from other sources to convert units.
52
+ For example, CPU utilization percentage is calculated by netdata, so netdata will convert ticks to percentage and
53
send the average percentage to the backend.
54
51
- - `sum` or `volume`: the sum of the interpolated values shown on the netdata graphs is sent to the backend.
52
- So, if netdata is configured to send data to the backend every 10 seconds, the sum of the 10 values shown on the
55
+ - `sum` or `volume`: the sum of the interpolated values shown on the netdata graphs is sent to the backend.
56
+ So, if netdata is configured to send data to the backend every 10 seconds, the sum of the 10 values shown on the
57
netdata charts will be used.
58
59
Time-series databases suggest to collect the raw values (`as-collected`). If you plan to invest on building your monitoring around a time-series database and you already know (or you will invest in learning) how to convert units and normalize the metrics in Grafana or other visualization tools, we suggest to use `as-collected`.
@@ -66,9 +70,9 @@ of `netdata.conf` from your netdata):
70
```
71
[backend]
72
enabled = yes | no
69
- type = graphite | opentsdb | json
73
+ type = graphite | opentsdb | json | kinesis
74
host tags = list of TAG=VALUE
71
- destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used
75
+ destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used, or a region for kinesis
76
data source = average | sum | as collected
77
prefix = netdata
78
hostname = my-name
@@ -82,7 +86,7 @@ of `netdata.conf` from your netdata):
86
87
- `enabled = yes | no`, enables or disables sending data to a backend
88
85
-- `type = graphite | opentsdb | json`, selects the backend type
89
+- `type = graphite | opentsdb | json | kinesis`, selects the backend type
90
91
- `destination = host1 host2 host3 ...`, accepts **a space separated list** of hostnames,
92
IPs (IPv4 and IPv6) and ports to connect to.
@@ -105,7 +109,7 @@ of `netdata.conf` from your netdata):
109
```
110
111
Example IPv6 and IPv4 together:
108
-
112
+
113
```
114
destination = [ffff:...:0001]:2003 10.11.12.1:2003
115
```
@@ -118,6 +122,8 @@ of `netdata.conf` from your netdata):
122
time-series database when it becomes available again. It can also be used to monitor / trace / debug
123
the metrics netdata generates.
124
125
+ For kinesis backend `destination` should be set to an AWS region (for example, `us-east-1`).
126
+
127
- `data source = as collected`, or `data source = average`, or `data source = sum`, selects the kind of
128
data that will be sent to the backend.
129
@@ -170,7 +176,7 @@ netdata provides 5 charts:
176
177
1. **Buffered metrics**, the number of metrics netdata added to the buffer for dispatching them to the
178
backend server.
173
-
179
+
180
2. **Buffered data size**, the amount of data (in KB) netdata added the buffer.
181
182
3. ~~**Backend latency**, the time the backend server needed to process the data netdata sent.
@@ -178,7 +184,7 @@ netdata provides 5 charts:
184
(this chart has been removed, because it only measures the time netdata needs to give the data
185
to the O/S - since the backend servers do not ack the reception, netdata does not have any means
186
to measure this properly).
181
-
187
+
188
4. **Backend operations**, the number of operations performed by netdata.
189
190
5. **Backend thread CPU usage**, the CPU resources consumed by the netdata thread, that is responsible
backends/aws_kinesis/Makefile.am
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
+
6
+dist_noinst_DATA = \
7
+ README.md \
8
+ $(NULL)
9
+
10
+dist_libconfig_DATA = \
11
+ aws_kinesis.conf \
12
+ $(NULL)
\ No newline at end of file
backends/aws_kinesis/README.md
new
+34
@@ -0,0 +1,34 @@
1
+# Using netdata with AWS Kinesis Data Streams
2
+
3
+## Prerequisites
4
+
5
+To use AWS Kinesis as a backend AWS SDK for C++ should be [installed](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) first. `libcrypto`, `libssl`, and `libcurl` are also required to compile netdata with Kinesis support enabled. Next, netdata should be re-installed from the source. The installer will detect that the required libraries are now available.
6
+
7
+If AWS SDK for C++ is being installed from sources, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the building process could take a very long time.
8
+
9
+## Configuration
10
+
11
+To enable data sending to the kinesis backend set the following options in `netdata.conf`:
12
+```
13
+[backend]
14
+ enabled = yes
15
+ type = kinesis
16
+ destination = us-east-1
17
+```
18
+set the `destination` option to an AWS region.
19
+
20
+In the netdata configuration directory run `./edit-config aws_kinesis.conf` and set AWS credentials and stream name:
21
+```
22
+# AWS credentials
23
+aws_access_key_id = your_access_key_id
24
+aws_secret_access_key = your_secret_access_key
25
+
26
+# destination stream
27
+stream name = your_stream_name
28
+```
29
+Alternatively, AWS credentials can be set for the *netdata* user using AWS SDK for C++ [standard methods](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html).
30
+
31
+A partition key for every record is computed automatically by the netdata with the purpose to distribute records across available shards evenly.
32
+
33
+
34
+[]()
backends/aws_kinesis/aws_kinesis.c
new
+101
@@ -0,0 +1,101 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#define BACKENDS_INTERNALS
4
+#include "aws_kinesis.h"
5
+
6
+#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2)
7
+
8
+// ----------------------------------------------------------------------------
9
+// kinesis backend
10
+
11
+// read the aws_kinesis.conf file
12
+int read_kinesis_conf(const char *path, char **access_key_id_p, char **secret_access_key_p, char **stream_name_p)
13
+{
14
+ char *access_key_id = *access_key_id_p;
15
+ char *secret_access_key = *secret_access_key_p;
16
+ char *stream_name = *stream_name_p;
17
+
18
+ if(unlikely(access_key_id)) freez(access_key_id);
19
+ if(unlikely(secret_access_key)) freez(secret_access_key);
20
+ if(unlikely(stream_name)) freez(stream_name);
21
+ access_key_id = NULL;
22
+ secret_access_key = NULL;
23
+ stream_name = NULL;
24
+
25
+ int line = 0;
26
+
27
+ char filename[FILENAME_MAX + 1];
28
+ snprintfz(filename, FILENAME_MAX, "%s/aws_kinesis.conf", path);
29
+
30
+ char buffer[CONFIG_FILE_LINE_MAX + 1], *s;
31
+
32
+ debug(D_BACKEND, "BACKEND: opening config file '%s'", filename);
33
+
34
+ FILE *fp = fopen(filename, "r");
35
+ if(!fp) {
36
+ return 1;
37
+ }
38
+
39
+ while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {
40
+ buffer[CONFIG_FILE_LINE_MAX] = '\0';
41
+ line++;
42
+
43
+ s = trim(buffer);
44
+ if(!s || *s == '#') {
45
+ debug(D_BACKEND, "BACKEND: ignoring line %d of file '%s', it is empty.", line, filename);
46
+ continue;
47
+ }
48
+
49
+ char *name = s;
50
+ char *value = strchr(s, '=');
51
+ if(unlikely(!value)) {
52
+ error("BACKEND: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename);
53
+ continue;
54
+ }
55
+ *value = '\0';
56
+ value++;
57
+
58
+ name = trim(name);
59
+ value = trim(value);
60
+
61
+ if(unlikely(!name || *name == '#')) {
62
+ error("BACKEND: ignoring line %d of file '%s', name is empty.", line, filename);
63
+ continue;
64
+ }
65
+
66
+ if(!value) value = "";
67
+
68
+ // strip quotes
69
+ if(*value == '"' || *value == '\'') {
70
+ value++;
71
+
72
+ s = value;
73
+ while(*s) s++;
74
+ if(s != value) s--;
75
+
76
+ if(*s == '"' || *s == '\'') *s = '\0';
77
+ }
78
+ if(name[0] == 'a' && name[4] == 'a' && !strcmp(name, "aws_access_key_id")) {
79
+ access_key_id = strdupz(value);
80
+ }
81
+ else if(name[0] == 'a' && name[4] == 's' && !strcmp(name, "aws_secret_access_key")) {
82
+ secret_access_key = strdupz(value);
83
+ }
84
+ else if(name[0] == 's' && !strcmp(name, "stream name")) {
85
+ stream_name = strdupz(value);
86
+ }
87
+ }
88
+
89
+ fclose(fp);
90
+
91
+ if(unlikely(!stream_name || !*stream_name)) {
92
+ error("BACKEND: stream name is a mandatory Kinesis parameter but it is not configured");
93
+ return 1;
94
+ }
95
+
96
+ *access_key_id_p = access_key_id;
97
+ *secret_access_key_p = secret_access_key;
98
+ *stream_name_p = stream_name;
99
+
100
+ return 0;
101
+}
backends/aws_kinesis/aws_kinesis.conf
new
+10
@@ -0,0 +1,10 @@
1
+# AWS Kinesis Data Streams backend configuration
2
+#
3
+# All options in this file are mandatory
4
+
5
+# AWS credentials
6
+aws_access_key_id =
7
+aws_secret_access_key =
8
+
9
+# destination stream
10
+stream name =
\ No newline at end of file
backends/aws_kinesis/aws_kinesis.h
new
+14
@@ -0,0 +1,14 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_BACKEND_KINESIS_H
4
+#define NETDATA_BACKEND_KINESIS_H
5
+
6
+#include "backends/backends.h"
7
+#include "aws_kinesis_put_record.h"
8
+
9
+#define KINESIS_PARTITION_KEY_MAX 256
10
+#define KINESIS_RECORD_MAX 1024 * 1024
11
+
12
+extern int read_kinesis_conf(const char *path, char **auth_key_id_p, char **secure_key_p, char **stream_name_p);
13
+
14
+#endif //NETDATA_BACKEND_KINESIS_H
backends/aws_kinesis/aws_kinesis_put_record.cc
new
+87
@@ -0,0 +1,87 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include <aws/core/Aws.h>
4
+#include <aws/core/client/ClientConfiguration.h>
5
+#include <aws/core/auth/AWSCredentials.h>
6
+#include <aws/core/utils/Outcome.h>
7
+#include <aws/kinesis/KinesisClient.h>
8
+#include <aws/kinesis/model/PutRecordRequest.h>
9
+#include "aws_kinesis_put_record.h"
10
+
11
+using namespace Aws;
12
+
13
+SDKOptions options;
14
+
15
+Kinesis::KinesisClient *client;
16
+
17
+struct request_outcome {
18
+ Kinesis::Model::PutRecordOutcomeCallable future_outcome;
19
+ size_t data_len;
20
+};
21
+
22
+Vector<request_outcome> request_outcomes;
23
+
24
+void kinesis_init(const char *region, const char *access_key_id, const char *secret_key, const long timeout) {
25
+ InitAPI(options);
26
+
27
+ Client::ClientConfiguration config;
28
+
29
+ config.region = region;
30
+ config.requestTimeoutMs = timeout;
31
+ config.connectTimeoutMs = timeout;
32
+
33
+ if(access_key_id && *access_key_id && secret_key && *secret_key) {
34
+ client = New<Kinesis::KinesisClient>("client", Auth::AWSCredentials(access_key_id, secret_key), config);
35
+ } else {
36
+ client = New<Kinesis::KinesisClient>("client", config);
37
+ }
38
+}
39
+
40
+void kinesis_shutdown() {
41
+ Delete(client);
42
+
43
+ ShutdownAPI(options);
44
+}
45
+
46
+int kinesis_put_record(const char *stream_name, const char *partition_key,
47
+ const char *data, size_t data_len) {
48
+ Kinesis::Model::PutRecordRequest request;
49
+
50
+ request.SetStreamName(stream_name);
51
+ request.SetPartitionKey(partition_key);
52
+ request.SetData(Utils::ByteBuffer((unsigned char*) data, data_len));
53
+
54
+ request_outcomes.push_back({client->PutRecordCallable(request), data_len});
55
+
56
+ return 0;
57
+}
58
+
59
+int kinesis_get_result(char *error_message, size_t *sent_bytes, size_t *lost_bytes) {
60
+ Kinesis::Model::PutRecordOutcome outcome;
61
+ *sent_bytes = 0;
62
+ *lost_bytes = 0;
63
+
64
+ for(auto request_outcome = request_outcomes.begin(); request_outcome != request_outcomes.end(); ) {
65
+ std::future_status status = request_outcome->future_outcome.wait_for(std::chrono::microseconds(100));
66
+
67
+ if(status == std::future_status::ready || status == std::future_status::deferred) {
68
+ outcome = request_outcome->future_outcome.get();
69
+ *sent_bytes += request_outcome->data_len;
70
+
71
+ if(!outcome.IsSuccess()) {
72
+ *lost_bytes += request_outcome->data_len;
73
+ outcome.GetError().GetMessage().copy(error_message, ERROR_LINE_MAX);
74
+ }
75
+
76
+ request_outcomes.erase(request_outcome);
77
+ } else {
78
+ ++request_outcome;
79
+ }
80
+ }
81
+
82
+ if(*lost_bytes) {
83
+ return 1;
84
+ }
85
+
86
+ return 0;
87
+}
\ No newline at end of file
backends/aws_kinesis/aws_kinesis_put_record.h
new
+25
@@ -0,0 +1,25 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_BACKEND_KINESIS_PUT_RECORD_H
4
+#define NETDATA_BACKEND_KINESIS_PUT_RECORD_H
5
+
6
+#define ERROR_LINE_MAX 1023
7
+
8
+#ifdef __cplusplus
9
+extern "C" {
10
+#endif
11
+
12
+void kinesis_init(const char *region, const char *access_key_id, const char *secret_key, const long timeout);
13
+
14
+void kinesis_shutdown();
15
+
16
+int kinesis_put_record(const char *stream_name, const char *partition_key,
17
+ const char *data, size_t data_len);
18
+
19
+int kinesis_get_result(char *error_message, size_t *sent_bytes, size_t *lost_bytes);
20
+
21
+#ifdef __cplusplus
22
+}
23
+#endif
24
+
25
+#endif //NETDATA_BACKEND_KINESIS_PUT_RECORD_H
backends/backends.c
+179
-72
@@ -238,6 +238,11 @@ void *backends_main(void *ptr) {
238
int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS) = NULL;
239
int (*backend_response_checker)(BUFFER *) = NULL;
240
241
+#if HAVE_KINESIS
242
+ int do_kinesis = 0;
243
+ char *kinesis_auth_key_id = NULL, *kinesis_secure_key = NULL, *kinesis_stream_name = NULL;
244
+#endif
245
+
246
// ------------------------------------------------------------------------
247
// collect configuration options
248
@@ -263,7 +268,6 @@ void *backends_main(void *ptr) {
268
charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
269
hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
270
266
-
271
// ------------------------------------------------------------------------
272
// validate configuration options
273
// and prepare for sending data to our backend
@@ -316,6 +320,26 @@ void *backends_main(void *ptr) {
320
backend_request_formatter = format_dimension_stored_json_plaintext;
321
322
}
323
+#if HAVE_KINESIS
324
+ else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
325
+
326
+ do_kinesis = 1;
327
+
328
+ if(unlikely(read_kinesis_conf(netdata_configured_user_config_dir, &kinesis_auth_key_id, &kinesis_secure_key, &kinesis_stream_name))) {
329
+ error("BACKEND: kinesis backend type is set but cannot read its configuration from %s/aws_kinesis.conf", netdata_configured_user_config_dir);
330
+ goto cleanup;
331
+ }
332
+
333
+ kinesis_init(destination, kinesis_auth_key_id, kinesis_secure_key, timeout.tv_sec * 1000 + timeout.tv_usec / 1000);
334
+
335
+ backend_response_checker = process_json_response;
336
+ if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
337
+ backend_request_formatter = format_dimension_collected_json_plaintext;
338
+ else
339
+ backend_request_formatter = format_dimension_stored_json_plaintext;
340
+
341
+ }
342
+#endif /* HAVE_KINESIS */
343
else {
344
error("BACKEND: Unknown backend type '%s'", type);
345
goto cleanup;
@@ -481,6 +505,7 @@ void *backends_main(void *ptr) {
505
chart_sent_bytes =
506
chart_sent_metrics =
507
chart_lost_metrics =
508
+ chart_receptions =
509
chart_transmission_successes =
510
chart_transmission_failures =
511
chart_data_lost_events =
@@ -497,104 +522,177 @@ void *backends_main(void *ptr) {
522
// to add incrementally data to buffer
523
after = before;
524
500
- // ------------------------------------------------------------------------
501
- // if we are connected, receive a response, without blocking
525
+#if HAVE_KINESIS
526
+ if(do_kinesis) {
527
+ unsigned long long partition_key_seq = 0;
528
503
- if(likely(sock != -1)) {
504
- errno = 0;
529
+ size_t buffer_len = buffer_strlen(b);
530
+ size_t sent = 0;
531
506
- // loop through to collect all data
507
- while(sock != -1 && errno != EWOULDBLOCK) {
508
- buffer_need_bytes(response, 4096);
532
+ while(sent < buffer_len) {
533
+ char partition_key[KINESIS_PARTITION_KEY_MAX + 1];
534
+ snprintf(partition_key, KINESIS_PARTITION_KEY_MAX, "netdata_%llu", partition_key_seq++);
535
+ size_t partition_key_len = strnlen(partition_key, KINESIS_PARTITION_KEY_MAX);
536
510
- ssize_t r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
511
- if(likely(r > 0)) {
512
- // we received some data
513
- response->len += r;
514
- chart_received_bytes += r;
515
- chart_receptions++;
537
+ const char *first_char = buffer_tostring(b) + sent;
538
+
539
+ size_t record_len = 0;
540
+
541
+ // split buffer into chunks of maximum allowed size
542
+ if(buffer_len - sent < KINESIS_RECORD_MAX - partition_key_len) {
543
+ record_len = buffer_len - sent;
544
}
517
- else if(r == 0) {
518
- error("BACKEND: '%s' closed the socket", destination);
519
- close(sock);
520
- sock = -1;
545
+ else {
546
+ record_len = KINESIS_RECORD_MAX - partition_key_len;
547
+ while(*(first_char + record_len) != '\n' && record_len) record_len--;
548
+ }
549
+
550
+ char error_message[ERROR_LINE_MAX + 1] = "";
551
+
552
+ debug(D_BACKEND, "BACKEND: kinesis_put_record(): dest = %s, id = %s, key = %s, stream = %s, partition_key = %s, \
553
+ buffer = %zu, record = %zu", destination, kinesis_auth_key_id, kinesis_secure_key, kinesis_stream_name,
554
+ partition_key, buffer_len, record_len);
555
+
556
+ kinesis_put_record(kinesis_stream_name, partition_key, first_char, record_len);
557
+
558
+ sent += record_len;
559
+ chart_transmission_successes++;
560
+
561
+ size_t sent_bytes = 0, lost_bytes = 0;
562
+
563
+ if(unlikely(kinesis_get_result(error_message, &sent_bytes, &lost_bytes))) {
564
+ // oops! we couldn't send (all or some of the) data
565
+ error("BACKEND: %s", error_message);
566
+ error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zu bytes.",
567
+ destination, sent_bytes, sent_bytes - lost_bytes);
568
+
569
+ chart_transmission_failures++;
570
+ chart_data_lost_events++;
571
+ chart_lost_bytes += lost_bytes;
572
+
573
+ // estimate the number of lost metrics
574
+ chart_lost_metrics += (collected_number)(chart_buffered_metrics
575
+ * (buffer_len && (lost_bytes > buffer_len) ? (double)lost_bytes / buffer_len : 1));
576
+
577
+ break;
578
}
579
else {
523
- // failed to receive data
524
- if(errno != EAGAIN && errno != EWOULDBLOCK) {
525
- error("BACKEND: cannot receive data from backend '%s'.", destination);
526
- }
580
+ chart_receptions++;
581
}
582
+
583
+ if(unlikely(netdata_exit)) break;
584
}
585
530
- // if we received data, process them
531
- if(buffer_strlen(response))
532
- backend_response_checker(response);
586
+ chart_sent_bytes += sent;
587
+ if(likely(sent == buffer_len))
588
+ chart_sent_metrics = chart_buffered_metrics;
589
+
590
+ buffer_flush(b);
591
}
592
+ else {
593
+#else
594
+ {
595
+#endif /* HAVE_KINESIS */
596
+
597
+ // ------------------------------------------------------------------------
598
+ // if we are connected, receive a response, without blocking
599
+
600
+ if(likely(sock != -1)) {
601
+ errno = 0;
602
+
603
+ // loop through to collect all data
604
+ while(sock != -1 && errno != EWOULDBLOCK) {
605
+ buffer_need_bytes(response, 4096);
606
+
607
+ ssize_t r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
608
+ if(likely(r > 0)) {
609
+ // we received some data
610
+ response->len += r;
611
+ chart_received_bytes += r;
612
+ chart_receptions++;
613
+ }
614
+ else if(r == 0) {
615
+ error("BACKEND: '%s' closed the socket", destination);
616
+ close(sock);
617
+ sock = -1;
618
+ }
619
+ else {
620
+ // failed to receive data
621
+ if(errno != EAGAIN && errno != EWOULDBLOCK) {
622
+ error("BACKEND: cannot receive data from backend '%s'.", destination);
623
+ }
624
+ }
625
+ }
626
535
- // ------------------------------------------------------------------------
536
- // if we are not connected, connect to a backend server
627
+ // if we received data, process them
628
+ if(buffer_strlen(response))
629
+ backend_response_checker(response);
630
+ }
631
538
- if(unlikely(sock == -1)) {
539
- // usec_t start_ut = now_monotonic_usec();
540
- size_t reconnects = 0;
632
+ // ------------------------------------------------------------------------
633
+ // if we are not connected, connect to a backend server
634
542
- sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
635
+ if(unlikely(sock == -1)) {
636
+ // usec_t start_ut = now_monotonic_usec();
637
+ size_t reconnects = 0;
638
544
- chart_backend_reconnects += reconnects;
545
- // chart_backend_latency += now_monotonic_usec() - start_ut;
546
- }
639
+ sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
640
548
- if(unlikely(netdata_exit)) break;
641
+ chart_backend_reconnects += reconnects;
642
+ // chart_backend_latency += now_monotonic_usec() - start_ut;
643
+ }
644
550
- // ------------------------------------------------------------------------
551
- // if we are connected, send our buffer to the backend server
552
-
553
- if(likely(sock != -1)) {
554
- size_t len = buffer_strlen(b);
555
- // usec_t start_ut = now_monotonic_usec();
556
- int flags = 0;
557
-#ifdef MSG_NOSIGNAL
558
- flags += MSG_NOSIGNAL;
559
-#endif
645
+ if(unlikely(netdata_exit)) break;
646
561
- ssize_t written = send(sock, buffer_tostring(b), len, flags);
562
- // chart_backend_latency += now_monotonic_usec() - start_ut;
563
- if(written != -1 && (size_t)written == len) {
564
- // we sent the data successfully
565
- chart_transmission_successes++;
566
- chart_sent_bytes += written;
567
- chart_sent_metrics = chart_buffered_metrics;
647
+ // ------------------------------------------------------------------------
648
+ // if we are connected, send our buffer to the backend server
649
569
- // reset the failures count
570
- failures = 0;
650
+ if(likely(sock != -1)) {
651
+ size_t len = buffer_strlen(b);
652
+ // usec_t start_ut = now_monotonic_usec();
653
+ int flags = 0;
654
+ #ifdef MSG_NOSIGNAL
655
+ flags += MSG_NOSIGNAL;
656
+ #endif
657
+
658
+ ssize_t written = send(sock, buffer_tostring(b), len, flags);
659
+ // chart_backend_latency += now_monotonic_usec() - start_ut;
660
+ if(written != -1 && (size_t)written == len) {
661
+ // we sent the data successfully
662
+ chart_transmission_successes++;
663
+ chart_sent_bytes += written;
664
+ chart_sent_metrics = chart_buffered_metrics;
665
572
- // empty the buffer
573
- buffer_flush(b);
666
+ // reset the failures count
667
+ failures = 0;
668
+
669
+ // empty the buffer
670
+ buffer_flush(b);
671
+ }
672
+ else {
673
+ // oops! we couldn't send (all or some of the) data
674
+ error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written);
675
+ chart_transmission_failures++;
676
+
677
+ if(written != -1)
678
+ chart_sent_bytes += written;
679
+
680
+ // increment the counter we check for data loss
681
+ failures++;
682
+
683
+ // close the socket - we will re-open it next time
684
+ close(sock);
685
+ sock = -1;
686
+ }
687
}
688
else {
576
- // oops! we couldn't send (all or some of the) data
577
- error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written);
689
+ error("BACKEND: failed to update database backend '%s'", destination);
690
chart_transmission_failures++;
691
580
- if(written != -1)
581
- chart_sent_bytes += written;
582
-
692
// increment the counter we check for data loss
693
failures++;
585
-
586
- // close the socket - we will re-open it next time
587
- close(sock);
588
- sock = -1;
694
}
695
}
591
- else {
592
- error("BACKEND: failed to update database backend '%s'", destination);
593
- chart_transmission_failures++;
594
-
595
- // increment the counter we check for data loss
596
- failures++;
597
- }
696
697
if(failures > buffer_on_failures) {
698
// too bad! we are going to lose data
@@ -651,6 +749,15 @@ void *backends_main(void *ptr) {
749
}
750
751
cleanup:
752
+#if HAVE_KINESIS
753
+ if(do_kinesis) {
754
+ kinesis_shutdown();
755
+ freez(kinesis_auth_key_id);
756
+ freez(kinesis_secure_key);
757
+ freez(kinesis_stream_name);
758
+ }
759
+#endif
760
+
761
if(sock != -1)
762
close(sock);
763
backends/backends.h
+4
@@ -49,4 +49,8 @@ extern int discard_response(BUFFER *b, const char *backend);
49
#include "backends/json/json.h"
50
#include "backends/opentsdb/opentsdb.h"
51
52
+#if HAVE_KINESIS
53
+#include "backends/aws_kinesis/aws_kinesis.h"
54
+#endif
55
+
56
#endif /* NETDATA_BACKENDS_H */
configure.ac
+135
-12
@@ -37,6 +37,7 @@ m4_ifdef([AM_SILENT_RULES], [
37
AC_CANONICAL_HOST
38
AC_PROG_CC
39
AM_PROG_CC_C_O
40
+AC_PROG_CXX
41
AC_PROG_INSTALL
42
PKG_PROG_PKG_CONFIG
43
AC_USE_SYSTEM_EXTENSIONS
@@ -69,6 +70,12 @@ AC_ARG_ENABLE(
70
,
71
[enable_plugin_xenstat="detect"]
72
)
73
+AC_ARG_ENABLE(
74
+ [backend-kinesis],
75
+ [AS_HELP_STRING([--enable-backend-xenstat], [enable kinesis backend @<:@default autodetect@:>@])],
76
+ ,
77
+ [enable_backend_kinesis="detect"]
78
+)
79
80
AC_ARG_ENABLE(
81
[pedantic],
@@ -250,7 +257,7 @@ AC_MSG_CHECKING([if zlib should be used])
257
if test "${with_zlib}" != "no" -a "${have_zlib}" = "yes"; then
258
with_zlib="yes"
259
AC_DEFINE([NETDATA_WITH_ZLIB], [1], [zlib usability])
253
- OPTIONAL_ZLIB_CLFAGS="${ZLIB_CFLAGS}"
260
+ OPTIONAL_ZLIB_CFLAGS="${ZLIB_CFLAGS}"
261
OPTIONAL_ZLIB_LIBS="${ZLIB_LIBS}"
262
else
263
with_zlib="no"
@@ -268,7 +275,7 @@ PKG_CHECK_MODULES(
275
[AC_MSG_ERROR([libuuid required but not found. Try installing 'uuid-dev' or 'libuuid-devel'.])]
276
)
277
AC_DEFINE([NETDATA_WITH_UUID], [1], [uuid usability])
271
-OPTIONAL_UUID_CLFAGS="${UUID_CFLAGS}"
278
+OPTIONAL_UUID_CFLAGS="${UUID_CFLAGS}"
279
OPTIONAL_UUID_LIBS="${UUID_LIBS}"
280
281
@@ -342,7 +349,7 @@ AC_MSG_CHECKING([if libcap should be used])
349
if test "${with_libcap}" != "no" -a "${have_libcap}" = "yes"; then
350
with_libcap="yes"
351
AC_DEFINE([HAVE_CAPABILITY], [1], [libcap usability])
345
- OPTIONAL_LIBCAP_CLFAGS="${LIBCAP_CFLAGS}"
352
+ OPTIONAL_LIBCAP_CFLAGS="${LIBCAP_CFLAGS}"
353
OPTIONAL_LIBCAP_LIBS="${LIBCAP_LIBS}"
354
else
355
with_libcap="no"
@@ -403,7 +410,7 @@ AC_MSG_CHECKING([if freeipmi.plugin should be enabled])
410
if test "${enable_plugin_freeipmi}" != "no" -a "${have_ipmimonitoring}" = "yes"; then
411
enable_plugin_freeipmi="yes"
412
AC_DEFINE([HAVE_FREEIPMI], [1], [ipmimonitoring usability])
406
- OPTIONAL_IPMIMONITORING_CLFAGS="${IPMIMONITORING_CFLAGS}"
413
+ OPTIONAL_IPMIMONITORING_CFLAGS="${IPMIMONITORING_CFLAGS}"
414
OPTIONAL_IPMIMONITORING_LIBS="${IPMIMONITORING_LIBS}"
415
else
416
enable_plugin_freeipmi="no"
@@ -451,7 +458,7 @@ if test "${enable_plugin_cups}" != "no" -a "${have_cups}" = "yes"; then
458
CUPS_CFLAGS="${CUPS_CFLAGS} `$CUPSCONFIG --cflags`"
459
CUPS_LIBS="${CUPS_LIBS} `$CUPSCONFIG --libs`"
460
454
- OPTIONAL_CUPS_CLFAGS="${CUPS_CFLAGS}"
461
+ OPTIONAL_CUPS_CFLAGS="${CUPS_CFLAGS}"
462
OPTIONAL_CUPS_LIBS="${CUPS_LIBS}"
463
else
464
enable_plugin_cups="no"
@@ -500,7 +507,7 @@ if test "${enable_plugin_nfacct}" != "no" -a "${have_libnetfilter_acct}" = "yes"
507
enable_plugin_nfacct="yes"
508
AC_DEFINE([HAVE_LIBMNL], [1], [libmnl usability])
509
AC_DEFINE([HAVE_LIBNETFILTER_ACCT], [1], [libnetfilter_acct usability])
503
- OPTIONAL_NFACCT_CLFAGS="${NFACCT_CFLAGS} ${LIBMNL_CFLAGS}"
510
+ OPTIONAL_NFACCT_CFLAGS="${NFACCT_CFLAGS} ${LIBMNL_CFLAGS}"
511
OPTIONAL_NFACCT_LIBS="${NFACCT_LIBS} ${LIBMNL_LIBS}"
512
else
513
enable_plugin_nfacct="no"
@@ -567,7 +574,7 @@ if test "${enable_plugin_xenstat}" != "no" -a "${have_libxenstat}" = "yes" -a "$
574
AC_DEFINE([HAVE_LIBXENSTAT], [1], [libxenstat usability])
575
AC_DEFINE([HAVE_LIBXENLIGHT], [1], [libxenlight usability])
576
AC_DEFINE([HAVE_LIBYAJL], [1], [libyajl usability])
570
- OPTIONAL_XENSTAT_CLFAGS="${XENLIGHT_CFLAGS} ${YAJL_CFLAGS}"
577
+ OPTIONAL_XENSTAT_CFLAGS="${XENLIGHT_CFLAGS} ${YAJL_CFLAGS}"
578
OPTIONAL_XENSTAT_LIBS="-lxenstat ${XENLIGHT_LIBS} ${YAJL_LIBS}"
579
else
580
enable_plugin_xenstat="no"
@@ -576,6 +583,117 @@ AC_MSG_RESULT([${enable_plugin_xenstat}])
583
AM_CONDITIONAL([ENABLE_PLUGIN_XENSTAT], [test "${enable_plugin_xenstat}" = "yes"])
584
585
586
+# -----------------------------------------------------------------------------
587
+# AWS Kinesis backend - libaws-cpp-sdk-kinesis, libaws-cpp-sdk-core, libssl, libcrypto, libcurl
588
+
589
+PKG_CHECK_MODULES(
590
+ [LIBCRYPTO],
591
+ [libcrypto],
592
+ [AC_CHECK_LIB(
593
+ [crypto],
594
+ [CRYPTO_new_ex_data],
595
+ [have_libcrypto=yes],
596
+ [have_libcrypto=no]
597
+ )],
598
+ [have_libcrypto=no]
599
+)
600
+
601
+PKG_CHECK_MODULES(
602
+ [LIBSSL],
603
+ [libssl],
604
+ [AC_CHECK_LIB(
605
+ [ssl],
606
+ [SSL_connect],
607
+ [have_libssl=yes],
608
+ [have_libssl=no]
609
+ )],
610
+ [have_libssl=no]
611
+)
612
+
613
+PKG_CHECK_MODULES(
614
+ [LIBCURL],
615
+ [libcurl],
616
+ [AC_CHECK_LIB(
617
+ [curl],
618
+ [curl_easy_init],
619
+ [have_libcurl=yes],
620
+ [have_libcurl=no]
621
+ )],
622
+ [have_libcurl=no]
623
+)
624
+
625
+AC_CHECK_LIB(
626
+ [aws-cpp-sdk-core],
627
+ [cJSON_free],
628
+ [have_libaws_cpp_sdk_core=yes],
629
+ [have_libaws_cpp_sdk_core=no],
630
+ [${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}]
631
+)
632
+
633
+AC_MSG_CHECKING([for Aws::Kinesis::Model::PutRecordRequest in -laws-cpp-sdk-kinesis])
634
+
635
+if test "${have_libaws_cpp_sdk_core}" = "yes" -a "${have_libcrypto}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcurl}" = "yes"; then
636
+ AC_LANG_SAVE
637
+ AC_LANG_CPLUSPLUS
638
+ save_LIBS="${LIBS}"
639
+ LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
640
+ save_CXXFLAGS="${CXXFLAGS}"
641
+ CXXFLAGS="${CXXFLAGS} -std=c++11"
642
+
643
+ AC_TRY_LINK(
644
+ [
645
+ #include <aws/core/Aws.h>
646
+ #include <aws/core/client/ClientConfiguration.h>
647
+ #include <aws/core/auth/AWSCredentials.h>
648
+ #include <aws/core/utils/Outcome.h>
649
+ #include <aws/kinesis/KinesisClient.h>
650
+ #include <aws/kinesis/model/PutRecordRequest.h>
651
+ ],
652
+ [Aws::Kinesis::Model::PutRecordRequest request;],
653
+ [have_libaws_cpp_sdk_kinesis=yes],
654
+ [have_libaws_cpp_sdk_kinesis=no]
655
+ )
656
+
657
+ LIBS="${save_LIBS}"
658
+ CXXFLAGS="${save_CXXFLAGS}"
659
+ AC_LANG_RESTORE
660
+else
661
+ have_libaws_cpp_sdk_kinesis=no
662
+fi
663
+
664
+AC_MSG_RESULT([${have_libaws_cpp_sdk_kinesis}])
665
+
666
+test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \
667
+ AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK])
668
+
669
+test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" != "yes" && \
670
+ AC_MSG_ERROR([libaws-cpp-sdk-core required but not found. try installing AWS C++ SDK])
671
+
672
+test "${enable_backend_kinesis}" = "yes" -a "${have_libcurl}" != "yes" && \
673
+ AC_MSG_ERROR([libcurl required but not found])
674
+
675
+test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \
676
+ AC_MSG_ERROR([libssl required but not found])
677
+
678
+test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \
679
+ AC_MSG_ERROR([libcrypto required but not found])
680
+
681
+AC_MSG_CHECKING([if kinesis backend should be enabled])
682
+if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" = "yes" \
683
+ -a "${have_libcurl}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcrypto}" = "yes"; then
684
+ enable_backend_kinesis="yes"
685
+ AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])
686
+ OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"
687
+ CXX11FLAG="-std=c++11"
688
+ OPTIONAL_KINESIS_LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
689
+else
690
+ enable_backend_kinesis="no"
691
+fi
692
+
693
+AC_MSG_RESULT([${enable_backend_kinesis}])
694
+AM_CONDITIONAL([ENABLE_BACKEND_KINESIS], [test "${enable_backend_kinesis}" = "yes"])
695
+
696
+
697
# -----------------------------------------------------------------------------
698
# check for setns() - cgroup-network
699
@@ -600,7 +718,7 @@ if test "${enable_lto}" != "no"; then
718
fi
719
if test "${have_lto}" = "yes"; then
720
oCFLAGS="${CFLAGS}"
603
- CFLAGS="${CFLAGS} -flto ${OPTIONAL_MATH_CLFAGS} ${OPTIONAL_NFACCT_CLFAGS} ${OPTIONAL_ZLIB_CLFAGS} ${OPTIONAL_UUID_CLFAGS} ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CLFAGS} ${OPTIONAL_XENSTAT_FLAGS}"
721
+ CFLAGS="${CFLAGS} -flto ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} ${OPTIONAL_KINESIS_CFLAGS}"
722
ac_cv_c_lto_cross_compile="${enable_lto}"
723
test "${ac_cv_c_lto_cross_compile}" != "yes" && ac_cv_c_lto_cross_compile="no"
724
AC_C_LTO
@@ -647,6 +765,8 @@ AC_SUBST([logdir])
765
AC_SUBST([pluginsdir])
766
AC_SUBST([webdir])
767
768
+CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
769
+
770
CPPFLAGS="\
771
-DTARGET_OS=${build_target_id} \
772
-DVARLIB_DIR=\"\\\"${varlibdir}\\\"\" \
@@ -659,13 +779,13 @@ CPPFLAGS="\
779
-DWEB_DIR=\"\\\"${webdir}\\\"\" \
780
"
781
662
-AC_SUBST([OPTIONAL_MATH_CLFAGS])
782
+AC_SUBST([OPTIONAL_MATH_CFLAGS])
783
AC_SUBST([OPTIONAL_MATH_LIBS])
664
-AC_SUBST([OPTIONAL_NFACCT_CLFAGS])
784
+AC_SUBST([OPTIONAL_NFACCT_CFLAGS])
785
AC_SUBST([OPTIONAL_NFACCT_LIBS])
666
-AC_SUBST([OPTIONAL_ZLIB_CLFAGS])
786
+AC_SUBST([OPTIONAL_ZLIB_CFLAGS])
787
AC_SUBST([OPTIONAL_ZLIB_LIBS])
668
-AC_SUBST([OPTIONAL_UUID_CLFAGS])
788
+AC_SUBST([OPTIONAL_UUID_CFLAGS])
789
AC_SUBST([OPTIONAL_UUID_LIBS])
790
AC_SUBST([OPTIONAL_LIBCAP_CFLAGS])
791
AC_SUBST([OPTIONAL_LIBCAP_LIBS])
@@ -675,6 +795,8 @@ AC_SUBST([OPTIONAL_CUPS_CFLAGS])
795
AC_SUBST([OPTIONAL_CUPS_LIBS])
796
AC_SUBST([OPTIONAL_XENSTAT_CFLAGS])
797
AC_SUBST([OPTIONAL_XENSTAT_LIBS])
798
+AC_SUBST([OPTIONAL_KINESIS_CFLAGS])
799
+AC_SUBST([OPTIONAL_KINESIS_LIBS])
800
801
802
AC_CONFIG_FILES([
@@ -685,6 +807,7 @@ AC_CONFIG_FILES([
807
backends/Makefile
808
backends/opentsdb/Makefile
809
backends/prometheus/Makefile
810
+ backends/aws_kinesis/Makefile
811
collectors/Makefile
812
collectors/apps.plugin/Makefile
813
collectors/cgroups.plugin/Makefile
netdata-installer.sh
+5
@@ -156,6 +156,9 @@ USAGE: ${PROGRAM} [options]
156
--disable-plugin-nfacct
157
--enable-plugin-xenstat Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available
158
--disable-plugin-xenstat Disable the xenstat plugin.
159
+ --enable-backend-kinesis Enable AWS Kinesis backend. Default: enable it when libaws_cpp_sdk_kinesis and libraries
160
+ it depends on are available.
161
+ --disable-backend-kinesis
162
--enable-lto Enable Link-Time-Optimization. Default: enabled
163
--disable-lto
164
--disable-x86-sse Disable SSE instructions. By default SSE optimizations are enabled.
@@ -201,6 +204,8 @@ while [ -n "${1}" ]; do
204
"--disable-plugin-nfacct") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-plugin-nfacct/} --disable-plugin-nfacct";;
205
"--enable-plugin-xenstat") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-plugin-xenstat/} --enable-plugin-xenstat";;
206
"--disable-plugin-xenstat") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-plugin-xenstat/} --disable-plugin-xenstat";;
207
+ "--enable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-kinesis/} --enable-backend-kinesis";;
208
+ "--disable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-kinesis/} --disable-backend-kinesis";;
209
"--enable-lto") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-lto/} --enable-lto";;
210
"--disable-lto") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-lto/} --disable-lto";;
211
"--disable-x86-sse") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-x86-sse/} --disable-x86-sse";;