Add MongoDB backend (#6524)
* Add mongodb backend skeleton * Send data to the backend * Send metrics as separate JSON documents * Add a configuration file * Send all metrics in a batch * Update the documentation * Free configuration strings on exit * Make socket timeout configurable
Vladimir Kobal committed
Aug 14, 2019 at 11:03 UTC
6262597923df989d4b9fc216f1645b006059da19
15 files changed
+526
-51
CMakeLists.txt
+28
@@ -115,6 +115,7 @@ set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_
115
#set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${JUDY_INCLUDE_DIRS})
116
set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} "-lJudy")
117
set(CMAKE_REQUIRED_LIBRARIES "Judy")
118
+include(CheckSymbolExists)
119
check_symbol_exists("JudyLLast" "Judy.h" HAVE_JUDY)
120
IF(HAVE_JUDY)
121
message(STATUS "Judy library found")
@@ -271,6 +272,15 @@ pkg_check_modules(SNAPPY snappy)
272
# ${SNAPPY_INCLUDE_DIRS}
273
274
275
+# -----------------------------------------------------------------------------
276
+# Detect libmongoc
277
+
278
+find_package(libmongoc-1.0)
279
+# later we use:
280
+# ${MONGOC_LIBRARIES}
281
+# ${MONGOC_INCLUDE_DIRS}
282
+
283
+
284
# -----------------------------------------------------------------------------
285
# netdata files
286
@@ -594,6 +604,11 @@ set(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES
604
backends/prometheus/remote_write/remote_write.h
605
)
606
607
+set(MONGODB_BACKEND_FILES
608
+ backends/mongodb/mongodb.c
609
+ backends/mongodb/mongodb.h
610
+ )
611
+
612
set(DAEMON_FILES
613
daemon/common.c
614
daemon/common.h
@@ -681,6 +696,19 @@ ELSE()
696
message(STATUS "prometheus remote write backend: disabled (requires protobuf and snappy libraries)")
697
ENDIF()
698
699
+# -----------------------------------------------------------------------------
700
+# mongodb backend
701
+
702
+IF(libmongoc-1.0_FOUND)
703
+ message(STATUS "mongodb backend: enabled")
704
+
705
+ list(APPEND NETDATA_FILES ${MONGODB_BACKEND_FILES})
706
+ list(APPEND NETDATA_COMMON_LIBRARIES ${MONGOC_LIBRARIES})
707
+ list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${MONGOC_INCLUDE_DIRS})
708
+ELSE()
709
+ message(STATUS "mongodb backend: disabled (requires mongoc library)")
710
+ENDIF()
711
+
712
# -----------------------------------------------------------------------------
713
# netdata
714
Makefile.am
+10
@@ -458,6 +458,11 @@ PROMETHEUS_REMOTE_WRITE_BACKEND_FILES = \
458
backends/prometheus/remote_write/remote_write.proto \
459
$(NULL)
460
461
+MONGODB_BACKEND_FILES = \
462
+ backends/mongodb/mongodb.c \
463
+ backends/mongodb/mongodb.h \
464
+ $(NULL)
465
+
466
DAEMON_FILES = \
467
daemon/common.c \
468
daemon/common.h \
@@ -615,3 +620,8 @@ backends/prometheus/remote_write/remote_write.pb.h: backends/prometheus/remote_w
620
$(PROTOC) --proto_path=$(srcdir) --cpp_out=$(builddir) $^
621
622
endif
623
+
624
+if ENABLE_BACKEND_MONGODB
625
+ netdata_SOURCES += $(MONGODB_BACKEND_FILES)
626
+ netdata_LDADD += $(OPTIONAL_MONGOC_LIBS)
627
+endif
README.md
+1
-1
@@ -324,7 +324,7 @@ This is what you should expect from Netdata:
324
- **Notifications**: [alerta.io](health/notifications/alerta/), [amazon sns](health/notifications/awssns/), [discordapp.com](health/notifications/discord/), [email](health/notifications/email/), [flock.com](health/notifications/flock/), [irc](health/notifications/irc/), [kavenegar.com](health/notifications/kavenegar/), [messagebird.com](health/notifications/messagebird/), [pagerduty.com](health/notifications/pagerduty/), [prowl](health/notifications/prowl/), [pushbullet.com](health/notifications/pushbullet/), [pushover.net](health/notifications/pushover/), [rocket.chat](health/notifications/rocketchat/), [slack.com](health/notifications/slack/), [smstools3](health/notifications/smstools3/), [syslog](health/notifications/syslog/), [telegram.org](health/notifications/telegram/), [twilio.com](health/notifications/twilio/), [web](health/notifications/web/) and [custom notifications](health/notifications/custom/).
325
326
### Integrations
327
-- **time-series dbs** - can archive its metrics to **Graphite**, **OpenTSDB**, **Prometheus**, **AWS Kinesis**, **JSON document DBs**, in the same or lower resolution (lower: to prevent it from congesting these servers due to the amount of data collected). Netdata also supports **Prometheus remote write API** which allows storing metrics to **Elasticsearch**, **Gnocchi**, **InfluxDB**, **Kafka**, **PostgreSQL/TimescaleDB**, **Splunk**, **VictoriaMetrics** and a lot of other [storage providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
327
+- **time-series dbs** - can archive its metrics to **Graphite**, **OpenTSDB**, **Prometheus**, **AWS Kinesis**, **MongoDB**, **JSON document DBs**, in the same or lower resolution (lower: to prevent it from congesting these servers due to the amount of data collected). Netdata also supports **Prometheus remote write API** which allows storing metrics to **Elasticsearch**, **Gnocchi**, **InfluxDB**, **Kafka**, **PostgreSQL/TimescaleDB**, **Splunk**, **VictoriaMetrics** and a lot of other [storage providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
328
329
## Visualization
330
backends/Makefile.am
+1
@@ -9,6 +9,7 @@ SUBDIRS = \
9
opentsdb \
10
prometheus \
11
aws_kinesis \
12
+ mongodb \
13
$(NULL)
14
15
dist_noinst_DATA = \
backends/README.md
+9
-2
@@ -44,6 +44,10 @@ X seconds (though, it can send them per second if you need it to).
44
45
metrics are sent to the service in `JSON` format.
46
47
+ - **MongoDB**
48
+
49
+ metrics are sent to the database in `JSON` format.
50
+
51
2. Only one backend may be active at a time.
52
53
3. Netdata can filter metrics (at the chart level), to send only a subset of the collected metrics.
@@ -78,7 +82,7 @@ of `netdata.conf` from your Netdata):
82
```
83
[backend]
84
enabled = yes | no
81
- type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | prometheus_remote_write | json | kinesis
85
+ type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | prometheus_remote_write | json | kinesis | mongodb
86
host tags = list of TAG=VALUE
87
destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used, or a region for kinesis
88
data source = average | sum | as collected
@@ -94,7 +98,7 @@ of `netdata.conf` from your Netdata):
98
99
- `enabled = yes | no`, enables or disables sending data to a backend
100
97
-- `type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | json | kinesis`, selects the backend type
101
+- `type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | json | kinesis | mongodb`, selects the backend type
102
103
- `destination = host1 host2 host3 ...`, accepts **a space separated list** of hostnames,
104
IPs (IPv4 and IPv6) and ports to connect to.
@@ -132,6 +136,9 @@ of `netdata.conf` from your Netdata):
136
137
For kinesis backend `destination` should be set to an AWS region (for example, `us-east-1`).
138
139
+ The MongoDB backend doesn't use the `destination` option for its configuration. It uses the `mongodb.conf`
140
+ [configuration file](mongodb/README.md) instead.
141
+
142
- `data source = as collected`, or `data source = average`, or `data source = sum`, selects the kind of
143
data that will be sent to the backend.
144
backends/aws_kinesis/aws_kinesis.c
+4
-11
@@ -63,18 +63,11 @@ int read_kinesis_conf(const char *path, char **access_key_id_p, char **secret_ac
63
continue;
64
}
65
66
- if(!value) value = "";
66
+ if(!value)
67
+ value = "";
68
+ else
69
+ value = strip_quotes(value);
70
68
- // strip quotes
69
- if(*value == '"' || *value == '\'') {
70
- value++;
71
-
72
- s = value;
73
- while(*s) s++;
74
- if(s != value) s--;
75
-
76
- if(*s == '"' || *s == '\'') *s = '\0';
77
- }
71
if(name[0] == 'a' && name[4] == 'a' && !strcmp(name, "aws_access_key_id")) {
72
access_key_id = strdupz(value);
73
}
backends/backends.c
+140
-29
@@ -249,12 +249,11 @@ static void backends_main_cleanup(void *ptr) {
249
/**
250
* Set Kinesis variables
251
*
252
- * Set the variables necessaries to work with this specific backend.
252
+ * Set the variables necessary to work with this specific backend.
253
*
254
- * @param default_port the default port of the backend
254
+ * @param default_port the default port of the backend
255
* @param brc function called to check the result.
256
- * @param brf function called to format the msessage to the backend
257
- * @param type the backend string selector.
256
+ * @param brf function called to format the message to the backend
257
*/
258
void backend_set_kinesis_variables(int *default_port,
259
backend_response_checker_t brc,
@@ -278,12 +277,11 @@ void backend_set_kinesis_variables(int *default_port,
277
/**
278
* Set Prometheus variables
279
*
281
- * Set the variables necessaries to work with this specific backend.
280
+ * Set the variables necessary to work with this specific backend.
281
*
283
- * @param default_port the default port of the backend
282
+ * @param default_port the default port of the backend
283
* @param brc function called to check the result.
285
- * @param brf function called to format the msessage to the backend
286
- * @param type the backend string selector.
284
+ * @param brf function called to format the message to the backend
285
*/
286
void backend_set_prometheus_variables(int *default_port,
287
backend_response_checker_t brc,
@@ -300,15 +298,42 @@ void backend_set_prometheus_variables(int *default_port,
298
#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
299
}
300
301
+/**
302
+ * Set MongoDB variables
303
+ *
304
+ * Set the variables necessary to work with this specific backend.
305
+ *
306
+ * @param default_port the default port of the backend
307
+ * @param brc function called to check the result.
308
+ * @param brf function called to format the message to the backend
309
+ */
310
+void backend_set_mongodb_variables(int *default_port,
311
+ backend_response_checker_t brc,
312
+ backend_request_formatter_t brf)
313
+{
314
+ (void)default_port;
315
+#ifndef HAVE_MONGOC
316
+ (void)brc;
317
+ (void)brf;
318
+#endif
319
+
320
+#if HAVE_MONGOC
321
+ *brc = process_json_response;
322
+ if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
323
+ *brf = format_dimension_collected_json_plaintext;
324
+ else
325
+ *brf = format_dimension_stored_json_plaintext;
326
+#endif
327
+}
328
+
329
/**
330
* Set JSON variables
331
*
306
- * Set the variables necessaries to work with this specific backend.
332
+ * Set the variables necessary to work with this specific backend.
333
*
308
- * @param default_port the default port of the backend
334
+ * @param default_port the default port of the backend
335
* @param brc function called to check the result.
310
- * @param brf function called to format the msessage to the backend
311
- * @param type the backend string selector.
336
+ * @param brf function called to format the message to the backend
337
*/
338
void backend_set_json_variables(int *default_port,
339
backend_response_checker_t brc,
@@ -326,12 +351,11 @@ void backend_set_json_variables(int *default_port,
351
/**
352
* Set OpenTSDB HTTP variables
353
*
329
- * Set the variables necessaries to work with this specific backend.
354
+ * Set the variables necessary to work with this specific backend.
355
*
331
- * @param default_port the default port of the backend
356
+ * @param default_port the default port of the backend
357
* @param brc function called to check the result.
333
- * @param brf function called to format the msessage to the backend
334
- * @param type the backend string selector.
358
+ * @param brf function called to format the message to the backend
359
*/
360
void backend_set_opentsdb_http_variables(int *default_port,
361
backend_response_checker_t brc,
@@ -350,12 +374,11 @@ void backend_set_opentsdb_http_variables(int *default_port,
374
/**
375
* Set OpenTSDB Telnet variables
376
*
353
- * Set the variables necessaries to work with this specific backend.
377
+ * Set the variables necessary to work with this specific backend.
378
*
355
- * @param default_port the default port of the backend
379
+ * @param default_port the default port of the backend
380
* @param brc function called to check the result.
357
- * @param brf function called to format the msessage to the backend
358
- * @param type the backend string selector.
381
+ * @param brf function called to format the message to the backend
382
*/
383
void backend_set_opentsdb_telnet_variables(int *default_port,
384
backend_response_checker_t brc,
@@ -373,12 +396,11 @@ void backend_set_opentsdb_telnet_variables(int *default_port,
396
/**
397
* Set Graphite variables
398
*
376
- * Set the variables necessaries to work with this specific backend.
399
+ * Set the variables necessary to work with this specific backend.
400
*
378
- * @param default_port the default port of the backend
401
+ * @param default_port the default port of the backend
402
* @param brc function called to check the result.
380
- * @param brf function called to format the msessage to the backend
381
- * @param type the backend string selector.
403
+ * @param brf function called to format the message to the backend
404
*/
405
void backend_set_graphite_variables(int *default_port,
406
backend_response_checker_t brc,
@@ -421,6 +443,9 @@ BACKEND_TYPE backend_select_type(const char *type) {
443
else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
444
return BACKEND_TYPE_KINESIS;
445
}
446
+ else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) {
447
+ return BACKEND_TYPE_MONGODB;
448
+ }
449
450
return BACKEND_TYPE_UNKNOWN;
451
}
@@ -450,7 +475,18 @@ void *backends_main(void *ptr) {
475
476
#if ENABLE_PROMETHEUS_REMOTE_WRITE
477
int do_prometheus_remote_write = 0;
453
- BUFFER *http_request_header = buffer_create(1);
478
+ BUFFER *http_request_header = NULL;
479
+#endif
480
+
481
+#if HAVE_MONGOC
482
+ int do_mongodb = 0;
483
+ char *mongodb_uri = NULL;
484
+ char *mongodb_database = NULL;
485
+ char *mongodb_collection = NULL;
486
+
487
+ // set the default socket timeout in ms
488
+ int32_t mongodb_default_socket_timeout = (int32_t)(global_backend_update_every >= 2)?(global_backend_update_every * MSEC_PER_SEC - 500):1000;
489
+
490
#endif
491
492
#ifdef ENABLE_HTTPS
@@ -524,6 +560,7 @@ void *backends_main(void *ptr) {
560
#if ENABLE_PROMETHEUS_REMOTE_WRITE
561
do_prometheus_remote_write = 1;
562
563
+ http_request_header = buffer_create(1);
564
init_write_request();
565
#else
566
error("BACKEND: Prometheus remote write support isn't compiled");
@@ -547,6 +584,30 @@ void *backends_main(void *ptr) {
584
backend_set_kinesis_variables(&default_port,&backend_response_checker,&backend_request_formatter);
585
break;
586
}
587
+ case BACKEND_TYPE_MONGODB: {
588
+#if HAVE_MONGOC
589
+ if(unlikely(read_mongodb_conf(netdata_configured_user_config_dir,
590
+ &mongodb_uri,
591
+ &mongodb_database,
592
+ &mongodb_collection))) {
593
+ error("BACKEND: mongodb backend type is set but cannot read its configuration from %s/mongodb.conf",
594
+ netdata_configured_user_config_dir);
595
+ goto cleanup;
596
+ }
597
+
598
+ if(likely(!mongodb_init(mongodb_uri, mongodb_database, mongodb_collection, mongodb_default_socket_timeout))) {
599
+ backend_set_mongodb_variables(&default_port, &backend_response_checker, &backend_request_formatter);
600
+ do_mongodb = 1;
601
+ }
602
+ else {
603
+ error("BACKEND: cannot initialize MongoDB backend");
604
+ goto cleanup;
605
+ }
606
+#else
607
+ error("BACKEND: MongoDB support isn't compiled");
608
+#endif // HAVE_MONGOC
609
+ break;
610
+ }
611
case BACKEND_TYPE_GRAPHITE: {
612
backend_set_graphite_variables(&default_port,&backend_response_checker,&backend_request_formatter);
613
break;
@@ -834,12 +895,53 @@ void *backends_main(void *ptr) {
895
chart_sent_metrics = chart_buffered_metrics;
896
897
buffer_flush(b);
837
- }
838
- else {
839
-#else
840
- {
898
+ } else
899
#endif /* HAVE_KINESIS */
900
901
+#if HAVE_MONGOC
902
+ if(do_mongodb) {
903
+ size_t buffer_len = buffer_strlen(b);
904
+ size_t sent = 0;
905
+
906
+ while(sent < buffer_len) {
907
+ const char *first_char = buffer_tostring(b);
908
+
909
+ debug(D_BACKEND, "BACKEND: mongodb_insert(): uri = %s, database = %s, collection = %s, \
910
+ buffer = %zu", mongodb_uri, mongodb_database, mongodb_collection, buffer_len);
911
+
912
+ if(likely(!mongodb_insert((char *)first_char, (size_t)chart_buffered_metrics))) {
913
+ sent += buffer_len;
914
+ chart_transmission_successes++;
915
+ chart_receptions++;
916
+ }
917
+ else {
918
+ // oops! we couldn't send (all or some of the) data
919
+ error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zu bytes.",
920
+ mongodb_uri, buffer_len, 0UL);
921
+
922
+ chart_transmission_failures++;
923
+ chart_data_lost_events++;
924
+ chart_lost_bytes += buffer_len;
925
+
926
+ // estimate the number of lost metrics
927
+ chart_lost_metrics += (collected_number)chart_buffered_metrics;
928
+
929
+ break;
930
+ }
931
+
932
+ if(unlikely(netdata_exit)) break;
933
+ }
934
+
935
+ chart_sent_bytes += sent;
936
+ if(likely(sent == buffer_len))
937
+ chart_sent_metrics = chart_buffered_metrics;
938
+
939
+ buffer_flush(b);
940
+ } else
941
+#endif /* HAVE_MONGOC */
942
+
943
+ {
944
+
945
// ------------------------------------------------------------------------
946
// if we are connected, receive a response, without blocking
947
@@ -1106,6 +1208,15 @@ cleanup:
1208
protocol_buffers_shutdown();
1209
#endif
1210
1211
+#if HAVE_MONGOC
1212
+ if(do_mongodb) {
1213
+ mongodb_cleanup();
1214
+ freez(mongodb_uri);
1215
+ freez(mongodb_database);
1216
+ freez(mongodb_collection);
1217
+ }
1218
+#endif
1219
+
1220
if(sock != -1)
1221
close(sock);
1222
backends/backends.h
+28
-7
@@ -16,13 +16,14 @@ typedef enum backend_options {
16
} BACKEND_OPTIONS;
17
18
typedef enum backend_types {
19
- BACKEND_TYPE_UNKNOWN, //Invalid type
20
- BACKEND_TYPE_GRAPHITE, //Send plain text to Graphite
21
- BACKEND_TYPE_OPENTSDB_USING_TELNET, //Send data to OpenTSDB using telnet API
22
- BACKEND_TYPE_OPENTSDB_USING_HTTP, //Send data to OpenTSDB using HTTP API
23
- BACKEND_TYPE_JSON, //Stores the data using JSON.
24
- BACKEND_TYPE_PROMETEUS, //The user selected to use Prometheus backend
25
- BACKEND_TYPE_KINESIS //Send message to AWS Kinesis
19
+ BACKEND_TYPE_UNKNOWN, // Invalid type
20
+ BACKEND_TYPE_GRAPHITE, // Send plain text to Graphite
21
+ BACKEND_TYPE_OPENTSDB_USING_TELNET, // Send data to OpenTSDB using telnet API
22
+ BACKEND_TYPE_OPENTSDB_USING_HTTP, // Send data to OpenTSDB using HTTP API
23
+ BACKEND_TYPE_JSON, // Stores the data using JSON.
24
+ BACKEND_TYPE_PROMETEUS, // The user selected to use Prometheus backend
25
+ BACKEND_TYPE_KINESIS, // Send message to AWS Kinesis
26
+ BACKEND_TYPE_MONGODB // Send data to MongoDB collection
27
} BACKEND_TYPE;
28
29
@@ -56,6 +57,22 @@ extern calculated_number backend_calculate_value_from_stored_data(
57
extern size_t backend_name_copy(char *d, const char *s, size_t usable);
58
extern int discard_response(BUFFER *b, const char *backend);
59
60
+static inline char *strip_quotes(char *str) {
61
+ if(*str == '"' || *str == '\'') {
62
+ char *s;
63
+
64
+ str++;
65
+
66
+ s = str;
67
+ while(*s) s++;
68
+ if(s != str) s--;
69
+
70
+ if(*s == '"' || *s == '\'') *s = '\0';
71
+ }
72
+
73
+ return str;
74
+}
75
+
76
#endif // BACKENDS_INTERNALS
77
78
#include "backends/prometheus/backend_prometheus.h"
@@ -71,4 +88,8 @@ extern int discard_response(BUFFER *b, const char *backend);
88
#include "backends/prometheus/remote_write/remote_write.h"
89
#endif
90
91
+#if HAVE_MONGOC
92
+#include "backends/mongodb/mongodb.h"
93
+#endif
94
+
95
#endif /* NETDATA_BACKENDS_H */
backends/mongodb/Makefile.am
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
+
6
+dist_noinst_DATA = \
7
+ README.md \
8
+ $(NULL)
9
+
10
+dist_libconfig_DATA = \
11
+ mongodb.conf \
12
+ $(NULL)
backends/mongodb/README.md
new
+31
@@ -0,0 +1,31 @@
1
+# MongoDB backend
2
+
3
+## Prerequisites
4
+
5
+To use MongoDB as a backend, `libmongoc` 1.7.0 or higher should be [installed](http://mongoc.org/libmongoc/current/installing.html) first. Next, Netdata should be re-installed from the source. The installer will detect that the required libraries are now available.
6
+
7
+## Configuration
8
+
9
+To enable data sending to the MongoDB backend set the following options in `netdata.conf`:
10
+```
11
+[backend]
12
+ enabled = yes
13
+ type = mongodb
14
+```
15
+
16
+In the Netdata configuration directory run `./edit-config mongodb.conf` and set [MongoDB URI](https://docs.mongodb.com/manual/reference/connection-string/), database name, and collection name:
17
+```
18
+# URI
19
+uri = mongodb://<hostname>
20
+
21
+# database name
22
+database = your_database_name
23
+
24
+# collection name
25
+collection = your_collection_name
26
+```
27
+
28
+The default socket timeout depends on the backend update interval. The timeout is 500 ms shorter than the interval (but not less than 1000 ms). You can alter the timeout using the `sockettimeoutms` MongoDB URI option.
29
+
30
+
31
+[]()
backends/mongodb/mongodb.c
new
+189
@@ -0,0 +1,189 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#define BACKENDS_INTERNALS
4
+#include "mongodb.h"
5
+#include <mongoc.h>
6
+
7
+#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2)
8
+
9
+mongoc_client_t *mongodb_client;
10
+mongoc_collection_t *mongodb_collection;
11
+
12
+int mongodb_init(const char *uri_string,
13
+ const char *database_string,
14
+ const char *collection_string,
15
+ int32_t default_socket_timeout) {
16
+ mongoc_uri_t *uri;
17
+ bson_error_t error;
18
+
19
+ mongoc_init();
20
+
21
+ uri = mongoc_uri_new_with_error(uri_string, &error);
22
+ if(unlikely(!uri)) {
23
+ error("BACKEND: failed to parse URI: %s. Error message: %s", uri_string, error.message);
24
+ return 1;
25
+ }
26
+
27
+ int32_t socket_timeout = mongoc_uri_get_option_as_int32(uri, MONGOC_URI_SOCKETTIMEOUTMS, default_socket_timeout);
28
+ if(!mongoc_uri_set_option_as_int32(uri, MONGOC_URI_SOCKETTIMEOUTMS, socket_timeout)) {
29
+ error("BACKEND: failed to set %s to the value %d", MONGOC_URI_SOCKETTIMEOUTMS, socket_timeout);
30
+ return 1;
31
+ };
32
+
33
+ mongodb_client = mongoc_client_new_from_uri(uri);
34
+ if(unlikely(!mongodb_client)) {
35
+ error("BACKEND: failed to create a new client");
36
+ return 1;
37
+ }
38
+
39
+ if(!mongoc_client_set_appname(mongodb_client, "netdata")) {
40
+ error("BACKEND: failed to set client appname");
41
+ };
42
+
43
+ mongodb_collection = mongoc_client_get_collection(mongodb_client, database_string, collection_string);
44
+
45
+ mongoc_uri_destroy(uri);
46
+
47
+ return 0;
48
+}
49
+
50
+void free_bson(bson_t **insert, size_t n_documents) {
51
+ size_t i;
52
+
53
+ for(i = 0; i < n_documents; i++)
54
+ bson_destroy(insert[i]);
55
+
56
+ free(insert);
57
+}
58
+
59
+int mongodb_insert(char *data, size_t n_metrics) {
60
+ bson_t **insert = calloc(n_metrics, sizeof(bson_t *));
61
+ bson_error_t error;
62
+ char *start = data, *end = data;
63
+ size_t n_documents = 0;
64
+
65
+ while(*end && n_documents <= n_metrics) {
66
+ while(*end && *end != '\n') end++;
67
+
68
+ if(likely(*end)) {
69
+ *end = '\0';
70
+ end++;
71
+ }
72
+ else {
73
+ break;
74
+ }
75
+
76
+ insert[n_documents] = bson_new_from_json((const uint8_t *)start, -1, &error);
77
+
78
+ if(unlikely(!insert[n_documents])) {
79
+ error("BACKEND: %s", error.message);
80
+ free_bson(insert, n_documents);
81
+ return 1;
82
+ }
83
+
84
+ start = end;
85
+
86
+ n_documents++;
87
+ }
88
+
89
+ if(unlikely(!mongoc_collection_insert_many(mongodb_collection, (const bson_t **)insert, n_documents, NULL, NULL, &error))) {
90
+ error("BACKEND: %s", error.message);
91
+ free_bson(insert, n_documents);
92
+ return 1;
93
+ }
94
+
95
+ free_bson(insert, n_documents);
96
+
97
+ return 0;
98
+}
99
+
100
+void mongodb_cleanup() {
101
+ mongoc_collection_destroy(mongodb_collection);
102
+ mongoc_client_destroy(mongodb_client);
103
+ mongoc_cleanup();
104
+
105
+ return;
106
+}
107
+
108
+int read_mongodb_conf(const char *path, char **uri_p, char **database_p, char **collection_p) {
109
+ char *uri = *uri_p;
110
+ char *database = *database_p;
111
+ char *collection = *collection_p;
112
+
113
+ if(unlikely(uri)) freez(uri);
114
+ if(unlikely(database)) freez(database);
115
+ if(unlikely(collection)) freez(collection);
116
+ uri = NULL;
117
+ database = NULL;
118
+ collection = NULL;
119
+
120
+ int line = 0;
121
+
122
+ char filename[FILENAME_MAX + 1];
123
+ snprintfz(filename, FILENAME_MAX, "%s/mongodb.conf", path);
124
+
125
+ char buffer[CONFIG_FILE_LINE_MAX + 1], *s;
126
+
127
+ debug(D_BACKEND, "BACKEND: opening config file '%s'", filename);
128
+
129
+ FILE *fp = fopen(filename, "r");
130
+ if(!fp) {
131
+ return 1;
132
+ }
133
+
134
+ while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {
135
+ buffer[CONFIG_FILE_LINE_MAX] = '\0';
136
+ line++;
137
+
138
+ s = trim(buffer);
139
+ if(!s || *s == '#') {
140
+ debug(D_BACKEND, "BACKEND: ignoring line %d of file '%s', it is empty.", line, filename);
141
+ continue;
142
+ }
143
+
144
+ char *name = s;
145
+ char *value = strchr(s, '=');
146
+ if(unlikely(!value)) {
147
+ error("BACKEND: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename);
148
+ continue;
149
+ }
150
+ *value = '\0';
151
+ value++;
152
+
153
+ name = trim(name);
154
+ value = trim(value);
155
+
156
+ if(unlikely(!name || *name == '#')) {
157
+ error("BACKEND: ignoring line %d of file '%s', name is empty.", line, filename);
158
+ continue;
159
+ }
160
+
161
+ if(!value)
162
+ value = "";
163
+ else
164
+ value = strip_quotes(value);
165
+
166
+ if(name[0] == 'u' && !strcmp(name, "uri")) {
167
+ uri = strdupz(value);
168
+ }
169
+ else if(name[0] == 'd' && !strcmp(name, "database")) {
170
+ database = strdupz(value);
171
+ }
172
+ else if(name[0] == 'c' && !strcmp(name, "collection")) {
173
+ collection = strdupz(value);
174
+ }
175
+ }
176
+
177
+ fclose(fp);
178
+
179
+ if(unlikely(!collection || !*collection)) {
180
+ error("BACKEND: collection name is a mandatory MongoDB parameter, but it is not configured");
181
+ return 1;
182
+ }
183
+
184
+ *uri_p = uri;
185
+ *database_p = database;
186
+ *collection_p = collection;
187
+
188
+ return 0;
189
+}
backends/mongodb/mongodb.conf
new
+12
@@ -0,0 +1,12 @@
1
+# MongoDB backend configuration
2
+#
3
+# All options in this file are mandatory
4
+
5
+# URI
6
+uri =
7
+
8
+# database name
9
+database =
10
+
11
+# collection name
12
+collection =
backends/mongodb/mongodb.h
new
+16
@@ -0,0 +1,16 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_BACKEND_MONGODB_H
4
+#define NETDATA_BACKEND_MONGODB_H
5
+
6
+#include "backends/backends.h"
7
+
8
+extern int mongodb_init(const char *uri_string, const char *database_string, const char *collection_string, const int32_t socket_timeout);
9
+
10
+extern int mongodb_insert(char *data, size_t n_metrics);
11
+
12
+extern void mongodb_cleanup();
13
+
14
+extern int read_mongodb_conf(const char *path, char **uri_p, char **database_p, char **collection_p);
15
+
16
+#endif //NETDATA_BACKEND_MONGODB_H
configure.ac
+41
-1
@@ -82,6 +82,12 @@ AC_ARG_ENABLE(
82
,
83
[enable_backend_prometheus_remote_write="detect"]
84
)
85
+AC_ARG_ENABLE(
86
+ [backend-mongodb],
87
+ [AS_HELP_STRING([--enable-backend-mongodb], [enable mongodb backend @<:@default autodetect@:>@])],
88
+ ,
89
+ [enable_backend_mongodb="detect"]
90
+)
91
AC_ARG_ENABLE(
92
[pedantic],
93
[AS_HELP_STRING([--enable-pedantic], [enable pedantic compiler warnings @<:@default disabled@:>@])],
@@ -964,6 +970,33 @@ AC_MSG_RESULT([${enable_backend_prometheus_remote_write}])
970
AM_CONDITIONAL([ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE], [test "${enable_backend_prometheus_remote_write}" = "yes"])
971
972
973
+# -----------------------------------------------------------------------------
974
+# MongoDB backend - libmongoc
975
+
976
+PKG_CHECK_MODULES(
977
+ [LIBMONGOC],
978
+ [libmongoc-1.0 >= 1.7],
979
+ [have_libmongoc=yes],
980
+ [have_libmongoc=no]
981
+)
982
+
983
+test "${enable_backend_mongodb}" = "yes" -a "${have_libmongoc}" != "yes" && \
984
+ AC_MSG_ERROR([libmongoc required but not found. Try installing `mongoc`.])
985
+
986
+AC_MSG_CHECKING([if mongodb backend should be enabled])
987
+if test "${enable_backend_mongodb}" != "no" -a "${have_libmongoc}" = "yes"; then
988
+ enable_backend_mongodb="yes"
989
+ AC_DEFINE([HAVE_MONGOC], [1], [libmongoc usability])
990
+ OPTIONAL_MONGOC_CFLAGS="${LIBMONGOC_CFLAGS}"
991
+ OPTIONAL_MONGOC_LIBS="${LIBMONGOC_LIBS}"
992
+else
993
+ enable_backend_mongodb="no"
994
+fi
995
+
996
+AC_MSG_RESULT([${enable_backend_mongodb}])
997
+AM_CONDITIONAL([ENABLE_BACKEND_MONGODB], [test "${enable_backend_mongodb}" = "yes"])
998
+
999
+
1000
# -----------------------------------------------------------------------------
1001
# check for setns() - cgroup-network
1002
@@ -988,7 +1021,7 @@ if test "${enable_lto}" != "no"; then
1021
fi
1022
if test "${have_lto}" = "yes"; then
1023
oCFLAGS="${CFLAGS}"
991
- CFLAGS="${CFLAGS} -flto ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS}"
1024
+ CFLAGS="${CFLAGS} -flto"
1025
ac_cv_c_lto_cross_compile="${enable_lto}"
1026
test "${ac_cv_c_lto_cross_compile}" != "yes" && ac_cv_c_lto_cross_compile="no"
1027
AC_C_LTO
@@ -1037,6 +1070,10 @@ AC_SUBST([logdir])
1070
AC_SUBST([pluginsdir])
1071
AC_SUBST([webdir])
1072
1073
+CFLAGS="${CFLAGS} ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} \
1074
+ ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \
1075
+ ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} ${OPTIONAL_MONGOC_CFLAGS}"
1076
+
1077
CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
1078
1079
CPPFLAGS="\
@@ -1076,6 +1113,8 @@ AC_SUBST([OPTIONAL_KINESIS_CFLAGS])
1113
AC_SUBST([OPTIONAL_KINESIS_LIBS])
1114
AC_SUBST([OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS])
1115
AC_SUBST([OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS])
1116
+AC_SUBST([OPTIONAL_MONGOC_CFLAGS])
1117
+AC_SUBST([OPTIONAL_MONGOC_LIBS])
1118
1119
1120
AC_CONFIG_FILES([
@@ -1088,6 +1127,7 @@ AC_CONFIG_FILES([
1127
backends/prometheus/Makefile
1128
backends/prometheus/remote_write/Makefile
1129
backends/aws_kinesis/Makefile
1130
+ backends/mongodb/Makefile
1131
collectors/Makefile
1132
collectors/apps.plugin/Makefile
1133
collectors/cgroups.plugin/Makefile
netdata-installer.sh
+4
@@ -168,6 +168,8 @@ USAGE: ${PROGRAM} [options]
168
--enable-backend-prometheus-remote-write Enable Prometheus remote write backend. Default: enable it when libprotobuf and
169
libsnappy are available.
170
--disable-backend-prometheus-remote-write
171
+ --enable-backend-mongodb Enable MongoDB backend. Default: enable it when libmongoc is available.
172
+ --disable-backend-mongodb
173
--enable-lto Enable Link-Time-Optimization. Default: enabled
174
--disable-lto
175
--disable-x86-sse Disable SSE instructions. By default SSE optimizations are enabled.
@@ -218,6 +220,8 @@ while [ -n "${1}" ]; do
220
"--disable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-kinesis/} --disable-backend-kinesis";;
221
"--enable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-prometheus-remote-write/} --enable-backend-prometheus-remote-write";;
222
"--disable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-prometheus-remote-write/} --disable-backend-prometheus-remote-write";;
223
+ "--enable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-mongodb/} --enable-backend-mongodb";;
224
+ "--disable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-mongodb/} --disable-backend-mongodb";;
225
"--enable-lto") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-lto/} --enable-lto";;
226
"--disable-lto") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-lto/} --disable-lto";;
227
"--disable-x86-sse") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-x86-sse/} --disable-x86-sse";;