@cryptotaxi247 / netdata-1 / commits / ca1799280

Backend and SSL! (#6220)

* SSL_backend Begin of the encryptation of backend! * SSL_backend changing opentsdb! * SSL_backend fix HTTP message with JSON! * SSL_backend HTTP API done! * SSL_fix_format preparing to connect with proxy! * SSL_backend wip SSL send/receive ! * SSL_backend working with proxy * SSL_backend removing comments! * SSL_backend docummentation! * SSL_backend review]! * SSL_backend organizing! * Alarm_backend remove comments! * SSL_backend! * SSL_backend typedef! * SSL_backend bring switch! * SSL_backend commiting format changes! * SSL_backend fix github parser! * SSL_Backend fix format! * SSL_backend switch everything! * SSL_backend reviewing! * SSL_backend comments! * SSL_backend indentation! * SSL_backend indentation 3! * SSL_backend documentation! * SSL_backend hidden pointer! * SSL_backend missing space * SSL_backend change documentation! * SSL_backend change documentation 2!

thiagoftsm committed Jun 27, 2019 at 11:20 UTC ca1799280d51e85f85ff436fca121e6d2a241bb3
10 files changed +583 -103
backends/README.md
+3 -3
@@ -22,7 +22,7 @@ X seconds (though, it can send them per second if you need it to).
22 metrics are sent to the backend server as `prefix.hostname.chart.dimension`. `prefix` is
23 configured below, `hostname` is the hostname of the machine (can also be configured).
24
25 - - **opentsdb** (`telnet interface`, used by **OpenTSDB**, **InfluxDB**, **KairosDB**, etc)
25 + - **opentsdb** (`telnet or HTTP interfaces`, used by **OpenTSDB**, **InfluxDB**, **KairosDB**, etc)
26
27 metrics are sent to opentsdb as `prefix.chart.dimension` with tag `host=hostname`.
28
@@ -76,7 +76,7 @@ of `netdata.conf` from your netdata):
76 ```
77 [backend]
78 enabled = yes | no
79 - type = graphite | opentsdb | json | prometheus_remote_write | kinesis
79 + type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | prometheus_remote_write | json | kinesis
80 host tags = list of TAG=VALUE
81 destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used, or a region for kinesis
82 data source = average | sum | as collected
@@ -92,7 +92,7 @@ of `netdata.conf` from your netdata):
92
93 - `enabled = yes | no`, enables or disables sending data to a backend
94
95 -- `type = graphite | opentsdb | json | kinesis`, selects the backend type
95 +- `type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | json | kinesis`, selects the backend type
96
97 - `destination = host1 host2 host3 ...`, accepts **a space separated list** of hostnames,
98 IPs (IPv4 and IPv6) and ports to connect to.
backends/backends.c
+321 -83
@@ -246,6 +246,194 @@ static void backends_main_cleanup(void *ptr) {
246 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
247 }
248
249 +/**
250 + * Set Kinesis variables
251 + *
252 + * Set the variables necessaries to work with this specific backend.
253 + *
254 + * @param default_port the default port of the backend
255 + * @param brc function called to check the result.
256 + * @param brf function called to format the msessage to the backend
257 + * @param type the backend string selector.
258 + */
259 +void backend_set_kinesis_variables(int *default_port,
260 + backend_response_checker_t brc,
261 + backend_request_formatter_t brf)
262 +{
263 + (void)default_port;
264 +#ifndef HAVE_KINESIS
265 + (void)brc;
266 + (void)brf;
267 +#endif
268 +
269 +#if HAVE_KINESIS
270 + *brc = process_json_response;
271 + if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
272 + *brf = format_dimension_collected_json_plaintext;
273 + else
274 + *brf = format_dimension_stored_json_plaintext;
275 +#endif
276 +}
277 +
278 +/**
279 + * Set Prometheus variables
280 + *
281 + * Set the variables necessaries to work with this specific backend.
282 + *
283 + * @param default_port the default port of the backend
284 + * @param brc function called to check the result.
285 + * @param brf function called to format the msessage to the backend
286 + * @param type the backend string selector.
287 + */
288 +void backend_set_prometheus_variables(int *default_port,
289 + backend_response_checker_t brc,
290 + backend_request_formatter_t brf)
291 +{
292 + (void)default_port;
293 + (void)brf;
294 +#ifndef ENABLE_PROMETHEUS_REMOTE_WRITE
295 + (void)brc;
296 +#endif
297 +
298 +#if ENABLE_PROMETHEUS_REMOTE_WRITE
299 + *brc = process_prometheus_remote_write_response;
300 +#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
301 +}
302 +
303 +/**
304 + * Set JSON variables
305 + *
306 + * Set the variables necessaries to work with this specific backend.
307 + *
308 + * @param default_port the default port of the backend
309 + * @param brc function called to check the result.
310 + * @param brf function called to format the msessage to the backend
311 + * @param type the backend string selector.
312 + */
313 +void backend_set_json_variables(int *default_port,
314 + backend_response_checker_t brc,
315 + backend_request_formatter_t brf)
316 +{
317 + *default_port = 5448;
318 + *brc = process_json_response;
319 +
320 + if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
321 + *brf = format_dimension_collected_json_plaintext;
322 + else
323 + *brf = format_dimension_stored_json_plaintext;
324 +}
325 +
326 +/**
327 + * Set OpenTSDB HTTP variables
328 + *
329 + * Set the variables necessaries to work with this specific backend.
330 + *
331 + * @param default_port the default port of the backend
332 + * @param brc function called to check the result.
333 + * @param brf function called to format the msessage to the backend
334 + * @param type the backend string selector.
335 + */
336 +void backend_set_opentsdb_http_variables(int *default_port,
337 + backend_response_checker_t brc,
338 + backend_request_formatter_t brf)
339 +{
340 + *default_port = 4242;
341 + *brc = process_opentsdb_response;
342 +
343 + if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
344 + *brf = format_dimension_collected_opentsdb_http;
345 + else
346 + *brf = format_dimension_stored_opentsdb_http;
347 +
348 +}
349 +
350 +/**
351 + * Set OpenTSDB Telnet variables
352 + *
353 + * Set the variables necessaries to work with this specific backend.
354 + *
355 + * @param default_port the default port of the backend
356 + * @param brc function called to check the result.
357 + * @param brf function called to format the msessage to the backend
358 + * @param type the backend string selector.
359 + */
360 +void backend_set_opentsdb_telnet_variables(int *default_port,
361 + backend_response_checker_t brc,
362 + backend_request_formatter_t brf)
363 +{
364 + *default_port = 4242;
365 + *brc = process_opentsdb_response;
366 +
367 + if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
368 + *brf = format_dimension_collected_opentsdb_telnet;
369 + else
370 + *brf = format_dimension_stored_opentsdb_telnet;
371 +}
372 +
373 +/**
374 + * Set Graphite variables
375 + *
376 + * Set the variables necessaries to work with this specific backend.
377 + *
378 + * @param default_port the default port of the backend
379 + * @param brc function called to check the result.
380 + * @param brf function called to format the msessage to the backend
381 + * @param type the backend string selector.
382 + */
383 +void backend_set_graphite_variables(int *default_port,
384 + backend_response_checker_t brc,
385 + backend_request_formatter_t brf)
386 +{
387 + *default_port = 2003;
388 + *brc = process_graphite_response;
389 +
390 + if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
391 + *brf = format_dimension_collected_graphite_plaintext;
392 + else
393 + *brf = format_dimension_stored_graphite_plaintext;
394 +}
395 +
396 +/**
397 + * Select Type
398 + *
399 + * Select the backedn type based in the user input
400 + *
401 + * @param type is the string that defines the backend type
402 + *
403 + * @return It returns the backend id.
404 + */
405 +BACKEND_TYPE backend_select_type(const char *type) {
406 + if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
407 + return BACKEND_TYPE_GRAPHITE;
408 + }
409 + else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
410 + return BACKEND_TYPE_OPENTSDB_USING_TELNET;
411 + }
412 + else if(!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
413 + return BACKEND_TYPE_OPENTSDB_USING_HTTP;
414 + }
415 + else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
416 + return BACKEND_TYPE_JSON;
417 + }
418 + else if (!strcmp(type, "prometheus_remote_write")) {
419 + return BACKEND_TYPE_PROMETEUS;
420 + }
421 + else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
422 + return BACKEND_TYPE_KINESIS;
423 + }
424 +
425 + return BACKEND_TYPE_UNKNOWN;
426 +}
427 +
428 +/**
429 + * Backend main
430 + *
431 + * The main thread used to control the backedns.
432 + *
433 + * @param ptr a pointer to netdata_static_structure.
434 + *
435 + * @return It always return NULL.
436 + */
437 void *backends_main(void *ptr) {
438 netdata_thread_cleanup_push(backends_main_cleanup, ptr);
439
@@ -265,6 +453,9 @@ void *backends_main(void *ptr) {
453 BUFFER *http_request_header = buffer_create(1);
454 #endif
455
456 +#ifdef ENABLE_HTTPS
457 + struct netdata_ssl opentsdb_ssl = {NULL , NETDATA_SSL_START};
458 +#endif
459
460 // ------------------------------------------------------------------------
461 // collect configuration options
@@ -313,74 +504,64 @@ void *backends_main(void *ptr) {
504
505 // ------------------------------------------------------------------------
506 // select the backend type
316 -
317 - if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
318 -
319 - default_port = 2003;
320 - backend_response_checker = process_graphite_response;
321 -
322 - if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
323 - backend_request_formatter = format_dimension_collected_graphite_plaintext;
324 - else
325 - backend_request_formatter = format_dimension_stored_graphite_plaintext;
326 -
327 - }
328 - else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
329 -
330 - default_port = 4242;
331 - backend_response_checker = process_opentsdb_response;
332 -
333 - if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
334 - backend_request_formatter = format_dimension_collected_opentsdb_telnet;
335 - else
336 - backend_request_formatter = format_dimension_stored_opentsdb_telnet;
337 -
507 + BACKEND_TYPE work_type = backend_select_type(type);
508 + if (work_type == BACKEND_TYPE_UNKNOWN) {
509 + error("BACKEND: Unknown backend type '%s'", type);
510 + goto cleanup;
511 }
339 - else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
340 -
341 - default_port = 5448;
342 - backend_response_checker = process_json_response;
512
344 - if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
345 - backend_request_formatter = format_dimension_collected_json_plaintext;
346 - else
347 - backend_request_formatter = format_dimension_stored_json_plaintext;
348 -
349 - }
350 - else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
351 -#if HAVE_KINESIS
352 - do_kinesis = 1;
353 -
354 - if(unlikely(read_kinesis_conf(netdata_configured_user_config_dir, &kinesis_auth_key_id, &kinesis_secure_key, &kinesis_stream_name))) {
355 - error("BACKEND: kinesis backend type is set but cannot read its configuration from %s/aws_kinesis.conf", netdata_configured_user_config_dir);
356 - goto cleanup;
513 + switch (work_type) {
514 + case BACKEND_TYPE_OPENTSDB_USING_HTTP: {
515 +#ifdef ENABLE_HTTPS
516 + if (!strcmp(type, "opentsdb:https")) {
517 + security_start_ssl(NETDATA_SSL_CONTEXT_OPENTSDB);
518 + }
519 +#endif
520 + backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);
521 + break;
522 }
523 + case BACKEND_TYPE_PROMETEUS: {
524 +#if ENABLE_PROMETHEUS_REMOTE_WRITE
525 + do_prometheus_remote_write = 1;
526
359 - kinesis_init(destination, kinesis_auth_key_id, kinesis_secure_key, timeout.tv_sec * 1000 + timeout.tv_usec / 1000);
360 -
361 - backend_response_checker = process_json_response;
362 - if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
363 - backend_request_formatter = format_dimension_collected_json_plaintext;
364 - else
365 - backend_request_formatter = format_dimension_stored_json_plaintext;
527 + init_write_request();
528 #else
367 - error("AWS Kinesis support isn't compiled");
368 -#endif /* HAVE_KINESIS */
369 - }
370 - else if (!strcmp(type, "prometheus_remote_write")) {
371 -#if ENABLE_PROMETHEUS_REMOTE_WRITE
372 - do_prometheus_remote_write = 1;
529 + error("BACKEND: Prometheus remote write support isn't compiled");
530 +#endif // ENABLE_PROMETHEUS_REMOTE_WRITE
531 + backend_set_prometheus_variables(&default_port,&backend_response_checker,&backend_request_formatter);
532 + break;
533 + }
534 + case BACKEND_TYPE_KINESIS: {
535 +#if HAVE_KINESIS
536 + do_kinesis = 1;
537
374 - backend_response_checker = process_prometheus_remote_write_response;
538 + if(unlikely(read_kinesis_conf(netdata_configured_user_config_dir, &kinesis_auth_key_id, &kinesis_secure_key, &kinesis_stream_name))) {
539 + error("BACKEND: kinesis backend type is set but cannot read its configuration from %s/aws_kinesis.conf", netdata_configured_user_config_dir);
540 + goto cleanup;
541 + }
542
376 - init_write_request();
543 + kinesis_init(destination, kinesis_auth_key_id, kinesis_secure_key, timeout.tv_sec * 1000 + timeout.tv_usec / 1000);
544 #else
378 - error("Prometheus remote write support isn't compiled");
379 -#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
380 - }
381 - else {
382 - error("BACKEND: Unknown backend type '%s'", type);
383 - goto cleanup;
545 + error("BACKEND: AWS Kinesis support isn't compiled");
546 +#endif // HAVE_KINESIS
547 + backend_set_kinesis_variables(&default_port,&backend_response_checker,&backend_request_formatter);
548 + break;
549 + }
550 + case BACKEND_TYPE_GRAPHITE: {
551 + backend_set_graphite_variables(&default_port,&backend_response_checker,&backend_request_formatter);
552 + break;
553 + }
554 + case BACKEND_TYPE_OPENTSDB_USING_TELNET: {
555 + backend_set_opentsdb_telnet_variables(&default_port,&backend_response_checker,&backend_request_formatter);
556 + break;
557 + }
558 + case BACKEND_TYPE_JSON: {
559 + backend_set_json_variables(&default_port,&backend_response_checker,&backend_request_formatter);
560 + break;
561 + }
562 + case BACKEND_TYPE_UNKNOWN: {
563 + break;
564 + }
565 }
566
567 #if ENABLE_PROMETHEUS_REMOTE_WRITE
@@ -393,25 +574,25 @@ void *backends_main(void *ptr) {
574 }
575
576
396 - // ------------------------------------------------------------------------
397 - // prepare the charts for monitoring the backend operation
577 +// ------------------------------------------------------------------------
578 +// prepare the charts for monitoring the backend operation
579
580 struct rusage thread;
581
582 collected_number
402 - chart_buffered_metrics = 0,
403 - chart_lost_metrics = 0,
404 - chart_sent_metrics = 0,
405 - chart_buffered_bytes = 0,
406 - chart_received_bytes = 0,
407 - chart_sent_bytes = 0,
408 - chart_receptions = 0,
409 - chart_transmission_successes = 0,
410 - chart_transmission_failures = 0,
411 - chart_data_lost_events = 0,
412 - chart_lost_bytes = 0,
413 - chart_backend_reconnects = 0;
414 - // chart_backend_latency = 0;
583 + chart_buffered_metrics = 0,
584 + chart_lost_metrics = 0,
585 + chart_sent_metrics = 0,
586 + chart_buffered_bytes = 0,
587 + chart_received_bytes = 0,
588 + chart_sent_bytes = 0,
589 + chart_receptions = 0,
590 + chart_transmission_successes = 0,
591 + chart_transmission_failures = 0,
592 + chart_data_lost_events = 0,
593 + chart_lost_bytes = 0,
594 + chart_backend_reconnects = 0;
595 + // chart_backend_latency = 0;
596
597 RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, global_backend_update_every, RRDSET_TYPE_LINE);
598 rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -432,12 +613,12 @@ void *backends_main(void *ptr) {
613 rrddim_add(chart_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
614
615 /*
435 - * this is misleading - we can only measure the time we need to send data
436 - * this time is not related to the time required for the data to travel to
437 - * the backend database and the time that server needed to process them
438 - *
439 - * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
440 - *
616 + * this is misleading - we can only measure the time we need to send data
617 + * this time is not related to the time required for the data to travel to
618 + * the backend database and the time that server needed to process them
619 + *
620 + * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
621 + *
622 RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, global_backend_update_every, RRDSET_TYPE_AREA);
623 rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
624 */
@@ -668,7 +849,16 @@ void *backends_main(void *ptr) {
849 while(sock != -1 && errno != EWOULDBLOCK) {
850 buffer_need_bytes(response, 4096);
851
671 - ssize_t r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
852 + ssize_t r;
853 +#ifdef ENABLE_HTTPS
854 + if(opentsdb_ssl.conn && !opentsdb_ssl.flags) {
855 + r = SSL_read(opentsdb_ssl.conn, &response->buffer[response->len], response->size - response->len);
856 + } else {
857 + r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
858 + }
859 +#else
860 + r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
861 +#endif
862 if(likely(r > 0)) {
863 // we received some data
864 response->len += r;
@@ -701,7 +891,37 @@ void *backends_main(void *ptr) {
891 size_t reconnects = 0;
892
893 sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
894 +#ifdef ENABLE_HTTPS
895 + if(sock != -1) {
896 + if(netdata_opentsdb_ctx) {
897 + if(!opentsdb_ssl.conn) {
898 + opentsdb_ssl.conn = SSL_new(netdata_opentsdb_ctx);
899 + if(!opentsdb_ssl.conn) {
900 + error("Failed to allocate SSL structure %d.", sock);
901 + opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
902 + }
903 + } else {
904 + SSL_clear(opentsdb_ssl.conn);
905 + }
906 + }
907
908 + if(opentsdb_ssl.conn) {
909 + if(SSL_set_fd(opentsdb_ssl.conn, sock) != 1) {
910 + error("Failed to set the socket to the SSL on socket fd %d.", host->rrdpush_sender_socket);
911 + opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
912 + } else {
913 + opentsdb_ssl.flags = NETDATA_SSL_HANDSHAKE_COMPLETE;
914 + SSL_set_connect_state(opentsdb_ssl.conn);
915 + int err = SSL_connect(opentsdb_ssl.conn);
916 + if (err != 1) {
917 + err = SSL_get_error(opentsdb_ssl.conn, err);
918 + error("SSL cannot connect with the server: %s ", ERR_error_string((long)SSL_get_error(opentsdb_ssl.conn, err), NULL));
919 + opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
920 + } //TODO: check certificate here
921 + }
922 + }
923 + }
924 +#endif
925 chart_backend_reconnects += reconnects;
926 // chart_backend_latency += now_monotonic_usec() - start_ut;
927 }
@@ -756,7 +976,17 @@ void *backends_main(void *ptr) {
976 }
977 #endif
978
759 - ssize_t written = send(sock, buffer_tostring(b), len, flags);
979 + ssize_t written;
980 +#ifdef ENABLE_HTTPS
981 + if(opentsdb_ssl.conn && !opentsdb_ssl.flags) {
982 + written = SSL_write(opentsdb_ssl.conn, buffer_tostring(b), len);
983 + } else {
984 + written = send(sock, buffer_tostring(b), len, flags);
985 + }
986 +#else
987 + written = send(sock, buffer_tostring(b), len, flags);
988 +#endif
989 +
990 // chart_backend_latency += now_monotonic_usec() - start_ut;
991 if(written != -1 && (size_t)written == len) {
992 // we sent the data successfully
@@ -883,6 +1113,14 @@ cleanup:
1113 buffer_free(b);
1114 buffer_free(response);
1115
1116 +#ifdef ENABLE_HTTPS
1117 + if(netdata_opentsdb_ctx) {
1118 + if(opentsdb_ssl.conn) {
1119 + SSL_free(opentsdb_ssl.conn);
1120 + }
1121 + }
1122 +#endif
1123 +
1124 netdata_thread_cleanup_pop(1);
1125 return NULL;
1126 }
backends/backends.h
+14
@@ -15,6 +15,20 @@ typedef enum backend_options {
15 BACKEND_OPTION_SEND_NAMES = (1 << 16)
16 } BACKEND_OPTIONS;
17
18 +typedef enum backend_types {
19 + BACKEND_TYPE_UNKNOWN, //Invalid type
20 + BACKEND_TYPE_GRAPHITE, //Send plain text to Graphite
21 + BACKEND_TYPE_OPENTSDB_USING_TELNET, //Send data to OpenTSDB using telnet API
22 + BACKEND_TYPE_OPENTSDB_USING_HTTP, //Send data to OpenTSDB using HTTP API
23 + BACKEND_TYPE_JSON, //Stores the data using JSON.
24 + BACKEND_TYPE_PROMETEUS, //The user selected to use Prometheus backend
25 + BACKEND_TYPE_KINESIS //Send message to AWS Kinesis
26 +} BACKEND_TYPE;
27 +
28 +
29 +typedef int (**backend_response_checker_t)(BUFFER *);
30 +typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS);
31 +
32 #define BACKEND_OPTIONS_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM)
33 #define BACKEND_OPTIONS_DATA_SOURCE(backend_options) (backend_options & BACKEND_OPTIONS_SOURCE_BITS)
34
backends/opentsdb/README.md new
+26
@@ -0,0 +1,26 @@
1 +# OpenTSDB with HTTP
2 +
3 +Since version 1.16 the Netdata has the feature to communicate with OpenTSDB using HTTP API. To enable this channel
4 +it is necessary to set the following options in your netdata.conf
5 +
6 +```
7 +[backend]
8 + type = opentsdb:http
9 + destination = localhost:4242
10 +```
11 +
12 +, in this example we are considering that OpenTSDB is running with its default port (4242).
13 +
14 +## HTTPS
15 +
16 +Netdata also supports sending the metrics using SSL/TLS, but OpenTDSB does not have support to safety connections,
17 +so it will be necessary to configure a reverse-proxy to enable the HTTPS communication. After to configure your proxy the
18 +following changes must be done in the netdata.conf:
19 +
20 +```
21 +[backend]
22 + type = opentsdb:https
23 + destination = localhost:8082
24 +```
25 +
26 +In this example we used the port 8082 for our reverse proxy.
backends/opentsdb/opentsdb.c
+115
@@ -80,6 +80,7 @@ int format_dimension_stored_opentsdb_telnet(
80
81 return 1;
82 }
83 +
84 return 0;
85 }
86
@@ -87,4 +88,118 @@ int process_opentsdb_response(BUFFER *b) {
88 return discard_response(b, "opentsdb");
89 }
90
91 +static inline void opentsdb_build_message(BUFFER *b, char *message, const char *hostname, int length) {
92 + buffer_sprintf(
93 + b
94 + , "POST /api/put HTTP/1.1\r\n"
95 + "Host: %s\r\n"
96 + "Content-Type: application/json\r\n"
97 + "Content-Length: %d\r\n"
98 + "\r\n"
99 + "%s"
100 + , hostname
101 + , length
102 + , message
103 + );
104 +}
105 +
106 +int format_dimension_collected_opentsdb_http(
107 + BUFFER *b // the buffer to write data to
108 + , const char *prefix // the prefix to use
109 + , RRDHOST *host // the host this chart comes from
110 + , const char *hostname // the hostname (to override host->hostname)
111 + , RRDSET *st // the chart
112 + , RRDDIM *rd // the dimension
113 + , time_t after // the start timestamp
114 + , time_t before // the end timestamp
115 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
116 +) {
117 + (void)host;
118 + (void)after;
119 + (void)before;
120 +
121 + char message[1024];
122 + char chart_name[RRD_ID_LENGTH_MAX + 1];
123 + char dimension_name[RRD_ID_LENGTH_MAX + 1];
124 + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
125 + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
126 +
127 + int length = snprintfz(message
128 + , sizeof(message)
129 + , "{"
130 + " \"metric\": \"%s.%s.%s\","
131 + " \"timestamp\": %llu,"
132 + " \"value\": "COLLECTED_NUMBER_FORMAT ","
133 + " \"tags\": {"
134 + " \"host\": \"%s%s%s\""
135 + " }"
136 + "}"
137 + , prefix
138 + , chart_name
139 + , dimension_name
140 + , (unsigned long long)rd->last_collected_time.tv_sec
141 + , rd->last_collected_value
142 + , hostname
143 + , (host->tags)?" ":""
144 + , (host->tags)?host->tags:""
145 + );
146 +
147 + if(length > 0) {
148 + opentsdb_build_message(b, message, hostname, length);
149 + }
150 +
151 + return 1;
152 +}
153
154 +int format_dimension_stored_opentsdb_http(
155 + BUFFER *b // the buffer to write data to
156 + , const char *prefix // the prefix to use
157 + , RRDHOST *host // the host this chart comes from
158 + , const char *hostname // the hostname (to override host->hostname)
159 + , RRDSET *st // the chart
160 + , RRDDIM *rd // the dimension
161 + , time_t after // the start timestamp
162 + , time_t before // the end timestamp
163 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
164 +) {
165 + (void)host;
166 +
167 + time_t first_t = after, last_t = before;
168 + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);
169 +
170 + if(!isnan(value)) {
171 + char chart_name[RRD_ID_LENGTH_MAX + 1];
172 + char dimension_name[RRD_ID_LENGTH_MAX + 1];
173 + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
174 + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
175 +
176 + char message[1024];
177 + int length = snprintfz(message
178 + , sizeof(message)
179 + , "{"
180 + " \"metric\": \"%s.%s.%s\","
181 + " \"timestamp\": %llu,"
182 + " \"value\": "CALCULATED_NUMBER_FORMAT ","
183 + " \"tags\": {"
184 + " \"host\": \"%s%s%s\""
185 + " }"
186 + "}"
187 + , prefix
188 + , chart_name
189 + , dimension_name
190 + , (unsigned long long)last_t
191 + , value
192 + , hostname
193 + , (host->tags)?" ":""
194 + , (host->tags)?host->tags:""
195 + );
196 +
197 + if(length > 0) {
198 + opentsdb_build_message(b, message, hostname, length);
199 + }
200 +
201 + return 1;
202 + }
203 +
204 + return 0;
205 +}
backends/opentsdb/opentsdb.h
+23
@@ -31,5 +31,28 @@ extern int format_dimension_stored_opentsdb_telnet(
31
32 extern int process_opentsdb_response(BUFFER *b);
33
34 +int format_dimension_collected_opentsdb_http(
35 + BUFFER *b // the buffer to write data to
36 + , const char *prefix // the prefix to use
37 + , RRDHOST *host // the host this chart comes from
38 + , const char *hostname // the hostname (to override host->hostname)
39 + , RRDSET *st // the chart
40 + , RRDDIM *rd // the dimension
41 + , time_t after // the start timestamp
42 + , time_t before // the end timestamp
43 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
44 +);
45 +
46 +int format_dimension_stored_opentsdb_http(
47 + BUFFER *b // the buffer to write data to
48 + , const char *prefix // the prefix to use
49 + , RRDHOST *host // the host this chart comes from
50 + , const char *hostname // the hostname (to override host->hostname)
51 + , RRDSET *st // the chart
52 + , RRDDIM *rd // the dimension
53 + , time_t after // the start timestamp
54 + , time_t before // the end timestamp
55 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
56 +);
57
58 #endif //NETDATA_BACKEND_OPENTSDB_H
libnetdata/socket/security.c
+73 -14
@@ -2,6 +2,7 @@
2
3 #ifdef ENABLE_HTTPS
4
5 +SSL_CTX *netdata_opentsdb_ctx=NULL;
6 SSL_CTX *netdata_client_ctx=NULL;
7 SSL_CTX *netdata_srv_ctx=NULL;
8 const char *security_key=NULL;
@@ -10,6 +11,15 @@ int netdata_use_ssl_on_stream = NETDATA_SSL_OPTIONAL;
11 int netdata_use_ssl_on_http = NETDATA_SSL_FORCE; //We force SSL due safety reasons
12 int netdata_validate_server = NETDATA_SSL_VALID_CERTIFICATE;
13
14 +/**
15 + * Info Callback
16 + *
17 + * Function used as callback for the OpenSSL Library
18 + *
19 + * @param ssl a pointer to the SSL structure of the client
20 + * @param where the variable with the flags set.
21 + * @param ret the return of the caller
22 + */
23 static void security_info_callback(const SSL *ssl, int where, int ret) {
24 (void)ssl;
25 if (where & SSL_CB_ALERT) {
@@ -17,6 +27,11 @@ static void security_info_callback(const SSL *ssl, int where, int ret) {
27 }
28 }
29
30 +/**
31 + * OpenSSL Library
32 + *
33 + * Starts the openssl library for the Netdata.
34 + */
35 void security_openssl_library()
36 {
37 #if OPENSSL_VERSION_NUMBER < 0x10100000L
@@ -36,6 +51,13 @@ void security_openssl_library()
51 #endif
52 }
53
54 +/**
55 + * OpenSSL common options
56 + *
57 + * Clients and SERVER have common options, this function is responsible to set them in the context.
58 + *
59 + * @param ctx
60 + */
61 void security_openssl_common_options(SSL_CTX *ctx) {
62 #if OPENSSL_VERSION_NUMBER >= 0x10100000L
63 static char *ciphers = {"ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"};
@@ -55,10 +77,15 @@ void security_openssl_common_options(SSL_CTX *ctx) {
77 error("SSL error. cannot set the cipher list");
78 }
79 #endif
58 -
59 -
80 }
81
82 +/**
83 + * Initialize Openssl Client
84 + *
85 + * Starts the client context with TLS 1.2.
86 + *
87 + * @return It returns the context on success or NULL otherwise
88 + */
89 static SSL_CTX * security_initialize_openssl_client() {
90 SSL_CTX *ctx;
91 #if OPENSSL_VERSION_NUMBER < 0x10100000L
@@ -66,11 +93,20 @@ static SSL_CTX * security_initialize_openssl_client() {
93 #else
94 ctx = SSL_CTX_new(TLS_client_method());
95 #endif
69 - security_openssl_common_options(ctx);
96 + if(ctx) {
97 + security_openssl_common_options(ctx);
98 + }
99
100 return ctx;
101 }
102
103 +/**
104 + * Initialize OpenSSL server
105 + *
106 + * Starts the server context with TLS 1.2 and load the certificate.
107 + *
108 + * @return It returns the context on success or NULL otherwise
109 + */
110 static SSL_CTX * security_initialize_openssl_server() {
111 SSL_CTX *ctx;
112 char lerror[512];
@@ -116,18 +152,36 @@ static SSL_CTX * security_initialize_openssl_server() {
152 return ctx;
153 }
154
119 -void security_start_ssl(int type) {
120 - if (!type) {
121 - struct stat statbuf;
122 - if (stat(security_key,&statbuf) || stat(security_cert,&statbuf)) {
123 - info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n");
124 - return;
155 +/**
156 + * Start SSL
157 + *
158 + * Call the correct function to start the SSL context.
159 + *
160 + * @param selector informs the context that must be initialized, the following list has the valid values:
161 + * NETDATA_SSL_CONTEXT_SERVER - the server context
162 + * NETDATA_SSL_CONTEXT_STREAMING - Starts the streaming context.
163 + * NETDATA_SSL_CONTEXT_OPENTSDB - Starts the OpenTSDB contextv
164 + */
165 +void security_start_ssl(int selector) {
166 + switch (selector) {
167 + case NETDATA_SSL_CONTEXT_SERVER: {
168 + struct stat statbuf;
169 + if (stat(security_key,&statbuf) || stat(security_cert,&statbuf)) {
170 + info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n");
171 + return;
172 + }
173 +
174 + netdata_srv_ctx = security_initialize_openssl_server();
175 + break;
176 + }
177 + case NETDATA_SSL_CONTEXT_STREAMING: {
178 + netdata_client_ctx = security_initialize_openssl_client();
179 + break;
180 + }
181 + case NETDATA_SSL_CONTEXT_OPENTSDB: {
182 + netdata_opentsdb_ctx = security_initialize_openssl_client();
183 + break;
184 }
126 -
127 - netdata_srv_ctx = security_initialize_openssl_server();
128 - }
129 - else {
130 - netdata_client_ctx = security_initialize_openssl_client();
185 }
186 }
187
@@ -142,6 +196,11 @@ void security_clean_openssl() {
196 SSL_CTX_free(netdata_client_ctx);
197 }
198
199 + if ( netdata_opentsdb_ctx )
200 + {
201 + SSL_CTX_free(netdata_opentsdb_ctx);
202 + }
203 +
204 #if OPENSSL_VERSION_NUMBER < 0x10100000L
205 ERR_free_strings();
206 #endif
libnetdata/socket/security.h
+6 -1
@@ -11,6 +11,10 @@
11 # define NETDATA_SSL_INVALID_CERTIFICATE 64 //Accepts invalid certificate
12 # define NETDATA_SSL_VALID_CERTIFICATE 128 //Accepts invalid certificate
13
14 +#define NETDATA_SSL_CONTEXT_SERVER 0
15 +#define NETDATA_SSL_CONTEXT_STREAMING 1
16 +#define NETDATA_SSL_CONTEXT_OPENTSDB 2
17 +
18 # ifdef ENABLE_HTTPS
19
20 # include <openssl/ssl.h>
@@ -24,6 +28,7 @@ struct netdata_ssl{
28 int flags;
29 };
30
31 +extern SSL_CTX *netdata_opentsdb_ctx;
32 extern SSL_CTX *netdata_client_ctx;
33 extern SSL_CTX *netdata_srv_ctx;
34 extern const char *security_key;
@@ -34,7 +39,7 @@ extern int netdata_validate_server;
39
40 void security_openssl_library();
41 void security_clean_openssl();
37 -void security_start_ssl(int type);
42 +void security_start_ssl(int selector);
43 int security_process_accept(SSL *ssl,int msg);
44 int security_test_certificate(SSL *ssl);
45
streaming/rrdpush.c
+1 -1
@@ -651,7 +651,7 @@ void *rrdpush_sender_thread(void *ptr) {
651
652 #ifdef ENABLE_HTTPS
653 if (netdata_use_ssl_on_stream & NETDATA_SSL_FORCE ){
654 - security_start_ssl(1);
654 + security_start_ssl(NETDATA_SSL_CONTEXT_STREAMING);
655 }
656 #endif
657
web/server/static/static-threaded.c
+1 -1
@@ -458,7 +458,7 @@ void *socket_listen_main_static_threaded(void *ptr) {
458 fatal("LISTENER: no listen sockets available.");
459
460 #ifdef ENABLE_HTTPS
461 - security_start_ssl(0);
461 + security_start_ssl(NETDATA_SSL_CONTEXT_SERVER);
462 #endif
463 // 6 threads is the optimal value
464 // since 6 are the parallel connections browsers will do