Implement the main flow for the Exporting Engine (#7149)
* Add top level tests * Add a skeleton for preparing buffers * Initialize graphite instance * Prepare buffers for all instances * Add Grafite collected value formatter * Add support for exporting.conf read and parsing * - Use new exporting_config instead of netdata_config * Implement Grafite worker * Disable exporting engine compilation if libuv is not available * Add mutex locks - Configure connectors as connector_<type> in sections of exporting.conf - Change exporting_select_type to check for connector_ fields * - Override exporting_config structure if there no exporting.conf so that look ups don't fail and we maintain backwards compatibility * Separate fixtures in unit tests * Test exporting_discard_responce * Test response receiving * Test buffer sending * Test simple connector worker - Instance section has the format connector:instance_name e.g graphite:my_graphite_instance - Connectors with : in their name e.g graphite:plaintext are reserved So graphite:plaintext is not accepted because it would activate an instance with name "plaintext" It should be graphite:plaintext:instance_name * - Enable the add_connector_instance to cleanup the internal structure by passing NULL,not NULL arguments * Implement configurable update interval - Add additional check to verify instance uniqueness across connectors * Add host and chart filters * Add the value calculation over a database series * Add the calculated over stored data graphite connector * Add tests for graphite connector * Add JSON connector * Add tests for JSON formatting functions * Add OpenTSDB connector * Add tests for the OpenTSDB connector * Add temporaty notes to the documentation
Vladimir Kobal committed
Dec 12, 2019 at 21:41 UTC
6f270819121afb743d783f9a72786e367d1048a3
44 files changed
+3832
-47
CMakeLists.txt
+62
-1
@@ -31,6 +31,7 @@ IF("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
31
set(CXX_FORMAT_SECURITY "-Werror=format-security")
32
set(CXX_STACK_PROTECTOR "-fstack-protector-all")
33
set(CXX_FLAGS_DEBUG "-O0")
34
+ set(CMAKE_C_STANDARD 99)
35
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1 ${CXX_FORMAT_SIGNEDNESS} ${CXX_FORMAT_SECURITY} ${CXX_STACK_PROTECTOR} ${CXX_FLAGS_DEBUG}")
36
ELSE()
37
message(STATUS "building for: release")
@@ -600,6 +601,23 @@ set(BACKENDS_PLUGIN_FILES
601
backends/prometheus/backend_prometheus.h
602
)
603
604
+set(EXPORTING_ENGINE_FILES
605
+ exporting/exporting_engine.c
606
+ exporting/exporting_engine.h
607
+ exporting/graphite/graphite.c
608
+ exporting/graphite/graphite.h
609
+ exporting/json/json.c
610
+ exporting/json/json.h
611
+ exporting/opentsdb/opentsdb.c
612
+ exporting/opentsdb/opentsdb.h
613
+ exporting/read_config.c
614
+ exporting/init_connectors.c
615
+ exporting/process_data.c
616
+ exporting/check_filters.c
617
+ exporting/send_data.c
618
+ exporting/send_internal_metrics.c
619
+ )
620
+
621
set(KINESIS_BACKEND_FILES
622
backends/aws_kinesis/aws_kinesis.c
623
backends/aws_kinesis/aws_kinesis.h
@@ -639,6 +657,7 @@ set(NETDATA_FILES
657
${DAEMON_FILES}
658
${API_PLUGIN_FILES}
659
${BACKENDS_PLUGIN_FILES}
660
+ ${EXPORTING_ENGINE_FILES}
661
${CHECKS_PLUGIN_FILES}
662
${HEALTH_PLUGIN_FILES}
663
${IDLEJITTER_PLUGIN_FILES}
@@ -916,6 +935,48 @@ if(BUILD_TESTING)
935
target_link_libraries(storage_number_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
936
add_test(NAME test_storage_number COMMAND storage_number_testdriver)
937
938
+ set(EXPORTING_ENGINE_TEST_FILES
939
+ exporting/tests/test_exporting_engine.c
940
+ exporting/tests/test_exporting_engine.h
941
+ exporting/tests/exporting_fixtures.c
942
+ exporting/tests/exporting_doubles.c
943
+ exporting/tests/netdata_doubles.c
944
+ exporting/tests/system_doubles.c
945
+ )
946
+ set(TEST_NAME exporting_engine)
947
+ add_executable(${TEST_NAME}_testdriver ${EXPORTING_ENGINE_TEST_FILES} ${EXPORTING_ENGINE_FILES})
948
+ target_compile_options(
949
+ ${TEST_NAME}_testdriver
950
+ PRIVATE
951
+ -DUNIT_TESTING
952
+ )
953
+ target_link_options(
954
+ ${TEST_NAME}_testdriver
955
+ PRIVATE
956
+ -Wl,--wrap=read_exporting_config
957
+ -Wl,--wrap=init_connectors
958
+ -Wl,--wrap=mark_scheduled_instances
959
+ -Wl,--wrap=rrdhost_is_exportable
960
+ -Wl,--wrap=rrdset_is_exportable
961
+ -Wl,--wrap=exporting_calculate_value_from_stored_data
962
+ -Wl,--wrap=prepare_buffers
963
+ -Wl,--wrap=notify_workers
964
+ -Wl,--wrap=send_internal_metrics
965
+ -Wl,--wrap=now_realtime_sec
966
+ -Wl,--wrap=uv_thread_create
967
+ -Wl,--wrap=uv_mutex_lock
968
+ -Wl,--wrap=uv_mutex_unlock
969
+ -Wl,--wrap=uv_cond_signal
970
+ -Wl,--wrap=uv_cond_wait
971
+ -Wl,--wrap=strdupz
972
+ -Wl,--wrap=info_int
973
+ -Wl,--wrap=recv
974
+ -Wl,--wrap=send
975
+ -Wl,--wrap=connect_to_one_of
976
+ )
977
+ target_link_libraries(${TEST_NAME}_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
978
+ add_test(NAME test_${TEST_NAME} COMMAND ${TEST_NAME}_testdriver)
979
+
980
set(WEB_API_TEST_FILES
981
web/api/tests/web_api.c
982
web/server/web_client.c
@@ -969,11 +1030,11 @@ if(BUILD_TESTING)
1030
set_target_properties(
1031
str2ld_testdriver
1032
storage_number_testdriver
1033
+ exporting_engine_testdriver
1034
web_api_testdriver
1035
valid_urls_testdriver
1036
PROPERTIES RUNTIME_OUTPUT_DIRECTORY tests
1037
)
1038
977
-
1039
endif()
1040
endif()
Makefile.am
+67
@@ -93,6 +93,7 @@ SUBDIRS += \
93
collectors \
94
daemon \
95
database \
96
+ exporting \
97
health \
98
libnetdata \
99
registry \
@@ -453,6 +454,23 @@ BACKENDS_PLUGIN_FILES = \
454
backends/prometheus/backend_prometheus.h \
455
$(NULL)
456
457
+EXPORTING_ENGINE_FILES = \
458
+ exporting/exporting_engine.c \
459
+ exporting/exporting_engine.h \
460
+ exporting/graphite/graphite.c \
461
+ exporting/graphite/graphite.h \
462
+ exporting/json/json.c \
463
+ exporting/json/json.h \
464
+ exporting/opentsdb/opentsdb.c \
465
+ exporting/opentsdb/opentsdb.h \
466
+ exporting/read_config.c \
467
+ exporting/init_connectors.c \
468
+ exporting/process_data.c \
469
+ exporting/check_filters.c \
470
+ exporting/send_data.c \
471
+ exporting/send_internal_metrics.c \
472
+ $(NULL)
473
+
474
KINESIS_BACKEND_FILES = \
475
backends/aws_kinesis/aws_kinesis.c \
476
backends/aws_kinesis/aws_kinesis.h \
@@ -527,6 +545,12 @@ if LINUX
545
546
endif
547
548
+if ENABLE_EXPORTING
549
+ NETDATA_FILES += \
550
+ $(EXPORTING_ENGINE_FILES) \
551
+ $(NULL)
552
+endif
553
+
554
NETDATA_COMMON_LIBS = \
555
$(OPTIONAL_MATH_LIBS) \
556
$(OPTIONAL_ZLIB_LIBS) \
@@ -666,6 +690,7 @@ if ENABLE_UNITTESTS
690
check_PROGRAMS = \
691
libnetdata/tests/str2ld_testdriver \
692
libnetdata/storage_number/tests/storage_number_testdriver \
693
+ exporting/tests/exporting_engine_testdriver \
694
web/api/tests/web_api_testdriver \
695
web/api/tests/valid_urls_testdriver \
696
$(NULL)
@@ -729,4 +754,46 @@ if ENABLE_UNITTESTS
754
$(LIBNETDATA_FILES) \
755
$(NULL)
756
libnetdata_storage_number_tests_storage_number_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
757
+
758
+ EXPORTING_ENGINE_TEST_FILES = \
759
+ exporting/tests/test_exporting_engine.c \
760
+ exporting/tests/test_exporting_engine.h \
761
+ exporting/tests/exporting_fixtures.c \
762
+ exporting/tests/exporting_doubles.c \
763
+ exporting/tests/netdata_doubles.c \
764
+ exporting/tests/system_doubles.c \
765
+ $(NULL)
766
+ exporting_tests_exporting_engine_testdriver_SOURCES = \
767
+ $(EXPORTING_ENGINE_TEST_FILES) \
768
+ $(EXPORTING_ENGINE_FILES) \
769
+ $(LIBNETDATA_FILES) \
770
+ $(NULL)
771
+ exporting_tests_exporting_engine_testdriver_CFLAGS = \
772
+ $(AM_CFLAGS) \
773
+ -DUNIT_TESTING \
774
+ $(NULL)
775
+ exporting_tests_exporting_engine_testdriver_LDFLAGS = \
776
+ -Wl,--wrap=read_exporting_config \
777
+ -Wl,--wrap=init_connectors \
778
+ -Wl,--wrap=mark_scheduled_instances \
779
+ -Wl,--wrap=rrdhost_is_exportable \
780
+ -Wl,--wrap=rrdset_is_exportable \
781
+ -Wl,--wrap=exporting_calculate_value_from_stored_data \
782
+ -Wl,--wrap=prepare_buffers \
783
+ -Wl,--wrap=notify_workers \
784
+ -Wl,--wrap=send_internal_metrics \
785
+ -Wl,--wrap=now_realtime_sec \
786
+ -Wl,--wrap=uv_thread_create \
787
+ -Wl,--wrap=uv_mutex_lock \
788
+ -Wl,--wrap=uv_mutex_unlock \
789
+ -Wl,--wrap=uv_cond_signal \
790
+ -Wl,--wrap=uv_cond_wait \
791
+ -Wl,--wrap=strdupz \
792
+ -Wl,--wrap=info_int \
793
+ -Wl,--wrap=recv \
794
+ -Wl,--wrap=send \
795
+ -Wl,--wrap=connect_to_one_of \
796
+ $(TEST_LDFLAGS) \
797
+ $(NULL)
798
+ exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
799
endif
backends/backends.c
+15
-12
@@ -269,9 +269,9 @@ void backend_set_kinesis_variables(int *default_port,
269
#if HAVE_KINESIS
270
*brc = process_json_response;
271
if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
272
- *brf = format_dimension_collected_json_plaintext;
272
+ *brf = backends_format_dimension_collected_json_plaintext;
273
else
274
- *brf = format_dimension_stored_json_plaintext;
274
+ *brf = backends_format_dimension_stored_json_plaintext;
275
#endif
276
}
277
@@ -321,9 +321,9 @@ void backend_set_mongodb_variables(int *default_port,
321
#if HAVE_MONGOC
322
*brc = process_json_response;
323
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
324
- *brf = format_dimension_collected_json_plaintext;
324
+ *brf = backends_format_dimension_collected_json_plaintext;
325
else
326
- *brf = format_dimension_stored_json_plaintext;
326
+ *brf = backends_format_dimension_stored_json_plaintext;
327
#endif
328
}
329
@@ -344,9 +344,9 @@ void backend_set_json_variables(int *default_port,
344
*brc = process_json_response;
345
346
if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
347
- *brf = format_dimension_collected_json_plaintext;
347
+ *brf = backends_format_dimension_collected_json_plaintext;
348
else
349
- *brf = format_dimension_stored_json_plaintext;
349
+ *brf = backends_format_dimension_stored_json_plaintext;
350
}
351
352
/**
@@ -366,9 +366,9 @@ void backend_set_opentsdb_http_variables(int *default_port,
366
*brc = process_opentsdb_response;
367
368
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
369
- *brf = format_dimension_collected_opentsdb_http;
369
+ *brf = backends_format_dimension_collected_opentsdb_http;
370
else
371
- *brf = format_dimension_stored_opentsdb_http;
371
+ *brf = backends_format_dimension_stored_opentsdb_http;
372
373
}
374
@@ -389,9 +389,9 @@ void backend_set_opentsdb_telnet_variables(int *default_port,
389
*brc = process_opentsdb_response;
390
391
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
392
- *brf = format_dimension_collected_opentsdb_telnet;
392
+ *brf = backends_format_dimension_collected_opentsdb_telnet;
393
else
394
- *brf = format_dimension_stored_opentsdb_telnet;
394
+ *brf = backends_format_dimension_stored_opentsdb_telnet;
395
}
396
397
/**
@@ -411,9 +411,9 @@ void backend_set_graphite_variables(int *default_port,
411
*brc = process_graphite_response;
412
413
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
414
- *brf = format_dimension_collected_graphite_plaintext;
414
+ *brf = backends_format_dimension_collected_graphite_plaintext;
415
else
416
- *brf = format_dimension_stored_graphite_plaintext;
416
+ *brf = backends_format_dimension_stored_graphite_plaintext;
417
}
418
419
/**
@@ -624,6 +624,9 @@ void *backends_main(void *ptr) {
624
case BACKEND_TYPE_UNKNOWN: {
625
break;
626
}
627
+ default: {
628
+ break;
629
+ }
630
}
631
632
#if ENABLE_PROMETHEUS_REMOTE_WRITE
backends/backends.h
+6
-1
@@ -23,9 +23,13 @@ typedef enum backend_types {
23
BACKEND_TYPE_JSON, // Stores the data using JSON.
24
BACKEND_TYPE_PROMETEUS, // The user selected to use Prometheus backend
25
BACKEND_TYPE_KINESIS, // Send message to AWS Kinesis
26
- BACKEND_TYPE_MONGODB // Send data to MongoDB collection
26
+ BACKEND_TYPE_MONGODB, // Send data to MongoDB collection
27
+ BACKEND_TYPE_NUM // Number of backend types
28
} BACKEND_TYPE;
29
30
+#ifdef ENABLE_EXPORTING
31
+#include "exporting/exporting_engine.h"
32
+#endif
33
34
typedef int (**backend_response_checker_t)(BUFFER *);
35
typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS);
@@ -38,6 +42,7 @@ extern BACKEND_OPTIONS global_backend_options;
42
extern const char *global_backend_prefix;
43
44
extern void *backends_main(void *ptr);
45
+BACKEND_TYPE backend_select_type(const char *type);
46
47
extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options);
48
backends/graphite/graphite.c
+2
-2
@@ -6,7 +6,7 @@
6
// ----------------------------------------------------------------------------
7
// graphite backend
8
9
-int format_dimension_collected_graphite_plaintext(
9
+int backends_format_dimension_collected_graphite_plaintext(
10
BUFFER *b // the buffer to write data to
11
, const char *prefix // the prefix to use
12
, RRDHOST *host // the host this chart comes from
@@ -42,7 +42,7 @@ int format_dimension_collected_graphite_plaintext(
42
return 1;
43
}
44
45
-int format_dimension_stored_graphite_plaintext(
45
+int backends_format_dimension_stored_graphite_plaintext(
46
BUFFER *b // the buffer to write data to
47
, const char *prefix // the prefix to use
48
, RRDHOST *host // the host this chart comes from
backends/graphite/graphite.h
+2
-2
@@ -6,7 +6,7 @@
6
7
#include "backends/backends.h"
8
9
-extern int format_dimension_collected_graphite_plaintext(
9
+extern int backends_format_dimension_collected_graphite_plaintext(
10
BUFFER *b // the buffer to write data to
11
, const char *prefix // the prefix to use
12
, RRDHOST *host // the host this chart comes from
@@ -18,7 +18,7 @@ extern int format_dimension_collected_graphite_plaintext(
18
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
19
);
20
21
-extern int format_dimension_stored_graphite_plaintext(
21
+extern int backends_format_dimension_stored_graphite_plaintext(
22
BUFFER *b // the buffer to write data to
23
, const char *prefix // the prefix to use
24
, RRDHOST *host // the host this chart comes from
backends/json/json.c
+2
-2
@@ -6,7 +6,7 @@
6
// ----------------------------------------------------------------------------
7
// json backend
8
9
-int format_dimension_collected_json_plaintext(
9
+int backends_format_dimension_collected_json_plaintext(
10
BUFFER *b // the buffer to write data to
11
, const char *prefix // the prefix to use
12
, RRDHOST *host // the host this chart comes from
@@ -74,7 +74,7 @@ int format_dimension_collected_json_plaintext(
74
return 1;
75
}
76
77
-int format_dimension_stored_json_plaintext(
77
+int backends_format_dimension_stored_json_plaintext(
78
BUFFER *b // the buffer to write data to
79
, const char *prefix // the prefix to use
80
, RRDHOST *host // the host this chart comes from
backends/json/json.h
+2
-2
@@ -5,7 +5,7 @@
5
6
#include "backends/backends.h"
7
8
-extern int format_dimension_collected_json_plaintext(
8
+extern int backends_format_dimension_collected_json_plaintext(
9
BUFFER *b // the buffer to write data to
10
, const char *prefix // the prefix to use
11
, RRDHOST *host // the host this chart comes from
@@ -17,7 +17,7 @@ extern int format_dimension_collected_json_plaintext(
17
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
18
);
19
20
-extern int format_dimension_stored_json_plaintext(
20
+extern int backends_format_dimension_stored_json_plaintext(
21
BUFFER *b // the buffer to write data to
22
, const char *prefix // the prefix to use
23
, RRDHOST *host // the host this chart comes from
backends/opentsdb/opentsdb.c
+4
-4
@@ -6,7 +6,7 @@
6
// ----------------------------------------------------------------------------
7
// opentsdb backend
8
9
-int format_dimension_collected_opentsdb_telnet(
9
+int backends_format_dimension_collected_opentsdb_telnet(
10
BUFFER *b // the buffer to write data to
11
, const char *prefix // the prefix to use
12
, RRDHOST *host // the host this chart comes from
@@ -42,7 +42,7 @@ int format_dimension_collected_opentsdb_telnet(
42
return 1;
43
}
44
45
-int format_dimension_stored_opentsdb_telnet(
45
+int backends_format_dimension_stored_opentsdb_telnet(
46
BUFFER *b // the buffer to write data to
47
, const char *prefix // the prefix to use
48
, RRDHOST *host // the host this chart comes from
@@ -103,7 +103,7 @@ static inline void opentsdb_build_message(BUFFER *b, char *message, const char *
103
);
104
}
105
106
-int format_dimension_collected_opentsdb_http(
106
+int backends_format_dimension_collected_opentsdb_http(
107
BUFFER *b // the buffer to write data to
108
, const char *prefix // the prefix to use
109
, RRDHOST *host // the host this chart comes from
@@ -151,7 +151,7 @@ int format_dimension_collected_opentsdb_http(
151
return 1;
152
}
153
154
-int format_dimension_stored_opentsdb_http(
154
+int backends_format_dimension_stored_opentsdb_http(
155
BUFFER *b // the buffer to write data to
156
, const char *prefix // the prefix to use
157
, RRDHOST *host // the host this chart comes from
backends/opentsdb/opentsdb.h
+4
-4
@@ -5,7 +5,7 @@
5
6
#include "backends/backends.h"
7
8
-extern int format_dimension_collected_opentsdb_telnet(
8
+extern int backends_format_dimension_collected_opentsdb_telnet(
9
BUFFER *b // the buffer to write data to
10
, const char *prefix // the prefix to use
11
, RRDHOST *host // the host this chart comes from
@@ -17,7 +17,7 @@ extern int format_dimension_collected_opentsdb_telnet(
17
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
18
);
19
20
-extern int format_dimension_stored_opentsdb_telnet(
20
+extern int backends_format_dimension_stored_opentsdb_telnet(
21
BUFFER *b // the buffer to write data to
22
, const char *prefix // the prefix to use
23
, RRDHOST *host // the host this chart comes from
@@ -31,7 +31,7 @@ extern int format_dimension_stored_opentsdb_telnet(
31
32
extern int process_opentsdb_response(BUFFER *b);
33
34
-int format_dimension_collected_opentsdb_http(
34
+int backends_format_dimension_collected_opentsdb_http(
35
BUFFER *b // the buffer to write data to
36
, const char *prefix // the prefix to use
37
, RRDHOST *host // the host this chart comes from
@@ -43,7 +43,7 @@ int format_dimension_collected_opentsdb_http(
43
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
44
);
45
46
-int format_dimension_stored_opentsdb_http(
46
+int backends_format_dimension_stored_opentsdb_http(
47
BUFFER *b // the buffer to write data to
48
, const char *prefix // the prefix to use
49
, RRDHOST *host // the host this chart comes from
configure.ac
+20
@@ -36,6 +36,7 @@ m4_ifdef([AM_SILENT_RULES], [
36
])
37
AC_CANONICAL_HOST
38
AC_PROG_CC
39
+AC_PROG_CC_C99
40
AM_PROG_CC_C_O
41
AC_PROG_CXX
42
AC_PROG_INSTALL
@@ -407,6 +408,20 @@ fi
408
AC_MSG_RESULT([${enable_https}])
409
AM_CONDITIONAL([ENABLE_HTTPS], [test "${enable_https}" = "yes"])
410
411
+# -----------------------------------------------------------------------------
412
+# Exporting engine
413
+AC_MSG_CHECKING([if netdata exporting engine should be used])
414
+if test "${UV_LIBS}"; then
415
+ enable_exporting_engine="yes"
416
+ AC_DEFINE([ENABLE_EXPORTING], [1], [netdata exporting engine usability])
417
+ OPTIONAL_UV_CFLAGS="${UV_CFLAGS}"
418
+ OPTIONAL_UV_LIBS="${UV_LIBS}"
419
+else
420
+ enable_exporting_engine="no"
421
+fi
422
+AC_MSG_RESULT([${enable_exporting_engine}])
423
+AM_CONDITIONAL([ENABLE_EXPORTING], [test "${enable_exporting_engine}" = "yes"])
424
+
425
# -----------------------------------------------------------------------------
426
# JSON-C
427
test "${enable_jsonc}" = "yes" -a -z "${JSONC_LIBS}" && \
@@ -1209,6 +1224,11 @@ AC_CONFIG_FILES([
1224
database/Makefile
1225
database/engine/Makefile
1226
diagrams/Makefile
1227
+ exporting/Makefile
1228
+ exporting/graphite/Makefile
1229
+ exporting/json/Makefile
1230
+ exporting/opentsdb/Makefile
1231
+ exporting/tests/Makefile
1232
health/Makefile
1233
health/notifications/Makefile
1234
libnetdata/Makefile
daemon/common.h
-1
@@ -27,7 +27,6 @@
27
28
#define config_generate(buffer, only_changed) appconfig_generate(&netdata_config, buffer, only_changed)
29
30
-
30
// ----------------------------------------------------------------------------
31
// netdata include files
32
daemon/main.c
+4
@@ -75,6 +75,9 @@ struct netdata_static_thread static_threads[] = {
75
76
// common plugins for all systems
77
{"BACKENDS", NULL, NULL, 1, NULL, NULL, backends_main},
78
+#ifdef ENABLE_EXPORTING
79
+ {"EXPORTING", NULL, NULL, 1, NULL, NULL, exporting_main},
80
+#endif
81
{"WEB_SERVER[static1]", NULL, NULL, 0, NULL, NULL, socket_listen_main_static_threaded},
82
{"STREAM", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread},
83
@@ -1105,6 +1108,7 @@ int main(int argc, char **argv) {
1108
if(!config_loaded)
1109
load_netdata_conf(NULL, 0);
1110
1111
+
1112
// ------------------------------------------------------------------------
1113
// initialize netdata
1114
{
database/rrd.h
+2
@@ -408,6 +408,7 @@ struct rrdset {
408
// it goes around in a round-robin fashion
409
410
RRDSET_FLAGS flags; // configuration flags
411
+ RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances
412
413
int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored
414
// netdata will interpolate values for gaps lower than this
@@ -627,6 +628,7 @@ struct rrdhost {
628
const char *timezone; // the timezone of the host
629
630
RRDHOST_FLAGS flags; // flags about this RRDHOST
631
+ RRDHOST_FLAGS *exporting_flags; // array of flags for exporting connector instances
632
633
int rrd_update_every; // the update frequency of the host
634
long rrd_history_entries; // the number of history entries for the host's charts
database/rrdhost.c
+2
@@ -597,6 +597,8 @@ void rrdhost_free(RRDHOST *host) {
597
while(host->rrdset_root)
598
rrdset_free(host->rrdset_root);
599
600
+ freez(host->exporting_flags);
601
+
602
while(host->alarms)
603
rrdcalc_unlink_and_free(host, host->alarms);
604
database/rrdset.c
+2
@@ -328,6 +328,8 @@ void rrdset_free(RRDSET *st) {
328
// ------------------------------------------------------------------------
329
// free its children structures
330
331
+ freez(st->exporting_flags);
332
+
333
while(st->variables) rrdsetvar_free(st->variables);
334
while(st->alarms) rrdsetcalc_unlink(st->alarms);
335
while(st->dimensions) rrddim_free(st, st->dimensions);
exporting/Makefile.am
new
+15
@@ -0,0 +1,15 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
+
6
+SUBDIRS = \
7
+ tests \
8
+ graphite \
9
+ json \
10
+ opentsdb \
11
+ $(NULL)
12
+
13
+dist_noinst_DATA = \
14
+ README.md \
15
+ $(NULL)
exporting/README.md
new
+45
@@ -0,0 +1,45 @@
1
+# Exporting metrics to external databases (experimental)
2
+
3
+The exporting engine is an update for the former [backends](../backends/). It's still work in progress. It has a
4
+modular structure and supports metric exporting via multiple exporting connector instances at the same time. You can
5
+have different update intervals and filters configured for every exporting connector instance. The exporting engine has
6
+its own configuration file `exporting.conf`. Configuration is almost similar to [backends](../backends/#configuration).
7
+The only difference is that the type of a connector should be specified in a section name before a colon and a name after
8
+the colon. At the moment only four types of connectors are supported: `graphite`, `json`, `opentsdb`, `opentsdb:http`.
9
+
10
+An example configuration:
11
+```conf
12
+[exporting:global]
13
+enabled = yes
14
+
15
+[graphite:my_instance1]
16
+enabled = yes
17
+destination = localhost:2003
18
+data source = sum
19
+update every = 5
20
+send charts matching = system.load
21
+
22
+[json:my_instance2]
23
+enabled = yes
24
+destination = localhost:5448
25
+data source = as collected
26
+update every = 2
27
+send charts matching = system.active_processes
28
+
29
+[opentsdb:my_instance3]
30
+enabled = yes
31
+destination = localhost:4242
32
+data source = sum
33
+update every = 10
34
+send charts matching = system.cpu
35
+
36
+[opentsdb:http:my_instance4]
37
+enabled = yes
38
+destination = localhost:4243
39
+data source = average
40
+update every = 3
41
+send charts matching = system.active_processes
42
+
43
+```
44
+
45
+[](<>)
exporting/check_filters.c
new
+78
@@ -0,0 +1,78 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "exporting_engine.h"
4
+
5
+/**
6
+ * Check if the connector instance should export the host metrics
7
+ *
8
+ * @param instance an exporting connector instance.
9
+ * @param host a data collecting host.
10
+ * @return Returns 1 if the connector instance should export the host metrics
11
+ */
12
+int rrdhost_is_exportable(struct instance *instance, RRDHOST *host)
13
+{
14
+ if (host->exporting_flags == NULL)
15
+ host->exporting_flags = callocz(instance->connector->engine->instance_num, sizeof(size_t));
16
+
17
+ RRDHOST_FLAGS *flags = &host->exporting_flags[instance->index];
18
+
19
+ if (unlikely((*flags & (RRDHOST_FLAG_BACKEND_SEND | RRDHOST_FLAG_BACKEND_DONT_SEND)) == 0)) {
20
+ char *host_name = (host == localhost) ? "localhost" : host->hostname;
21
+
22
+ if (!instance->config.hosts_pattern || simple_pattern_matches(instance->config.hosts_pattern, host_name)) {
23
+ *flags |= RRDHOST_FLAG_BACKEND_SEND;
24
+ info("enabled exporting of host '%s' for instance '%s'", host_name, instance->config.name);
25
+ } else {
26
+ *flags |= RRDHOST_FLAG_BACKEND_DONT_SEND;
27
+ info("disabled exporting of host '%s' for instance '%s'", host_name, instance->config.name);
28
+ }
29
+ }
30
+
31
+ if (likely(*flags & RRDHOST_FLAG_BACKEND_SEND))
32
+ return 1;
33
+ else
34
+ return 0;
35
+}
36
+
37
+/**
38
+ * Check if the connector instance should export the chart
39
+ *
40
+ * @param instance an exporting connector instance.
41
+ * @param st a chart.
42
+ * @return Returns 1 if the connector instance should export the chart
43
+ */
44
+int rrdset_is_exportable(struct instance *instance, RRDSET *st)
45
+{
46
+ RRDHOST *host = st->rrdhost;
47
+
48
+ if (st->exporting_flags == NULL)
49
+ st->exporting_flags = callocz(instance->connector->engine->instance_num, sizeof(size_t));
50
+
51
+ RRDSET_FLAGS *flags = &st->exporting_flags[instance->index];
52
+
53
+ if(unlikely(*flags & RRDSET_FLAG_BACKEND_IGNORE))
54
+ return 0;
55
+
56
+ if(unlikely(!(*flags & RRDSET_FLAG_BACKEND_SEND))) {
57
+ // we have not checked this chart
58
+ if(simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name))
59
+ *flags |= RRDSET_FLAG_BACKEND_SEND;
60
+ else {
61
+ *flags |= RRDSET_FLAG_BACKEND_IGNORE;
62
+ debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, host->hostname);
63
+ return 0;
64
+ }
65
+ }
66
+
67
+ if(unlikely(!rrdset_is_available_for_backends(st))) {
68
+ debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.", st->id, host->hostname);
69
+ return 0;
70
+ }
71
+
72
+ if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
73
+ debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode));
74
+ return 0;
75
+ }
76
+
77
+ return 1;
78
+}
exporting/exporting_engine.c
new
+60
@@ -0,0 +1,60 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "exporting_engine.h"
4
+
5
+/**
6
+ * Exporting engine main
7
+ *
8
+ * The main thread used to control the exporting engine.
9
+ *
10
+ * @param ptr a pointer to netdata_static_structure.
11
+ *
12
+ * @return It always returns NULL.
13
+ */
14
+void *exporting_main(void *ptr)
15
+{
16
+ (void)ptr;
17
+
18
+ struct engine *engine = read_exporting_config();
19
+ if (!engine) {
20
+ info("EXPORTING: no exporting connectors configured");
21
+ return NULL;
22
+ }
23
+
24
+ if (init_connectors(engine) != 0) {
25
+ error("EXPORTING: cannot initialize exporting connectors");
26
+ return NULL;
27
+ }
28
+
29
+ usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC;
30
+ heartbeat_t hb;
31
+ heartbeat_init(&hb);
32
+
33
+ while (!netdata_exit) {
34
+ heartbeat_next(&hb, step_ut);
35
+ engine->now = now_realtime_sec();
36
+
37
+ if (mark_scheduled_instances(engine)) {
38
+ if (prepare_buffers(engine) != 0) {
39
+ error("EXPORTING: cannot prepare data to send");
40
+ return NULL;
41
+ }
42
+ }
43
+
44
+ if (notify_workers(engine) != 0) {
45
+ error("EXPORTING: cannot communicate with exporting connector instance working threads");
46
+ return NULL;
47
+ }
48
+
49
+ if (send_internal_metrics(engine) != 0) {
50
+ error("EXPORTING: cannot send metrics for the operation of exporting engine");
51
+ return NULL;
52
+ }
53
+
54
+#ifdef UNIT_TESTING
55
+ break;
56
+#endif
57
+ }
58
+
59
+ return NULL;
60
+}
exporting/exporting_engine.h
new
+185
@@ -0,0 +1,185 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EXPORTING_ENGINE_H
4
+#define NETDATA_EXPORTING_ENGINE_H 1
5
+
6
+#include "daemon/common.h"
7
+
8
+#include <uv.h>
9
+
10
+#define exporter_get(section, name, value) expconfig_get(&exporting_config, section, name, value)
11
+#define exporter_get_number(section, name, value) expconfig_get_number(&exporting_config, section, name, value)
12
+#define exporter_get_boolean(section, name, value) expconfig_get_boolean(&exporting_config, section, name, value)
13
+
14
+extern struct config exporting_config;
15
+
16
+#define EXPORTER_DATA_SOURCE "data source"
17
+#define EXPORTER_DATA_SOURCE_DEFAULT "average"
18
+
19
+#define EXPORTER_DESTINATION "destination"
20
+#define EXPORTER_DESTINATION_DEFAULT "localhost"
21
+
22
+#define EXPORTER_UPDATE_EVERY "update every"
23
+#define EXPORTER_UPDATE_EVERY_DEFAULT 10
24
+
25
+#define EXPORTER_BUF_ONFAIL "buffer on failures"
26
+#define EXPORTER_BUF_ONFAIL_DEFAULT 10
27
+
28
+#define EXPORTER_TIMEOUT_MS "timeout ms"
29
+#define EXPORTER_TIMEOUT_MS_DEFAULT 10000
30
+
31
+#define EXPORTER_SEND_CHART_MATCH "send charts matching"
32
+#define EXPORTER_SEND_CHART_MATCH_DEFAULT "*"
33
+
34
+#define EXPORTER_SEND_HOST_MATCH "send hosts matching"
35
+#define EXPORTER_SEND_HOST_MATCH_DEFAULT "localhost *"
36
+
37
+#define EXPORTER_SEND_NAMES "send names instead of ids"
38
+#define EXPORTER_SEND_NAMES_DEFAULT CONFIG_BOOLEAN_YES
39
+
40
+typedef enum exporting_options {
41
+ EXPORTING_OPTION_NONE = 0,
42
+
43
+ EXPORTING_SOURCE_DATA_AS_COLLECTED = (1 << 0),
44
+ EXPORTING_SOURCE_DATA_AVERAGE = (1 << 1),
45
+ EXPORTING_SOURCE_DATA_SUM = (1 << 2),
46
+
47
+ EXPORTING_OPTION_SEND_NAMES = (1 << 16)
48
+} EXPORTING_OPTIONS;
49
+
50
+#define EXPORTING_OPTIONS_SOURCE_BITS \
51
+ (EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_SOURCE_DATA_SUM)
52
+#define EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) (exporting_options & EXPORTING_OPTIONS_SOURCE_BITS)
53
+
54
+struct engine;
55
+
56
+struct instance_config {
57
+ const char *name;
58
+ const char *destination;
59
+
60
+ int update_every;
61
+ int buffer_on_failures;
62
+ long timeoutms;
63
+
64
+ EXPORTING_OPTIONS options;
65
+ SIMPLE_PATTERN *charts_pattern;
66
+ SIMPLE_PATTERN *hosts_pattern;
67
+
68
+ void *connector_specific_config;
69
+};
70
+
71
+struct simple_connector_config {
72
+ int default_port;
73
+};
74
+
75
+struct connector_config {
76
+ BACKEND_TYPE type;
77
+ void *connector_specific_config;
78
+};
79
+
80
+struct engine_config {
81
+ const char *prefix;
82
+ const char *hostname;
83
+ int update_every;
84
+};
85
+
86
+struct stats {
87
+ collected_number chart_buffered_metrics;
88
+ collected_number chart_lost_metrics;
89
+ collected_number chart_sent_metrics;
90
+ collected_number chart_buffered_bytes;
91
+ collected_number chart_received_bytes;
92
+ collected_number chart_sent_bytes;
93
+ collected_number chart_receptions;
94
+ collected_number chart_transmission_successes;
95
+ collected_number chart_transmission_failures;
96
+ collected_number chart_data_lost_events;
97
+ collected_number chart_lost_bytes;
98
+ collected_number chart_reconnects;
99
+};
100
+
101
+struct instance {
102
+ struct instance_config config;
103
+ void *buffer;
104
+ struct stats stats;
105
+
106
+ int scheduled;
107
+ int skip_host;
108
+ int skip_chart;
109
+
110
+ time_t after;
111
+ time_t before;
112
+
113
+ uv_thread_t thread;
114
+ uv_mutex_t mutex;
115
+ uv_cond_t cond_var;
116
+
117
+ int (*start_batch_formatting)(struct instance *instance);
118
+ int (*start_host_formatting)(struct instance *instance, RRDHOST *host);
119
+ int (*start_chart_formatting)(struct instance *instance, RRDSET *st);
120
+ int (*metric_formatting)(struct instance *instance, RRDDIM *rd);
121
+ int (*end_chart_formatting)(struct instance *instance, RRDSET *st);
122
+ int (*end_host_formatting)(struct instance *instance, RRDHOST *host);
123
+ int (*end_batch_formatting)(struct instance *instance);
124
+
125
+ size_t index;
126
+ struct instance *next;
127
+ struct connector *connector;
128
+};
129
+
130
+struct connector {
131
+ struct connector_config config;
132
+
133
+ void (*worker)(void *instance_p);
134
+
135
+ struct instance *instance_root;
136
+ struct connector *next;
137
+ struct engine *engine;
138
+};
139
+
140
+struct engine {
141
+ struct engine_config config;
142
+
143
+ size_t instance_num;
144
+ time_t now;
145
+
146
+ struct connector *connector_root;
147
+};
148
+
149
+void *exporting_main(void *ptr);
150
+
151
+struct engine *read_exporting_config();
152
+BACKEND_TYPE exporting_select_type(const char *type);
153
+
154
+int init_connectors(struct engine *engine);
155
+
156
+int mark_scheduled_instances(struct engine *engine);
157
+int prepare_buffers(struct engine *engine);
158
+int notify_workers(struct engine *engine);
159
+
160
+size_t exporting_name_copy(char *dst, const char *src, size_t max_len);
161
+
162
+int rrdhost_is_exportable(struct instance *instance, RRDHOST *host);
163
+int rrdset_is_exportable(struct instance *instance, RRDSET *st);
164
+
165
+calculated_number exporting_calculate_value_from_stored_data(
166
+ struct instance *instance,
167
+ RRDDIM *rd,
168
+ time_t *last_timestamp);
169
+
170
+int start_batch_formatting(struct engine *engine);
171
+int start_host_formatting(struct engine *engine, RRDHOST *host);
172
+int start_chart_formatting(struct engine *engine, RRDSET *st);
173
+int metric_formatting(struct engine *engine, RRDDIM *rd);
174
+int end_chart_formatting(struct engine *engine, RRDSET *st);
175
+int end_host_formatting(struct engine *engine, RRDHOST *host);
176
+int end_batch_formatting(struct engine *engine);
177
+
178
+int exporting_discard_response(BUFFER *buffer, struct instance *instance);
179
+void simple_connector_receive_response(int *sock, struct instance *instance);
180
+void simple_connector_send_buffer(int *sock, int *failures, struct instance *instance);
181
+void simple_connector_worker(void *instance_p);
182
+
183
+int send_internal_metrics(struct engine *engine);
184
+
185
+#endif /* NETDATA_EXPORTING_ENGINE_H */
exporting/graphite/Makefile.am
new
+4
@@ -0,0 +1,4 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
exporting/graphite/graphite.c
new
+138
@@ -0,0 +1,138 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "graphite.h"
4
+
5
+/**
6
+ * Initialize Graphite connector
7
+ *
8
+ * @param instance a connector data structure.
9
+ * @return Always returns 0.
10
+ */
11
+int init_graphite_connector(struct connector *connector)
12
+{
13
+ connector->worker = simple_connector_worker;
14
+
15
+ struct simple_connector_config *connector_specific_config = mallocz(sizeof(struct simple_connector_config));
16
+ connector->config.connector_specific_config = (void *)connector_specific_config;
17
+ connector_specific_config->default_port = 2003;
18
+
19
+ return 0;
20
+}
21
+
22
+/**
23
+ * Initialize Graphite connector instance
24
+ *
25
+ * @param instance an instance data structure.
26
+ * @return Returns 0 on success, 1 on failure.
27
+ */
28
+int init_graphite_instance(struct instance *instance)
29
+{
30
+ instance->start_batch_formatting = NULL;
31
+ instance->start_host_formatting = NULL;
32
+ instance->start_chart_formatting = NULL;
33
+
34
+ if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
35
+ instance->metric_formatting = format_dimension_collected_graphite_plaintext;
36
+ else
37
+ instance->metric_formatting = format_dimension_stored_graphite_plaintext;
38
+
39
+ instance->end_chart_formatting = NULL;
40
+ instance->end_host_formatting = NULL;
41
+ instance->end_batch_formatting = NULL;
42
+
43
+ instance->buffer = (void *)buffer_create(0);
44
+ if (!instance->buffer) {
45
+ error("EXPORTING: cannot create buffer for graphite exporting connector instance %s", instance->config.name);
46
+ return 1;
47
+ }
48
+ uv_mutex_init(&instance->mutex);
49
+ uv_cond_init(&instance->cond_var);
50
+
51
+ return 0;
52
+}
53
+
54
+/**
55
+ * Format dimension using collected data for Graphite connector
56
+ *
57
+ * @param instance an instance data structure.
58
+ * @param rd a dimension.
59
+ * @return Always returns 0.
60
+ */
61
+int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd)
62
+{
63
+ struct engine *engine = instance->connector->engine;
64
+ RRDSET *st = rd->rrdset;
65
+ RRDHOST *host = st->rrdhost;
66
+
67
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
68
+ exporting_name_copy(
69
+ chart_name,
70
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
71
+ RRD_ID_LENGTH_MAX);
72
+
73
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
74
+ exporting_name_copy(
75
+ dimension_name,
76
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
77
+ RRD_ID_LENGTH_MAX);
78
+
79
+ buffer_sprintf(
80
+ instance->buffer,
81
+ "%s.%s.%s.%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n",
82
+ engine->config.prefix,
83
+ engine->config.hostname,
84
+ chart_name,
85
+ dimension_name,
86
+ (host->tags) ? ";" : "",
87
+ (host->tags) ? host->tags : "",
88
+ rd->last_collected_value,
89
+ (unsigned long long)rd->last_collected_time.tv_sec);
90
+
91
+ return 0;
92
+}
93
+
94
+/**
95
+ * Format dimension using a calculated value from stored data for Graphite connector
96
+ *
97
+ * @param instance an instance data structure.
98
+ * @param rd a dimension.
99
+ * @return Always returns 0.
100
+ */
101
+int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM *rd)
102
+{
103
+ struct engine *engine = instance->connector->engine;
104
+ RRDSET *st = rd->rrdset;
105
+ RRDHOST *host = st->rrdhost;
106
+
107
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
108
+ exporting_name_copy(
109
+ chart_name,
110
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
111
+ RRD_ID_LENGTH_MAX);
112
+
113
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
114
+ exporting_name_copy(
115
+ dimension_name,
116
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
117
+ RRD_ID_LENGTH_MAX);
118
+
119
+ time_t last_t;
120
+ calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
121
+
122
+ if(isnan(value))
123
+ return 0;
124
+
125
+ buffer_sprintf(
126
+ instance->buffer,
127
+ "%s.%s.%s.%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
128
+ engine->config.prefix,
129
+ engine->config.hostname,
130
+ chart_name,
131
+ dimension_name,
132
+ (host->tags) ? ";" : "",
133
+ (host->tags) ? host->tags : "",
134
+ value,
135
+ (unsigned long long)last_t);
136
+
137
+ return 0;
138
+}
exporting/graphite/graphite.h
new
+13
@@ -0,0 +1,13 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EXPORTING_GRAPHITE_H
4
+#define NETDATA_EXPORTING_GRAPHITE_H
5
+
6
+#include "exporting/exporting_engine.h"
7
+
8
+int init_graphite_connector(struct connector *connector);
9
+int init_graphite_instance(struct instance *instance);
10
+int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd);
11
+int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM *rd);
12
+
13
+#endif //NETDATA_EXPORTING_GRAPHITE_H
exporting/init_connectors.c
new
+72
@@ -0,0 +1,72 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "exporting_engine.h"
4
+#include "graphite/graphite.h"
5
+#include "json/json.h"
6
+#include "opentsdb/opentsdb.h"
7
+
8
+/**
9
+ * Initialize connectors
10
+ *
11
+ * @param engine an engine data structure.
12
+ * @return Returns 0 on success, 1 on failure.
13
+ */
14
+int init_connectors(struct engine *engine)
15
+{
16
+ engine->now = now_realtime_sec();
17
+
18
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
19
+ switch (connector->config.type) {
20
+ case BACKEND_TYPE_GRAPHITE:
21
+ if (init_graphite_connector(connector) != 0)
22
+ return 1;
23
+ break;
24
+ case BACKEND_TYPE_JSON:
25
+ if (init_json_connector(connector) != 0)
26
+ return 1;
27
+ break;
28
+ case BACKEND_TYPE_OPENTSDB_USING_TELNET:
29
+ if (init_opentsdb_connector(connector) != 0)
30
+ return 1;
31
+ break;
32
+ case BACKEND_TYPE_OPENTSDB_USING_HTTP:
33
+ if (init_opentsdb_connector(connector) != 0)
34
+ return 1;
35
+ break;
36
+ default:
37
+ error("EXPORTING: unknown exporting connector type");
38
+ return 1;
39
+ }
40
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
41
+ instance->index = engine->instance_num++;
42
+ instance->after = engine->now;
43
+
44
+ switch (connector->config.type) {
45
+ case BACKEND_TYPE_GRAPHITE:
46
+ if (init_graphite_instance(instance) != 0)
47
+ return 1;
48
+ break;
49
+ case BACKEND_TYPE_JSON:
50
+ if (init_json_instance(instance) != 0)
51
+ return 1;
52
+ break;
53
+ case BACKEND_TYPE_OPENTSDB_USING_TELNET:
54
+ if (init_opentsdb_telnet_instance(instance) != 0)
55
+ return 1;
56
+ break;
57
+ case BACKEND_TYPE_OPENTSDB_USING_HTTP:
58
+ if (init_opentsdb_http_instance(instance) != 0)
59
+ return 1;
60
+ break;
61
+ default:
62
+ error("EXPORTING: unknown exporting connector type");
63
+ return 1;
64
+ }
65
+
66
+ // dispatch the instance worker thread
67
+ uv_thread_create(&instance->thread, connector->worker, instance);
68
+ }
69
+ }
70
+
71
+ return 0;
72
+}
exporting/json/Makefile.am
new
+4
@@ -0,0 +1,4 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
exporting/json/json.c
new
+194
@@ -0,0 +1,194 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "json.h"
4
+
5
+/**
6
+ * Initialize JSON connector
7
+ *
8
+ * @param instance a connector data structure.
9
+ * @return Always returns 0.
10
+ */
11
+int init_json_connector(struct connector *connector)
12
+{
13
+ connector->worker = simple_connector_worker;
14
+
15
+ struct simple_connector_config *connector_specific_config = mallocz(sizeof(struct simple_connector_config));
16
+ connector->config.connector_specific_config = (void *)connector_specific_config;
17
+ connector_specific_config->default_port = 5448;
18
+
19
+ return 0;
20
+}
21
+
22
+/**
23
+ * Initialize JSON connector instance
24
+ *
25
+ * @param instance an instance data structure.
26
+ * @return Returns 0 on success, 1 on failure.
27
+ */
28
+int init_json_instance(struct instance *instance)
29
+{
30
+ instance->start_batch_formatting = NULL;
31
+ instance->start_host_formatting = NULL;
32
+ instance->start_chart_formatting = NULL;
33
+
34
+ if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
35
+ instance->metric_formatting = format_dimension_collected_json_plaintext;
36
+ else
37
+ instance->metric_formatting = format_dimension_stored_json_plaintext;
38
+
39
+ instance->end_chart_formatting = NULL;
40
+ instance->end_host_formatting = NULL;
41
+ instance->end_batch_formatting = NULL;
42
+
43
+ instance->buffer = (void *)buffer_create(0);
44
+ if (!instance->buffer) {
45
+ error("EXPORTING: cannot create buffer for json exporting connector instance %s", instance->config.name);
46
+ return 1;
47
+ }
48
+ uv_mutex_init(&instance->mutex);
49
+ uv_cond_init(&instance->cond_var);
50
+
51
+ return 0;
52
+}
53
+
54
+/**
55
+ * Format dimension using collected data for JSON connector
56
+ *
57
+ * @param instance an instance data structure.
58
+ * @param rd a dimension.
59
+ * @return Always returns 0.
60
+ */
61
+int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM *rd)
62
+{
63
+ struct engine *engine = instance->connector->engine;
64
+ RRDSET *st = rd->rrdset;
65
+ RRDHOST *host = st->rrdhost;
66
+
67
+ const char *tags_pre = "", *tags_post = "", *tags = host->tags;
68
+ if (!tags)
69
+ tags = "";
70
+
71
+ if (*tags) {
72
+ if (*tags == '{' || *tags == '[' || *tags == '"') {
73
+ tags_pre = "\"host_tags\":";
74
+ tags_post = ",";
75
+ } else {
76
+ tags_pre = "\"host_tags\":\"";
77
+ tags_post = "\",";
78
+ }
79
+ }
80
+
81
+ buffer_sprintf(
82
+ instance->buffer,
83
+ "{"
84
+ "\"prefix\":\"%s\","
85
+ "\"hostname\":\"%s\","
86
+ "%s%s%s"
87
+
88
+ "\"chart_id\":\"%s\","
89
+ "\"chart_name\":\"%s\","
90
+ "\"chart_family\":\"%s\","
91
+ "\"chart_context\": \"%s\","
92
+ "\"chart_type\":\"%s\","
93
+ "\"units\": \"%s\","
94
+
95
+ "\"id\":\"%s\","
96
+ "\"name\":\"%s\","
97
+ "\"value\":" COLLECTED_NUMBER_FORMAT ","
98
+
99
+ "\"timestamp\": %llu}\n",
100
+ engine->config.prefix,
101
+ engine->config.hostname,
102
+ tags_pre,
103
+ tags,
104
+ tags_post,
105
+
106
+ st->id,
107
+ st->name,
108
+ st->family,
109
+ st->context,
110
+ st->type,
111
+ st->units,
112
+
113
+ rd->id,
114
+ rd->name,
115
+ rd->last_collected_value,
116
+
117
+ (unsigned long long)rd->last_collected_time.tv_sec);
118
+
119
+ return 0;
120
+}
121
+
122
+/**
123
+ * Format dimension using a calculated value from stored data for JSON connector
124
+ *
125
+ * @param instance an instance data structure.
126
+ * @param rd a dimension.
127
+ * @return Always returns 0.
128
+ */
129
+int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd)
130
+{
131
+ struct engine *engine = instance->connector->engine;
132
+ RRDSET *st = rd->rrdset;
133
+ RRDHOST *host = st->rrdhost;
134
+
135
+ time_t last_t;
136
+ calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
137
+
138
+ if(isnan(value))
139
+ return 0;
140
+
141
+ const char *tags_pre = "", *tags_post = "", *tags = host->tags;
142
+ if (!tags)
143
+ tags = "";
144
+
145
+ if (*tags) {
146
+ if (*tags == '{' || *tags == '[' || *tags == '"') {
147
+ tags_pre = "\"host_tags\":";
148
+ tags_post = ",";
149
+ } else {
150
+ tags_pre = "\"host_tags\":\"";
151
+ tags_post = "\",";
152
+ }
153
+ }
154
+
155
+ buffer_sprintf(
156
+ instance->buffer,
157
+ "{"
158
+ "\"prefix\":\"%s\","
159
+ "\"hostname\":\"%s\","
160
+ "%s%s%s"
161
+
162
+ "\"chart_id\":\"%s\","
163
+ "\"chart_name\":\"%s\","
164
+ "\"chart_family\":\"%s\","
165
+ "\"chart_context\": \"%s\","
166
+ "\"chart_type\":\"%s\","
167
+ "\"units\": \"%s\","
168
+
169
+ "\"id\":\"%s\","
170
+ "\"name\":\"%s\","
171
+ "\"value\":" CALCULATED_NUMBER_FORMAT ","
172
+
173
+ "\"timestamp\": %llu}\n",
174
+ engine->config.prefix,
175
+ engine->config.hostname,
176
+ tags_pre,
177
+ tags,
178
+ tags_post,
179
+
180
+ st->id,
181
+ st->name,
182
+ st->family,
183
+ st->context,
184
+ st->type,
185
+ st->units,
186
+
187
+ rd->id,
188
+ rd->name,
189
+ value,
190
+
191
+ (unsigned long long)last_t);
192
+
193
+ return 0;
194
+}
exporting/json/json.h
new
+13
@@ -0,0 +1,13 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EXPORTING_JSON_H
4
+#define NETDATA_EXPORTING_JSON_H
5
+
6
+#include "exporting/exporting_engine.h"
7
+
8
+int init_json_connector(struct connector *connector);
9
+int init_json_instance(struct instance *instance);
10
+int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM *rd);
11
+int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd);
12
+
13
+#endif //NETDATA_EXPORTING_JSON_H
exporting/opentsdb/Makefile.am
new
+4
@@ -0,0 +1,4 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
exporting/opentsdb/opentsdb.c
new
+305
@@ -0,0 +1,305 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "opentsdb.h"
4
+
5
+/**
6
+ * Initialize OpenTSDB connector
7
+ *
8
+ * @param instance a connector data structure.
9
+ * @return Always returns 0.
10
+ */
11
+int init_opentsdb_connector(struct connector *connector)
12
+{
13
+ connector->worker = simple_connector_worker;
14
+
15
+ struct simple_connector_config *connector_specific_config = mallocz(sizeof(struct simple_connector_config));
16
+ connector->config.connector_specific_config = (void *)connector_specific_config;
17
+ connector_specific_config->default_port = 4242;
18
+
19
+ return 0;
20
+}
21
+
22
+/**
23
+ * Initialize OpenTSDB telnet connector instance
24
+ *
25
+ * @param instance an instance data structure.
26
+ * @return Returns 0 on success, 1 on failure.
27
+ */
28
+int init_opentsdb_telnet_instance(struct instance *instance)
29
+{
30
+ instance->start_batch_formatting = NULL;
31
+ instance->start_host_formatting = NULL;
32
+ instance->start_chart_formatting = NULL;
33
+
34
+ if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
35
+ instance->metric_formatting = format_dimension_collected_opentsdb_telnet;
36
+ else
37
+ instance->metric_formatting = format_dimension_stored_opentsdb_telnet;
38
+
39
+ instance->end_chart_formatting = NULL;
40
+ instance->end_host_formatting = NULL;
41
+ instance->end_batch_formatting = NULL;
42
+
43
+ instance->buffer = (void *)buffer_create(0);
44
+ if (!instance->buffer) {
45
+ error("EXPORTING: cannot create buffer for opentsdb telnet exporting connector instance %s", instance->config.name);
46
+ return 1;
47
+ }
48
+ uv_mutex_init(&instance->mutex);
49
+ uv_cond_init(&instance->cond_var);
50
+
51
+ return 0;
52
+}
53
+
54
+/**
55
+ * Initialize OpenTSDB HTTP connector instance
56
+ *
57
+ * @param instance an instance data structure.
58
+ * @return Returns 0 on success, 1 on failure.
59
+ */
60
+int init_opentsdb_http_instance(struct instance *instance)
61
+{
62
+ instance->start_batch_formatting = NULL;
63
+ instance->start_host_formatting = NULL;
64
+ instance->start_chart_formatting = NULL;
65
+
66
+ if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
67
+ instance->metric_formatting = format_dimension_collected_opentsdb_http;
68
+ else
69
+ instance->metric_formatting = format_dimension_stored_opentsdb_http;
70
+
71
+ instance->end_chart_formatting = NULL;
72
+ instance->end_host_formatting = NULL;
73
+ instance->end_batch_formatting = NULL;
74
+
75
+ instance->buffer = (void *)buffer_create(0);
76
+ if (!instance->buffer) {
77
+ error("EXPORTING: cannot create buffer for opentsdb HTTP exporting connector instance %s", instance->config.name);
78
+ return 1;
79
+ }
80
+ uv_mutex_init(&instance->mutex);
81
+ uv_cond_init(&instance->cond_var);
82
+
83
+ return 0;
84
+}
85
+
86
+/**
87
+ * Format dimension using collected data for OpenTSDB telnet connector
88
+ *
89
+ * @param instance an instance data structure.
90
+ * @param rd a dimension.
91
+ * @return Always returns 0.
92
+ */
93
+int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM *rd)
94
+{
95
+ struct engine *engine = instance->connector->engine;
96
+ RRDSET *st = rd->rrdset;
97
+ RRDHOST *host = st->rrdhost;
98
+
99
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
100
+ exporting_name_copy(
101
+ chart_name,
102
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
103
+ RRD_ID_LENGTH_MAX);
104
+
105
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
106
+ exporting_name_copy(
107
+ dimension_name,
108
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
109
+ RRD_ID_LENGTH_MAX);
110
+
111
+ buffer_sprintf(
112
+ instance->buffer,
113
+ "put %s.%s.%s %llu " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n",
114
+ engine->config.prefix,
115
+ chart_name,
116
+ dimension_name,
117
+ (unsigned long long)rd->last_collected_time.tv_sec,
118
+ rd->last_collected_value,
119
+ engine->config.hostname,
120
+ (host->tags) ? " " : "",
121
+ (host->tags) ? host->tags : "");
122
+
123
+ return 0;
124
+}
125
+
126
+/**
127
+ * Format dimension using a calculated value from stored data for OpenTSDB telnet connector
128
+ *
129
+ * @param instance an instance data structure.
130
+ * @param rd a dimension.
131
+ * @return Always returns 0.
132
+ */
133
+int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *rd)
134
+{
135
+ struct engine *engine = instance->connector->engine;
136
+ RRDSET *st = rd->rrdset;
137
+ RRDHOST *host = st->rrdhost;
138
+
139
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
140
+ exporting_name_copy(
141
+ chart_name,
142
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
143
+ RRD_ID_LENGTH_MAX);
144
+
145
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
146
+ exporting_name_copy(
147
+ dimension_name,
148
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
149
+ RRD_ID_LENGTH_MAX);
150
+
151
+ time_t last_t;
152
+ calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
153
+
154
+ if(isnan(value))
155
+ return 0;
156
+
157
+ buffer_sprintf(
158
+ instance->buffer,
159
+ "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n",
160
+ engine->config.prefix,
161
+ chart_name,
162
+ dimension_name,
163
+ (unsigned long long)last_t,
164
+ value,
165
+ engine->config.hostname,
166
+ (host->tags) ? " " : "",
167
+ (host->tags) ? host->tags : "");
168
+
169
+ return 0;
170
+}
171
+
172
+/**
173
+ * Prepare an HTTP message for OpenTSDB HTTP connector
174
+ *
175
+ * @param buffer a buffer to write the message to.
176
+ * @param message the body of the message.
177
+ * @param hostname the name of the host that sends the message.
178
+ * @param length the length of the message body.
179
+ */
180
+static inline void opentsdb_build_message(BUFFER *buffer, char *message, const char *hostname, int length)
181
+{
182
+ buffer_sprintf(
183
+ buffer,
184
+ "POST /api/put HTTP/1.1\r\n"
185
+ "Host: %s\r\n"
186
+ "Content-Type: application/json\r\n"
187
+ "Content-Length: %d\r\n"
188
+ "\r\n"
189
+ "%s",
190
+ hostname,
191
+ length,
192
+ message);
193
+}
194
+
195
+/**
196
+ * Format dimension using collected data for OpenTSDB HTTP connector
197
+ *
198
+ * @param instance an instance data structure.
199
+ * @param rd a dimension.
200
+ * @return Always returns 0.
201
+ */
202
+int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *rd)
203
+{
204
+ struct engine *engine = instance->connector->engine;
205
+ RRDSET *st = rd->rrdset;
206
+ RRDHOST *host = st->rrdhost;
207
+
208
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
209
+ exporting_name_copy(
210
+ chart_name,
211
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
212
+ RRD_ID_LENGTH_MAX);
213
+
214
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
215
+ exporting_name_copy(
216
+ dimension_name,
217
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
218
+ RRD_ID_LENGTH_MAX);
219
+
220
+ char message[1024];
221
+ int length = snprintfz(
222
+ message,
223
+ sizeof(message),
224
+ "{"
225
+ " \"metric\": \"%s.%s.%s\","
226
+ " \"timestamp\": %llu,"
227
+ " \"value\": " COLLECTED_NUMBER_FORMAT ","
228
+ " \"tags\": {"
229
+ " \"host\": \"%s%s%s\""
230
+ " }"
231
+ "}",
232
+ engine->config.prefix,
233
+ chart_name,
234
+ dimension_name,
235
+ (unsigned long long)rd->last_collected_time.tv_sec,
236
+ rd->last_collected_value,
237
+ engine->config.hostname,
238
+ (host->tags) ? " " : "",
239
+ (host->tags) ? host->tags : "");
240
+
241
+ if (length > 0) {
242
+ opentsdb_build_message(instance->buffer, message, engine->config.hostname, length);
243
+ }
244
+
245
+ return 0;
246
+}
247
+
248
+/**
249
+ * Format dimension using a calculated value from stored data for OpenTSDB HTTP connector
250
+ *
251
+ * @param instance an instance data structure.
252
+ * @param rd a dimension.
253
+ * @return Always returns 0.
254
+ */
255
+int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
256
+{
257
+ struct engine *engine = instance->connector->engine;
258
+ RRDSET *st = rd->rrdset;
259
+ RRDHOST *host = st->rrdhost;
260
+
261
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
262
+ exporting_name_copy(
263
+ chart_name,
264
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
265
+ RRD_ID_LENGTH_MAX);
266
+
267
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
268
+ exporting_name_copy(
269
+ dimension_name,
270
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
271
+ RRD_ID_LENGTH_MAX);
272
+
273
+ time_t last_t;
274
+ calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
275
+
276
+ if(isnan(value))
277
+ return 0;
278
+
279
+ char message[1024];
280
+ int length = snprintfz(
281
+ message,
282
+ sizeof(message),
283
+ "{"
284
+ " \"metric\": \"%s.%s.%s\","
285
+ " \"timestamp\": %llu,"
286
+ " \"value\": " CALCULATED_NUMBER_FORMAT ","
287
+ " \"tags\": {"
288
+ " \"host\": \"%s%s%s\""
289
+ " }"
290
+ "}",
291
+ engine->config.prefix,
292
+ chart_name,
293
+ dimension_name,
294
+ (unsigned long long)last_t,
295
+ value,
296
+ engine->config.hostname,
297
+ (host->tags) ? " " : "",
298
+ (host->tags) ? host->tags : "");
299
+
300
+ if (length > 0) {
301
+ opentsdb_build_message(instance->buffer, message, engine->config.hostname, length);
302
+ }
303
+
304
+ return 0;
305
+}
exporting/opentsdb/opentsdb.h
new
+18
@@ -0,0 +1,18 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EXPORTING_OPENTSDB_H
4
+#define NETDATA_EXPORTING_OPENTSDB_H
5
+
6
+#include "exporting/exporting_engine.h"
7
+
8
+int init_opentsdb_connector(struct connector *connector);
9
+int init_opentsdb_telnet_instance(struct instance *instance);
10
+int init_opentsdb_http_instance(struct instance *instance);
11
+
12
+int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM *rd);
13
+int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *rd);
14
+
15
+int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *rd);
16
+int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd);
17
+
18
+#endif //NETDATA_EXPORTING_OPENTSDB_H
exporting/process_data.c
new
+404
@@ -0,0 +1,404 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "exporting_engine.h"
4
+
5
+/**
6
+ * Normalize chart and dimension names
7
+ *
8
+ * Substitute '_' for any special character except '.'.
9
+ *
10
+ * @param dst where to copy name to.
11
+ * @param src where to copy name from.
12
+ * @param max_len the maximum size of copied name.
13
+ * @return Returns the size of the copied name.
14
+ */
15
+size_t exporting_name_copy(char *dst, const char *src, size_t max_len)
16
+{
17
+ size_t n;
18
+
19
+ for (n = 0; *src && n < max_len; dst++, src++, n++) {
20
+ char c = *src;
21
+
22
+ if (c != '.' && !isalnum(c))
23
+ *dst = '_';
24
+ else
25
+ *dst = c;
26
+ }
27
+ *dst = '\0';
28
+
29
+ return n;
30
+}
31
+
32
+/**
33
+ * Mark scheduled instances
34
+ *
35
+ * Any instance can have its own update interval. On every exporting engine update only those instances are picked,
36
+ * which are scheduled for the update.
37
+ *
38
+ * @param engine an engine data structure.
39
+ * @return Returns 1 if there are instances to process
40
+ */
41
+int mark_scheduled_instances(struct engine *engine)
42
+{
43
+ int instances_were_scheduled = 0;
44
+
45
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
46
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
47
+ if (engine->now % instance->config.update_every < localhost->rrd_update_every) {
48
+ instance->scheduled = 1;
49
+ instances_were_scheduled = 1;
50
+ instance->before = engine->now;
51
+ }
52
+ }
53
+ }
54
+
55
+ return instances_were_scheduled;
56
+}
57
+
58
+/**
59
+ * Calculate the SUM or AVERAGE of a dimension, for any timeframe
60
+ *
61
+ * May return NAN if the database does not have any value in the give timeframe.
62
+ *
63
+ * @param instance an instance data structure.
64
+ * @param rd a dimension(metric) in the Netdata database.
65
+ * @param last_timestamp the timestamp that should be reported to the exporting connector instance.
66
+ * @return Returns the value, calculated over the given period.
67
+ */
68
+calculated_number exporting_calculate_value_from_stored_data(
69
+ struct instance *instance,
70
+ RRDDIM *rd,
71
+ time_t *last_timestamp)
72
+{
73
+ RRDSET *st = rd->rrdset;
74
+ RRDHOST *host = st->rrdhost;
75
+ time_t after = instance->after;
76
+ time_t before = instance->before;
77
+
78
+ // find the edges of the rrd database for this chart
79
+ time_t first_t = rd->state->query_ops.oldest_time(rd);
80
+ time_t last_t = rd->state->query_ops.latest_time(rd);
81
+ time_t update_every = st->update_every;
82
+ struct rrddim_query_handle handle;
83
+ storage_number n;
84
+
85
+ // step back a little, to make sure we have complete data collection
86
+ // for all metrics
87
+ after -= update_every * 2;
88
+ before -= update_every * 2;
89
+
90
+ // align the time-frame
91
+ after = after - (after % update_every);
92
+ before = before - (before % update_every);
93
+
94
+ // for before, loose another iteration
95
+ // the latest point will be reported the next time
96
+ before -= update_every;
97
+
98
+ if (unlikely(after > before))
99
+ // this can happen when update_every > before - after
100
+ after = before;
101
+
102
+ if (unlikely(after < first_t))
103
+ after = first_t;
104
+
105
+ if (unlikely(before > last_t))
106
+ before = last_t;
107
+
108
+ if (unlikely(before < first_t || after > last_t)) {
109
+ // the chart has not been updated in the wanted timeframe
110
+ debug(
111
+ D_BACKEND,
112
+ "EXPORTING: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu",
113
+ host->hostname,
114
+ st->id,
115
+ rd->id,
116
+ (unsigned long)after,
117
+ (unsigned long)before,
118
+ (unsigned long)first_t,
119
+ (unsigned long)last_t);
120
+ return NAN;
121
+ }
122
+
123
+ *last_timestamp = before;
124
+
125
+ size_t counter = 0;
126
+ calculated_number sum = 0;
127
+
128
+ for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
129
+ time_t curr_t;
130
+ n = rd->state->query_ops.next_metric(&handle, &curr_t);
131
+
132
+ if (unlikely(!does_storage_number_exist(n))) {
133
+ // not collected
134
+ continue;
135
+ }
136
+
137
+ calculated_number value = unpack_storage_number(n);
138
+ sum += value;
139
+
140
+ counter++;
141
+ }
142
+ rd->state->query_ops.finalize(&handle);
143
+ if (unlikely(!counter)) {
144
+ debug(
145
+ D_BACKEND,
146
+ "EXPORTING: %s.%s.%s: no values stored in database for range %lu to %lu",
147
+ host->hostname,
148
+ st->id,
149
+ rd->id,
150
+ (unsigned long)after,
151
+ (unsigned long)before);
152
+ return NAN;
153
+ }
154
+
155
+ if (unlikely(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM))
156
+ return sum;
157
+
158
+ return sum / (calculated_number)counter;
159
+}
160
+
161
+/**
162
+ * Start batch formatting for every connector instance's buffer
163
+ *
164
+ * @param engine an engine data structure.
165
+ * @return Returns 0 on success, 1 on failure.
166
+ */
167
+int start_batch_formatting(struct engine *engine)
168
+{
169
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
170
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
171
+ if (instance->scheduled) {
172
+ uv_mutex_lock(&instance->mutex);
173
+ if (instance->start_batch_formatting && instance->start_batch_formatting(instance) != 0) {
174
+ error("EXPORTING: cannot start batch formatting for %s", instance->config.name);
175
+ return 1;
176
+ }
177
+ }
178
+ }
179
+ }
180
+
181
+ return 0;
182
+}
183
+
184
+/**
185
+ * Start host formatting for every connector instance's buffer
186
+ *
187
+ * @param engine an engine data structure.
188
+ * @param host a data collecting host.
189
+ * @return Returns 0 on success, 1 on failure.
190
+ */
191
+int start_host_formatting(struct engine *engine, RRDHOST *host)
192
+{
193
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
194
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
195
+ if (instance->scheduled) {
196
+ if (rrdhost_is_exportable(instance, host)) {
197
+ if (instance->start_host_formatting && instance->start_host_formatting(instance, host) != 0) {
198
+ error("EXPORTING: cannot start host formatting for %s", instance->config.name);
199
+ return 1;
200
+ }
201
+ } else {
202
+ instance->skip_host = 1;
203
+ }
204
+ }
205
+ }
206
+ }
207
+
208
+ return 0;
209
+}
210
+
211
+/**
212
+ * Start chart formatting for every connector instance's buffer
213
+ *
214
+ * @param engine an engine data structure.
215
+ * @param a chart.
216
+ * @return Returns 0 on success, 1 on failure.
217
+ */
218
+int start_chart_formatting(struct engine *engine, RRDSET *st)
219
+{
220
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
221
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
222
+ if (instance->scheduled && !instance->skip_host) {
223
+ if (rrdset_is_exportable(instance, st)) {
224
+ if (instance->start_chart_formatting && instance->start_chart_formatting(instance, st) != 0) {
225
+ error("EXPORTING: cannot start chart formatting for %s", instance->config.name);
226
+ return 1;
227
+ }
228
+ } else {
229
+ instance->skip_chart = 1;
230
+ }
231
+ }
232
+ }
233
+ }
234
+
235
+ return 0;
236
+}
237
+
238
+/**
239
+ * Format metric for every connector instance's buffer
240
+ *
241
+ * @param engine an engine data structure.
242
+ * @param rd a dimension(metric) in the Netdata database.
243
+ * @return Returns 0 on success, 1 on failure.
244
+ */
245
+int metric_formatting(struct engine *engine, RRDDIM *rd)
246
+{
247
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
248
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
249
+ if (instance->scheduled && !instance->skip_host && !instance->skip_chart) {
250
+ if (instance->metric_formatting && instance->metric_formatting(instance, rd) != 0) {
251
+ error("EXPORTING: cannot format metric for %s", instance->config.name);
252
+ return 1;
253
+ }
254
+ instance->stats.chart_buffered_metrics++;
255
+ }
256
+ }
257
+ }
258
+
259
+ return 0;
260
+}
261
+
262
+/**
263
+ * End chart formatting for every connector instance's buffer
264
+ *
265
+ * @param engine an engine data structure.
266
+ * @param a chart.
267
+ * @return Returns 0 on success, 1 on failure.
268
+ */
269
+int end_chart_formatting(struct engine *engine, RRDSET *st)
270
+{
271
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
272
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
273
+ if (instance->scheduled && !instance->skip_host && !instance->skip_chart) {
274
+ if (instance->end_chart_formatting && instance->end_chart_formatting(instance, st) != 0) {
275
+ error("EXPORTING: cannot end chart formatting for %s", instance->config.name);
276
+ return 1;
277
+ }
278
+ }
279
+ instance->skip_chart = 0;
280
+ }
281
+ }
282
+
283
+ return 0;
284
+}
285
+
286
+/**
287
+ * End host formatting for every connector instance's buffer
288
+ *
289
+ * @param engine an engine data structure.
290
+ * @param host a data collecting host.
291
+ * @return Returns 0 on success, 1 on failure.
292
+ */
293
+int end_host_formatting(struct engine *engine, RRDHOST *host)
294
+{
295
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
296
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
297
+ if (instance->scheduled && !instance->skip_host) {
298
+ if (instance->end_host_formatting && instance->end_host_formatting(instance, host) != 0) {
299
+ error("EXPORTING: cannot end host formatting for %s", instance->config.name);
300
+ return 1;
301
+ }
302
+ }
303
+ instance->skip_host = 0;
304
+ }
305
+ }
306
+
307
+ return 0;
308
+}
309
+
310
+/**
311
+ * End batch formatting for every connector instance's buffer
312
+ *
313
+ * @param engine an engine data structure.
314
+ * @return Returns 0 on success, 1 on failure.
315
+ */
316
+int end_batch_formatting(struct engine *engine)
317
+{
318
+ for (struct connector *connector = engine->connector_root; connector; connector = connector->next) {
319
+ for (struct instance *instance = connector->instance_root; instance; instance = instance->next) {
320
+ if (instance->scheduled) {
321
+ if (instance->end_batch_formatting && instance->end_batch_formatting(instance) != 0) {
322
+ error("EXPORTING: cannot end batch formatting for %s", instance->config.name);
323
+ return 1;
324
+ }
325
+ uv_mutex_unlock(&instance->mutex);
326
+ uv_cond_signal(&instance->cond_var);
327
+
328
+ instance->scheduled = 0;
329
+ instance->after = instance->before;
330
+ }
331
+ }
332
+ }
333
+
334
+ return 0;
335
+}
336
+
337
+/**
338
+ * Prepare buffers
339
+ *
340
+ * Walk through the Netdata database and fill buffers for every scheduled exporting connector instance according to
341
+ * configured rules.
342
+ *
343
+ * @param engine an engine data structure.
344
+ * @return Returns 0 on success, 1 on failure.
345
+ */
346
+int prepare_buffers(struct engine *engine)
347
+{
348
+ netdata_thread_disable_cancelability();
349
+ rrd_rdlock();
350
+ if (start_batch_formatting(engine) != 0)
351
+ return 1;
352
+
353
+ RRDHOST *host;
354
+ rrdhost_foreach_read(host)
355
+ {
356
+ rrdhost_rdlock(host);
357
+ if (start_host_formatting(engine, host) != 0)
358
+ return 1;
359
+ RRDSET *st;
360
+ rrdset_foreach_read(st, host)
361
+ {
362
+ rrdset_rdlock(st);
363
+ if (start_chart_formatting(engine, st) != 0)
364
+ return 1;
365
+
366
+ RRDDIM *rd;
367
+ rrddim_foreach_read(rd, st)
368
+ {
369
+ if (metric_formatting(engine, rd) != 0)
370
+ return 1;
371
+ }
372
+
373
+ if (end_chart_formatting(engine, st) != 0)
374
+ return 1;
375
+ rrdset_unlock(st);
376
+ }
377
+
378
+ if (end_host_formatting(engine, host) != 0)
379
+ return 1;
380
+ rrdhost_unlock(host);
381
+ }
382
+
383
+ if (end_batch_formatting(engine) != 0)
384
+ return 1;
385
+ rrd_unlock();
386
+ netdata_thread_enable_cancelability();
387
+
388
+ return 0;
389
+}
390
+
391
+/**
392
+ * Notify workers
393
+ *
394
+ * Notify exporting connector instance working threads that data is ready to send.
395
+ *
396
+ * @param engine an engine data structure.
397
+ * @return Returns 0 on success, 1 on failure.
398
+ */
399
+int notify_workers(struct engine *engine)
400
+{
401
+ (void)engine;
402
+
403
+ return 0;
404
+}
exporting/read_config.c
new
+354
@@ -0,0 +1,354 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "exporting_engine.h"
4
+
5
+struct config exporting_config = {.sections = NULL,
6
+ .mutex = NETDATA_MUTEX_INITIALIZER,
7
+ .index = {.avl_tree = {.root = NULL, .compar = appconfig_section_compare},
8
+ .rwlock = AVL_LOCK_INITIALIZER}};
9
+
10
+
11
+static _CONNECTOR_INSTANCE *find_instance(const char *section)
12
+{
13
+ _CONNECTOR_INSTANCE *local_ci;
14
+
15
+ local_ci = add_connector_instance(NULL, NULL); // Get root section
16
+ if (unlikely(!local_ci))
17
+ return local_ci;
18
+
19
+ if (!section)
20
+ return local_ci;
21
+
22
+ while (local_ci) {
23
+ if (!strcmp(local_ci->instance_name, section))
24
+ break;
25
+ local_ci = local_ci->next;
26
+ }
27
+ return local_ci;
28
+}
29
+
30
+char *expconfig_get(struct config *root, const char *section, const char *name, const char *default_value)
31
+{
32
+ _CONNECTOR_INSTANCE *local_ci;
33
+
34
+ if (!strcmp(section, CONFIG_SECTION_EXPORTING))
35
+ return appconfig_get(root, CONFIG_SECTION_EXPORTING, name, default_value);
36
+
37
+ local_ci = find_instance(section);
38
+
39
+ if (!local_ci)
40
+ return NULL; // TODO: Check if it is meaningful to return default_value
41
+
42
+ return appconfig_get(
43
+ root,
44
+ local_ci->instance_name,
45
+ name,
46
+ appconfig_get(
47
+ root, local_ci->connector_name, name, appconfig_get(root, CONFIG_SECTION_EXPORTING, name, default_value)));
48
+}
49
+
50
+int expconfig_get_boolean(struct config *root, const char *section, const char *name, int default_value)
51
+{
52
+ _CONNECTOR_INSTANCE *local_ci;
53
+
54
+ if (!strcmp(section, CONFIG_SECTION_EXPORTING))
55
+ return appconfig_get_boolean(root, CONFIG_SECTION_EXPORTING, name, default_value);
56
+
57
+ local_ci = find_instance(section);
58
+
59
+ if (!local_ci)
60
+ return 0; // TODO: Check if it is meaningful to return default_value
61
+
62
+ return appconfig_get_boolean(
63
+ root,
64
+ local_ci->instance_name,
65
+ name,
66
+ appconfig_get_boolean(
67
+ root,
68
+ local_ci->connector_name,
69
+ name,
70
+ appconfig_get_boolean(root, CONFIG_SECTION_EXPORTING, name, default_value)));
71
+}
72
+
73
+long long expconfig_get_number(struct config *root, const char *section, const char *name, long long default_value)
74
+{
75
+ _CONNECTOR_INSTANCE *local_ci;
76
+
77
+ if (!strcmp(section, CONFIG_SECTION_EXPORTING))
78
+ return appconfig_get_number(root, CONFIG_SECTION_EXPORTING, name, default_value);
79
+
80
+ local_ci = find_instance(section);
81
+
82
+ if (!local_ci)
83
+ return 0; // TODO: Check if it is meaningful to return default_value
84
+
85
+ return appconfig_get_number(
86
+ root,
87
+ local_ci->instance_name,
88
+ name,
89
+ appconfig_get_number(
90
+ root,
91
+ local_ci->connector_name,
92
+ name,
93
+ appconfig_get_number(root, CONFIG_SECTION_EXPORTING, name, default_value)));
94
+}
95
+
96
+/*
97
+ * Get the next connector instance that we need to activate
98
+ *
99
+ * @param @target_ci will be filled with instance name and connector name
100
+ *
101
+ * @return - 1 if more connectors to be fetched, 0 done
102
+ *
103
+ */
104
+
105
+int get_connector_instance(struct connector_instance *target_ci)
106
+{
107
+ static _CONNECTOR_INSTANCE *local_ci = NULL;
108
+ _CONNECTOR_INSTANCE *global_connector_instance;
109
+
110
+ global_connector_instance = find_instance(NULL); // Fetch head of instances
111
+
112
+ if (unlikely(!global_connector_instance))
113
+ return 0;
114
+
115
+ if (target_ci == NULL) {
116
+ local_ci = NULL;
117
+ return 1;
118
+ }
119
+ if (local_ci == NULL)
120
+ local_ci = global_connector_instance;
121
+ else {
122
+ local_ci = local_ci->next;
123
+ if (local_ci == NULL)
124
+ return 0;
125
+ }
126
+
127
+ strcpy(target_ci->instance_name, local_ci->instance_name);
128
+ strcpy(target_ci->connector_name, local_ci->connector_name);
129
+
130
+ return 1;
131
+}
132
+
133
+/**
134
+ * Select Type
135
+ *
136
+ * Select the connector type based on the user input
137
+ *
138
+ * @param type is the string that defines the connector type
139
+ *
140
+ * @return It returns the connector id.
141
+ */
142
+BACKEND_TYPE exporting_select_type(const char *type)
143
+{
144
+ if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
145
+ return BACKEND_TYPE_GRAPHITE;
146
+ } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
147
+ return BACKEND_TYPE_OPENTSDB_USING_TELNET;
148
+ } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
149
+ return BACKEND_TYPE_OPENTSDB_USING_HTTP;
150
+ } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
151
+ return BACKEND_TYPE_JSON;
152
+ } else if (!strcmp(type, "prometheus_remote_write")) {
153
+ return BACKEND_TYPE_PROMETEUS;
154
+ } else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
155
+ return BACKEND_TYPE_KINESIS;
156
+ } else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext"))
157
+ return BACKEND_TYPE_MONGODB;
158
+
159
+ return BACKEND_TYPE_UNKNOWN;
160
+}
161
+
162
+EXPORTING_OPTIONS exporting_parse_data_source(const char *data_source, EXPORTING_OPTIONS exporting_options) {
163
+ if(!strcmp(data_source, "raw") || !strcmp(data_source, "as collected") || !strcmp(data_source, "as-collected") || !strcmp(data_source, "as_collected") || !strcmp(data_source, "ascollected")) {
164
+ exporting_options |= EXPORTING_SOURCE_DATA_AS_COLLECTED;
165
+ exporting_options &= ~(EXPORTING_OPTIONS_SOURCE_BITS ^ EXPORTING_SOURCE_DATA_AS_COLLECTED);
166
+ }
167
+ else if(!strcmp(data_source, "average")) {
168
+ exporting_options |= EXPORTING_SOURCE_DATA_AVERAGE;
169
+ exporting_options &= ~(EXPORTING_OPTIONS_SOURCE_BITS ^ EXPORTING_SOURCE_DATA_AVERAGE);
170
+ }
171
+ else if(!strcmp(data_source, "sum") || !strcmp(data_source, "volume")) {
172
+ exporting_options |= EXPORTING_SOURCE_DATA_SUM;
173
+ exporting_options &= ~(EXPORTING_OPTIONS_SOURCE_BITS ^ EXPORTING_SOURCE_DATA_SUM);
174
+ }
175
+ else {
176
+ error("EXPORTING: invalid data data_source method '%s'.", data_source);
177
+ }
178
+
179
+ return exporting_options;
180
+}
181
+
182
+/**
183
+ * Read configuration
184
+ *
185
+ * Based on read configuration an engine data structure is filled with exporting connector instances.
186
+ *
187
+ * @return Returns a filled engine data structure or NULL if there are no connector instances configured.
188
+ */
189
+struct engine *read_exporting_config()
190
+{
191
+ int instances_to_activate = 0;
192
+ int exporting_config_exists = 0;
193
+
194
+ static struct engine *engine = NULL;
195
+ struct connector_instance_list {
196
+ struct connector_instance local_ci;
197
+ struct connector_instance_list *next;
198
+ };
199
+ struct connector_instance local_ci;
200
+ struct connector_instance_list *tmp_ci_list, *tmp_ci_list1;
201
+ struct connector_instance_list **ci_list;
202
+
203
+ if (unlikely(engine))
204
+ return engine;
205
+
206
+ char *filename = strdupz_path_subpath(netdata_configured_user_config_dir, EXPORTING_CONF);
207
+
208
+ exporting_config_exists = appconfig_load(&exporting_config, filename, 0);
209
+ if (!exporting_config_exists) {
210
+ info("CONFIG: cannot load user exporting config '%s'. Will try the stock version.", filename);
211
+ freez(filename);
212
+
213
+ filename = strdupz_path_subpath(netdata_configured_stock_config_dir, EXPORTING_CONF);
214
+ exporting_config_exists = appconfig_load(&exporting_config, filename, 0);
215
+ if (!exporting_config_exists)
216
+ info("CONFIG: cannot load stock exporting config '%s'. Running with internal defaults.", filename);
217
+ }
218
+
219
+ freez(filename);
220
+
221
+ // Will build a list of instances per connector
222
+ // TODO: change BACKEND to EXPORTING
223
+ ci_list = callocz(BACKEND_TYPE_NUM, sizeof(struct connector_instance_list *));
224
+
225
+ while (get_connector_instance(&local_ci)) {
226
+ BACKEND_TYPE backend_type;
227
+
228
+ info("Processing connector instance (%s)", local_ci.instance_name);
229
+ if (exporter_get_boolean(local_ci.instance_name, "enabled", 0)) {
230
+ backend_type = exporting_select_type(local_ci.connector_name);
231
+
232
+ info(
233
+ "Instance (%s) on connector (%s) is enabled and scheduled for activation",
234
+ local_ci.instance_name,
235
+ local_ci.connector_name);
236
+
237
+ tmp_ci_list = (struct connector_instance_list *)callocz(1, sizeof(struct connector_instance_list));
238
+ memcpy(&tmp_ci_list->local_ci, &local_ci, sizeof(local_ci));
239
+ tmp_ci_list->next = ci_list[backend_type];
240
+ ci_list[backend_type] = tmp_ci_list;
241
+ instances_to_activate++;
242
+ } else
243
+ info("Instance (%s) on connector (%s) is not enabled", local_ci.instance_name, local_ci.connector_name);
244
+ }
245
+
246
+ if (unlikely(!instances_to_activate)) {
247
+ info("No connector instances to activate");
248
+ freez(ci_list);
249
+ return NULL;
250
+ }
251
+
252
+ engine = (struct engine *)calloc(1, sizeof(struct engine));
253
+ // TODO: Check and fill engine fields if actually needed
254
+
255
+ if (exporting_config_exists) {
256
+ engine->config.hostname =
257
+ strdupz(exporter_get(CONFIG_SECTION_EXPORTING, "hostname", netdata_configured_hostname));
258
+ engine->config.prefix = strdupz(exporter_get(CONFIG_SECTION_EXPORTING, "prefix", "netdata"));
259
+ engine->config.update_every =
260
+ exporter_get_number(CONFIG_SECTION_EXPORTING, EXPORTER_UPDATE_EVERY, EXPORTER_UPDATE_EVERY_DEFAULT);
261
+ }
262
+
263
+ for (size_t i = 0; i < BACKEND_TYPE_NUM; i++) {
264
+ // For each connector build list
265
+ tmp_ci_list = ci_list[i];
266
+
267
+ // If we have a list of instances for this connector then build it
268
+ if (tmp_ci_list) {
269
+ struct connector *tmp_connector;
270
+
271
+ tmp_connector = (struct connector *)calloc(1, sizeof(struct connector));
272
+ tmp_connector->next = engine->connector_root;
273
+ engine->connector_root = tmp_connector;
274
+
275
+ tmp_connector->config.type = i;
276
+ tmp_connector->engine = engine;
277
+
278
+ while (tmp_ci_list) {
279
+ struct instance *tmp_instance;
280
+ char *instance_name;
281
+
282
+ info("Instance %s on %s", tmp_ci_list->local_ci.instance_name, tmp_ci_list->local_ci.connector_name);
283
+
284
+ tmp_instance = (struct instance *)calloc(1, sizeof(struct instance));
285
+ tmp_instance->connector = engine->connector_root;
286
+ tmp_instance->next = engine->connector_root->instance_root;
287
+ engine->connector_root->instance_root = tmp_instance;
288
+ tmp_instance->connector = engine->connector_root;
289
+
290
+ instance_name = tmp_ci_list->local_ci.instance_name;
291
+
292
+ tmp_instance->config.name = strdupz(tmp_ci_list->local_ci.instance_name);
293
+
294
+ tmp_instance->config.destination =
295
+ strdupz(exporter_get(instance_name, EXPORTER_DESTINATION, EXPORTER_DESTINATION_DEFAULT));
296
+
297
+ tmp_instance->config.update_every =
298
+ exporter_get_number(instance_name, EXPORTER_UPDATE_EVERY, EXPORTER_UPDATE_EVERY_DEFAULT);
299
+
300
+ tmp_instance->config.buffer_on_failures =
301
+ exporter_get_number(instance_name, EXPORTER_BUF_ONFAIL, EXPORTER_BUF_ONFAIL_DEFAULT);
302
+
303
+ tmp_instance->config.timeoutms =
304
+ exporter_get_number(instance_name, EXPORTER_TIMEOUT_MS, EXPORTER_TIMEOUT_MS_DEFAULT);
305
+
306
+ tmp_instance->config.charts_pattern = simple_pattern_create(
307
+ exporter_get(instance_name, EXPORTER_SEND_CHART_MATCH, EXPORTER_SEND_CHART_MATCH_DEFAULT),
308
+ NULL,
309
+ SIMPLE_PATTERN_EXACT);
310
+
311
+ tmp_instance->config.hosts_pattern = simple_pattern_create(
312
+ exporter_get(instance_name, EXPORTER_SEND_HOST_MATCH, EXPORTER_SEND_HOST_MATCH_DEFAULT),
313
+ NULL,
314
+ SIMPLE_PATTERN_EXACT);
315
+
316
+ char *data_source =
317
+ exporter_get(instance_name, EXPORTER_DATA_SOURCE, EXPORTER_DATA_SOURCE_DEFAULT);
318
+
319
+ tmp_instance->config.options = exporting_parse_data_source(data_source, tmp_instance->config.options);
320
+
321
+ if (exporter_get_boolean(instance_name, EXPORTER_SEND_NAMES, EXPORTER_SEND_NAMES_DEFAULT))
322
+ tmp_instance->config.options |= EXPORTING_OPTION_SEND_NAMES;
323
+ else
324
+ tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;
325
+
326
+#ifdef NETDATA_INTERNAL_CHECKS
327
+ info(
328
+ " Dest=[%s], upd=[%d], buffer=[%d] timeout=[%ld] options=[%u]",
329
+ tmp_instance->config.destination,
330
+ tmp_instance->config.update_every,
331
+ tmp_instance->config.buffer_on_failures,
332
+ tmp_instance->config.timeoutms,
333
+ tmp_instance->config.options);
334
+#endif
335
+
336
+ if (unlikely(!exporting_config_exists) && !engine->config.hostname) {
337
+ engine->config.hostname =
338
+ strdupz(config_get(instance_name, "hostname", netdata_configured_hostname));
339
+ engine->config.prefix = strdupz(config_get(instance_name, "prefix", "netdata"));
340
+ engine->config.update_every =
341
+ config_get_number(instance_name, EXPORTER_UPDATE_EVERY, EXPORTER_UPDATE_EVERY_DEFAULT);
342
+ }
343
+
344
+ tmp_ci_list1 = tmp_ci_list->next;
345
+ freez(tmp_ci_list);
346
+ tmp_ci_list = tmp_ci_list1;
347
+ }
348
+ }
349
+ }
350
+
351
+ freez(ci_list);
352
+
353
+ return engine;
354
+}
exporting/send_data.c
new
+203
@@ -0,0 +1,203 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "exporting_engine.h"
4
+
5
+/**
6
+ * Discard response
7
+ *
8
+ * Discards a response received by an exporting connector instance after logging a sample of it to error.log
9
+ *
10
+ * @param buffer buffer with response data.
11
+ * @param instance an instance data structure.
12
+ * @return Always returns 0.
13
+ */
14
+int exporting_discard_response(BUFFER *buffer, struct instance *instance) {
15
+ char sample[1024];
16
+ const char *s = buffer_tostring(buffer);
17
+ char *d = sample, *e = &sample[sizeof(sample) - 1];
18
+
19
+ for(; *s && d < e ;s++) {
20
+ char c = *s;
21
+ if(unlikely(!isprint(c))) c = ' ';
22
+ *d++ = c;
23
+ }
24
+ *d = '\0';
25
+
26
+ info(
27
+ "EXPORTING: received %zu bytes from %s connector instance. Ignoring them. Sample: '%s'",
28
+ buffer_strlen(buffer),
29
+ instance->config.name,
30
+ sample);
31
+ buffer_flush(buffer);
32
+ return 0;
33
+}
34
+
35
+/**
36
+ * Receive response
37
+ *
38
+ * @param sock communication socket.
39
+ * @param instance an instance data structure.
40
+ */
41
+void simple_connector_receive_response(int *sock, struct instance *instance)
42
+{
43
+ static BUFFER *response = NULL;
44
+ if (!response)
45
+ response = buffer_create(1);
46
+
47
+ struct stats *stats = &instance->stats;
48
+
49
+ errno = 0;
50
+
51
+ // loop through to collect all data
52
+ while (*sock != -1 && errno != EWOULDBLOCK) {
53
+ buffer_need_bytes(response, 4096);
54
+
55
+ ssize_t r;
56
+ r = recv(*sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
57
+ if (likely(r > 0)) {
58
+ // we received some data
59
+ response->len += r;
60
+ stats->chart_received_bytes += r;
61
+ stats->chart_receptions++;
62
+ } else if (r == 0) {
63
+ error("EXPORTING: '%s' closed the socket", instance->config.destination);
64
+ close(*sock);
65
+ *sock = -1;
66
+ } else {
67
+ // failed to receive data
68
+ if (errno != EAGAIN && errno != EWOULDBLOCK) {
69
+ error("EXPORTING: cannot receive data from '%s'.", instance->config.destination);
70
+ }
71
+ }
72
+#ifdef UNIT_TESTING
73
+ break;
74
+#endif
75
+ }
76
+
77
+ // if we received data, process them
78
+ if (buffer_strlen(response))
79
+ exporting_discard_response(response, instance);
80
+}
81
+
82
+/**
83
+ * Send buffer to a server
84
+ *
85
+ * @param sock communication socket.
86
+ * @param failures the number of communication failures.
87
+ * @param instance an instance data structure.
88
+ */
89
+void simple_connector_send_buffer(int *sock, int *failures, struct instance *instance)
90
+{
91
+ BUFFER *buffer = (BUFFER *)instance->buffer;
92
+ size_t len = buffer_strlen(buffer);
93
+
94
+ int flags = 0;
95
+#ifdef MSG_NOSIGNAL
96
+ flags += MSG_NOSIGNAL;
97
+#endif
98
+
99
+ struct stats *stats = &instance->stats;
100
+
101
+ ssize_t written;
102
+ written = send(*sock, buffer_tostring(buffer), len, flags);
103
+
104
+ if(written != -1 && (size_t)written == len) {
105
+ // we sent the data successfully
106
+ stats->chart_transmission_successes++;
107
+ stats->chart_sent_bytes += written;
108
+ stats->chart_sent_metrics = stats->chart_buffered_metrics;
109
+
110
+ // reset the failures count
111
+ *failures = 0;
112
+
113
+ // empty the buffer
114
+ buffer_flush(buffer);
115
+ }
116
+ else {
117
+ // oops! we couldn't send (all or some of the) data
118
+ error(
119
+ "EXPORTING: failed to write data to '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.",
120
+ instance->config.destination,
121
+ len,
122
+ written);
123
+ stats->chart_transmission_failures++;
124
+
125
+ if(written != -1)
126
+ stats->chart_sent_bytes += written;
127
+
128
+ // increment the counter we check for data loss
129
+ (*failures)++;
130
+
131
+ // close the socket - we will re-open it next time
132
+ close(*sock);
133
+ *sock = -1;
134
+ }
135
+}
136
+
137
+/**
138
+ * Simple connector worker
139
+ *
140
+ * Runs in a separate thread for every instance.
141
+ *
142
+ * @param instance_p an instance data structure.
143
+ */
144
+void simple_connector_worker(void *instance_p)
145
+{
146
+ struct instance *instance = (struct instance*)instance_p;
147
+
148
+ struct simple_connector_config *connector_specific_config = instance->connector->config.connector_specific_config;
149
+ struct stats *stats = &instance->stats;
150
+
151
+ int sock = -1;
152
+ struct timeval timeout = {.tv_sec = (instance->config.timeoutms * 1000) / 1000000,
153
+ .tv_usec = (instance->config.timeoutms * 1000) % 1000000};
154
+ int failures = 0;
155
+
156
+ while(!netdata_exit) {
157
+ // ------------------------------------------------------------------------
158
+ // if we are connected, receive a response, without blocking
159
+
160
+ if(likely(sock != -1))
161
+ simple_connector_receive_response(&sock, instance);
162
+
163
+ // ------------------------------------------------------------------------
164
+ // if we are not connected, connect to a data collecting server
165
+
166
+ if(unlikely(sock == -1)) {
167
+ size_t reconnects = 0;
168
+
169
+ sock = connect_to_one_of(
170
+ instance->config.destination,
171
+ connector_specific_config->default_port,
172
+ &timeout,
173
+ &reconnects,
174
+ NULL,
175
+ 0);
176
+ stats->chart_reconnects += reconnects;
177
+ }
178
+
179
+ if(unlikely(netdata_exit)) break;
180
+
181
+ // ------------------------------------------------------------------------
182
+ // if we are connected, send our buffer to the data collecting server
183
+
184
+ uv_mutex_lock(&instance->mutex);
185
+ uv_cond_wait(&instance->cond_var, &instance->mutex);
186
+
187
+ if (likely(sock != -1)) {
188
+ simple_connector_send_buffer(&sock, &failures, instance);
189
+ } else {
190
+ error("EXPORTING: failed to update '%s'", instance->config.destination);
191
+ stats->chart_transmission_failures++;
192
+
193
+ // increment the counter we check for data loss
194
+ failures++;
195
+ }
196
+
197
+ uv_mutex_unlock(&instance->mutex);
198
+
199
+#ifdef UNIT_TESTING
200
+ break;
201
+#endif
202
+ }
203
+}
exporting/send_internal_metrics.c
new
+18
@@ -0,0 +1,18 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "exporting_engine.h"
4
+
5
+/**
6
+ * Send internal metrics
7
+ *
8
+ * Send performance metrics for the operation of exporting engine itself to the Netdata database.
9
+ *
10
+ * @param engine an engine data structure.
11
+ * @return Returns 0 on success, 1 on failure.
12
+ */
13
+int send_internal_metrics(struct engine *engine)
14
+{
15
+ (void)engine;
16
+
17
+ return 0;
18
+}
exporting/tests/Makefile.am
new
+4
@@ -0,0 +1,4 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
exporting/tests/exporting_doubles.c
new
+162
@@ -0,0 +1,162 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "test_exporting_engine.h"
4
+
5
+struct engine *__real_read_exporting_config();
6
+struct engine *__wrap_read_exporting_config()
7
+{
8
+ function_called();
9
+ return mock_ptr_type(struct engine *);
10
+}
11
+
12
+struct engine *__mock_read_exporting_config()
13
+{
14
+ struct engine *engine = calloc(1, sizeof(struct engine));
15
+ engine->config.prefix = strdupz("netdata");
16
+ engine->config.hostname = strdupz("test-host");
17
+ engine->config.update_every = 3;
18
+
19
+ engine->connector_root = calloc(1, sizeof(struct connector));
20
+ engine->connector_root->config.type = BACKEND_TYPE_GRAPHITE;
21
+ engine->connector_root->engine = engine;
22
+
23
+ engine->connector_root->instance_root = calloc(1, sizeof(struct instance));
24
+ struct instance *instance = engine->connector_root->instance_root;
25
+ instance->connector = engine->connector_root;
26
+ instance->config.name = strdupz("instance_name");
27
+ instance->config.destination = strdupz("localhost");
28
+ instance->config.update_every = 1;
29
+ instance->config.buffer_on_failures = 10;
30
+ instance->config.timeoutms = 10000;
31
+ instance->config.charts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT);
32
+ instance->config.hosts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT);
33
+ instance->config.options = EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_OPTION_SEND_NAMES;
34
+
35
+ return engine;
36
+}
37
+
38
+int __real_init_connectors(struct engine *engine);
39
+int __wrap_init_connectors(struct engine *engine)
40
+{
41
+ function_called();
42
+ check_expected_ptr(engine);
43
+ return mock_type(int);
44
+}
45
+
46
+int __real_mark_scheduled_instances(struct engine *engine);
47
+int __wrap_mark_scheduled_instances(struct engine *engine)
48
+{
49
+ function_called();
50
+ check_expected_ptr(engine);
51
+ return mock_type(int);
52
+}
53
+
54
+calculated_number __real_exporting_calculate_value_from_stored_data(
55
+ struct instance *instance,
56
+ RRDDIM *rd,
57
+ time_t *last_timestamp);
58
+calculated_number __wrap_exporting_calculate_value_from_stored_data(
59
+ struct instance *instance,
60
+ RRDDIM *rd,
61
+ time_t *last_timestamp)
62
+{
63
+ (void)instance;
64
+ (void)rd;
65
+
66
+ *last_timestamp = 15052;
67
+
68
+ function_called();
69
+ return mock_type(calculated_number);
70
+}
71
+
72
+int __real_prepare_buffers(struct engine *engine);
73
+int __wrap_prepare_buffers(struct engine *engine)
74
+{
75
+ function_called();
76
+ check_expected_ptr(engine);
77
+ return mock_type(int);
78
+}
79
+
80
+int __wrap_notify_workers(struct engine *engine)
81
+{
82
+ function_called();
83
+ check_expected_ptr(engine);
84
+ return mock_type(int);
85
+}
86
+
87
+int __wrap_send_internal_metrics(struct engine *engine)
88
+{
89
+ function_called();
90
+ check_expected_ptr(engine);
91
+ return mock_type(int);
92
+}
93
+
94
+int __wrap_rrdhost_is_exportable(struct instance *instance, RRDHOST *host)
95
+{
96
+ function_called();
97
+ check_expected_ptr(instance);
98
+ check_expected_ptr(host);
99
+ return mock_type(int);
100
+}
101
+
102
+int __wrap_rrdset_is_exportable(struct instance *instance, RRDSET *st)
103
+{
104
+ function_called();
105
+ check_expected_ptr(instance);
106
+ check_expected_ptr(st);
107
+ return mock_type(int);
108
+}
109
+
110
+int __mock_start_batch_formatting(struct instance *instance)
111
+{
112
+ function_called();
113
+ check_expected_ptr(instance);
114
+ return mock_type(int);
115
+}
116
+
117
+int __mock_start_host_formatting(struct instance *instance, RRDHOST *host)
118
+{
119
+ function_called();
120
+ check_expected_ptr(instance);
121
+ check_expected_ptr(host);
122
+ return mock_type(int);
123
+}
124
+
125
+int __mock_start_chart_formatting(struct instance *instance, RRDSET *st)
126
+{
127
+ function_called();
128
+ check_expected_ptr(instance);
129
+ check_expected_ptr(st);
130
+ return mock_type(int);
131
+}
132
+
133
+int __mock_metric_formatting(struct instance *instance, RRDDIM *rd)
134
+{
135
+ function_called();
136
+ check_expected_ptr(instance);
137
+ check_expected_ptr(rd);
138
+ return mock_type(int);
139
+}
140
+
141
+int __mock_end_chart_formatting(struct instance *instance, RRDSET *st)
142
+{
143
+ function_called();
144
+ check_expected_ptr(instance);
145
+ check_expected_ptr(st);
146
+ return mock_type(int);
147
+}
148
+
149
+int __mock_end_host_formatting(struct instance *instance, RRDHOST *host)
150
+{
151
+ function_called();
152
+ check_expected_ptr(instance);
153
+ check_expected_ptr(host);
154
+ return mock_type(int);
155
+}
156
+
157
+int __mock_end_batch_formatting(struct instance *instance)
158
+{
159
+ function_called();
160
+ check_expected_ptr(instance);
161
+ return mock_type(int);
162
+}
exporting/tests/exporting_fixtures.c
new
+110
@@ -0,0 +1,110 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "test_exporting_engine.h"
4
+
5
+int setup_configured_engine(void **state)
6
+{
7
+ struct engine *engine = __mock_read_exporting_config();
8
+
9
+ *state = engine;
10
+
11
+ return 0;
12
+}
13
+
14
+int teardown_configured_engine(void **state)
15
+{
16
+ struct engine *engine = *state;
17
+
18
+ struct instance *instance = engine->connector_root->instance_root;
19
+ free((void *)instance->config.destination);
20
+ free((void *)instance->config.name);
21
+ simple_pattern_free(instance->config.charts_pattern);
22
+ simple_pattern_free(instance->config.hosts_pattern);
23
+ free(instance);
24
+
25
+ free(engine->connector_root);
26
+
27
+ free((void *)engine->config.prefix);
28
+ free((void *)engine->config.hostname);
29
+ free(engine);
30
+
31
+ return 0;
32
+}
33
+
34
+int setup_rrdhost()
35
+{
36
+ localhost = calloc(1, sizeof(RRDHOST));
37
+
38
+ localhost->rrd_update_every = 1;
39
+
40
+ localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2");
41
+
42
+ localhost->rrdset_root = calloc(1, sizeof(RRDSET));
43
+ RRDSET *st = localhost->rrdset_root;
44
+ st->rrdhost = localhost;
45
+ strcpy(st->id, "chart_id");
46
+ st->name = strdupz("chart_name");
47
+ st->flags |= RRDSET_FLAG_ENABLED;
48
+ st->rrd_memory_mode |= RRD_MEMORY_MODE_SAVE;
49
+ st->update_every = 1;
50
+
51
+ localhost->rrdset_root->dimensions = calloc(1, sizeof(RRDDIM));
52
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
53
+ rd->rrdset = st;
54
+ rd->id = strdupz("dimension_id");
55
+ rd->name = strdupz("dimension_name");
56
+ rd->last_collected_value = 123000321;
57
+ rd->last_collected_time.tv_sec = 15051;
58
+ rd->next = NULL;
59
+
60
+ rd->state = calloc(1, sizeof(*rd->state));
61
+ rd->state->query_ops.oldest_time = __mock_rrddim_query_oldest_time;
62
+ rd->state->query_ops.latest_time = __mock_rrddim_query_latest_time;
63
+ rd->state->query_ops.init = __mock_rrddim_query_init;
64
+ rd->state->query_ops.is_finished = __mock_rrddim_query_is_finished;
65
+ rd->state->query_ops.next_metric = __mock_rrddim_query_next_metric;
66
+ rd->state->query_ops.finalize = __mock_rrddim_query_finalize;
67
+
68
+ return 0;
69
+}
70
+
71
+int teardown_rrdhost()
72
+{
73
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
74
+ free((void *)rd->name);
75
+ free((void *)rd->id);
76
+ free(rd->state);
77
+ free(rd);
78
+
79
+ RRDSET *st = localhost->rrdset_root;
80
+ free((void *)st->name);
81
+ free(st);
82
+
83
+ free((void *)localhost->tags);
84
+ free(localhost);
85
+
86
+ return 0;
87
+}
88
+
89
+int setup_initialized_engine(void **state)
90
+{
91
+ setup_configured_engine(state);
92
+
93
+ struct engine *engine = *state;
94
+ init_connectors_in_tests(engine);
95
+
96
+ setup_rrdhost();
97
+
98
+ return 0;
99
+}
100
+
101
+int teardown_initialized_engine(void **state)
102
+{
103
+ struct engine *engine = *state;
104
+
105
+ teardown_rrdhost();
106
+ buffer_free(engine->connector_root->instance_root->buffer);
107
+ teardown_configured_engine(state);
108
+
109
+ return 0;
110
+}
exporting/tests/netdata_doubles.c
new
+134
@@ -0,0 +1,134 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "test_exporting_engine.h"
4
+
5
+// Use memomy allocation functions guarded by CMocka in strdupz
6
+const char *__wrap_strdupz(const char *s)
7
+{
8
+ char *duplicate = malloc(sizeof(char) * (strlen(s) + 1));
9
+ strcpy(duplicate, s);
10
+
11
+ return duplicate;
12
+}
13
+
14
+time_t __wrap_now_realtime_sec(void)
15
+{
16
+ function_called();
17
+ return mock_type(time_t);
18
+}
19
+
20
+void __wrap_info_int(const char *file, const char *function, const unsigned long line, const char *fmt, ...)
21
+{
22
+ (void)file;
23
+ (void)function;
24
+ (void)line;
25
+
26
+ function_called();
27
+
28
+ va_list args;
29
+
30
+ va_start(args, fmt);
31
+ vsnprintf(log_line, MAX_LOG_LINE, fmt, args);
32
+ va_end(args);
33
+}
34
+
35
+int __wrap_connect_to_one_of(
36
+ const char *destination,
37
+ int default_port,
38
+ struct timeval *timeout,
39
+ size_t *reconnects_counter,
40
+ char *connected_to,
41
+ size_t connected_to_size)
42
+{
43
+ (void)timeout;
44
+
45
+ function_called();
46
+
47
+ check_expected(destination);
48
+ check_expected_ptr(default_port);
49
+ // TODO: check_expected_ptr(timeout);
50
+ check_expected(reconnects_counter);
51
+ check_expected(connected_to);
52
+ check_expected(connected_to_size);
53
+
54
+ return mock_type(int);
55
+}
56
+
57
+void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line)
58
+{
59
+ (void)host;
60
+ (void)file;
61
+ (void)function;
62
+ (void)line;
63
+}
64
+
65
+void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line)
66
+{
67
+ (void)st;
68
+ (void)file;
69
+ (void)function;
70
+ (void)line;
71
+}
72
+
73
+void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line)
74
+{
75
+ (void)file;
76
+ (void)function;
77
+ (void)line;
78
+}
79
+
80
+const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
81
+{
82
+ (void)id;
83
+ return RRD_MEMORY_MODE_NONE_NAME;
84
+}
85
+
86
+time_t __mock_rrddim_query_oldest_time(RRDDIM *rd)
87
+{
88
+ (void)rd;
89
+
90
+ function_called();
91
+ return mock_type(time_t);
92
+}
93
+
94
+time_t __mock_rrddim_query_latest_time(RRDDIM *rd)
95
+{
96
+ (void)rd;
97
+
98
+ function_called();
99
+ return mock_type(time_t);
100
+}
101
+
102
+void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time)
103
+{
104
+ (void)rd;
105
+ (void)handle;
106
+
107
+ function_called();
108
+ check_expected(start_time);
109
+ check_expected(end_time);
110
+}
111
+
112
+int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
113
+{
114
+ (void)handle;
115
+
116
+ function_called();
117
+ return mock_type(int);
118
+}
119
+
120
+storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time)
121
+{
122
+ (void)handle;
123
+ (void)current_time;
124
+
125
+ function_called();
126
+ return mock_type(storage_number);
127
+}
128
+
129
+void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
130
+{
131
+ (void)handle;
132
+
133
+ function_called();
134
+}
exporting/tests/system_doubles.c
new
+61
@@ -0,0 +1,61 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "test_exporting_engine.h"
4
+
5
+void __wrap_uv_thread_create(uv_thread_t thread, void (*worker)(void *arg), void *arg)
6
+{
7
+ function_called();
8
+
9
+ check_expected_ptr(thread);
10
+ check_expected_ptr(worker);
11
+ check_expected_ptr(arg);
12
+}
13
+
14
+void __wrap_uv_mutex_lock(uv_mutex_t *mutex)
15
+{
16
+ (void)mutex;
17
+}
18
+
19
+void __wrap_uv_mutex_unlock(uv_mutex_t *mutex)
20
+{
21
+ (void)mutex;
22
+}
23
+
24
+void __wrap_uv_cond_signal(uv_cond_t *cond_var)
25
+{
26
+ (void)cond_var;
27
+}
28
+
29
+void __wrap_uv_cond_wait(uv_cond_t *cond_var, uv_mutex_t *mutex)
30
+{
31
+ (void)cond_var;
32
+ (void)mutex;
33
+}
34
+
35
+ssize_t __wrap_recv(int sockfd, void *buf, size_t len, int flags)
36
+{
37
+ function_called();
38
+
39
+ check_expected(sockfd);
40
+ check_expected_ptr(buf);
41
+ check_expected(len);
42
+ check_expected(flags);
43
+
44
+ char *mock_string = "Test recv";
45
+ strcpy(buf, mock_string);
46
+
47
+ return strlen(mock_string);
48
+}
49
+
50
+ssize_t __wrap_send(int sockfd, const void *buf, size_t len, int flags)
51
+{
52
+ function_called();
53
+
54
+ check_expected(sockfd);
55
+ check_expected_ptr(buf);
56
+ check_expected_ptr(buf);
57
+ check_expected(len);
58
+ check_expected(flags);
59
+
60
+ return strlen(buf);
61
+}
exporting/tests/test_exporting_engine.c
new
+738
@@ -0,0 +1,738 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "test_exporting_engine.h"
4
+#include "libnetdata/required_dummies.h"
5
+
6
+RRDHOST *localhost;
7
+netdata_rwlock_t rrd_rwlock;
8
+
9
+// global variables needed by read_exporting_config()
10
+struct config netdata_config;
11
+char *netdata_configured_user_config_dir = ".";
12
+char *netdata_configured_stock_config_dir = ".";
13
+char *netdata_configured_hostname = "test_host";
14
+
15
+char log_line[MAX_LOG_LINE + 1];
16
+
17
+void init_connectors_in_tests(struct engine *engine)
18
+{
19
+ expect_function_call(__wrap_now_realtime_sec);
20
+ will_return(__wrap_now_realtime_sec, 2);
21
+
22
+ expect_function_call(__wrap_uv_thread_create);
23
+
24
+ expect_value(__wrap_uv_thread_create, thread, &engine->connector_root->instance_root->thread);
25
+ expect_value(__wrap_uv_thread_create, worker, simple_connector_worker);
26
+ expect_value(__wrap_uv_thread_create, arg, engine->connector_root->instance_root);
27
+
28
+ assert_int_equal(__real_init_connectors(engine), 0);
29
+
30
+ assert_int_equal(engine->now, 2);
31
+ assert_int_equal(engine->connector_root->instance_root->after, 2);
32
+}
33
+
34
+static void test_exporting_engine(void **state)
35
+{
36
+ struct engine *engine = *state;
37
+
38
+ expect_function_call(__wrap_read_exporting_config);
39
+ will_return(__wrap_read_exporting_config, engine);
40
+
41
+ expect_function_call(__wrap_init_connectors);
42
+ expect_memory(__wrap_init_connectors, engine, engine, sizeof(struct engine));
43
+ will_return(__wrap_init_connectors, 0);
44
+
45
+ expect_function_call(__wrap_now_realtime_sec);
46
+ will_return(__wrap_now_realtime_sec, 2);
47
+
48
+ expect_function_call(__wrap_mark_scheduled_instances);
49
+ expect_memory(__wrap_mark_scheduled_instances, engine, engine, sizeof(struct engine));
50
+ will_return(__wrap_mark_scheduled_instances, 1);
51
+
52
+ expect_function_call(__wrap_prepare_buffers);
53
+ expect_memory(__wrap_prepare_buffers, engine, engine, sizeof(struct engine));
54
+ will_return(__wrap_prepare_buffers, 0);
55
+
56
+ expect_function_call(__wrap_notify_workers);
57
+ expect_memory(__wrap_notify_workers, engine, engine, sizeof(struct engine));
58
+ will_return(__wrap_notify_workers, 0);
59
+
60
+ expect_function_call(__wrap_send_internal_metrics);
61
+ expect_memory(__wrap_send_internal_metrics, engine, engine, sizeof(struct engine));
62
+ will_return(__wrap_send_internal_metrics, 0);
63
+
64
+ void *ptr = malloc(sizeof(int));
65
+ assert_ptr_equal(exporting_main(ptr), NULL);
66
+ assert_int_equal(engine->now, 2);
67
+ free(ptr);
68
+}
69
+
70
+static void test_read_exporting_config(void **state)
71
+{
72
+ struct engine *engine = __mock_read_exporting_config(); // TODO: use real read_exporting_config() function
73
+ *state = engine;
74
+
75
+ assert_ptr_not_equal(engine, NULL);
76
+ assert_string_equal(engine->config.prefix, "netdata");
77
+ assert_string_equal(engine->config.hostname, "test-host");
78
+ assert_int_equal(engine->config.update_every, 3);
79
+ assert_int_equal(engine->instance_num, 0);
80
+
81
+ struct connector *connector = engine->connector_root;
82
+ assert_ptr_not_equal(connector, NULL);
83
+ assert_ptr_equal(connector->next, NULL);
84
+ assert_ptr_equal(connector->engine, engine);
85
+ assert_int_equal(connector->config.type, BACKEND_TYPE_GRAPHITE);
86
+
87
+ struct instance *instance = connector->instance_root;
88
+ assert_ptr_not_equal(instance, NULL);
89
+ assert_ptr_equal(instance->next, NULL);
90
+ assert_ptr_equal(instance->connector, connector);
91
+ assert_string_equal(instance->config.destination, "localhost");
92
+ assert_int_equal(instance->config.update_every, 1);
93
+ assert_int_equal(instance->config.buffer_on_failures, 10);
94
+ assert_int_equal(instance->config.timeoutms, 10000);
95
+ assert_true(simple_pattern_matches(instance->config.charts_pattern, "any_chart"));
96
+ assert_true(simple_pattern_matches(instance->config.hosts_pattern, "anyt_host"));
97
+ assert_int_equal(instance->config.options, EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_OPTION_SEND_NAMES);
98
+
99
+ teardown_configured_engine(state);
100
+}
101
+
102
+static void test_init_connectors(void **state)
103
+{
104
+ struct engine *engine = *state;
105
+
106
+ init_connectors_in_tests(engine);
107
+
108
+ assert_int_equal(engine->instance_num, 1);
109
+
110
+ struct connector *connector = engine->connector_root;
111
+ assert_ptr_equal(connector->next, NULL);
112
+ assert_ptr_equal(connector->worker, simple_connector_worker);
113
+
114
+ struct simple_connector_config *connector_specific_config = connector->config.connector_specific_config;
115
+ assert_int_equal(connector_specific_config->default_port, 2003);
116
+
117
+ struct instance *instance = connector->instance_root;
118
+ assert_ptr_equal(instance->next, NULL);
119
+ assert_int_equal(instance->index, 0);
120
+ assert_ptr_equal(instance->start_batch_formatting, NULL);
121
+ assert_ptr_equal(instance->start_host_formatting, NULL);
122
+ assert_ptr_equal(instance->start_chart_formatting, NULL);
123
+ assert_ptr_equal(instance->metric_formatting, format_dimension_collected_graphite_plaintext);
124
+ assert_ptr_equal(instance->end_chart_formatting, NULL);
125
+ assert_ptr_equal(instance->end_host_formatting, NULL);
126
+ assert_ptr_equal(instance->end_batch_formatting, NULL);
127
+
128
+ BUFFER *buffer = instance->buffer;
129
+ assert_ptr_not_equal(buffer, NULL);
130
+ buffer_sprintf(buffer, "%s", "graphite test");
131
+ assert_string_equal(buffer_tostring(buffer), "graphite test");
132
+}
133
+
134
+static void test_init_graphite_instance(void **state)
135
+{
136
+ struct engine *engine = *state;
137
+ struct connector *connector = engine->connector_root;
138
+ struct instance *instance = connector->instance_root;
139
+
140
+ init_graphite_connector(connector);
141
+ assert_int_equal(
142
+ ((struct simple_connector_config *)(connector->config.connector_specific_config))->default_port, 2003);
143
+ freez(connector->config.connector_specific_config);
144
+
145
+ instance->config.options = EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_OPTION_SEND_NAMES;
146
+ assert_int_equal(init_graphite_instance(instance), 0);
147
+ assert_ptr_equal(instance->metric_formatting, format_dimension_collected_graphite_plaintext);
148
+ assert_ptr_not_equal(instance->buffer, NULL);
149
+ buffer_free(instance->buffer);
150
+
151
+ instance->config.options = EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_OPTION_SEND_NAMES;
152
+ assert_int_equal(init_graphite_instance(instance), 0);
153
+ assert_ptr_equal(instance->metric_formatting, format_dimension_stored_graphite_plaintext);
154
+}
155
+
156
+static void test_init_json_instance(void **state)
157
+{
158
+ struct engine *engine = *state;
159
+ struct connector *connector = engine->connector_root;
160
+ struct instance *instance = connector->instance_root;
161
+
162
+ init_json_connector(connector);
163
+ assert_int_equal(
164
+ ((struct simple_connector_config *)(connector->config.connector_specific_config))->default_port, 5448);
165
+ freez(connector->config.connector_specific_config);
166
+
167
+ instance->config.options = EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_OPTION_SEND_NAMES;
168
+ assert_int_equal(init_json_instance(instance), 0);
169
+ assert_ptr_equal(instance->metric_formatting, format_dimension_collected_json_plaintext);
170
+ assert_ptr_not_equal(instance->buffer, NULL);
171
+ buffer_free(instance->buffer);
172
+
173
+ instance->config.options = EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_OPTION_SEND_NAMES;
174
+ assert_int_equal(init_json_instance(instance), 0);
175
+ assert_ptr_equal(instance->metric_formatting, format_dimension_stored_json_plaintext);
176
+}
177
+
178
+static void test_init_opentsdb_connector(void **state)
179
+{
180
+ struct engine *engine = *state;
181
+ struct connector *connector = engine->connector_root;
182
+
183
+ init_opentsdb_connector(connector);
184
+ assert_int_equal(
185
+ ((struct simple_connector_config *)(connector->config.connector_specific_config))->default_port, 4242);
186
+ freez(connector->config.connector_specific_config);
187
+}
188
+
189
+static void test_init_opentsdb_telnet_instance(void **state)
190
+{
191
+ struct engine *engine = *state;
192
+ struct connector *connector = engine->connector_root;
193
+ struct instance *instance = connector->instance_root;
194
+
195
+ instance->config.options = EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_OPTION_SEND_NAMES;
196
+ assert_int_equal(init_opentsdb_telnet_instance(instance), 0);
197
+ assert_ptr_equal(instance->metric_formatting, format_dimension_collected_opentsdb_telnet);
198
+ assert_ptr_not_equal(instance->buffer, NULL);
199
+ buffer_free(instance->buffer);
200
+
201
+ instance->config.options = EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_OPTION_SEND_NAMES;
202
+ assert_int_equal(init_opentsdb_telnet_instance(instance), 0);
203
+ assert_ptr_equal(instance->metric_formatting, format_dimension_stored_opentsdb_telnet);
204
+}
205
+
206
+static void test_init_opentsdb_http_instance(void **state)
207
+{
208
+ struct engine *engine = *state;
209
+ struct connector *connector = engine->connector_root;
210
+ struct instance *instance = connector->instance_root;
211
+
212
+ instance->config.options = EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_OPTION_SEND_NAMES;
213
+ assert_int_equal(init_opentsdb_http_instance(instance), 0);
214
+ assert_ptr_equal(instance->metric_formatting, format_dimension_collected_opentsdb_http);
215
+ assert_ptr_not_equal(instance->buffer, NULL);
216
+ buffer_free(instance->buffer);
217
+
218
+ instance->config.options = EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_OPTION_SEND_NAMES;
219
+ assert_int_equal(init_opentsdb_http_instance(instance), 0);
220
+ assert_ptr_equal(instance->metric_formatting, format_dimension_stored_opentsdb_http);
221
+}
222
+
223
+static void test_mark_scheduled_instances(void **state)
224
+{
225
+ struct engine *engine = *state;
226
+
227
+ assert_int_equal(__real_mark_scheduled_instances(engine), 1);
228
+
229
+ struct instance *instance = engine->connector_root->instance_root;
230
+ assert_int_equal(instance->scheduled, 1);
231
+ assert_int_equal(instance->before, 2);
232
+}
233
+
234
+static void test_rrdhost_is_exportable(void **state)
235
+{
236
+ struct engine *engine = *state;
237
+ struct instance *instance = engine->connector_root->instance_root;
238
+
239
+ expect_function_call(__wrap_info_int);
240
+
241
+ assert_ptr_equal(localhost->exporting_flags, NULL);
242
+
243
+ assert_int_equal(__real_rrdhost_is_exportable(instance, localhost), 1);
244
+
245
+ assert_string_equal(log_line, "enabled exporting of host 'localhost' for instance 'instance_name'");
246
+
247
+ assert_ptr_not_equal(localhost->exporting_flags, NULL);
248
+ assert_int_equal(localhost->exporting_flags[0], RRDHOST_FLAG_BACKEND_SEND);
249
+}
250
+
251
+static void test_false_rrdhost_is_exportable(void **state)
252
+{
253
+ struct engine *engine = *state;
254
+ struct instance *instance = engine->connector_root->instance_root;
255
+
256
+ simple_pattern_free(instance->config.hosts_pattern);
257
+ instance->config.hosts_pattern = simple_pattern_create("!*", NULL, SIMPLE_PATTERN_EXACT);
258
+
259
+ expect_function_call(__wrap_info_int);
260
+
261
+ assert_ptr_equal(localhost->exporting_flags, NULL);
262
+
263
+ assert_int_equal(__real_rrdhost_is_exportable(instance, localhost), 0);
264
+
265
+ assert_string_equal(log_line, "disabled exporting of host 'localhost' for instance 'instance_name'");
266
+
267
+ assert_ptr_not_equal(localhost->exporting_flags, NULL);
268
+ assert_int_equal(localhost->exporting_flags[0], RRDHOST_FLAG_BACKEND_DONT_SEND);
269
+}
270
+
271
+static void test_rrdset_is_exportable(void **state)
272
+{
273
+ struct engine *engine = *state;
274
+ struct instance *instance = engine->connector_root->instance_root;
275
+ RRDSET *st = localhost->rrdset_root;
276
+
277
+ assert_ptr_equal(st->exporting_flags, NULL);
278
+
279
+ assert_int_equal(__real_rrdset_is_exportable(instance, st), 1);
280
+
281
+ assert_ptr_not_equal(st->exporting_flags, NULL);
282
+ assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_SEND);
283
+}
284
+
285
+static void test_false_rrdset_is_exportable(void **state)
286
+{
287
+ struct engine *engine = *state;
288
+ struct instance *instance = engine->connector_root->instance_root;
289
+ RRDSET *st = localhost->rrdset_root;
290
+
291
+ simple_pattern_free(instance->config.charts_pattern);
292
+ instance->config.charts_pattern = simple_pattern_create("!*", NULL, SIMPLE_PATTERN_EXACT);
293
+
294
+ assert_ptr_equal(st->exporting_flags, NULL);
295
+
296
+ assert_int_equal(__real_rrdset_is_exportable(instance, st), 0);
297
+
298
+ assert_ptr_not_equal(st->exporting_flags, NULL);
299
+ assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_IGNORE);
300
+}
301
+
302
+static void test_exporting_calculate_value_from_stored_data(void **state)
303
+{
304
+ struct engine *engine = *state;
305
+ struct instance *instance = engine->connector_root->instance_root;
306
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
307
+ time_t timestamp;
308
+
309
+ instance->after = 3;
310
+ instance->before = 10;
311
+
312
+ expect_function_call(__mock_rrddim_query_oldest_time);
313
+ will_return(__mock_rrddim_query_oldest_time, 1);
314
+
315
+ expect_function_call(__mock_rrddim_query_latest_time);
316
+ will_return(__mock_rrddim_query_latest_time, 2);
317
+
318
+ expect_function_call(__mock_rrddim_query_init);
319
+ expect_value(__mock_rrddim_query_init, start_time, 1);
320
+ expect_value(__mock_rrddim_query_init, end_time, 2);
321
+
322
+ expect_function_call(__mock_rrddim_query_is_finished);
323
+ will_return(__mock_rrddim_query_is_finished, 0);
324
+ expect_function_call(__mock_rrddim_query_next_metric);
325
+ will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_EXISTS));
326
+
327
+ expect_function_call(__mock_rrddim_query_is_finished);
328
+ will_return(__mock_rrddim_query_is_finished, 0);
329
+ expect_function_call(__mock_rrddim_query_next_metric);
330
+ will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_EXISTS));
331
+
332
+ expect_function_call(__mock_rrddim_query_is_finished);
333
+ will_return(__mock_rrddim_query_is_finished, 1);
334
+
335
+ expect_function_call(__mock_rrddim_query_finalize);
336
+
337
+ assert_int_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, ×tamp), 36);
338
+}
339
+
340
+static void test_prepare_buffers(void **state)
341
+{
342
+ struct engine *engine = *state;
343
+ struct instance *instance = engine->connector_root->instance_root;
344
+
345
+ instance->start_batch_formatting = __mock_start_batch_formatting;
346
+ instance->start_host_formatting = __mock_start_host_formatting;
347
+ instance->start_chart_formatting = __mock_start_chart_formatting;
348
+ instance->metric_formatting = __mock_metric_formatting;
349
+ instance->end_chart_formatting = __mock_end_chart_formatting;
350
+ instance->end_host_formatting = __mock_end_host_formatting;
351
+ instance->end_batch_formatting = __mock_end_batch_formatting;
352
+ __real_mark_scheduled_instances(engine);
353
+
354
+ expect_function_call(__mock_start_batch_formatting);
355
+ expect_value(__mock_start_batch_formatting, instance, instance);
356
+ will_return(__mock_start_batch_formatting, 0);
357
+
358
+ expect_function_call(__wrap_rrdhost_is_exportable);
359
+ expect_value(__wrap_rrdhost_is_exportable, instance, instance);
360
+ expect_value(__wrap_rrdhost_is_exportable, host, localhost);
361
+ will_return(__wrap_rrdhost_is_exportable, 1);
362
+
363
+ expect_function_call(__mock_start_host_formatting);
364
+ expect_value(__mock_start_host_formatting, instance, instance);
365
+ expect_value(__mock_start_host_formatting, host, localhost);
366
+ will_return(__mock_start_host_formatting, 0);
367
+
368
+ RRDSET *st = localhost->rrdset_root;
369
+ expect_function_call(__wrap_rrdset_is_exportable);
370
+ expect_value(__wrap_rrdset_is_exportable, instance, instance);
371
+ expect_value(__wrap_rrdset_is_exportable, st, st);
372
+ will_return(__wrap_rrdset_is_exportable, 1);
373
+
374
+ expect_function_call(__mock_start_chart_formatting);
375
+ expect_value(__mock_start_chart_formatting, instance, instance);
376
+ expect_value(__mock_start_chart_formatting, st, st);
377
+ will_return(__mock_start_chart_formatting, 0);
378
+
379
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
380
+ expect_function_call(__mock_metric_formatting);
381
+ expect_value(__mock_metric_formatting, instance, instance);
382
+ expect_value(__mock_metric_formatting, rd, rd);
383
+ will_return(__mock_metric_formatting, 0);
384
+
385
+ expect_function_call(__mock_end_chart_formatting);
386
+ expect_value(__mock_end_chart_formatting, instance, instance);
387
+ expect_value(__mock_end_chart_formatting, st, st);
388
+ will_return(__mock_end_chart_formatting, 0);
389
+
390
+ expect_function_call(__mock_end_host_formatting);
391
+ expect_value(__mock_end_host_formatting, instance, instance);
392
+ expect_value(__mock_end_host_formatting, host, localhost);
393
+ will_return(__mock_end_host_formatting, 0);
394
+
395
+ expect_function_call(__mock_end_batch_formatting);
396
+ expect_value(__mock_end_batch_formatting, instance, instance);
397
+ will_return(__mock_end_batch_formatting, 0);
398
+
399
+ assert_int_equal(__real_prepare_buffers(engine), 0);
400
+
401
+ assert_int_equal(instance->stats.chart_buffered_metrics, 1);
402
+
403
+ // check with NULL functions
404
+ instance->start_batch_formatting = NULL;
405
+ instance->start_host_formatting = NULL;
406
+ instance->start_chart_formatting = NULL;
407
+ instance->metric_formatting = NULL;
408
+ instance->end_chart_formatting = NULL;
409
+ instance->end_host_formatting = NULL;
410
+ instance->end_batch_formatting = NULL;
411
+ assert_int_equal(__real_prepare_buffers(engine), 0);
412
+
413
+ assert_int_equal(instance->scheduled, 0);
414
+ assert_int_equal(instance->after, 2);
415
+}
416
+
417
+static void test_exporting_name_copy(void **state)
418
+{
419
+ (void)state;
420
+
421
+ char *source_name = "test.name-with/special#characters_";
422
+ char destination_name[RRD_ID_LENGTH_MAX + 1];
423
+
424
+ assert_int_equal(exporting_name_copy(destination_name, source_name, RRD_ID_LENGTH_MAX), 34);
425
+ assert_string_equal(destination_name, "test.name_with_special_characters_");
426
+}
427
+
428
+static void test_format_dimension_collected_graphite_plaintext(void **state)
429
+{
430
+ struct engine *engine = *state;
431
+
432
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
433
+ assert_int_equal(format_dimension_collected_graphite_plaintext(engine->connector_root->instance_root, rd), 0);
434
+ assert_string_equal(
435
+ buffer_tostring(engine->connector_root->instance_root->buffer),
436
+ "netdata.test-host.chart_name.dimension_name;TAG1=VALUE1 TAG2=VALUE2 123000321 15051\n");
437
+}
438
+
439
+static void test_format_dimension_stored_graphite_plaintext(void **state)
440
+{
441
+ struct engine *engine = *state;
442
+
443
+ expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
444
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
445
+
446
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
447
+ assert_int_equal(format_dimension_stored_graphite_plaintext(engine->connector_root->instance_root, rd), 0);
448
+ assert_string_equal(
449
+ buffer_tostring(engine->connector_root->instance_root->buffer),
450
+ "netdata.test-host.chart_name.dimension_name;TAG1=VALUE1 TAG2=VALUE2 690565856.0000000 15052\n");
451
+}
452
+
453
+static void test_format_dimension_collected_json_plaintext(void **state)
454
+{
455
+ struct engine *engine = *state;
456
+
457
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
458
+ assert_int_equal(format_dimension_collected_json_plaintext(engine->connector_root->instance_root, rd), 0);
459
+ assert_string_equal(
460
+ buffer_tostring(engine->connector_root->instance_root->buffer),
461
+ "{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\","
462
+ "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\","
463
+ "\"chart_context\": \"(null)\",\"chart_type\":\"(null)\",\"units\": \"(null)\",\"id\":\"dimension_id\","
464
+ "\"name\":\"dimension_name\",\"value\":123000321,\"timestamp\": 15051}\n");
465
+}
466
+
467
+static void test_format_dimension_stored_json_plaintext(void **state)
468
+{
469
+ struct engine *engine = *state;
470
+
471
+ expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
472
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
473
+
474
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
475
+ assert_int_equal(format_dimension_stored_json_plaintext(engine->connector_root->instance_root, rd), 0);
476
+ assert_string_equal(
477
+ buffer_tostring(engine->connector_root->instance_root->buffer),
478
+ "{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\","
479
+ "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\"," \
480
+ "\"chart_context\": \"(null)\",\"chart_type\":\"(null)\",\"units\": \"(null)\",\"id\":\"dimension_id\","
481
+ "\"name\":\"dimension_name\",\"value\":690565856.0000000,\"timestamp\": 15052}\n");
482
+}
483
+
484
+static void test_format_dimension_collected_opentsdb_telnet(void **state)
485
+{
486
+ struct engine *engine = *state;
487
+
488
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
489
+ assert_int_equal(format_dimension_collected_opentsdb_telnet(engine->connector_root->instance_root, rd), 0);
490
+ assert_string_equal(
491
+ buffer_tostring(engine->connector_root->instance_root->buffer),
492
+ "put netdata.chart_name.dimension_name 15051 123000321 host=test-host TAG1=VALUE1 TAG2=VALUE2\n");
493
+}
494
+
495
+static void test_format_dimension_stored_opentsdb_telnet(void **state)
496
+{
497
+ struct engine *engine = *state;
498
+
499
+ expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
500
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
501
+
502
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
503
+ assert_int_equal(format_dimension_stored_opentsdb_telnet(engine->connector_root->instance_root, rd), 0);
504
+ assert_string_equal(
505
+ buffer_tostring(engine->connector_root->instance_root->buffer),
506
+ "put netdata.chart_name.dimension_name 15052 690565856.0000000 host=test-host TAG1=VALUE1 TAG2=VALUE2\n");
507
+}
508
+
509
+static void test_format_dimension_collected_opentsdb_http(void **state)
510
+{
511
+ struct engine *engine = *state;
512
+
513
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
514
+ assert_int_equal(format_dimension_collected_opentsdb_http(engine->connector_root->instance_root, rd), 0);
515
+ assert_string_equal(
516
+ buffer_tostring(engine->connector_root->instance_root->buffer),
517
+ "POST /api/put HTTP/1.1\r\n"
518
+ "Host: test-host\r\n"
519
+ "Content-Type: application/json\r\n"
520
+ "Content-Length: 153\r\n\r\n"
521
+ "{ \"metric\": \"netdata.chart_name.dimension_name\", "
522
+ "\"timestamp\": 15051, "
523
+ "\"value\": 123000321, "
524
+ "\"tags\": { \"host\": \"test-host TAG1=VALUE1 TAG2=VALUE2\" }}");
525
+}
526
+
527
+static void test_format_dimension_stored_opentsdb_http(void **state)
528
+{
529
+ struct engine *engine = *state;
530
+
531
+ expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
532
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
533
+
534
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
535
+ assert_int_equal(format_dimension_stored_opentsdb_http(engine->connector_root->instance_root, rd), 0);
536
+ assert_string_equal(
537
+ buffer_tostring(engine->connector_root->instance_root->buffer),
538
+ "POST /api/put HTTP/1.1\r\n"
539
+ "Host: test-host\r\n"
540
+ "Content-Type: application/json\r\n"
541
+ "Content-Length: 161\r\n\r\n"
542
+ "{ \"metric\": \"netdata.chart_name.dimension_name\", "
543
+ "\"timestamp\": 15052, "
544
+ "\"value\": 690565856.0000000, "
545
+ "\"tags\": { \"host\": \"test-host TAG1=VALUE1 TAG2=VALUE2\" }}");
546
+}
547
+
548
+static void test_exporting_discard_response(void **state)
549
+{
550
+ struct engine *engine = *state;
551
+
552
+ BUFFER *response = buffer_create(0);
553
+ buffer_sprintf(response, "Test response");
554
+
555
+ expect_function_call(__wrap_info_int);
556
+
557
+ assert_int_equal(exporting_discard_response(response, engine->connector_root->instance_root), 0);
558
+
559
+ assert_string_equal(
560
+ log_line,
561
+ "EXPORTING: received 13 bytes from instance_name connector instance. Ignoring them. Sample: 'Test response'");
562
+
563
+ assert_int_equal(buffer_strlen(response), 0);
564
+
565
+ buffer_free(response);
566
+}
567
+
568
+static void test_simple_connector_receive_response(void **state)
569
+{
570
+ struct engine *engine = *state;
571
+ struct instance *instance = engine->connector_root->instance_root;
572
+ struct stats *stats = &instance->stats;
573
+
574
+ int sock = 1;
575
+
576
+ expect_function_call(__wrap_recv);
577
+ expect_value(__wrap_recv, sockfd, 1);
578
+ expect_not_value(__wrap_recv, buf, 0);
579
+ expect_value(__wrap_recv, len, 4096);
580
+ expect_value(__wrap_recv, flags, MSG_DONTWAIT);
581
+
582
+ expect_function_call(__wrap_info_int);
583
+
584
+ simple_connector_receive_response(&sock, instance);
585
+
586
+ assert_string_equal(
587
+ log_line,
588
+ "EXPORTING: received 9 bytes from instance_name connector instance. Ignoring them. Sample: 'Test recv'");
589
+
590
+ assert_int_equal(stats->chart_received_bytes, 9);
591
+ assert_int_equal(stats->chart_receptions, 1);
592
+ assert_int_equal(sock, 1);
593
+}
594
+
595
+static void test_simple_connector_send_buffer(void **state)
596
+{
597
+ struct engine *engine = *state;
598
+ struct instance *instance = engine->connector_root->instance_root;
599
+ struct stats *stats = &instance->stats;
600
+ BUFFER *buffer = instance->buffer;
601
+
602
+ int sock = 1;
603
+ int failures = 3;
604
+
605
+ __real_mark_scheduled_instances(engine);
606
+
607
+ expect_function_call(__wrap_rrdhost_is_exportable);
608
+ expect_value(__wrap_rrdhost_is_exportable, instance, instance);
609
+ expect_value(__wrap_rrdhost_is_exportable, host, localhost);
610
+ will_return(__wrap_rrdhost_is_exportable, 1);
611
+
612
+ RRDSET *st = localhost->rrdset_root;
613
+ expect_function_call(__wrap_rrdset_is_exportable);
614
+ expect_value(__wrap_rrdset_is_exportable, instance, instance);
615
+ expect_value(__wrap_rrdset_is_exportable, st, st);
616
+ will_return(__wrap_rrdset_is_exportable, 1);
617
+
618
+ __real_prepare_buffers(engine);
619
+
620
+ expect_function_call(__wrap_send);
621
+ expect_value(__wrap_send, sockfd, 1);
622
+ expect_value(__wrap_send, buf, buffer_tostring(buffer));
623
+ expect_string(
624
+ __wrap_send, buf, "netdata.test-host.chart_name.dimension_name;TAG1=VALUE1 TAG2=VALUE2 123000321 15051\n");
625
+ expect_value(__wrap_send, len, 84);
626
+ expect_value(__wrap_send, flags, MSG_NOSIGNAL);
627
+
628
+ simple_connector_send_buffer(&sock, &failures, instance);
629
+
630
+ assert_int_equal(failures, 0);
631
+ assert_int_equal(stats->chart_transmission_successes, 1);
632
+ assert_int_equal(stats->chart_sent_bytes, 84);
633
+ assert_int_equal(stats->chart_sent_metrics, 1);
634
+ assert_int_equal(stats->chart_transmission_failures, 0);
635
+
636
+ assert_int_equal(buffer_strlen(buffer), 0);
637
+
638
+ assert_int_equal(sock, 1);
639
+}
640
+
641
+static void test_simple_connector_worker(void **state)
642
+{
643
+ struct engine *engine = *state;
644
+ struct instance *instance = engine->connector_root->instance_root;
645
+ BUFFER *buffer = instance->buffer;
646
+
647
+ __real_mark_scheduled_instances(engine);
648
+
649
+ expect_function_call(__wrap_rrdhost_is_exportable);
650
+ expect_value(__wrap_rrdhost_is_exportable, instance, instance);
651
+ expect_value(__wrap_rrdhost_is_exportable, host, localhost);
652
+ will_return(__wrap_rrdhost_is_exportable, 1);
653
+
654
+ RRDSET *st = localhost->rrdset_root;
655
+ expect_function_call(__wrap_rrdset_is_exportable);
656
+ expect_value(__wrap_rrdset_is_exportable, instance, instance);
657
+ expect_value(__wrap_rrdset_is_exportable, st, st);
658
+ will_return(__wrap_rrdset_is_exportable, 1);
659
+
660
+ __real_prepare_buffers(engine);
661
+
662
+ expect_function_call(__wrap_connect_to_one_of);
663
+ expect_string(__wrap_connect_to_one_of, destination, "localhost");
664
+ expect_value(__wrap_connect_to_one_of, default_port, 2003);
665
+ expect_not_value(__wrap_connect_to_one_of, reconnects_counter, 0);
666
+ expect_value(__wrap_connect_to_one_of, connected_to, 0);
667
+ expect_value(__wrap_connect_to_one_of, connected_to_size, 0);
668
+ will_return(__wrap_connect_to_one_of, 2);
669
+
670
+ expect_function_call(__wrap_send);
671
+ expect_value(__wrap_send, sockfd, 2);
672
+ expect_value(__wrap_send, buf, buffer_tostring(buffer));
673
+ expect_string(
674
+ __wrap_send, buf, "netdata.test-host.chart_name.dimension_name;TAG1=VALUE1 TAG2=VALUE2 123000321 15051\n");
675
+ expect_value(__wrap_send, len, 84);
676
+ expect_value(__wrap_send, flags, MSG_NOSIGNAL);
677
+
678
+ simple_connector_worker(instance);
679
+}
680
+
681
+int main(void)
682
+{
683
+ const struct CMUnitTest tests[] = {
684
+ cmocka_unit_test_setup_teardown(test_exporting_engine, setup_initialized_engine, teardown_initialized_engine),
685
+ cmocka_unit_test(test_read_exporting_config),
686
+ cmocka_unit_test_setup_teardown(test_init_connectors, setup_configured_engine, teardown_configured_engine),
687
+ cmocka_unit_test_setup_teardown(
688
+ test_init_graphite_instance, setup_configured_engine, teardown_configured_engine),
689
+ cmocka_unit_test_setup_teardown(
690
+ test_init_json_instance, setup_configured_engine, teardown_configured_engine),
691
+ cmocka_unit_test_setup_teardown(
692
+ test_init_opentsdb_connector, setup_configured_engine, teardown_configured_engine),
693
+ cmocka_unit_test_setup_teardown(
694
+ test_init_opentsdb_telnet_instance, setup_configured_engine, teardown_configured_engine),
695
+ cmocka_unit_test_setup_teardown(
696
+ test_init_opentsdb_http_instance, setup_configured_engine, teardown_configured_engine),
697
+ cmocka_unit_test_setup_teardown(
698
+ test_mark_scheduled_instances, setup_initialized_engine, teardown_initialized_engine),
699
+ cmocka_unit_test_setup_teardown(
700
+ test_rrdhost_is_exportable, setup_initialized_engine, teardown_initialized_engine),
701
+ cmocka_unit_test_setup_teardown(
702
+ test_false_rrdhost_is_exportable, setup_initialized_engine, teardown_initialized_engine),
703
+ cmocka_unit_test_setup_teardown(
704
+ test_rrdset_is_exportable, setup_initialized_engine, teardown_initialized_engine),
705
+ cmocka_unit_test_setup_teardown(
706
+ test_false_rrdset_is_exportable, setup_initialized_engine, teardown_initialized_engine),
707
+ cmocka_unit_test_setup_teardown(
708
+ test_exporting_calculate_value_from_stored_data, setup_initialized_engine, teardown_initialized_engine),
709
+ cmocka_unit_test_setup_teardown(test_prepare_buffers, setup_initialized_engine, teardown_initialized_engine),
710
+ cmocka_unit_test(test_exporting_name_copy),
711
+ cmocka_unit_test_setup_teardown(
712
+ test_format_dimension_collected_graphite_plaintext, setup_initialized_engine, teardown_initialized_engine),
713
+ cmocka_unit_test_setup_teardown(
714
+ test_format_dimension_stored_graphite_plaintext, setup_initialized_engine, teardown_initialized_engine),
715
+ cmocka_unit_test_setup_teardown(
716
+ test_format_dimension_collected_json_plaintext, setup_initialized_engine, teardown_initialized_engine),
717
+ cmocka_unit_test_setup_teardown(
718
+ test_format_dimension_stored_json_plaintext, setup_initialized_engine, teardown_initialized_engine),
719
+ cmocka_unit_test_setup_teardown(
720
+ test_format_dimension_collected_opentsdb_telnet, setup_initialized_engine, teardown_initialized_engine),
721
+ cmocka_unit_test_setup_teardown(
722
+ test_format_dimension_stored_opentsdb_telnet, setup_initialized_engine, teardown_initialized_engine),
723
+ cmocka_unit_test_setup_teardown(
724
+ test_format_dimension_collected_opentsdb_http, setup_initialized_engine, teardown_initialized_engine),
725
+ cmocka_unit_test_setup_teardown(
726
+ test_format_dimension_stored_opentsdb_http, setup_initialized_engine, teardown_initialized_engine),
727
+ cmocka_unit_test_setup_teardown(
728
+ test_exporting_discard_response, setup_initialized_engine, teardown_initialized_engine),
729
+ cmocka_unit_test_setup_teardown(
730
+ test_simple_connector_receive_response, setup_initialized_engine, teardown_initialized_engine),
731
+ cmocka_unit_test_setup_teardown(
732
+ test_simple_connector_send_buffer, setup_initialized_engine, teardown_initialized_engine),
733
+ cmocka_unit_test_setup_teardown(
734
+ test_simple_connector_worker, setup_initialized_engine, teardown_initialized_engine),
735
+ };
736
+
737
+ return cmocka_run_group_tests_name("exporting_engine", tests, NULL, NULL);
738
+}
exporting/tests/test_exporting_engine.h
new
+110
@@ -0,0 +1,110 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef TEST_EXPORTING_ENGINE_H
4
+#define TEST_EXPORTING_ENGINE_H 1
5
+
6
+#include "libnetdata/libnetdata.h"
7
+
8
+#include "exporting/exporting_engine.h"
9
+#include "exporting/graphite/graphite.h"
10
+#include "exporting/json/json.h"
11
+#include "exporting/opentsdb/opentsdb.h"
12
+
13
+#include <stdarg.h>
14
+#include <stddef.h>
15
+#include <setjmp.h>
16
+#include <stdint.h>
17
+#include <cmocka.h>
18
+
19
+#define MAX_LOG_LINE 1024
20
+extern char log_line[];
21
+
22
+// -----------------------------------------------------------------------
23
+// doubles for Netdata functions
24
+
25
+const char *__wrap_strdupz(const char *s);
26
+void __wrap_info_int(const char *file, const char *function, const unsigned long line, const char *fmt, ...);
27
+int __wrap_connect_to_one_of(
28
+ const char *destination,
29
+ int default_port,
30
+ struct timeval *timeout,
31
+ size_t *reconnects_counter,
32
+ char *connected_to,
33
+ size_t connected_to_size);
34
+void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);
35
+void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line);
36
+void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line);
37
+time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
38
+time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
39
+void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
40
+int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
41
+storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time);
42
+void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
43
+
44
+// -----------------------------------------------------------------------
45
+// wraps for system functions
46
+
47
+void __wrap_uv_thread_create(uv_thread_t thread, void (*worker)(void *arg), void *arg);
48
+void __wrap_uv_mutex_lock(uv_mutex_t *mutex);
49
+void __wrap_uv_mutex_unlock(uv_mutex_t *mutex);
50
+void __wrap_uv_cond_signal(uv_cond_t *cond_var);
51
+void __wrap_uv_cond_wait(uv_cond_t *cond_var, uv_mutex_t *mutex);
52
+ssize_t __wrap_recv(int sockfd, void *buf, size_t len, int flags);
53
+ssize_t __wrap_send(int sockfd, const void *buf, size_t len, int flags);
54
+
55
+// -----------------------------------------------------------------------
56
+// doubles and originals for exporting engine functions
57
+
58
+struct engine *__real_read_exporting_config();
59
+struct engine *__wrap_read_exporting_config();
60
+struct engine *__mock_read_exporting_config();
61
+
62
+int __real_init_connectors(struct engine *engine);
63
+int __wrap_init_connectors(struct engine *engine);
64
+
65
+int __real_mark_scheduled_instances(struct engine *engine);
66
+int __wrap_mark_scheduled_instances(struct engine *engine);
67
+
68
+calculated_number __real_exporting_calculate_value_from_stored_data(
69
+ struct instance *instance,
70
+ RRDDIM *rd,
71
+ time_t *last_timestamp);
72
+calculated_number __wrap_exporting_calculate_value_from_stored_data(
73
+ struct instance *instance,
74
+ RRDDIM *rd,
75
+ time_t *last_timestamp);
76
+
77
+int __real_prepare_buffers(struct engine *engine);
78
+int __wrap_prepare_buffers(struct engine *engine);
79
+
80
+int __wrap_notify_workers(struct engine *engine);
81
+
82
+int __wrap_send_internal_metrics(struct engine *engine);
83
+
84
+int __real_rrdhost_is_exportable(struct instance *instance, RRDHOST *host);
85
+int __wrap_rrdhost_is_exportable(struct instance *instance, RRDHOST *host);
86
+
87
+int __real_rrdset_is_exportable(struct instance *instance, RRDSET *st);
88
+int __wrap_rrdset_is_exportable(struct instance *instance, RRDSET *st);
89
+
90
+int __mock_start_batch_formatting(struct instance *instance);
91
+int __mock_start_host_formatting(struct instance *instance, RRDHOST *host);
92
+int __mock_start_chart_formatting(struct instance *instance, RRDSET *st);
93
+int __mock_metric_formatting(struct instance *instance, RRDDIM *rd);
94
+int __mock_end_chart_formatting(struct instance *instance, RRDSET *st);
95
+int __mock_end_host_formatting(struct instance *instance, RRDHOST *host);
96
+int __mock_end_batch_formatting(struct instance *instance);
97
+
98
+// -----------------------------------------------------------------------
99
+// fixtures
100
+
101
+int setup_configured_engine(void **state);
102
+int teardown_configured_engine(void **state);
103
+int setup_rrdhost();
104
+int teardown_rrdhost();
105
+int setup_initialized_engine(void **state);
106
+int teardown_initialized_engine(void **state);
107
+
108
+void init_connectors_in_tests(struct engine *engine);
109
+
110
+#endif /* TEST_EXPORTING_ENGINE_H */
libnetdata/config/appconfig.c
+158
-7
@@ -42,6 +42,89 @@ struct section {
42
// readers are protected using the rwlock in avl_tree_lock
43
};
44
45
+/*
46
+ * @Input:
47
+ * Connector / instance to add to an internal structure
48
+ * @Return
49
+ * The current head of the linked list of connector_instance
50
+ *
51
+ */
52
+
53
+_CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance)
54
+{
55
+ static struct _connector_instance *global_connector_instance = NULL;
56
+ struct _connector_instance *local_ci, *local_ci_tmp;
57
+
58
+ if (unlikely(!connector)) {
59
+ if (unlikely(!instance))
60
+ return global_connector_instance;
61
+
62
+ local_ci = global_connector_instance;
63
+ while (local_ci) {
64
+ local_ci_tmp = local_ci->next;
65
+ freez(local_ci);
66
+ local_ci = local_ci_tmp;
67
+ }
68
+ global_connector_instance = NULL;
69
+ return NULL;
70
+ }
71
+
72
+ local_ci = callocz(1, sizeof(struct _connector_instance));
73
+ local_ci->instance = instance;
74
+ local_ci->connector = connector;
75
+ strncpy(local_ci->instance_name, instance->name, CONFIG_MAX_NAME);
76
+ strncpy(local_ci->connector_name, connector->name, CONFIG_MAX_NAME);
77
+ local_ci->next = global_connector_instance;
78
+ global_connector_instance = local_ci;
79
+
80
+ return global_connector_instance;
81
+}
82
+
83
+int is_valid_connector(char *type, int check_reserved)
84
+{
85
+ int rc = 1;
86
+
87
+ if (unlikely(!type))
88
+ return 0;
89
+
90
+ if (!check_reserved) {
91
+ if (unlikely(is_valid_connector(type,1))) {
92
+ return 0;
93
+ }
94
+ //if (unlikely(*type == ':')
95
+ // return 0;
96
+ char *separator = strrchr(type, ':');
97
+ if (likely(separator)) {
98
+ *separator = '\0';
99
+ rc = separator - type;
100
+ } else
101
+ return 0;
102
+ }
103
+// else {
104
+// if (unlikely(is_valid_connector(type,1))) {
105
+// error("Section %s invalid -- reserved name", type);
106
+// return 0;
107
+// }
108
+// }
109
+
110
+ if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
111
+ return rc;
112
+ } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
113
+ return rc;
114
+ } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
115
+ return rc;
116
+ } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
117
+ return rc;
118
+ } else if (!strcmp(type, "prometheus_remote_write")) {
119
+ return rc;
120
+ } else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
121
+ return rc;
122
+ } else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) {
123
+ return rc;
124
+ }
125
+
126
+ return 0;
127
+}
128
129
// ----------------------------------------------------------------------------
130
// locking
@@ -241,6 +324,7 @@ cleanup:
324
return ret;
325
}
326
327
+
328
char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value)
329
{
330
struct config_option *cv;
@@ -440,6 +524,14 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
524
{
525
int line = 0;
526
struct section *co = NULL;
527
+ int is_exporter_config = 0;
528
+ int is_backend_config = 0;
529
+ int have_backend_config = 0;
530
+ int _backends = 0; // number of backend sections we have
531
+ char working_instance[CONFIG_MAX_NAME + 1];
532
+ char working_connector[CONFIG_MAX_NAME + 1];
533
+ struct section *working_connector_section = NULL;
534
+ int global_exporting_section = 0;
535
536
char buffer[CONFIG_FILE_LINE_MAX + 1], *s;
537
@@ -453,6 +545,8 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
545
return 0;
546
}
547
548
+ is_exporter_config = (strstr(filename, EXPORTING_CONF) != NULL);
549
+
550
while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {
551
buffer[CONFIG_FILE_LINE_MAX] = '\0';
552
line++;
@@ -469,6 +563,46 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
563
s[len - 1] = '\0';
564
s++;
565
566
+ if (is_exporter_config) {
567
+ global_exporting_section = !(strcmp(s, CONFIG_SECTION_EXPORTING));
568
+ if (unlikely(!global_exporting_section)) {
569
+ int rc;
570
+ rc = is_valid_connector(s, 0);
571
+ if (likely(rc)) {
572
+ strncpy(working_connector, s, CONFIG_MAX_NAME);
573
+ s = s + rc + 1;
574
+ if (unlikely(!(*s))) {
575
+ _backends++;
576
+ sprintf(buffer, "instance_%d", _backends);
577
+ s = buffer;
578
+ }
579
+ strncpy(working_instance, s, CONFIG_MAX_NAME);
580
+ working_connector_section = NULL;
581
+ if (unlikely(appconfig_section_find(root, working_instance))) {
582
+ error("Instance (%s) already exists", working_instance);
583
+ co = NULL;
584
+ continue;
585
+ }
586
+ } else {
587
+ co = NULL;
588
+ error("Section (%s) does not specify a valid connector", s);
589
+ continue;
590
+ }
591
+ }
592
+ }
593
+
594
+ is_backend_config = !(strcmp(s, CONFIG_SECTION_BACKEND));
595
+ if (!have_backend_config)
596
+ have_backend_config = is_backend_config;
597
+
598
+ if (is_backend_config) {
599
+ if (_backends) {
600
+ sprintf(buffer, CONFIG_SECTION_BACKEND "/%d", _backends);
601
+ s = buffer;
602
+ }
603
+ _backends++;
604
+ }
605
+
606
co = appconfig_section_find(root, s);
607
if(!co) co = appconfig_section_create(root, s);
608
@@ -502,15 +636,32 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
636
637
struct config_option *cv = appconfig_option_index_find(co, name, 0);
638
505
- if(!cv) cv = appconfig_value_create(co, name, value);
506
- else {
507
- if(((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) {
508
- debug(D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name);
639
+ if (!cv) {
640
+ cv = appconfig_value_create(co, name, value);
641
+ if (likely(is_exporter_config) && unlikely(!global_exporting_section)) {
642
+ if (unlikely(!working_connector_section)) {
643
+ working_connector_section = appconfig_section_find(root, working_connector);
644
+ if (!working_connector_section)
645
+ working_connector_section = appconfig_section_create(root, working_connector);
646
+ if (likely(working_connector_section)) {
647
+ add_connector_instance(working_connector_section, co);
648
+ }
649
+ }
650
+ }
651
+ } else {
652
+ if (((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) {
653
+ debug(
654
+ D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name);
655
freez(cv->value);
656
cv->value = strdupz(value);
511
- }
512
- else
513
- debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.", line, filename, co->name, cv->name);
657
+ } else
658
+ debug(
659
+ D_CONFIG,
660
+ "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.",
661
+ line,
662
+ filename,
663
+ co->name,
664
+ cv->name);
665
}
666
cv->flags |= CONFIG_VALUE_LOADED;
667
}
libnetdata/config/appconfig.h
+32
-9
@@ -82,15 +82,17 @@
82
83
#define CONFIG_FILENAME "netdata.conf"
84
85
-#define CONFIG_SECTION_GLOBAL "global"
86
-#define CONFIG_SECTION_WEB "web"
87
-#define CONFIG_SECTION_STATSD "statsd"
88
-#define CONFIG_SECTION_PLUGINS "plugins"
89
-#define CONFIG_SECTION_CLOUD "cloud"
90
-#define CONFIG_SECTION_REGISTRY "registry"
91
-#define CONFIG_SECTION_HEALTH "health"
92
-#define CONFIG_SECTION_BACKEND "backend"
93
-#define CONFIG_SECTION_STREAM "stream"
85
+#define CONFIG_SECTION_GLOBAL "global"
86
+#define CONFIG_SECTION_WEB "web"
87
+#define CONFIG_SECTION_STATSD "statsd"
88
+#define CONFIG_SECTION_PLUGINS "plugins"
89
+#define CONFIG_SECTION_CLOUD "cloud"
90
+#define CONFIG_SECTION_REGISTRY "registry"
91
+#define CONFIG_SECTION_HEALTH "health"
92
+#define CONFIG_SECTION_BACKEND "backend"
93
+#define CONFIG_SECTION_STREAM "stream"
94
+#define CONFIG_SECTION_EXPORTING "exporting:global"
95
+#define EXPORTING_CONF "exporting.conf"
96
97
// these are used to limit the configuration names and values lengths
98
// they are not enforced by config.c functions (they will strdup() all strings, no matter of their length)
@@ -103,6 +105,11 @@ struct config {
105
avl_tree_lock index;
106
};
107
108
+//struct connector_instance {
109
+// char instance_name[CONFIG_MAX_NAME + 1];
110
+// char connector_name[CONFIG_MAX_NAME + 1];
111
+//};
112
+
113
#define CONFIG_BOOLEAN_INVALID 100 // an invalid value to check for validity (used as default initialization when needed)
114
115
#define CONFIG_BOOLEAN_NO 0 // disabled
@@ -136,4 +143,20 @@ extern int appconfig_section_compare(void *a, void *b);
143
144
extern int config_parse_duration(const char* string, int* result);
145
146
+
147
+struct connector_instance {
148
+ char instance_name[CONFIG_MAX_NAME + 1];
149
+ char connector_name[CONFIG_MAX_NAME + 1];
150
+};
151
+
152
+typedef struct _connector_instance {
153
+ struct section *connector; // actual connector
154
+ struct section *instance; // This instance
155
+ char instance_name[CONFIG_MAX_NAME + 1];
156
+ char connector_name[CONFIG_MAX_NAME + 1];
157
+ struct _connector_instance *next; // Next instance
158
+} _CONNECTOR_INSTANCE;
159
+
160
+extern _CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance);
161
+
162
#endif /* NETDATA_CONFIG_H */