Add a Prometheus Remote Write connector to the exporting engine (#8292)
* Copy files from the Prometheus remote write backend * Update the documentation * Rename backend -> exporting * Add the connector to the Netdata build * Separate files for the remote write connector * Add an initializer and formatters * Read a connector specific configuration option * Add a separate function for header sending * Use labels instead of tags * Separate write request for every instance * Add unit tests
Vladimir Kobal committed
Mar 12, 2020 at 13:28 UTC
bc0ca9b1b3cbdd6d1a76d2150080dff79e116e24
35 files changed
+2407
-168
CMakeLists.txt
+23
-2
@@ -634,6 +634,8 @@ set(EXPORTING_ENGINE_FILES
634
exporting/json/json.h
635
exporting/opentsdb/opentsdb.c
636
exporting/opentsdb/opentsdb.h
637
+ exporting/prometheus/prometheus.c
638
+ exporting/prometheus/prometheus.h
639
exporting/read_config.c
640
exporting/init_connectors.c
641
exporting/process_data.c
@@ -642,6 +644,13 @@ set(EXPORTING_ENGINE_FILES
644
exporting/send_internal_metrics.c
645
)
646
647
+set(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES
648
+ exporting/prometheus/remote_write/remote_write.c
649
+ exporting/prometheus/remote_write/remote_write.h
650
+ exporting/prometheus/remote_write/remote_write_request.cc
651
+ exporting/prometheus/remote_write/remote_write_request.h
652
+ )
653
+
654
set(KINESIS_EXPORTING_FILES
655
exporting/aws_kinesis/aws_kinesis.c
656
exporting/aws_kinesis/aws_kinesis.h
@@ -753,9 +762,9 @@ IF(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)
762
message(STATUS "prometheus remote write backend: enabled")
763
764
find_package(Protobuf REQUIRED)
756
- protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS backends/prometheus/remote_write/remote_write.proto)
765
+ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS exporting/prometheus/remote_write/remote_write.proto)
766
758
- list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
767
+ list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
768
list(APPEND NETDATA_COMMON_LIBRARIES ${PROTOBUF_LIBRARIES} ${SNAPPY_LIBRARIES})
769
list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS} ${SNAPPY_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR})
770
list(APPEND NETDATA_COMMON_CFLAGS ${PROTOBUF_CFLAGS_OTHER} ${SNAPPY_CFLAGS_OTHER})
@@ -1021,6 +1030,17 @@ if(BUILD_TESTING)
1030
)
1031
set(TEST_NAME exporting_engine)
1032
set(KINESIS_LINK_OPTIONS)
1033
+ set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS)
1034
+if(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)
1035
+ list(APPEND EXPORTING_ENGINE_FILES ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
1036
+ list(
1037
+ APPEND PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS
1038
+ -Wl,--wrap=init_write_request
1039
+ -Wl,--wrap=add_host_info
1040
+ -Wl,--wrap=add_label
1041
+ -Wl,--wrap=add_metric
1042
+ )
1043
+endif()
1044
if(ENABLE_BACKEND_KINESIS)
1045
list(APPEND EXPORTING_ENGINE_FILES ${KINESIS_EXPORTING_FILES})
1046
list(
@@ -1062,6 +1082,7 @@ endif()
1082
-Wl,--wrap=send
1083
-Wl,--wrap=connect_to_one_of
1084
${KINESIS_LINK_OPTIONS}
1085
+ ${PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS}
1086
)
1087
target_link_libraries(${TEST_NAME}_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
1088
add_test(NAME test_${TEST_NAME} COMMAND ${TEST_NAME}_testdriver)
Makefile.am
+28
-5
@@ -492,6 +492,8 @@ EXPORTING_ENGINE_FILES = \
492
exporting/json/json.h \
493
exporting/opentsdb/opentsdb.c \
494
exporting/opentsdb/opentsdb.h \
495
+ exporting/prometheus/prometheus.c \
496
+ exporting/prometheus/prometheus.h \
497
exporting/read_config.c \
498
exporting/init_connectors.c \
499
exporting/process_data.c \
@@ -500,6 +502,14 @@ EXPORTING_ENGINE_FILES = \
502
exporting/send_internal_metrics.c \
503
$(NULL)
504
505
+PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES = \
506
+ exporting/prometheus/remote_write/remote_write.c \
507
+ exporting/prometheus/remote_write/remote_write.h \
508
+ exporting/prometheus/remote_write/remote_write_request.cc \
509
+ exporting/prometheus/remote_write/remote_write_request.h \
510
+ exporting/prometheus/remote_write/remote_write.proto \
511
+ $(NULL)
512
+
513
KINESIS_EXPORTING_FILES = \
514
exporting/aws_kinesis/aws_kinesis.c \
515
exporting/aws_kinesis/aws_kinesis.h \
@@ -517,7 +527,6 @@ KINESIS_BACKEND_FILES = \
527
PROMETHEUS_REMOTE_WRITE_BACKEND_FILES = \
528
backends/prometheus/remote_write/remote_write.cc \
529
backends/prometheus/remote_write/remote_write.h \
520
- backends/prometheus/remote_write/remote_write.proto \
530
$(NULL)
531
532
MONGODB_BACKEND_FILES = \
@@ -737,16 +746,19 @@ if ENABLE_BACKEND_KINESIS
746
endif
747
748
if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE
749
+if ENABLE_EXPORTING
750
+ netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
751
+endif
752
netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES)
753
netdata_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS)
754
BUILT_SOURCES = \
743
- backends/prometheus/remote_write/remote_write.pb.cc \
744
- backends/prometheus/remote_write/remote_write.pb.h \
755
+ exporting/prometheus/remote_write/remote_write.pb.cc \
756
+ exporting/prometheus/remote_write/remote_write.pb.h \
757
$(NULL)
758
nodist_netdata_SOURCES = $(BUILT_SOURCES)
759
748
-backends/prometheus/remote_write/remote_write.pb.cc \
749
-backends/prometheus/remote_write/remote_write.pb.h: backends/prometheus/remote_write/remote_write.proto
760
+exporting/prometheus/remote_write/remote_write.pb.cc \
761
+exporting/prometheus/remote_write/remote_write.pb.h: exporting/prometheus/remote_write/remote_write.proto
762
$(PROTOC) --proto_path=$(srcdir) --cpp_out=$(builddir) $^
763
764
endif
@@ -867,6 +879,17 @@ if ENABLE_UNITTESTS
879
$(TEST_LDFLAGS) \
880
$(NULL)
881
exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
882
+if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE
883
+ exporting_tests_exporting_engine_testdriver_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
884
+ exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS)
885
+ exporting_tests_exporting_engine_testdriver_LDFLAGS += \
886
+ -Wl,--wrap=init_write_request \
887
+ -Wl,--wrap=add_host_info \
888
+ -Wl,--wrap=add_label \
889
+ -Wl,--wrap=add_metric \
890
+ $(NULL)
891
+ nodist_exporting_tests_exporting_engine_testdriver_SOURCES = $(BUILT_SOURCES)
892
+endif
893
if ENABLE_BACKEND_KINESIS
894
exporting_tests_exporting_engine_testdriver_SOURCES += $(KINESIS_EXPORTING_FILES)
895
exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_KINESIS_LIBS)
backends/backends.c
+10
-10
@@ -295,7 +295,7 @@ void backend_set_prometheus_variables(int *default_port,
295
#endif
296
297
#if ENABLE_PROMETHEUS_REMOTE_WRITE
298
- *brc = process_prometheus_remote_write_response;
298
+ *brc = backends_process_prometheus_remote_write_response;
299
#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
300
}
301
@@ -439,7 +439,7 @@ BACKEND_TYPE backend_select_type(const char *type) {
439
return BACKEND_TYPE_JSON;
440
}
441
else if (!strcmp(type, "prometheus_remote_write")) {
442
- return BACKEND_TYPE_PROMETHEUS;
442
+ return BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE;
443
}
444
else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
445
return BACKEND_TYPE_KINESIS;
@@ -557,12 +557,12 @@ void *backends_main(void *ptr) {
557
backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);
558
break;
559
}
560
- case BACKEND_TYPE_PROMETHEUS: {
560
+ case BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE: {
561
#if ENABLE_PROMETHEUS_REMOTE_WRITE
562
do_prometheus_remote_write = 1;
563
564
http_request_header = buffer_create(1);
565
- init_write_request();
565
+ backends_init_write_request();
566
#else
567
error("BACKEND: Prometheus remote write support isn't compiled");
568
#endif // ENABLE_PROMETHEUS_REMOTE_WRITE
@@ -724,7 +724,7 @@ void *backends_main(void *ptr) {
724
725
#if ENABLE_PROMETHEUS_REMOTE_WRITE
726
if(do_prometheus_remote_write)
727
- clear_write_request();
727
+ backends_clear_write_request();
728
#endif
729
rrd_rdlock();
730
RRDHOST *host;
@@ -755,7 +755,7 @@ void *backends_main(void *ptr) {
755
756
#if ENABLE_PROMETHEUS_REMOTE_WRITE
757
if(do_prometheus_remote_write) {
758
- rrd_stats_remote_write_allmetrics_prometheus(
758
+ backends_rrd_stats_remote_write_allmetrics_prometheus(
759
host
760
, __hostname
761
, global_backend_prefix
@@ -1048,7 +1048,7 @@ void *backends_main(void *ptr) {
1048
1049
#if ENABLE_PROMETHEUS_REMOTE_WRITE
1050
if(do_prometheus_remote_write) {
1051
- size_t data_size = get_write_request_size();
1051
+ size_t data_size = backends_get_write_request_size();
1052
1053
if(unlikely(!data_size)) {
1054
error("BACKEND: write request size is out of range");
@@ -1057,7 +1057,7 @@ void *backends_main(void *ptr) {
1057
1058
buffer_flush(b);
1059
buffer_need_bytes(b, data_size);
1060
- if(unlikely(pack_write_request(b->buffer, &data_size))) {
1060
+ if(unlikely(backends_pack_write_request(b->buffer, &data_size))) {
1061
error("BACKEND: cannot pack write request");
1062
continue;
1063
}
@@ -1137,7 +1137,7 @@ void *backends_main(void *ptr) {
1137
if(do_prometheus_remote_write && failures) {
1138
(void) buffer_on_failures;
1139
failures = 0;
1140
- chart_lost_bytes = chart_buffered_bytes = get_write_request_size(); // estimated write request size
1140
+ chart_lost_bytes = chart_buffered_bytes = backends_get_write_request_size(); // estimated write request size
1141
chart_data_lost_events++;
1142
chart_lost_metrics = chart_buffered_metrics;
1143
} else
@@ -1209,7 +1209,7 @@ cleanup:
1209
#if ENABLE_PROMETHEUS_REMOTE_WRITE
1210
buffer_free(http_request_header);
1211
if(do_prometheus_remote_write)
1212
- protocol_buffers_shutdown();
1212
+ backends_protocol_buffers_shutdown();
1213
#endif
1214
1215
#if HAVE_MONGOC
backends/backends.h
+1
-1
@@ -21,7 +21,7 @@ typedef enum backend_types {
21
BACKEND_TYPE_OPENTSDB_USING_TELNET, // Send data to OpenTSDB using telnet API
22
BACKEND_TYPE_OPENTSDB_USING_HTTP, // Send data to OpenTSDB using HTTP API
23
BACKEND_TYPE_JSON, // Stores the data using JSON.
24
- BACKEND_TYPE_PROMETHEUS, // The user selected to use Prometheus backend
24
+ BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE, // The user selected to use Prometheus backend
25
BACKEND_TYPE_KINESIS, // Send message to AWS Kinesis
26
BACKEND_TYPE_MONGODB, // Send data to MongoDB collection
27
BACKEND_TYPE_NUM // Number of backend types
backends/prometheus/backend_prometheus.c
+44
-44
@@ -159,7 +159,7 @@ struct host_variables_callback_options {
159
RRDHOST *host;
160
BUFFER *wb;
161
BACKEND_OPTIONS backend_options;
162
- PROMETHEUS_OUTPUT_OPTIONS output_options;
162
+ BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options;
163
const char *prefix;
164
const char *labels;
165
time_t now;
@@ -174,14 +174,14 @@ static int print_host_variables(RRDVAR *rv, void *data) {
174
if(!opts->host_header_printed) {
175
opts->host_header_printed = 1;
176
177
- if(opts->output_options & PROMETHEUS_OUTPUT_HELP) {
177
+ if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP) {
178
buffer_sprintf(opts->wb, "\n# COMMENT global host and chart variables\n");
179
}
180
}
181
182
calculated_number value = rrdvar2number(rv);
183
if(isnan(value) || isinf(value)) {
184
- if(opts->output_options & PROMETHEUS_OUTPUT_HELP)
184
+ if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP)
185
buffer_sprintf(opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value))?"NAN":"INF");
186
187
return 0;
@@ -196,7 +196,7 @@ static int print_host_variables(RRDVAR *rv, void *data) {
196
197
prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));
198
199
- if(opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
199
+ if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
200
buffer_sprintf(opts->wb
201
, "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n"
202
, opts->prefix
@@ -223,7 +223,7 @@ static int print_host_variables(RRDVAR *rv, void *data) {
223
return 0;
224
}
225
226
-static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, BACKEND_OPTIONS backend_options, time_t after, time_t before, int allhosts, PROMETHEUS_OUTPUT_OPTIONS output_options) {
226
+static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, BACKEND_OPTIONS backend_options, time_t after, time_t before, int allhosts, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {
227
rrdhost_rdlock(host);
228
229
char hostname[PROMETHEUS_ELEMENT_MAX + 1];
@@ -231,13 +231,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
231
232
char labels[PROMETHEUS_LABELS_MAX + 1] = "";
233
if(allhosts) {
234
- if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
234
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
235
buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
236
else
237
buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);
238
239
if(host->tags && *(host->tags)) {
240
- if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
240
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS) {
241
buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);
242
243
// deprecated, exists only for compatibility with older queries
@@ -255,13 +255,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
255
snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
256
}
257
else {
258
- if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
258
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
259
buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
260
else
261
buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);
262
263
if(host->tags && *(host->tags)) {
264
- if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
264
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS) {
265
buffer_sprintf(wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
266
267
// deprecated, exists only for compatibility with older queries
@@ -277,7 +277,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
277
}
278
279
// send custom variables set for the host
280
- if(output_options & PROMETHEUS_OUTPUT_VARIABLES){
280
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_VARIABLES){
281
struct host_variables_callback_options opts = {
282
.host = host,
283
.wb = wb,
@@ -299,7 +299,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
299
char family[PROMETHEUS_ELEMENT_MAX + 1];
300
char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
301
302
- prometheus_label_copy(chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
302
+ prometheus_label_copy(chart, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
303
prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
304
prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
305
@@ -316,14 +316,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
316
homogeneous = 0;
317
}
318
else {
319
- if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE && !(output_options & PROMETHEUS_OUTPUT_HIDEUNITS))
320
- prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS);
319
+ if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE && !(output_options & BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS))
320
+ prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS);
321
}
322
323
- if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
323
+ if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
324
buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n"
325
, (homogeneous)?"homogeneous":"heterogeneous"
326
- , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
326
+ , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
327
, st->context
328
, st->family
329
, st->units
@@ -354,18 +354,18 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
354
// all the dimensions of the chart, has the same algorithm, multiplier and divisor
355
// we add all dimensions as labels
356
357
- prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
357
+ prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
358
359
- if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
359
+ if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
360
buffer_sprintf(wb
361
, "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"
362
, prefix
363
, context
364
, suffix
365
- , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
365
+ , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
366
, st->context
367
, st->family
368
- , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
368
+ , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
369
, rd->multiplier
370
, rd->divisor
371
, h
@@ -373,7 +373,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
373
, t
374
);
375
376
- if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
376
+ if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))
377
buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n"
378
, prefix
379
, context
@@ -381,7 +381,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
381
, t
382
);
383
384
- if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
384
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
385
buffer_sprintf(wb
386
, "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"
387
, prefix
@@ -411,19 +411,19 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
411
// the dimensions of the chart, do not have the same algorithm, multiplier or divisor
412
// we create a metric per dimension
413
414
- prometheus_name_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
414
+ prometheus_name_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
415
416
- if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
416
+ if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
417
buffer_sprintf(wb
418
, "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"
419
, prefix
420
, context
421
, dimension
422
, suffix
423
- , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
423
+ , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
424
, st->context
425
, st->family
426
- , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
426
+ , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
427
, rd->multiplier
428
, rd->divisor
429
, h
@@ -431,7 +431,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
431
, t
432
);
433
434
- if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
434
+ if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))
435
buffer_sprintf(wb, "# COMMENT TYPE %s_%s_%s%s %s\n"
436
, prefix
437
, context
@@ -440,7 +440,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
440
, t
441
);
442
443
- if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
443
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
444
buffer_sprintf(wb
445
, "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"
446
, prefix
@@ -480,21 +480,21 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
480
else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)
481
suffix = "_sum";
482
483
- prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
483
+ prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
484
485
- if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
485
+ if (unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
486
buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n"
487
, prefix
488
, context
489
, units
490
, suffix
491
- , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
491
+ , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
492
, st->units
493
, (unsigned long long)first_t
494
, (unsigned long long)last_t
495
);
496
497
- if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
497
+ if (unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))
498
buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n"
499
, prefix
500
, context
@@ -502,7 +502,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
502
, suffix
503
);
504
505
- if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
505
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
506
buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n"
507
, prefix
508
, context
@@ -581,7 +581,7 @@ inline static void remote_write_split_words(char *str, char **words, int max_wor
581
}
582
}
583
584
-void rrd_stats_remote_write_allmetrics_prometheus(
584
+void backends_rrd_stats_remote_write_allmetrics_prometheus(
585
RRDHOST *host
586
, const char *__hostname
587
, const char *prefix
@@ -595,7 +595,7 @@ void rrd_stats_remote_write_allmetrics_prometheus(
595
char hostname[PROMETHEUS_ELEMENT_MAX + 1];
596
prometheus_label_copy(hostname, __hostname, PROMETHEUS_ELEMENT_MAX);
597
598
- add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
598
+ backends_add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
599
600
if(host->tags && *(host->tags)) {
601
char tags[PROMETHEUS_LABELS_MAX + 1];
@@ -605,10 +605,10 @@ void rrd_stats_remote_write_allmetrics_prometheus(
605
606
remote_write_split_words(tags, words, PROMETHEUS_LABELS_MAX_NUMBER);
607
608
- add_host_info("netdata_host_tags_info", hostname, NULL, NULL, now_realtime_usec() / USEC_PER_MS);
608
+ backends_add_host_info("netdata_host_tags_info", hostname, NULL, NULL, now_realtime_usec() / USEC_PER_MS);
609
610
for(i = 0; words[i] != NULL && words[i + 1] != NULL && (i + 1) < PROMETHEUS_LABELS_MAX_NUMBER; i += 2) {
611
- add_tag(words[i], words[i + 1]);
611
+ backends_add_tag(words[i], words[i + 1]);
612
}
613
}
614
@@ -667,7 +667,7 @@ void rrd_stats_remote_write_allmetrics_prometheus(
667
prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
668
snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", prefix, context, suffix);
669
670
- add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));
670
+ backends_add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));
671
(*count_dims)++;
672
}
673
else {
@@ -677,7 +677,7 @@ void rrd_stats_remote_write_allmetrics_prometheus(
677
prometheus_name_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
678
snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", prefix, context, dimension, suffix);
679
680
- add_metric(name, chart, family, NULL, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));
680
+ backends_add_metric(name, chart, family, NULL, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));
681
(*count_dims)++;
682
}
683
}
@@ -697,7 +697,7 @@ void rrd_stats_remote_write_allmetrics_prometheus(
697
prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
698
snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", prefix, context, units, suffix);
699
700
- add_metric(name, chart, family, dimension, hostname, value, last_t * MSEC_PER_SEC);
700
+ backends_add_metric(name, chart, family, dimension, hostname, value, last_t * MSEC_PER_SEC);
701
(*count_dims)++;
702
}
703
}
@@ -710,7 +710,7 @@ void rrd_stats_remote_write_allmetrics_prometheus(
710
}
711
#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
712
713
-static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_OPTIONS backend_options, const char *server, time_t now, PROMETHEUS_OUTPUT_OPTIONS output_options) {
713
+static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_OPTIONS backend_options, const char *server, time_t now, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {
714
if(!server || !*server) server = "default";
715
716
time_t after = prometheus_server_last_access(server, host, now);
@@ -726,7 +726,7 @@ static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_O
726
after = now - global_backend_update_every;
727
}
728
729
- if(output_options & PROMETHEUS_OUTPUT_HELP) {
729
+ if(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP) {
730
char *mode;
731
if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
732
mode = "as collected";
@@ -751,7 +751,7 @@ static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_O
751
return after;
752
}
753
754
-void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options) {
754
+void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {
755
time_t before = now_realtime_sec();
756
757
// we start at the point we had stopped before
@@ -760,7 +760,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BU
760
rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, backend_options, after, before, 0, output_options);
761
}
762
763
-void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options) {
763
+void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {
764
time_t before = now_realtime_sec();
765
766
// we start at the point we had stopped before
@@ -774,7 +774,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFF
774
}
775
776
#if ENABLE_PROMETHEUS_REMOTE_WRITE
777
-int process_prometheus_remote_write_response(BUFFER *b) {
777
+int backends_process_prometheus_remote_write_response(BUFFER *b) {
778
if(unlikely(!b)) return 1;
779
780
const char *s = buffer_tostring(b);
backends/prometheus/backend_prometheus.h
+14
-14
@@ -5,22 +5,22 @@
5
6
#include "backends/backends.h"
7
8
-typedef enum prometheus_output_flags {
9
- PROMETHEUS_OUTPUT_NONE = 0,
10
- PROMETHEUS_OUTPUT_HELP = (1 << 0),
11
- PROMETHEUS_OUTPUT_TYPES = (1 << 1),
12
- PROMETHEUS_OUTPUT_NAMES = (1 << 2),
13
- PROMETHEUS_OUTPUT_TIMESTAMPS = (1 << 3),
14
- PROMETHEUS_OUTPUT_VARIABLES = (1 << 4),
15
- PROMETHEUS_OUTPUT_OLDUNITS = (1 << 5),
16
- PROMETHEUS_OUTPUT_HIDEUNITS = (1 << 6)
17
-} PROMETHEUS_OUTPUT_OPTIONS;
8
+typedef enum backends_prometheus_output_flags {
9
+ BACKENDS_PROMETHEUS_OUTPUT_NONE = 0,
10
+ BACKENDS_PROMETHEUS_OUTPUT_HELP = (1 << 0),
11
+ BACKENDS_PROMETHEUS_OUTPUT_TYPES = (1 << 1),
12
+ BACKENDS_PROMETHEUS_OUTPUT_NAMES = (1 << 2),
13
+ BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS = (1 << 3),
14
+ BACKENDS_PROMETHEUS_OUTPUT_VARIABLES = (1 << 4),
15
+ BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS = (1 << 5),
16
+ BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS = (1 << 6)
17
+} BACKENDS_PROMETHEUS_OUTPUT_OPTIONS;
18
19
-extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
20
-extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
19
+extern void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options);
20
+extern void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options);
21
22
#if ENABLE_PROMETHEUS_REMOTE_WRITE
23
-extern void rrd_stats_remote_write_allmetrics_prometheus(
23
+extern void backends_rrd_stats_remote_write_allmetrics_prometheus(
24
RRDHOST *host
25
, const char *__hostname
26
, const char *prefix
@@ -31,7 +31,7 @@ extern void rrd_stats_remote_write_allmetrics_prometheus(
31
, size_t *count_dims
32
, size_t *count_dims_skipped
33
);
34
-extern int process_prometheus_remote_write_response(BUFFER *b);
34
+extern int backends_process_prometheus_remote_write_response(BUFFER *b);
35
#endif
36
37
#endif //NETDATA_BACKEND_PROMETHEUS_H
backends/prometheus/remote_write/remote_write.cc
+11
-12
@@ -1,25 +1,24 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include <snappy.h>
4
-#include "remote_write.pb.h"
4
+#include "../../../exporting/prometheus/remote_write/remote_write.pb.h"
5
#include "remote_write.h"
6
7
using namespace prometheus;
8
9
+static google::protobuf::Arena arena;
10
+static WriteRequest *write_request;
11
10
-google::protobuf::Arena arena;
11
-WriteRequest *write_request;
12
-
13
-void init_write_request() {
12
+void backends_init_write_request() {
13
GOOGLE_PROTOBUF_VERIFY_VERSION;
14
write_request = google::protobuf::Arena::CreateMessage<WriteRequest>(&arena);
15
}
16
18
-void clear_write_request() {
17
+void backends_clear_write_request() {
18
write_request->clear_timeseries();
19
}
20
22
-void add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp) {
21
+void backends_add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp) {
22
TimeSeries *timeseries;
23
Sample *sample;
24
Label *label;
@@ -52,7 +51,7 @@ void add_host_info(const char *name, const char *instance, const char *applicati
51
}
52
53
// adds tag to the last created timeseries
55
-void add_tag(char *tag, char *value) {
54
+void backends_add_tag(char *tag, char *value) {
55
TimeSeries *timeseries;
56
Label *label;
57
@@ -63,7 +62,7 @@ void add_tag(char *tag, char *value) {
62
label->set_value(value);
63
}
64
66
-void add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp) {
65
+void backends_add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp) {
66
TimeSeries *timeseries;
67
Sample *sample;
68
Label *label;
@@ -97,7 +96,7 @@ void add_metric(const char *name, const char *chart, const char *family, const c
96
sample->set_timestamp(timestamp);
97
}
98
100
-size_t get_write_request_size(){
99
+size_t backends_get_write_request_size(){
100
#if GOOGLE_PROTOBUF_VERSION < 3001000
101
size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSize());
102
#else
@@ -107,7 +106,7 @@ size_t get_write_request_size(){
106
return (size < INT_MAX)?size:0;
107
}
108
110
-int pack_write_request(char *buffer, size_t *size) {
109
+int backends_pack_write_request(char *buffer, size_t *size) {
110
std::string uncompressed_write_request;
111
if(write_request->SerializeToString(&uncompressed_write_request) == false) return 1;
112
@@ -116,6 +115,6 @@ int pack_write_request(char *buffer, size_t *size) {
115
return 0;
116
}
117
119
-void protocol_buffers_shutdown() {
118
+void backends_protocol_buffers_shutdown() {
119
google::protobuf::ShutdownProtobufLibrary();
120
}
backends/prometheus/remote_write/remote_write.h
+8
-8
@@ -7,21 +7,21 @@
7
extern "C" {
8
#endif
9
10
-void init_write_request();
10
+void backends_init_write_request();
11
12
-void clear_write_request();
12
+void backends_clear_write_request();
13
14
-void add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp);
14
+void backends_add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp);
15
16
-void add_tag(char *tag, char *value);
16
+void backends_add_tag(char *tag, char *value);
17
18
-void add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp);
18
+void backends_add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp);
19
20
-size_t get_write_request_size();
20
+size_t backends_get_write_request_size();
21
22
-int pack_write_request(char *buffer, size_t *size);
22
+int backends_pack_write_request(char *buffer, size_t *size);
23
24
-void protocol_buffers_shutdown();
24
+void backends_protocol_buffers_shutdown();
25
26
#ifdef __cplusplus
27
}
configure.ac
+2
@@ -1316,6 +1316,8 @@ AC_CONFIG_FILES([
1316
exporting/graphite/Makefile
1317
exporting/json/Makefile
1318
exporting/opentsdb/Makefile
1319
+ exporting/prometheus/Makefile
1320
+ exporting/prometheus/remote_write/Makefile
1321
exporting/aws_kinesis/Makefile
1322
exporting/tests/Makefile
1323
health/Makefile
exporting/Makefile.am
+1
@@ -8,6 +8,7 @@ SUBDIRS = \
8
graphite \
9
json \
10
opentsdb \
11
+ prometheus \
12
aws_kinesis \
13
$(NULL)
14
exporting/aws_kinesis/aws_kinesis.c
+3
@@ -25,6 +25,9 @@ int init_aws_kinesis_instance(struct instance *instance)
25
instance->end_host_formatting = flush_host_labels;
26
instance->end_batch_formatting = NULL;
27
28
+ instance->send_header = NULL;
29
+ instance->check_response = NULL;
30
+
31
instance->buffer = (void *)buffer_create(0);
32
if (!instance->buffer) {
33
error("EXPORTING: cannot create buffer for AWS Kinesis exporting connector instance %s", instance->config.name);
exporting/exporting_engine.h
+9
-35
@@ -13,41 +13,8 @@
13
14
extern struct config exporting_config;
15
16
-#define EXPORTER_DATA_SOURCE "data source"
17
-#define EXPORTER_DATA_SOURCE_DEFAULT "average"
18
-
19
-#define EXPORTER_DESTINATION "destination"
20
-#define EXPORTER_DESTINATION_DEFAULT "localhost"
21
-
22
-#define EXPORTER_UPDATE_EVERY "update every"
23
-#define EXPORTER_UPDATE_EVERY_DEFAULT 10
24
-
25
-#define EXPORTER_BUF_ONFAIL "buffer on failures"
26
-#define EXPORTER_BUF_ONFAIL_DEFAULT 10
27
-
28
-#define EXPORTER_TIMEOUT_MS "timeout ms"
29
-#define EXPORTER_TIMEOUT_MS_DEFAULT 10000
30
-
31
-#define EXPORTER_SEND_CHART_MATCH "send charts matching"
32
-#define EXPORTER_SEND_CHART_MATCH_DEFAULT "*"
33
-
34
-#define EXPORTER_SEND_HOST_MATCH "send hosts matching"
35
-#define EXPORTER_SEND_HOST_MATCH_DEFAULT "localhost *"
36
-
37
-#define EXPORTER_SEND_CONFIGURED_LABELS "send configured labels"
38
-#define EXPORTER_SEND_CONFIGURED_LABELS_DEFAULT CONFIG_BOOLEAN_YES
39
-
40
-#define EXPORTER_SEND_AUTOMATIC_LABELS "send automatic labels"
41
-#define EXPORTER_SEND_AUTOMATIC_LABELS_DEFAULT CONFIG_BOOLEAN_NO
42
-
43
-#define EXPORTER_SEND_NAMES "send names instead of ids"
44
-#define EXPORTER_SEND_NAMES_DEFAULT CONFIG_BOOLEAN_YES
45
-
46
-#define EXPORTER_KINESIS_STREAM_NAME "stream name"
47
-#define EXPORTER_KINESIS_STREAM_NAME_DEFAULT "netdata"
48
-
49
-#define EXPORTER_AWS_ACCESS_KEY_ID "aws_access_key_id"
50
-#define EXPORTER_AWS_SECRET_ACCESS_KEY "aws_secret_access_key"
16
+#define EXPORTING_UPDATE_EVERY_OPTION_NAME "update every"
17
+#define EXPORTING_UPDATE_EVERY_DEFAULT 10
18
19
typedef enum exporting_options {
20
EXPORTING_OPTION_NONE = 0,
@@ -98,6 +65,10 @@ struct simple_connector_config {
65
int default_port;
66
};
67
68
+struct prometheus_remote_write_specific_config {
69
+ char *remote_write_path;
70
+};
71
+
72
struct aws_kinesis_specific_config {
73
char *stream_name;
74
char *auth_key_id;
@@ -152,6 +123,9 @@ struct instance {
123
int (*end_host_formatting)(struct instance *instance, RRDHOST *host);
124
int (*end_batch_formatting)(struct instance *instance);
125
126
+ int (*send_header)(int *sock, struct instance *instance);
127
+ int (*check_response)(BUFFER *buffer, struct instance *instance);
128
+
129
void *connector_specific_data;
130
131
size_t index;
exporting/graphite/graphite.c
+3
@@ -29,6 +29,9 @@ int init_graphite_instance(struct instance *instance)
29
instance->end_host_formatting = flush_host_labels;
30
instance->end_batch_formatting = NULL;
31
32
+ instance->send_header = NULL;
33
+ instance->check_response = exporting_discard_response;
34
+
35
instance->buffer = (void *)buffer_create(0);
36
if (!instance->buffer) {
37
error("EXPORTING: cannot create buffer for graphite exporting connector instance %s", instance->config.name);
exporting/init_connectors.c
+7
@@ -5,6 +5,7 @@
5
#include "json/json.h"
6
#include "opentsdb/opentsdb.h"
7
#include "aws_kinesis/aws_kinesis.h"
8
+#include "prometheus/remote_write/remote_write.h"
9
10
/**
11
* Initialize connectors
@@ -37,6 +38,12 @@ int init_connectors(struct engine *engine)
38
if (init_opentsdb_http_instance(instance) != 0)
39
return 1;
40
break;
41
+ case BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE:
42
+#if ENABLE_PROMETHEUS_REMOTE_WRITE
43
+ if (init_prometheus_remote_write_instance(instance) != 0)
44
+ return 1;
45
+#endif
46
+ break;
47
case BACKEND_TYPE_KINESIS:
48
#if HAVE_KINESIS
49
if (init_aws_kinesis_instance(instance) != 0)
exporting/json/json.c
+3
@@ -29,6 +29,9 @@ int init_json_instance(struct instance *instance)
29
instance->end_host_formatting = flush_host_labels;
30
instance->end_batch_formatting = NULL;
31
32
+ instance->send_header = NULL;
33
+ instance->check_response = exporting_discard_response;
34
+
35
instance->buffer = (void *)buffer_create(0);
36
if (!instance->buffer) {
37
error("EXPORTING: cannot create buffer for json exporting connector instance %s", instance->config.name);
exporting/opentsdb/opentsdb.c
+6
@@ -29,6 +29,9 @@ int init_opentsdb_telnet_instance(struct instance *instance)
29
instance->end_host_formatting = flush_host_labels;
30
instance->end_batch_formatting = NULL;
31
32
+ instance->send_header = NULL;
33
+ instance->check_response = exporting_discard_response;
34
+
35
instance->buffer = (void *)buffer_create(0);
36
if (!instance->buffer) {
37
error("EXPORTING: cannot create buffer for opentsdb telnet exporting connector instance %s", instance->config.name);
@@ -67,6 +70,9 @@ int init_opentsdb_http_instance(struct instance *instance)
70
instance->end_host_formatting = flush_host_labels;
71
instance->end_batch_formatting = NULL;
72
73
+ instance->send_header = NULL;
74
+ instance->check_response = exporting_discard_response;
75
+
76
instance->buffer = (void *)buffer_create(0);
77
if (!instance->buffer) {
78
error("EXPORTING: cannot create buffer for opentsdb HTTP exporting connector instance %s", instance->config.name);
exporting/prometheus/Makefile.am
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
+
6
+SUBDIRS = \
7
+ remote_write \
8
+ $(NULL)
9
+
10
+dist_noinst_DATA = \
11
+ README.md \
12
+ $(NULL)
exporting/prometheus/README.md
new
+452
@@ -0,0 +1,452 @@
1
+# Using Netdata with Prometheus
2
+
3
+> IMPORTANT: the format Netdata sends metrics to Prometheus has changed since Netdata v1.7. The new Prometheus exporting
4
+> connector for Netdata supports a lot more features and is aligned to the development of the rest of the Netdata
5
+> exporting connectors.
6
+
7
+Prometheus is a distributed monitoring system which offers a very simple setup along with a robust data model. Recently
8
+Netdata added support for Prometheus. I'm going to quickly show you how to install both Netdata and Prometheus on the
9
+same server. We can then use Grafana pointed at Prometheus to obtain long term metrics Netdata offers. I'm assuming we
10
+are starting at a fresh ubuntu shell (whether you'd like to follow along in a VM or a cloud instance is up to you).
11
+
12
+## Installing Netdata and Prometheus
13
+
14
+### Installing Netdata
15
+
16
+There are number of ways to install Netdata according to [Installation](../../packaging/installer/). The suggested way
17
+of installing the latest Netdata and keep it upgrade automatically. Using one line installation:
18
+
19
+```sh
20
+bash <(curl -Ss https://my-netdata.io/kickstart.sh)
21
+```
22
+
23
+At this point we should have Netdata listening on port 19999. Attempt to take your browser here:
24
+
25
+```sh
26
+http://your.netdata.ip:19999
27
+```
28
+
29
+_(replace `your.netdata.ip` with the IP or hostname of the server running Netdata)_
30
+
31
+### Installing Prometheus
32
+
33
+In order to install Prometheus we are going to introduce our own systemd startup script along with an example of
34
+prometheus.yaml configuration. Prometheus needs to be pointed to your server at a specific target url for it to scrape
35
+Netdata's api. Prometheus is always a pull model meaning Netdata is the passive client within this architecture.
36
+Prometheus always initiates the connection with Netdata.
37
+
38
+#### Download Prometheus
39
+
40
+```sh
41
+cd /tmp && curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest \
42
+| grep "browser_download_url.*linux-amd64.tar.gz" \
43
+| cut -d '"' -f 4 \
44
+| wget -qi -
45
+```
46
+
47
+#### Create prometheus system user
48
+
49
+```sh
50
+sudo useradd -r prometheus
51
+```
52
+
53
+#### Create prometheus directory
54
+
55
+```sh
56
+sudo mkdir /opt/prometheus
57
+sudo chown prometheus:prometheus /opt/prometheus
58
+```
59
+
60
+#### Untar prometheus directory
61
+
62
+```sh
63
+sudo tar -xvf /tmp/prometheus-*linux-amd64.tar.gz -C /opt/prometheus --strip=1
64
+```
65
+
66
+#### Install prometheus.yml
67
+
68
+We will use the following `prometheus.yml` file. Save it at `/opt/prometheus/prometheus.yml`.
69
+
70
+Make sure to replace `your.netdata.ip` with the IP or hostname of the host running Netdata.
71
+
72
+```yaml
73
+# my global config
74
+global:
75
+ scrape_interval: 5s # Set the scrape interval to every 5 seconds. Default is every 1 minute.
76
+ evaluation_interval: 5s # Evaluate rules every 5 seconds. The default is every 1 minute.
77
+ # scrape_timeout is set to the global default (10s).
78
+
79
+ # Attach these labels to any time series or alerts when communicating with
80
+ # external systems (federation, remote storage, Alertmanager).
81
+ external_labels:
82
+ monitor: 'codelab-monitor'
83
+
84
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
85
+rule_files:
86
+ # - "first.rules"
87
+ # - "second.rules"
88
+
89
+# A scrape configuration containing exactly one endpoint to scrape:
90
+# Here it's Prometheus itself.
91
+scrape_configs:
92
+ # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
93
+ - job_name: 'prometheus'
94
+
95
+ # metrics_path defaults to '/metrics'
96
+ # scheme defaults to 'http'.
97
+
98
+ static_configs:
99
+ - targets: ['0.0.0.0:9090']
100
+
101
+ - job_name: 'netdata-scrape'
102
+
103
+ metrics_path: '/api/v1/allmetrics'
104
+ params:
105
+ # format: prometheus | prometheus_all_hosts
106
+ # You can use `prometheus_all_hosts` if you want Prometheus to set the `instance` to your hostname instead of IP
107
+ format: [prometheus]
108
+ #
109
+ # sources: as-collected | raw | average | sum | volume
110
+ # default is: average
111
+ #source: [as-collected]
112
+ #
113
+ # server name for this prometheus - the default is the client IP
114
+ # for Netdata to uniquely identify it
115
+ #server: ['prometheus1']
116
+ honor_labels: true
117
+
118
+ static_configs:
119
+ - targets: ['{your.netdata.ip}:19999']
120
+```
121
+
122
+#### Install nodes.yml
123
+
124
+The following is completely optional, it will enable Prometheus to generate alerts from some NetData sources. Tweak the
125
+values to your own needs. We will use the following `nodes.yml` file below. Save it at `/opt/prometheus/nodes.yml`, and
126
+add a _- "nodes.yml"_ entry under the _rule_files:_ section in the example prometheus.yml file above.
127
+
128
+```yaml
129
+groups:
130
+- name: nodes
131
+
132
+ rules:
133
+ - alert: node_high_cpu_usage_70
134
+ expr: avg(rate(netdata_cpu_cpu_percentage_average{dimension="idle"}[1m])) by (job) > 70
135
+ for: 1m
136
+ annotations:
137
+ description: '{{ $labels.job }} on ''{{ $labels.job }}'' CPU usage is at {{ humanize $value }}%.'
138
+ summary: CPU alert for container node '{{ $labels.job }}'
139
+
140
+ - alert: node_high_memory_usage_70
141
+ expr: 100 / sum(netdata_system_ram_MB_average) by (job)
142
+ * sum(netdata_system_ram_MB_average{dimension=~"free|cached"}) by (job) < 30
143
+ for: 1m
144
+ annotations:
145
+ description: '{{ $labels.job }} memory usage is {{ humanize $value}}%.'
146
+ summary: Memory alert for container node '{{ $labels.job }}'
147
+
148
+ - alert: node_low_root_filesystem_space_20
149
+ expr: 100 / sum(netdata_disk_space_GB_average{family="/"}) by (job)
150
+ * sum(netdata_disk_space_GB_average{family="/",dimension=~"avail|cached"}) by (job) < 20
151
+ for: 1m
152
+ annotations:
153
+ description: '{{ $labels.job }} root filesystem space is {{ humanize $value}}%.'
154
+ summary: Root filesystem alert for container node '{{ $labels.job }}'
155
+
156
+ - alert: node_root_filesystem_fill_rate_6h
157
+ expr: predict_linear(netdata_disk_space_GB_average{family="/",dimension=~"avail|cached"}[1h], 6 * 3600) < 0
158
+ for: 1h
159
+ labels:
160
+ severity: critical
161
+ annotations:
162
+ description: Container node {{ $labels.job }} root filesystem is going to fill up in 6h.
163
+ summary: Disk fill alert for Swarm node '{{ $labels.job }}'
164
+```
165
+
166
+#### Install prometheus.service
167
+
168
+Save this service file as `/etc/systemd/system/prometheus.service`:
169
+
170
+```sh
171
+[Unit]
172
+Description=Prometheus Server
173
+AssertPathExists=/opt/prometheus
174
+
175
+[Service]
176
+Type=simple
177
+WorkingDirectory=/opt/prometheus
178
+User=prometheus
179
+Group=prometheus
180
+ExecStart=/opt/prometheus/prometheus --config.file=/opt/prometheus/prometheus.yml --log.level=info
181
+ExecReload=/bin/kill -SIGHUP $MAINPID
182
+ExecStop=/bin/kill -SIGINT $MAINPID
183
+
184
+[Install]
185
+WantedBy=multi-user.target
186
+```
187
+
188
+##### Start Prometheus
189
+
190
+```sh
191
+sudo systemctl start prometheus
192
+sudo systemctl enable prometheus
193
+```
194
+
195
+Prometheus should now start and listen on port 9090. Attempt to head there with your browser.
196
+
197
+If everything is working correctly when you fetch `http://your.prometheus.ip:9090` you will see a 'Status' tab. Click
198
+this and click on 'targets' We should see the Netdata host as a scraped target.
199
+
200
+---
201
+
202
+## Netdata support for Prometheus
203
+
204
+> IMPORTANT: the format Netdata sends metrics to Prometheus has changed since Netdata v1.6. The new format allows easier
205
+> queries for metrics and supports both `as collected` and normalized metrics.
206
+
207
+Before explaining the changes, we have to understand the key differences between Netdata and Prometheus.
208
+
209
+### understanding Netdata metrics
210
+
211
+#### charts
212
+
213
+Each chart in Netdata has several properties (common to all its metrics):
214
+
215
+- `chart_id` - uniquely identifies a chart.
216
+
217
+- `chart_name` - a more human friendly name for `chart_id`, also unique.
218
+
219
+- `context` - this is the template of the chart. All disk I/O charts have the same context, all mysql requests charts
220
+ have the same context, etc. This is used for alarm templates to match all the charts they should be attached to.
221
+
222
+- `family` groups a set of charts together. It is used as the submenu of the dashboard.
223
+
224
+- `units` is the units for all the metrics attached to the chart.
225
+
226
+#### dimensions
227
+
228
+Then each Netdata chart contains metrics called `dimensions`. All the dimensions of a chart have the same units of
229
+measurement, and are contextually in the same category (ie. the metrics for disk bandwidth are `read` and `write` and
230
+they are both in the same chart).
231
+
232
+### Netdata data source
233
+
234
+Netdata can send metrics to Prometheus from 3 data sources:
235
+
236
+- `as collected` or `raw` - this data source sends the metrics to Prometheus as they are collected. No conversion is
237
+ done by Netdata. The latest value for each metric is just given to Prometheus. This is the most preferred method by
238
+ Prometheus, but it is also the harder to work with. To work with this data source, you will need to understand how
239
+ to get meaningful values out of them.
240
+
241
+ The format of the metrics is: `CONTEXT{chart="CHART",family="FAMILY",dimension="DIMENSION"}`.
242
+
243
+ If the metric is a counter (`incremental` in Netdata lingo), `_total` is appended the context.
244
+
245
+ Unlike Prometheus, Netdata allows each dimension of a chart to have a different algorithm and conversion constants
246
+ (`multiplier` and `divisor`). In this case, that the dimensions of a charts are heterogeneous, Netdata will use this
247
+ format: `CONTEXT_DIMENSION{chart="CHART",family="FAMILY"}`
248
+
249
+- `average` - this data source uses the Netdata database to send the metrics to Prometheus as they are presented on
250
+ the Netdata dashboard. So, all the metrics are sent as gauges, at the units they are presented in the Netdata
251
+ dashboard charts. This is the easiest to work with.
252
+
253
+ The format of the metrics is: `CONTEXT_UNITS_average{chart="CHART",family="FAMILY",dimension="DIMENSION"}`.
254
+
255
+ When this source is used, Netdata keeps track of the last access time for each Prometheus server fetching the
256
+ metrics. This last access time is used at the subsequent queries of the same Prometheus server to identify the
257
+ time-frame the `average` will be calculated.
258
+
259
+ So, no matter how frequently Prometheus scrapes Netdata, it will get all the database data.
260
+ To identify each Prometheus server, Netdata uses by default the IP of the client fetching the metrics.
261
+
262
+ If there are multiple Prometheus servers fetching data from the same Netdata, using the same IP, each Prometheus
263
+ server can append `server=NAME` to the URL. Netdata will use this `NAME` to uniquely identify the Prometheus server.
264
+
265
+- `sum` or `volume`, is like `average` but instead of averaging the values, it sums them.
266
+
267
+ The format of the metrics is: `CONTEXT_UNITS_sum{chart="CHART",family="FAMILY",dimension="DIMENSION"}`. All the
268
+ other operations are the same with `average`.
269
+
270
+ To change the data source to `sum` or `as-collected` you need to provide the `source` parameter in the request URL.
271
+ e.g.: `http://your.netdata.ip:19999/api/v1/allmetrics?format=prometheus&help=yes&source=as-collected`
272
+
273
+ Keep in mind that early versions of Netdata were sending the metrics as: `CHART_DIMENSION{}`.
274
+
275
+### Querying Metrics
276
+
277
+Fetch with your web browser this URL:
278
+
279
+`http://your.netdata.ip:19999/api/v1/allmetrics?format=prometheus&help=yes`
280
+
281
+_(replace `your.netdata.ip` with the ip or hostname of your Netdata server)_
282
+
283
+Netdata will respond with all the metrics it sends to Prometheus.
284
+
285
+If you search that page for `"system.cpu"` you will find all the metrics Netdata is exporting to Prometheus for this
286
+chart. `system.cpu` is the chart name on the Netdata dashboard (on the Netdata dashboard all charts have a text heading
287
+such as : `Total CPU utilization (system.cpu)`. What we are interested here in the chart name: `system.cpu`).
288
+
289
+Searching for `"system.cpu"` reveals:
290
+
291
+```sh
292
+# COMMENT homogeneous chart "system.cpu", context "system.cpu", family "cpu", units "percentage"
293
+# COMMENT netdata_system_cpu_percentage_average: dimension "guest_nice", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
294
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="guest_nice"} 0.0000000 1500066662000
295
+# COMMENT netdata_system_cpu_percentage_average: dimension "guest", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
296
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="guest"} 1.7837326 1500066662000
297
+# COMMENT netdata_system_cpu_percentage_average: dimension "steal", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
298
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="steal"} 0.0000000 1500066662000
299
+# COMMENT netdata_system_cpu_percentage_average: dimension "softirq", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
300
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="softirq"} 0.5275442 1500066662000
301
+# COMMENT netdata_system_cpu_percentage_average: dimension "irq", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
302
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="irq"} 0.2260836 1500066662000
303
+# COMMENT netdata_system_cpu_percentage_average: dimension "user", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
304
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="user"} 2.3362762 1500066662000
305
+# COMMENT netdata_system_cpu_percentage_average: dimension "system", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
306
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 1.7961062 1500066662000
307
+# COMMENT netdata_system_cpu_percentage_average: dimension "nice", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
308
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="nice"} 0.0000000 1500066662000
309
+# COMMENT netdata_system_cpu_percentage_average: dimension "iowait", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
310
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="iowait"} 0.9671802 1500066662000
311
+# COMMENT netdata_system_cpu_percentage_average: dimension "idle", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive
312
+netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="idle"} 92.3630770 1500066662000
313
+```
314
+
315
+_(Netdata response for `system.cpu` with source=`average`)_
316
+
317
+In `average` or `sum` data sources, all values are normalized and are reported to Prometheus as gauges. Now, use the
318
+'expression' text form in Prometheus. Begin to type the metrics we are looking for: `netdata_system_cpu`. You should see
319
+that the text form begins to auto-fill as Prometheus knows about this metric.
320
+
321
+If the data source was `as collected`, the response would be:
322
+
323
+```sh
324
+# COMMENT homogeneous chart "system.cpu", context "system.cpu", family "cpu", units "percentage"
325
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "guest_nice", value * 1 / 1 delta gives percentage (counter)
326
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="guest_nice"} 0 1500066716438
327
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "guest", value * 1 / 1 delta gives percentage (counter)
328
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="guest"} 63945 1500066716438
329
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "steal", value * 1 / 1 delta gives percentage (counter)
330
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="steal"} 0 1500066716438
331
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "softirq", value * 1 / 1 delta gives percentage (counter)
332
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="softirq"} 8295 1500066716438
333
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "irq", value * 1 / 1 delta gives percentage (counter)
334
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="irq"} 4079 1500066716438
335
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "user", value * 1 / 1 delta gives percentage (counter)
336
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="user"} 116488 1500066716438
337
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "system", value * 1 / 1 delta gives percentage (counter)
338
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="system"} 35084 1500066716438
339
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "nice", value * 1 / 1 delta gives percentage (counter)
340
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="nice"} 505 1500066716438
341
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "iowait", value * 1 / 1 delta gives percentage (counter)
342
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="iowait"} 23314 1500066716438
343
+# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "idle", value * 1 / 1 delta gives percentage (counter)
344
+netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="idle"} 918470 1500066716438
345
+```
346
+
347
+_(Netdata response for `system.cpu` with source=`as-collected`)_
348
+
349
+For more information check Prometheus documentation.
350
+
351
+### Streaming data from upstream hosts
352
+
353
+The `format=prometheus` parameter only exports the host's Netdata metrics. If you are using the master/slave
354
+functionality of Netdata this ignores any upstream hosts - so you should consider using the below in your
355
+**prometheus.yml**:
356
+
357
+```yaml
358
+ metrics_path: '/api/v1/allmetrics'
359
+ params:
360
+ format: [prometheus_all_hosts]
361
+ honor_labels: true
362
+```
363
+
364
+This will report all upstream host data, and `honor_labels` will make Prometheus take note of the instance names
365
+provided.
366
+
367
+### Timestamps
368
+
369
+To pass the metrics through Prometheus pushgateway, Netdata supports the option `×tamps=no` to send the metrics
370
+without timestamps.
371
+
372
+## Netdata host variables
373
+
374
+Netdata collects various system configuration metrics, like the max number of TCP sockets supported, the max number of
375
+files allowed system-wide, various IPC sizes, etc. These metrics are not exposed to Prometheus by default.
376
+
377
+To expose them, append `variables=yes` to the Netdata URL.
378
+
379
+### TYPE and HELP
380
+
381
+To save bandwidth, and because Prometheus does not use them anyway, `# TYPE` and `# HELP` lines are suppressed. If
382
+wanted they can be re-enabled via `types=yes` and `help=yes`, e.g.
383
+`/api/v1/allmetrics?format=prometheus&types=yes&help=yes`
384
+
385
+### Names and IDs
386
+
387
+Netdata supports names and IDs for charts and dimensions. Usually IDs are unique identifiers as read by the system and
388
+names are human friendly labels (also unique).
389
+
390
+Most charts and metrics have the same ID and name, but in several cases they are different: disks with device-mapper,
391
+interrupts, QoS classes, statsd synthetic charts, etc.
392
+
393
+The default is controlled in `netdata.conf`:
394
+
395
+```conf
396
+[backend]
397
+ send names instead of ids = yes | no
398
+```
399
+
400
+You can overwrite it from Prometheus, by appending to the URL:
401
+
402
+- `&names=no` to get IDs (the old behaviour)
403
+- `&names=yes` to get names
404
+
405
+### Filtering metrics sent to Prometheus
406
+
407
+Netdata can filter the metrics it sends to Prometheus with this setting:
408
+
409
+```conf
410
+[backend]
411
+ send charts matching = *
412
+```
413
+
414
+This settings accepts a space separated list of [simple patterns](../../libnetdata/simple_pattern/README.md) to match
415
+the **charts** to be sent to Prometheus. Each pattern can use `*` as wildcard, any number of times (e.g `*a*b*c*` is
416
+valid). Patterns starting with `!` give a negative match (e.g `!*.bad users.* groups.*` will send all the users and
417
+groups except `bad` user and `bad` group). The order is important: the first match (positive or negative) left to right,
418
+is used.
419
+
420
+### Changing the prefix of Netdata metrics
421
+
422
+Netdata sends all metrics prefixed with `netdata_`. You can change this in `netdata.conf`, like this:
423
+
424
+```conf
425
+[backend]
426
+ prefix = netdata
427
+```
428
+
429
+It can also be changed from the URL, by appending `&prefix=netdata`.
430
+
431
+### Metric Units
432
+
433
+The default source `average` adds the unit of measurement to the name of each metric (e.g. `_KiB_persec`). To hide the
434
+units and get the same metric names as with the other sources, append to the URL `&hideunits=yes`.
435
+
436
+The units were standardized in v1.12, with the effect of changing the metric names. To get the metric names as they were
437
+before v1.12, append to the URL `&oldunits=yes`
438
+
439
+### Accuracy of `average` and `sum` data sources
440
+
441
+When the data source is set to `average` or `sum`, Netdata remembers the last access of each client accessing Prometheus
442
+metrics and uses this last access time to respond with the `average` or `sum` of all the entries in the database since
443
+that. This means that Prometheus servers are not losing data when they access Netdata with data source = `average` or
444
+`sum`.
445
+
446
+To uniquely identify each Prometheus server, Netdata uses the IP of the client accessing the metrics. If however the IP
447
+is not good enough for identifying a single Prometheus server (e.g. when Prometheus servers are accessing Netdata
448
+through a web proxy, or when multiple Prometheus servers are NATed to a single IP), each Prometheus may append
449
+`&server=NAME` to the URL. This `NAME` is used by Netdata to uniquely identify each Prometheus server and keep track of
450
+its last access time.
451
+
452
+[](<>)
exporting/prometheus/prometheus.c
new
+727
@@ -0,0 +1,727 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#define EXPORTINGS_INTERNALS
4
+#include "prometheus.h"
5
+
6
+// ----------------------------------------------------------------------------
7
+// PROMETHEUS
8
+// /api/v1/allmetrics?format=prometheus and /api/v1/allmetrics?format=prometheus_all_hosts
9
+
10
+inline int can_send_rrdset(struct instance *instance, RRDSET *st)
11
+{
12
+ RRDHOST *host = st->rrdhost;
13
+ (void)host;
14
+
15
+ if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE)))
16
+ return 0;
17
+
18
+ if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) {
19
+ // we have not checked this chart
20
+ if (simple_pattern_matches(instance->config.charts_pattern, st->id) ||
21
+ simple_pattern_matches(instance->config.charts_pattern, st->name))
22
+ rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND);
23
+ else {
24
+ rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE);
25
+ debug(
26
+ D_BACKEND,
27
+ "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.",
28
+ st->id,
29
+ host->hostname);
30
+ return 0;
31
+ }
32
+ }
33
+
34
+ if (unlikely(!rrdset_is_available_for_backends(st))) {
35
+ debug(
36
+ D_BACKEND,
37
+ "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.",
38
+ st->id,
39
+ host->hostname);
40
+ return 0;
41
+ }
42
+
43
+ if (unlikely(
44
+ st->rrd_memory_mode == RRD_MEMORY_MODE_NONE &&
45
+ !(BACKEND_OPTIONS_DATA_SOURCE(instance->config.options) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {
46
+ debug(
47
+ D_BACKEND,
48
+ "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.",
49
+ st->id,
50
+ host->hostname,
51
+ rrd_memory_mode_name(host->rrd_memory_mode));
52
+ return 0;
53
+ }
54
+
55
+ return 1;
56
+}
57
+
58
+static struct prometheus_server {
59
+ const char *server;
60
+ uint32_t hash;
61
+ RRDHOST *host;
62
+ time_t last_access;
63
+ struct prometheus_server *next;
64
+} *prometheus_server_root = NULL;
65
+
66
+static inline time_t prometheus_server_last_access(const char *server, RRDHOST *host, time_t now)
67
+{
68
+ static netdata_mutex_t prometheus_server_root_mutex = NETDATA_MUTEX_INITIALIZER;
69
+
70
+ uint32_t hash = simple_hash(server);
71
+
72
+ netdata_mutex_lock(&prometheus_server_root_mutex);
73
+
74
+ struct prometheus_server *ps;
75
+ for (ps = prometheus_server_root; ps; ps = ps->next) {
76
+ if (host == ps->host && hash == ps->hash && !strcmp(server, ps->server)) {
77
+ time_t last = ps->last_access;
78
+ ps->last_access = now;
79
+ netdata_mutex_unlock(&prometheus_server_root_mutex);
80
+ return last;
81
+ }
82
+ }
83
+
84
+ ps = callocz(1, sizeof(struct prometheus_server));
85
+ ps->server = strdupz(server);
86
+ ps->hash = hash;
87
+ ps->host = host;
88
+ ps->last_access = now;
89
+ ps->next = prometheus_server_root;
90
+ prometheus_server_root = ps;
91
+
92
+ netdata_mutex_unlock(&prometheus_server_root_mutex);
93
+ return 0;
94
+}
95
+
96
+inline size_t prometheus_name_copy(char *d, const char *s, size_t usable)
97
+{
98
+ size_t n;
99
+
100
+ for (n = 0; *s && n < usable; d++, s++, n++) {
101
+ register char c = *s;
102
+
103
+ if (!isalnum(c))
104
+ *d = '_';
105
+ else
106
+ *d = c;
107
+ }
108
+ *d = '\0';
109
+
110
+ return n;
111
+}
112
+
113
+inline size_t prometheus_label_copy(char *d, const char *s, size_t usable)
114
+{
115
+ size_t n;
116
+
117
+ // make sure we can escape one character without overflowing the buffer
118
+ usable--;
119
+
120
+ for (n = 0; *s && n < usable; d++, s++, n++) {
121
+ register char c = *s;
122
+
123
+ if (unlikely(c == '"' || c == '\\' || c == '\n')) {
124
+ *d++ = '\\';
125
+ n++;
126
+ }
127
+ *d = c;
128
+ }
129
+ *d = '\0';
130
+
131
+ return n;
132
+}
133
+
134
+inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits)
135
+{
136
+ const char *sorig = s;
137
+ char *ret = d;
138
+ size_t n;
139
+
140
+ // Fix for issue 5227
141
+ if (unlikely(showoldunits)) {
142
+ static struct {
143
+ const char *newunit;
144
+ uint32_t hash;
145
+ const char *oldunit;
146
+ } units[] = { { "KiB/s", 0, "kilobytes/s" },
147
+ { "MiB/s", 0, "MB/s" },
148
+ { "GiB/s", 0, "GB/s" },
149
+ { "KiB", 0, "KB" },
150
+ { "MiB", 0, "MB" },
151
+ { "GiB", 0, "GB" },
152
+ { "inodes", 0, "Inodes" },
153
+ { "percentage", 0, "percent" },
154
+ { "faults/s", 0, "page faults/s" },
155
+ { "KiB/operation", 0, "kilobytes per operation" },
156
+ { "milliseconds/operation", 0, "ms per operation" },
157
+ { NULL, 0, NULL } };
158
+ static int initialized = 0;
159
+ int i;
160
+
161
+ if (unlikely(!initialized)) {
162
+ for (i = 0; units[i].newunit; i++)
163
+ units[i].hash = simple_hash(units[i].newunit);
164
+ initialized = 1;
165
+ }
166
+
167
+ uint32_t hash = simple_hash(s);
168
+ for (i = 0; units[i].newunit; i++) {
169
+ if (unlikely(hash == units[i].hash && !strcmp(s, units[i].newunit))) {
170
+ // info("matched extension for filename '%s': '%s'", filename, last_dot);
171
+ s = units[i].oldunit;
172
+ sorig = s;
173
+ break;
174
+ }
175
+ }
176
+ }
177
+ *d++ = '_';
178
+ for (n = 1; *s && n < usable; d++, s++, n++) {
179
+ register char c = *s;
180
+
181
+ if (!isalnum(c))
182
+ *d = '_';
183
+ else
184
+ *d = c;
185
+ }
186
+
187
+ if (n == 2 && sorig[0] == '%') {
188
+ n = 0;
189
+ d = ret;
190
+ s = "_percent";
191
+ for (; *s && n < usable; n++)
192
+ *d++ = *s++;
193
+ } else if (n > 3 && sorig[n - 3] == '/' && sorig[n - 2] == 's') {
194
+ n = n - 2;
195
+ d -= 2;
196
+ s = "_persec";
197
+ for (; *s && n < usable; n++)
198
+ *d++ = *s++;
199
+ }
200
+
201
+ *d = '\0';
202
+
203
+ return ret;
204
+}
205
+
206
+struct host_variables_callback_options {
207
+ RRDHOST *host;
208
+ BUFFER *wb;
209
+ EXPORTING_OPTIONS exporting_options;
210
+ PROMETHEUS_OUTPUT_OPTIONS output_options;
211
+ const char *prefix;
212
+ const char *labels;
213
+ time_t now;
214
+ int host_header_printed;
215
+ char name[PROMETHEUS_VARIABLE_MAX + 1];
216
+};
217
+
218
+static int print_host_variables(RRDVAR *rv, void *data)
219
+{
220
+ struct host_variables_callback_options *opts = data;
221
+
222
+ if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) {
223
+ if (!opts->host_header_printed) {
224
+ opts->host_header_printed = 1;
225
+
226
+ if (opts->output_options & PROMETHEUS_OUTPUT_HELP) {
227
+ buffer_sprintf(opts->wb, "\n# COMMENT global host and chart variables\n");
228
+ }
229
+ }
230
+
231
+ calculated_number value = rrdvar2number(rv);
232
+ if (isnan(value) || isinf(value)) {
233
+ if (opts->output_options & PROMETHEUS_OUTPUT_HELP)
234
+ buffer_sprintf(
235
+ opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value)) ? "NAN" : "INF");
236
+
237
+ return 0;
238
+ }
239
+
240
+ char *label_pre = "";
241
+ char *label_post = "";
242
+ if (opts->labels && *opts->labels) {
243
+ label_pre = "{";
244
+ label_post = "}";
245
+ }
246
+
247
+ prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));
248
+
249
+ if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
250
+ buffer_sprintf(
251
+ opts->wb,
252
+ "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
253
+ opts->prefix,
254
+ opts->name,
255
+ label_pre,
256
+ opts->labels,
257
+ label_post,
258
+ value,
259
+ ((rv->last_updated) ? rv->last_updated : opts->now) * 1000ULL);
260
+ else
261
+ buffer_sprintf(
262
+ opts->wb,
263
+ "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n",
264
+ opts->prefix,
265
+ opts->name,
266
+ label_pre,
267
+ opts->labels,
268
+ label_post,
269
+ value);
270
+
271
+ return 1;
272
+ }
273
+
274
+ return 0;
275
+}
276
+
277
+static void rrd_stats_api_v1_charts_allmetrics_prometheus(
278
+ struct instance *instance,
279
+ RRDHOST *host,
280
+ BUFFER *wb,
281
+ const char *prefix,
282
+ EXPORTING_OPTIONS exporting_options,
283
+ time_t after,
284
+ time_t before,
285
+ int allhosts,
286
+ PROMETHEUS_OUTPUT_OPTIONS output_options)
287
+{
288
+ rrdhost_rdlock(host);
289
+
290
+ char hostname[PROMETHEUS_ELEMENT_MAX + 1];
291
+ prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
292
+
293
+ char labels[PROMETHEUS_LABELS_MAX + 1] = "";
294
+ if (allhosts) {
295
+ if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
296
+ buffer_sprintf(
297
+ wb,
298
+ "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
299
+ hostname,
300
+ host->program_name,
301
+ host->program_version,
302
+ now_realtime_usec() / USEC_PER_MS);
303
+ else
304
+ buffer_sprintf(
305
+ wb,
306
+ "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
307
+ hostname,
308
+ host->program_name,
309
+ host->program_version);
310
+
311
+ if (host->tags && *(host->tags)) {
312
+ if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
313
+ buffer_sprintf(
314
+ wb,
315
+ "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n",
316
+ hostname,
317
+ host->tags,
318
+ now_realtime_usec() / USEC_PER_MS);
319
+
320
+ // deprecated, exists only for compatibility with older queries
321
+ buffer_sprintf(
322
+ wb,
323
+ "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n",
324
+ hostname,
325
+ host->tags,
326
+ now_realtime_usec() / USEC_PER_MS);
327
+ } else {
328
+ buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, host->tags);
329
+
330
+ // deprecated, exists only for compatibility with older queries
331
+ buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, host->tags);
332
+ }
333
+ }
334
+
335
+ snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
336
+ } else {
337
+ if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
338
+ buffer_sprintf(
339
+ wb,
340
+ "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
341
+ hostname,
342
+ host->program_name,
343
+ host->program_version,
344
+ now_realtime_usec() / USEC_PER_MS);
345
+ else
346
+ buffer_sprintf(
347
+ wb,
348
+ "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
349
+ hostname,
350
+ host->program_name,
351
+ host->program_version);
352
+
353
+ if (host->tags && *(host->tags)) {
354
+ if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
355
+ buffer_sprintf(
356
+ wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
357
+
358
+ // deprecated, exists only for compatibility with older queries
359
+ buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
360
+ } else {
361
+ buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", host->tags);
362
+
363
+ // deprecated, exists only for compatibility with older queries
364
+ buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", host->tags);
365
+ }
366
+ }
367
+ }
368
+
369
+ // send custom variables set for the host
370
+ if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {
371
+ struct host_variables_callback_options opts = { .host = host,
372
+ .wb = wb,
373
+ .labels = (labels[0] == ',') ? &labels[1] : labels,
374
+ .exporting_options = exporting_options,
375
+ .output_options = output_options,
376
+ .prefix = prefix,
377
+ .now = now_realtime_sec(),
378
+ .host_header_printed = 0 };
379
+ foreach_host_variable_callback(host, print_host_variables, &opts);
380
+ }
381
+
382
+ // for each chart
383
+ RRDSET *st;
384
+ rrdset_foreach_read(st, host)
385
+ {
386
+ char chart[PROMETHEUS_ELEMENT_MAX + 1];
387
+ char context[PROMETHEUS_ELEMENT_MAX + 1];
388
+ char family[PROMETHEUS_ELEMENT_MAX + 1];
389
+
390
+ prometheus_label_copy(
391
+ chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX);
392
+ prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
393
+ prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
394
+
395
+ if (likely(can_send_rrdset(instance, st))) {
396
+ rrdset_rdlock(st);
397
+
398
+ char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
399
+
400
+ int as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED);
401
+ int homogeneous = 1;
402
+ if (as_collected) {
403
+ if (rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK))
404
+ rrdset_update_heterogeneous_flag(st);
405
+
406
+ if (rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS))
407
+ homogeneous = 0;
408
+ } else {
409
+ if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE &&
410
+ !(output_options & PROMETHEUS_OUTPUT_HIDEUNITS))
411
+ prometheus_units_copy(
412
+ units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS);
413
+ }
414
+
415
+ if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
416
+ buffer_sprintf(
417
+ wb,
418
+ "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n",
419
+ (homogeneous) ? "homogeneous" : "heterogeneous",
420
+ (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id,
421
+ st->context,
422
+ st->family,
423
+ st->units);
424
+
425
+ // for each dimension
426
+ RRDDIM *rd;
427
+ rrddim_foreach_read(rd, st)
428
+ {
429
+ if (rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
430
+ char dimension[PROMETHEUS_ELEMENT_MAX + 1];
431
+ char *suffix = "";
432
+
433
+ if (as_collected) {
434
+ // we need as-collected / raw data
435
+
436
+ if (unlikely(rd->last_collected_time.tv_sec < after))
437
+ continue;
438
+
439
+ const char *t = "gauge", *h = "gives";
440
+ if (rd->algorithm == RRD_ALGORITHM_INCREMENTAL ||
441
+ rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) {
442
+ t = "counter";
443
+ h = "delta gives";
444
+ suffix = "_total";
445
+ }
446
+
447
+ if (homogeneous) {
448
+ // all the dimensions of the chart, has the same algorithm, multiplier and divisor
449
+ // we add all dimensions as labels
450
+
451
+ prometheus_label_copy(
452
+ dimension,
453
+ (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
454
+ PROMETHEUS_ELEMENT_MAX);
455
+
456
+ if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
457
+ buffer_sprintf(
458
+ wb,
459
+ "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT
460
+ " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n",
461
+ prefix,
462
+ context,
463
+ suffix,
464
+ (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id,
465
+ st->context,
466
+ st->family,
467
+ (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
468
+ rd->multiplier,
469
+ rd->divisor,
470
+ h,
471
+ st->units,
472
+ t);
473
+
474
+ if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
475
+ buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n", prefix, context, suffix, t);
476
+
477
+ if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
478
+ buffer_sprintf(
479
+ wb,
480
+ "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT
481
+ " %llu\n",
482
+ prefix,
483
+ context,
484
+ suffix,
485
+ chart,
486
+ family,
487
+ dimension,
488
+ labels,
489
+ rd->last_collected_value,
490
+ timeval_msec(&rd->last_collected_time));
491
+ else
492
+ buffer_sprintf(
493
+ wb,
494
+ "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT
495
+ "\n",
496
+ prefix,
497
+ context,
498
+ suffix,
499
+ chart,
500
+ family,
501
+ dimension,
502
+ labels,
503
+ rd->last_collected_value);
504
+ } else {
505
+ // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
506
+ // we create a metric per dimension
507
+
508
+ prometheus_name_copy(
509
+ dimension,
510
+ (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
511
+ PROMETHEUS_ELEMENT_MAX);
512
+
513
+ if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
514
+ buffer_sprintf(
515
+ wb,
516
+ "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT
517
+ " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n",
518
+ prefix,
519
+ context,
520
+ dimension,
521
+ suffix,
522
+ (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id,
523
+ st->context,
524
+ st->family,
525
+ (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
526
+ rd->multiplier,
527
+ rd->divisor,
528
+ h,
529
+ st->units,
530
+ t);
531
+
532
+ if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
533
+ buffer_sprintf(
534
+ wb, "# COMMENT TYPE %s_%s_%s%s %s\n", prefix, context, dimension, suffix, t);
535
+
536
+ if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
537
+ buffer_sprintf(
538
+ wb,
539
+ "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n",
540
+ prefix,
541
+ context,
542
+ dimension,
543
+ suffix,
544
+ chart,
545
+ family,
546
+ labels,
547
+ rd->last_collected_value,
548
+ timeval_msec(&rd->last_collected_time));
549
+ else
550
+ buffer_sprintf(
551
+ wb,
552
+ "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT "\n",
553
+ prefix,
554
+ context,
555
+ dimension,
556
+ suffix,
557
+ chart,
558
+ family,
559
+ labels,
560
+ rd->last_collected_value);
561
+ }
562
+ } else {
563
+ // we need average or sum of the data
564
+
565
+ time_t first_t = after, last_t = before;
566
+ calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
567
+
568
+ if (!isnan(value) && !isinf(value)) {
569
+ if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
570
+ suffix = "_average";
571
+ else if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_SUM)
572
+ suffix = "_sum";
573
+
574
+ prometheus_label_copy(
575
+ dimension,
576
+ (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
577
+ PROMETHEUS_ELEMENT_MAX);
578
+
579
+ if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
580
+ buffer_sprintf(
581
+ wb,
582
+ "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n",
583
+ prefix,
584
+ context,
585
+ units,
586
+ suffix,
587
+ (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
588
+ st->units,
589
+ (unsigned long long)first_t,
590
+ (unsigned long long)last_t);
591
+
592
+ if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
593
+ buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n", prefix, context, units, suffix);
594
+
595
+ if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
596
+ buffer_sprintf(
597
+ wb,
598
+ "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
599
+ " %llu\n",
600
+ prefix,
601
+ context,
602
+ units,
603
+ suffix,
604
+ chart,
605
+ family,
606
+ dimension,
607
+ labels,
608
+ value,
609
+ last_t * MSEC_PER_SEC);
610
+ else
611
+ buffer_sprintf(
612
+ wb,
613
+ "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
614
+ "\n",
615
+ prefix,
616
+ context,
617
+ units,
618
+ suffix,
619
+ chart,
620
+ family,
621
+ dimension,
622
+ labels,
623
+ value);
624
+ }
625
+ }
626
+ }
627
+ }
628
+
629
+ rrdset_unlock(st);
630
+ }
631
+ }
632
+
633
+ rrdhost_unlock(host);
634
+}
635
+
636
+static inline time_t prometheus_preparation(
637
+ struct instance *instance,
638
+ RRDHOST *host,
639
+ BUFFER *wb,
640
+ EXPORTING_OPTIONS exporting_options,
641
+ const char *server,
642
+ time_t now,
643
+ PROMETHEUS_OUTPUT_OPTIONS output_options)
644
+{
645
+ if (!server || !*server)
646
+ server = "default";
647
+
648
+ time_t after = prometheus_server_last_access(server, host, now);
649
+
650
+ int first_seen = 0;
651
+ if (!after) {
652
+ after = now - instance->engine->config.update_every;
653
+ first_seen = 1;
654
+ }
655
+
656
+ if (after > now) {
657
+ // oops! this should never happen
658
+ after = now - instance->engine->config.update_every;
659
+ }
660
+
661
+ if (output_options & PROMETHEUS_OUTPUT_HELP) {
662
+ char *mode;
663
+ if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
664
+ mode = "as collected";
665
+ else if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
666
+ mode = "average";
667
+ else if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_SUM)
668
+ mode = "sum";
669
+ else
670
+ mode = "unknown";
671
+
672
+ buffer_sprintf(
673
+ wb,
674
+ "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s, time range %lu to %lu\n\n",
675
+ host->hostname,
676
+ (first_seen) ? "FIRST SEEN " : "",
677
+ server,
678
+ mode,
679
+ (unsigned long)((first_seen) ? 0 : (now - after)),
680
+ (first_seen) ? "never" : "seconds ago",
681
+ (unsigned long)after,
682
+ (unsigned long)now);
683
+ }
684
+
685
+ return after;
686
+}
687
+
688
+void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
689
+ struct instance *instance,
690
+ RRDHOST *host,
691
+ BUFFER *wb,
692
+ const char *server,
693
+ const char *prefix,
694
+ EXPORTING_OPTIONS exporting_options,
695
+ PROMETHEUS_OUTPUT_OPTIONS output_options)
696
+{
697
+ time_t before = now_realtime_sec();
698
+
699
+ // we start at the point we had stopped before
700
+ time_t after = prometheus_preparation(instance, host, wb, exporting_options, server, before, output_options);
701
+
702
+ rrd_stats_api_v1_charts_allmetrics_prometheus(
703
+ instance, host, wb, prefix, exporting_options, after, before, 0, output_options);
704
+}
705
+
706
+void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
707
+ struct instance *instance,
708
+ RRDHOST *host,
709
+ BUFFER *wb,
710
+ const char *server,
711
+ const char *prefix,
712
+ EXPORTING_OPTIONS exporting_options,
713
+ PROMETHEUS_OUTPUT_OPTIONS output_options)
714
+{
715
+ time_t before = now_realtime_sec();
716
+
717
+ // we start at the point we had stopped before
718
+ time_t after = prometheus_preparation(instance, host, wb, exporting_options, server, before, output_options);
719
+
720
+ rrd_rdlock();
721
+ rrdhost_foreach_read(host)
722
+ {
723
+ rrd_stats_api_v1_charts_allmetrics_prometheus(
724
+ instance, host, wb, prefix, exporting_options, after, before, 1, output_options);
725
+ }
726
+ rrd_unlock();
727
+}
exporting/prometheus/prometheus.h
new
+37
@@ -0,0 +1,37 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EXPORTING_PROMETHEUS_H
4
+#define NETDATA_EXPORTING_PROMETHEUS_H 1
5
+
6
+#include "exporting/exporting_engine.h"
7
+
8
+#define PROMETHEUS_ELEMENT_MAX 256
9
+#define PROMETHEUS_LABELS_MAX 1024
10
+#define PROMETHEUS_VARIABLE_MAX 256
11
+
12
+#define PROMETHEUS_LABELS_MAX_NUMBER 128
13
+
14
+typedef enum prometheus_output_flags {
15
+ PROMETHEUS_OUTPUT_NONE = 0,
16
+ PROMETHEUS_OUTPUT_HELP = (1 << 0),
17
+ PROMETHEUS_OUTPUT_TYPES = (1 << 1),
18
+ PROMETHEUS_OUTPUT_NAMES = (1 << 2),
19
+ PROMETHEUS_OUTPUT_TIMESTAMPS = (1 << 3),
20
+ PROMETHEUS_OUTPUT_VARIABLES = (1 << 4),
21
+ PROMETHEUS_OUTPUT_OLDUNITS = (1 << 5),
22
+ PROMETHEUS_OUTPUT_HIDEUNITS = (1 << 6)
23
+} PROMETHEUS_OUTPUT_OPTIONS;
24
+
25
+extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
26
+ struct instance *instance, RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
27
+ EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
28
+extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
29
+ struct instance *instance, RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
30
+ EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
31
+
32
+int can_send_rrdset(struct instance *instance, RRDSET *st);
33
+size_t prometheus_name_copy(char *d, const char *s, size_t usable);
34
+size_t prometheus_label_copy(char *d, const char *s, size_t usable);
35
+char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits);
36
+
37
+#endif //NETDATA_EXPORTING_PROMETHEUS_H
exporting/prometheus/remote_write/Makefile.am
new
+14
@@ -0,0 +1,14 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+AUTOMAKE_OPTIONS = subdir-objects
4
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
+
6
+CLEANFILES = \
7
+ remote_write.pb.cc \
8
+ remote_write.pb.h \
9
+ $(NULL)
10
+
11
+dist_noinst_DATA = \
12
+ remote_write.proto \
13
+ README.md \
14
+ $(NULL)
exporting/prometheus/remote_write/README.md
new
+41
@@ -0,0 +1,41 @@
1
+# Prometheus remote write exporting connector
2
+
3
+The Prometheus remote write exporting connector uses the exporting engine to send Netdata metrics to your choice of more
4
+than 20 external storage providers for long-term archiving and further analysis.
5
+
6
+## Prerequisites
7
+
8
+To use the Prometheus remote write API with [storage
9
+providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage), install
10
+[protobuf](https://developers.google.com/protocol-buffers/) and [snappy](https://github.com/google/snappy) libraries.
11
+Next, re-install Netdata from the source, which detects that the required libraries and
12
+utilities are now available.
13
+
14
+## Configuration
15
+
16
+To enable data exporting to a storage provider using the Prometheus remote write API, run `./edit-config exporting.conf`
17
+in the Netdata configuration directory and set the following options:
18
+
19
+```conf
20
+[remote_write:my_instance]
21
+ enabled = yes
22
+ destination = example.domain:example_port
23
+ remote write URL path = /receive
24
+```
25
+
26
+`remote write URL path` is used to set an endpoint path for the remote write protocol. The default value is `/receive`.
27
+For example, if your endpoint is `http://example.domain:example_port/storage/read`:
28
+
29
+```conf
30
+ destination = example.domain:example_port
31
+ remote write URL path = /storage/read
32
+```
33
+
34
+`buffered` and `lost` dimensions in the Netdata Exporting Connector Data Size operation monitoring chart estimate uncompressed
35
+buffer size on failures.
36
+
37
+## Notes
38
+
39
+The remote write exporting connector does not support `buffer on failures`
40
+
41
+[](<>)
exporting/prometheus/remote_write/remote_write.c
new
+320
@@ -0,0 +1,320 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "remote_write.h"
4
+
5
+static int as_collected;
6
+static int homogeneous;
7
+char context[PROMETHEUS_ELEMENT_MAX + 1];
8
+char chart[PROMETHEUS_ELEMENT_MAX + 1];
9
+char family[PROMETHEUS_ELEMENT_MAX + 1];
10
+char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
11
+
12
+/**
13
+ * Send header to a server
14
+ *
15
+ * @param sock a communication socket.
16
+ * @param instance an instance data structure.
17
+ * @return Returns 0 on success, 1 on failure.
18
+ */
19
+int prometheus_remote_write_send_header(int *sock, struct instance *instance)
20
+{
21
+ int flags = 0;
22
+#ifdef MSG_NOSIGNAL
23
+ flags += MSG_NOSIGNAL;
24
+#endif
25
+
26
+ struct prometheus_remote_write_specific_config *connector_specific_config =
27
+ instance->config.connector_specific_config;
28
+
29
+ static BUFFER *header;
30
+ if (!header)
31
+ header = buffer_create(0);
32
+
33
+ buffer_sprintf(
34
+ header,
35
+ "POST %s HTTP/1.1\r\n"
36
+ "Host: %s\r\n"
37
+ "Accept: */*\r\n"
38
+ "Content-Length: %zu\r\n"
39
+ "Content-Type: application/x-www-form-urlencoded\r\n\r\n",
40
+ connector_specific_config->remote_write_path,
41
+ instance->engine->config.hostname,
42
+ buffer_strlen((BUFFER *)instance->buffer));
43
+
44
+ size_t header_len = buffer_strlen(header);
45
+ ssize_t written = send(*sock, buffer_tostring(header), header_len, flags);
46
+
47
+ buffer_flush(header);
48
+
49
+ if (written != -1 && (size_t)written == header_len)
50
+ return 0;
51
+ else
52
+ return 1;
53
+}
54
+
55
+/**
56
+ * Process a responce received after Prometheus remote write connector had sent data
57
+ *
58
+ * @param buffer a response from a remote service.
59
+ * @param instance an instance data structure.
60
+ * @return Returns 0 on success, 1 on failure.
61
+ */
62
+int process_prometheus_remote_write_response(BUFFER *buffer, struct instance *instance)
63
+{
64
+ if (unlikely(!buffer))
65
+ return 1;
66
+
67
+ const char *s = buffer_tostring(buffer);
68
+ int len = buffer_strlen(buffer);
69
+
70
+ // do nothing with HTTP responses 200 or 204
71
+
72
+ while (!isspace(*s) && len) {
73
+ s++;
74
+ len--;
75
+ }
76
+ s++;
77
+ len--;
78
+
79
+ if (likely(len > 4 && (!strncmp(s, "200 ", 4) || !strncmp(s, "204 ", 4))))
80
+ return 0;
81
+ else
82
+ return exporting_discard_response(buffer, instance);
83
+}
84
+
85
+/**
86
+ * Initialize Prometheus Remote Write connector instance
87
+ *
88
+ * @param instance an instance data structure.
89
+ * @return Returns 0 on success, 1 on failure.
90
+ */
91
+int init_prometheus_remote_write_instance(struct instance *instance)
92
+{
93
+ instance->worker = simple_connector_worker;
94
+
95
+ instance->start_batch_formatting = NULL;
96
+ instance->start_host_formatting = format_host_prometheus_remote_write;
97
+ instance->start_chart_formatting = format_chart_prometheus_remote_write;
98
+ instance->metric_formatting = format_dimension_prometheus_remote_write;
99
+ instance->end_chart_formatting = NULL;
100
+ instance->end_host_formatting = NULL;
101
+ instance->end_batch_formatting = format_batch_prometheus_remote_write;
102
+
103
+ instance->send_header = prometheus_remote_write_send_header;
104
+ instance->check_response = process_prometheus_remote_write_response;
105
+
106
+ instance->buffer = (void *)buffer_create(0);
107
+ if (!instance->buffer) {
108
+ error("EXPORTING: cannot create buffer for AWS Kinesis exporting connector instance %s", instance->config.name);
109
+ return 1;
110
+ }
111
+ uv_mutex_init(&instance->mutex);
112
+ uv_cond_init(&instance->cond_var);
113
+
114
+ struct prometheus_remote_write_specific_data *connector_specific_data =
115
+ callocz(1, sizeof(struct prometheus_remote_write_specific_data));
116
+ instance->connector_specific_data = (void *)connector_specific_data;
117
+
118
+ connector_specific_data->write_request = init_write_request();
119
+
120
+ return 0;
121
+}
122
+
123
+/**
124
+ * Format host data for Prometheus Remote Write connector
125
+ *
126
+ * @param instance an instance data structure.
127
+ * @param host a data collecting host.
128
+ * @return Always returns 0.
129
+ */
130
+int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host)
131
+{
132
+ struct prometheus_remote_write_specific_data *connector_specific_data =
133
+ (struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
134
+
135
+ char hostname[PROMETHEUS_ELEMENT_MAX + 1];
136
+ prometheus_label_copy(hostname, instance->engine->config.hostname, PROMETHEUS_ELEMENT_MAX);
137
+
138
+ add_host_info(
139
+ connector_specific_data->write_request,
140
+ "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
141
+
142
+ if (unlikely(sending_labels_configured(instance))) {
143
+ rrdhost_check_rdlock(host);
144
+ netdata_rwlock_rdlock(&host->labels_rwlock);
145
+ for (struct label *label = host->labels; label; label = label->next) {
146
+ if (!should_send_label(instance, label))
147
+ continue;
148
+
149
+ char key[PROMETHEUS_ELEMENT_MAX + 1];
150
+ prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
151
+
152
+ char value[PROMETHEUS_ELEMENT_MAX + 1];
153
+ prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
154
+
155
+ add_label(connector_specific_data->write_request, key, value);
156
+ }
157
+ netdata_rwlock_unlock(&host->labels_rwlock);
158
+ }
159
+
160
+ return 0;
161
+}
162
+
163
+/**
164
+ * Format chart data for Prometheus Remote Write connector
165
+ *
166
+ * @param instance an instance data structure.
167
+ * @param st a chart.
168
+ * @return Always returns 0.
169
+ */
170
+int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st)
171
+{
172
+ prometheus_label_copy(
173
+ chart,
174
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
175
+ PROMETHEUS_ELEMENT_MAX);
176
+ prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
177
+ prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
178
+
179
+ if (likely(can_send_rrdset(instance, st))) {
180
+ as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED);
181
+ homogeneous = 1;
182
+ if (as_collected) {
183
+ if (rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK))
184
+ rrdset_update_heterogeneous_flag(st);
185
+
186
+ if (rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS))
187
+ homogeneous = 0;
188
+ } else {
189
+ if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE)
190
+ prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0);
191
+ }
192
+ }
193
+
194
+ return 0;
195
+}
196
+
197
+/**
198
+ * Format dimension data for Prometheus Remote Write connector
199
+ *
200
+ * @param instance an instance data structure.
201
+ * @param rd a dimension.
202
+ * @return Always returns 0.
203
+ */
204
+int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd)
205
+{
206
+ struct prometheus_remote_write_specific_data *connector_specific_data =
207
+ (struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
208
+
209
+ if (rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
210
+ char name[PROMETHEUS_LABELS_MAX + 1];
211
+ char dimension[PROMETHEUS_ELEMENT_MAX + 1];
212
+ char *suffix = "";
213
+
214
+ if (as_collected) {
215
+ // we need as-collected / raw data
216
+
217
+ if (unlikely(rd->last_collected_time.tv_sec < instance->after)) {
218
+ debug(
219
+ D_BACKEND,
220
+ "EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', "
221
+ "its last data collection (%lu) is not within our timeframe (%lu to %lu)",
222
+ rd->id, rd->rrdset->id,
223
+ instance->engine->config.hostname,
224
+ (unsigned long)rd->last_collected_time.tv_sec,
225
+ (unsigned long)instance->after,
226
+ (unsigned long)instance->before);
227
+ return 1;
228
+ }
229
+
230
+ if (homogeneous) {
231
+ // all the dimensions of the chart, has the same algorithm, multiplier and divisor
232
+ // we add all dimensions as labels
233
+
234
+ prometheus_label_copy(
235
+ dimension,
236
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
237
+ PROMETHEUS_ELEMENT_MAX);
238
+ snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->engine->config.prefix, context, suffix);
239
+
240
+ add_metric(
241
+ connector_specific_data->write_request,
242
+ name, chart, family, dimension, instance->engine->config.hostname,
243
+ rd->last_collected_value, timeval_msec(&rd->last_collected_time));
244
+ } else {
245
+ // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
246
+ // we create a metric per dimension
247
+
248
+ prometheus_name_copy(
249
+ dimension,
250
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
251
+ PROMETHEUS_ELEMENT_MAX);
252
+ snprintf(
253
+ name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", instance->engine->config.prefix, context, dimension,
254
+ suffix);
255
+
256
+ add_metric(
257
+ connector_specific_data->write_request,
258
+ name, chart, family, NULL, instance->engine->config.hostname,
259
+ rd->last_collected_value, timeval_msec(&rd->last_collected_time));
260
+ }
261
+ } else {
262
+ // we need average or sum of the data
263
+
264
+ time_t last_t = instance->before;
265
+ calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
266
+
267
+ if (!isnan(value) && !isinf(value)) {
268
+ if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE)
269
+ suffix = "_average";
270
+ else if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM)
271
+ suffix = "_sum";
272
+
273
+ prometheus_label_copy(
274
+ dimension,
275
+ (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
276
+ PROMETHEUS_ELEMENT_MAX);
277
+ snprintf(
278
+ name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", instance->engine->config.prefix, context, units, suffix);
279
+
280
+ add_metric(
281
+ connector_specific_data->write_request,
282
+ name, chart, family, dimension, instance->engine->config.hostname,
283
+ value, last_t * MSEC_PER_SEC);
284
+ }
285
+ }
286
+ }
287
+
288
+ return 0;
289
+}
290
+
291
+/**
292
+ * Format a batch for Prometheus Remote Write connector
293
+ *
294
+ * @param instance an instance data structure.
295
+ * @return Returns 0 on success, 1 on failure.
296
+ */
297
+int format_batch_prometheus_remote_write(struct instance *instance)
298
+{
299
+ struct prometheus_remote_write_specific_data *connector_specific_data =
300
+ (struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
301
+
302
+ size_t data_size = get_write_request_size(connector_specific_data->write_request);
303
+
304
+ if (unlikely(!data_size)) {
305
+ error("EXPORTING: write request size is out of range");
306
+ return 1;
307
+ }
308
+
309
+ BUFFER *buffer = instance->buffer;
310
+
311
+ buffer_need_bytes(buffer, data_size);
312
+ if (unlikely(pack_and_clear_write_request(connector_specific_data->write_request, buffer->buffer, &data_size))) {
313
+ error("EXPORTING: cannot pack write request");
314
+ return 1;
315
+ }
316
+ buffer->len = data_size;
317
+ instance->stats.chart_buffered_bytes = (collected_number)buffer_strlen(buffer);
318
+
319
+ return 0;
320
+}
exporting/prometheus/remote_write/remote_write.h
new
+20
@@ -0,0 +1,20 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EXPORTING_PROMETHEUS_REMOTE_WRITE_H
4
+#define NETDATA_EXPORTING_PROMETHEUS_REMOTE_WRITE_H
5
+
6
+#include "exporting/exporting_engine.h"
7
+#include "exporting/prometheus/prometheus.h"
8
+#include "remote_write_request.h"
9
+
10
+int init_prometheus_remote_write_instance(struct instance *instance);
11
+
12
+int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host);
13
+int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st);
14
+int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd);
15
+int format_batch_prometheus_remote_write(struct instance *instance);
16
+
17
+int prometheus_remote_write_send_header(int *sock, struct instance *instance);
18
+int process_prometheus_remote_write_response(BUFFER *buffer, struct instance *instance);
19
+
20
+#endif //NETDATA_EXPORTING_PROMETHEUS_REMOTE_WRITE_H
exporting/prometheus/remote_write/remote_write.proto
new
+29
@@ -0,0 +1,29 @@
1
+syntax = "proto3";
2
+package prometheus;
3
+
4
+option cc_enable_arenas = true;
5
+
6
+import "google/protobuf/descriptor.proto";
7
+
8
+message WriteRequest {
9
+ repeated TimeSeries timeseries = 1 [(nullable) = false];
10
+}
11
+
12
+message TimeSeries {
13
+ repeated Label labels = 1 [(nullable) = false];
14
+ repeated Sample samples = 2 [(nullable) = false];
15
+}
16
+
17
+message Label {
18
+ string name = 1;
19
+ string value = 2;
20
+}
21
+
22
+message Sample {
23
+ double value = 1;
24
+ int64 timestamp = 2;
25
+}
26
+
27
+extend google.protobuf.FieldOptions {
28
+ bool nullable = 65001;
29
+}
exporting/prometheus/remote_write/remote_write_request.cc
new
+186
@@ -0,0 +1,186 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include <snappy.h>
4
+#include "remote_write.pb.h"
5
+#include "remote_write_request.h"
6
+
7
+using namespace prometheus;
8
+
9
+google::protobuf::Arena arena;
10
+
11
+/**
12
+ * Initialize a write request
13
+ *
14
+ * @return Returns a new write request
15
+ */
16
+void *init_write_request()
17
+{
18
+ GOOGLE_PROTOBUF_VERIFY_VERSION;
19
+ WriteRequest *write_request = google::protobuf::Arena::CreateMessage<WriteRequest>(&arena);
20
+ return (void *)write_request;
21
+}
22
+
23
+/**
24
+ * Adds information about a host to a write request
25
+ *
26
+ * @param write_request_p the write request
27
+ * @param name the name of a metric which is used for providing the host information
28
+ * @param instance the name of the host itself
29
+ * @param application the name of a program which sends the information
30
+ * @param version the version of the program
31
+ * @param timestamp the timestamp for the metric in milliseconds
32
+ */
33
+void add_host_info(
34
+ void *write_request_p,
35
+ const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp)
36
+{
37
+ WriteRequest *write_request = (WriteRequest *)write_request_p;
38
+ TimeSeries *timeseries;
39
+ Sample *sample;
40
+ Label *label;
41
+
42
+ timeseries = write_request->add_timeseries();
43
+
44
+ label = timeseries->add_labels();
45
+ label->set_name("__name__");
46
+ label->set_value(name);
47
+
48
+ label = timeseries->add_labels();
49
+ label->set_name("instance");
50
+ label->set_value(instance);
51
+
52
+ if (application) {
53
+ label = timeseries->add_labels();
54
+ label->set_name("application");
55
+ label->set_value(application);
56
+ }
57
+
58
+ if (version) {
59
+ label = timeseries->add_labels();
60
+ label->set_name("version");
61
+ label->set_value(version);
62
+ }
63
+
64
+ sample = timeseries->add_samples();
65
+ sample->set_value(1);
66
+ sample->set_timestamp(timestamp);
67
+}
68
+
69
+/**
70
+ * Adds a label to the last created timeseries
71
+ *
72
+ * @param write_request_p the write request with the timeseries
73
+ * @param key the key of the label
74
+ * @param value the value of the label
75
+ */
76
+void add_label(void *write_request_p, char *key, char *value)
77
+{
78
+ WriteRequest *write_request = (WriteRequest *)write_request_p;
79
+ TimeSeries *timeseries;
80
+ Label *label;
81
+
82
+ timeseries = write_request->mutable_timeseries(write_request->timeseries_size() - 1);
83
+
84
+ label = timeseries->add_labels();
85
+ label->set_name(key);
86
+ label->set_value(value);
87
+}
88
+
89
+/**
90
+ * Adds a metric to a write request
91
+ *
92
+ * @param write_request_p the write request
93
+ * @param name the name of the metric
94
+ * @param chart the chart, the metric belongs to
95
+ * @param family the family, the metric belongs to
96
+ * @param dimension the dimension, the metric belongs to
97
+ * @param instance the name of the host, the metric belongs to
98
+ * @param value the value of the metric
99
+ * @param timestamp the timestamp for the metric in milliseconds
100
+ */
101
+void add_metric(
102
+ void *write_request_p,
103
+ const char *name, const char *chart, const char *family, const char *dimension, const char *instance,
104
+ const double value, const int64_t timestamp)
105
+{
106
+ WriteRequest *write_request = (WriteRequest *)write_request_p;
107
+ TimeSeries *timeseries;
108
+ Sample *sample;
109
+ Label *label;
110
+
111
+ timeseries = write_request->add_timeseries();
112
+
113
+ label = timeseries->add_labels();
114
+ label->set_name("__name__");
115
+ label->set_value(name);
116
+
117
+ label = timeseries->add_labels();
118
+ label->set_name("chart");
119
+ label->set_value(chart);
120
+
121
+ label = timeseries->add_labels();
122
+ label->set_name("family");
123
+ label->set_value(family);
124
+
125
+ if (dimension) {
126
+ label = timeseries->add_labels();
127
+ label->set_name("dimension");
128
+ label->set_value(dimension);
129
+ }
130
+
131
+ label = timeseries->add_labels();
132
+ label->set_name("instance");
133
+ label->set_value(instance);
134
+
135
+ sample = timeseries->add_samples();
136
+ sample->set_value(value);
137
+ sample->set_timestamp(timestamp);
138
+}
139
+
140
+/**
141
+ * Gets the size of a write request
142
+ *
143
+ * @param write_request_p the write request
144
+ * @return Returns the size of the write request
145
+ */
146
+size_t get_write_request_size(void *write_request_p)
147
+{
148
+ WriteRequest *write_request = (WriteRequest *)write_request_p;
149
+
150
+#if GOOGLE_PROTOBUF_VERSION < 3001000
151
+ size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSize());
152
+#else
153
+ size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSizeLong());
154
+#endif
155
+
156
+ return (size < INT_MAX) ? size : 0;
157
+}
158
+
159
+/**
160
+ * Packs a write request into a buffer and clears the request
161
+ *
162
+ * @param write_request_p the write request
163
+ * @param buffer a buffer, where compressed data is written
164
+ * @param size gets the size of the write request, returns the size of the compressed data
165
+ * @return Returns 0 on success, 1 on failure
166
+ */
167
+int pack_and_clear_write_request(void *write_request_p, char *buffer, size_t *size)
168
+{
169
+ WriteRequest *write_request = (WriteRequest *)write_request_p;
170
+ std::string uncompressed_write_request;
171
+
172
+ if (write_request->SerializeToString(&uncompressed_write_request) == false)
173
+ return 1;
174
+ write_request->clear_timeseries();
175
+ snappy::RawCompress(uncompressed_write_request.data(), uncompressed_write_request.size(), buffer, size);
176
+
177
+ return 0;
178
+}
179
+
180
+/**
181
+ * Shuts down the Protobuf library
182
+ */
183
+void protocol_buffers_shutdown()
184
+{
185
+ google::protobuf::ShutdownProtobufLibrary();
186
+}
exporting/prometheus/remote_write/remote_write_request.h
new
+37
@@ -0,0 +1,37 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EXPORTING_PROMETHEUS_REMOTE_WRITE_REQUEST_H
4
+#define NETDATA_EXPORTING_PROMETHEUS_REMOTE_WRITE_REQUEST_H
5
+
6
+#ifdef __cplusplus
7
+extern "C" {
8
+#endif
9
+
10
+struct prometheus_remote_write_specific_data {
11
+ void *write_request;
12
+};
13
+
14
+void *init_write_request();
15
+
16
+void add_host_info(
17
+ void *write_request_p,
18
+ const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp);
19
+
20
+void add_label(void *write_request_p, char *key, char *value);
21
+
22
+void add_metric(
23
+ void *write_request_p,
24
+ const char *name, const char *chart, const char *family, const char *dimension,
25
+ const char *instance, const double value, const int64_t timestamp);
26
+
27
+size_t get_write_request_size(void *write_request_p);
28
+
29
+int pack_and_clear_write_request(void *write_request_p, char *buffer, size_t *size);
30
+
31
+void protocol_buffers_shutdown();
32
+
33
+#ifdef __cplusplus
34
+}
35
+#endif
36
+
37
+#endif //NETDATA_EXPORTING_PROMETHEUS_REMOTE_WRITE_REQUEST_H
exporting/read_config.c
+33
-16
@@ -150,7 +150,7 @@ BACKEND_TYPE exporting_select_type(const char *type)
150
} else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
151
return BACKEND_TYPE_JSON;
152
} else if (!strcmp(type, "prometheus_remote_write")) {
153
- return BACKEND_TYPE_PROMETHEUS;
153
+ return BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE;
154
} else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
155
return BACKEND_TYPE_KINESIS;
156
} else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext"))
@@ -252,7 +252,7 @@ struct engine *read_exporting_config()
252
strdupz(exporter_get(CONFIG_SECTION_EXPORTING, "hostname", netdata_configured_hostname));
253
engine->config.prefix = strdupz(exporter_get(CONFIG_SECTION_EXPORTING, "prefix", "netdata"));
254
engine->config.update_every =
255
- exporter_get_number(CONFIG_SECTION_EXPORTING, EXPORTER_UPDATE_EVERY, EXPORTER_UPDATE_EVERY_DEFAULT);
255
+ exporter_get_number(CONFIG_SECTION_EXPORTING, EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
256
}
257
258
while (tmp_ci_list) {
@@ -266,6 +266,13 @@ struct engine *read_exporting_config()
266
goto next_connector_instance;
267
}
268
269
+#ifndef ENABLE_PROMETHEUS_REMOTE_WRITE
270
+ if (tmp_ci_list->backend_type == BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE) {
271
+ error("Prometheus Remote Write support isn't compiled");
272
+ goto next_connector_instance;
273
+ }
274
+#endif
275
+
276
#ifndef HAVE_KINESIS
277
if (tmp_ci_list->backend_type == BACKEND_TYPE_KINESIS) {
278
error("AWS Kinesis support isn't compiled");
@@ -285,49 +292,59 @@ struct engine *read_exporting_config()
292
tmp_instance->config.name = strdupz(tmp_ci_list->local_ci.instance_name);
293
294
tmp_instance->config.destination =
288
- strdupz(exporter_get(instance_name, EXPORTER_DESTINATION, EXPORTER_DESTINATION_DEFAULT));
295
+ strdupz(exporter_get(instance_name, "destination", "localhost"));
296
297
tmp_instance->config.update_every =
291
- exporter_get_number(instance_name, EXPORTER_UPDATE_EVERY, EXPORTER_UPDATE_EVERY_DEFAULT);
298
+ exporter_get_number(instance_name, EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
299
300
tmp_instance->config.buffer_on_failures =
294
- exporter_get_number(instance_name, EXPORTER_BUF_ONFAIL, EXPORTER_BUF_ONFAIL_DEFAULT);
301
+ exporter_get_number(instance_name, "buffer on failures", 10);
302
303
tmp_instance->config.timeoutms =
297
- exporter_get_number(instance_name, EXPORTER_TIMEOUT_MS, EXPORTER_TIMEOUT_MS_DEFAULT);
304
+ exporter_get_number(instance_name, "timeout ms", 10000);
305
306
tmp_instance->config.charts_pattern = simple_pattern_create(
300
- exporter_get(instance_name, EXPORTER_SEND_CHART_MATCH, EXPORTER_SEND_CHART_MATCH_DEFAULT),
307
+ exporter_get(instance_name, "send charts matching", "*"),
308
NULL,
309
SIMPLE_PATTERN_EXACT);
310
311
tmp_instance->config.hosts_pattern = simple_pattern_create(
305
- exporter_get(instance_name, EXPORTER_SEND_HOST_MATCH, EXPORTER_SEND_HOST_MATCH_DEFAULT),
312
+ exporter_get(instance_name, "send hosts matching", "localhost *"),
313
NULL,
314
SIMPLE_PATTERN_EXACT);
315
316
char *data_source =
310
- exporter_get(instance_name, EXPORTER_DATA_SOURCE, EXPORTER_DATA_SOURCE_DEFAULT);
317
+ exporter_get(instance_name, "data source", "average");
318
319
tmp_instance->config.options = exporting_parse_data_source(data_source, tmp_instance->config.options);
320
321
if (exporter_get_boolean(
315
- instance_name, EXPORTER_SEND_CONFIGURED_LABELS, EXPORTER_SEND_CONFIGURED_LABELS_DEFAULT))
322
+ instance_name, "send configured labels", CONFIG_BOOLEAN_YES))
323
tmp_instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
324
else
325
tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
326
327
if (exporter_get_boolean(
321
- instance_name, EXPORTER_SEND_AUTOMATIC_LABELS, EXPORTER_SEND_AUTOMATIC_LABELS_DEFAULT))
328
+ instance_name, "send automatic labels", CONFIG_BOOLEAN_NO))
329
tmp_instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
330
else
331
tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
332
326
- if (exporter_get_boolean(instance_name, EXPORTER_SEND_NAMES, EXPORTER_SEND_NAMES_DEFAULT))
333
+ if (exporter_get_boolean(instance_name, "send names instead of ids", CONFIG_BOOLEAN_YES))
334
tmp_instance->config.options |= EXPORTING_OPTION_SEND_NAMES;
335
else
336
tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;
337
338
+ if (tmp_instance->config.type == BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE) {
339
+ struct prometheus_remote_write_specific_config *connector_specific_config =
340
+ callocz(1, sizeof(struct prometheus_remote_write_specific_config));
341
+
342
+ tmp_instance->config.connector_specific_config = connector_specific_config;
343
+
344
+ connector_specific_config->remote_write_path = strdupz(exporter_get(
345
+ instance_name, "remote write URL path", "/receive"));
346
+ }
347
+
348
if (tmp_instance->config.type == BACKEND_TYPE_KINESIS) {
349
struct aws_kinesis_specific_config *connector_specific_config =
350
callocz(1, sizeof(struct aws_kinesis_specific_config));
@@ -335,13 +352,13 @@ struct engine *read_exporting_config()
352
tmp_instance->config.connector_specific_config = connector_specific_config;
353
354
connector_specific_config->stream_name = strdupz(exporter_get(
338
- instance_name, EXPORTER_KINESIS_STREAM_NAME, EXPORTER_KINESIS_STREAM_NAME_DEFAULT));
355
+ instance_name, "stream name", "netdata"));
356
357
connector_specific_config->auth_key_id = strdupz(exporter_get(
341
- instance_name, EXPORTER_AWS_ACCESS_KEY_ID, ""));
358
+ instance_name, "aws_access_key_id", ""));
359
360
connector_specific_config->secure_key = strdupz(exporter_get(
344
- instance_name, EXPORTER_AWS_SECRET_ACCESS_KEY, ""));
361
+ instance_name, "aws_secret_access_key", ""));
362
}
363
364
#ifdef NETDATA_INTERNAL_CHECKS
@@ -359,7 +376,7 @@ struct engine *read_exporting_config()
376
strdupz(config_get(instance_name, "hostname", netdata_configured_hostname));
377
engine->config.prefix = strdupz(config_get(instance_name, "prefix", "netdata"));
378
engine->config.update_every =
362
- config_get_number(instance_name, EXPORTER_UPDATE_EVERY, EXPORTER_UPDATE_EVERY_DEFAULT);
379
+ config_get_number(instance_name, EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
380
}
381
382
next_connector_instance:
exporting/send_data.c
+9
-3
@@ -76,7 +76,7 @@ void simple_connector_receive_response(int *sock, struct instance *instance)
76
77
// if we received data, process them
78
if (buffer_strlen(response))
79
- exporting_discard_response(response, instance);
79
+ instance->check_response(response, instance);
80
}
81
82
/**
@@ -98,8 +98,14 @@ void simple_connector_send_buffer(int *sock, int *failures, struct instance *ins
98
99
struct stats *stats = &instance->stats;
100
101
- ssize_t written;
102
- written = send(*sock, buffer_tostring(buffer), len, flags);
101
+ int ret = 0;
102
+ if (instance->send_header)
103
+ ret = instance->send_header(sock, instance);
104
+
105
+ ssize_t written = -1;
106
+
107
+ if (!ret)
108
+ written = send(*sock, buffer_tostring(buffer), len, flags);
109
110
if(written != -1 && (size_t)written == len) {
111
// we sent the data successfully
exporting/tests/exporting_doubles.c
+45
@@ -159,6 +159,51 @@ int __mock_end_batch_formatting(struct instance *instance)
159
return mock_type(int);
160
}
161
162
+#if ENABLE_PROMETHEUS_REMOTE_WRITE
163
+void *__wrap_init_write_request()
164
+{
165
+ function_called();
166
+ return mock_ptr_type(void *);
167
+}
168
+
169
+void __wrap_add_host_info(
170
+ void *write_request_p,
171
+ const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp)
172
+{
173
+ function_called();
174
+ check_expected_ptr(write_request_p);
175
+ check_expected_ptr(name);
176
+ check_expected_ptr(instance);
177
+ check_expected_ptr(application);
178
+ check_expected_ptr(version);
179
+ check_expected(timestamp);
180
+}
181
+
182
+void __wrap_add_label(void *write_request_p, char *key, char *value)
183
+{
184
+ function_called();
185
+ check_expected_ptr(write_request_p);
186
+ check_expected_ptr(key);
187
+ check_expected_ptr(value);
188
+}
189
+
190
+void __wrap_add_metric(
191
+ void *write_request_p,
192
+ const char *name, const char *chart, const char *family, const char *dimension,
193
+ const char *instance, const double value, const int64_t timestamp)
194
+{
195
+ function_called();
196
+ check_expected_ptr(write_request_p);
197
+ check_expected_ptr(name);
198
+ check_expected_ptr(chart);
199
+ check_expected_ptr(family);
200
+ check_expected_ptr(dimension);
201
+ check_expected_ptr(instance);
202
+ check_expected(value);
203
+ check_expected(timestamp);
204
+}
205
+#endif // ENABLE_PROMETHEUS_REMOTE_WRITE
206
+
207
#if HAVE_KINESIS
208
void __wrap_aws_sdk_init()
209
{
exporting/tests/exporting_fixtures.c
+1
@@ -65,6 +65,7 @@ int setup_rrdhost()
65
rd->name = strdupz("dimension_name");
66
rd->last_collected_value = 123000321;
67
rd->last_collected_time.tv_sec = 15051;
68
+ rd->collections_counter++;
69
rd->next = NULL;
70
71
rd->state = calloc(1, sizeof(*rd->state));
exporting/tests/netdata_doubles.c
+19
@@ -91,6 +91,25 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
91
return RRD_MEMORY_MODE_NONE_NAME;
92
}
93
94
+calculated_number rrdvar2number(RRDVAR *rv)
95
+{
96
+ (void)rv;
97
+ return 0;
98
+}
99
+
100
+int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data)
101
+{
102
+ (void)host;
103
+ (void)callback;
104
+ (void)data;
105
+ return 0;
106
+}
107
+
108
+void rrdset_update_heterogeneous_flag(RRDSET *st)
109
+{
110
+ (void)st;
111
+}
112
+
113
time_t __mock_rrddim_query_oldest_time(RRDDIM *rd)
114
{
115
(void)rd;
exporting/tests/test_exporting_engine.c
+217
-6
@@ -761,6 +761,200 @@ static void test_flush_host_labels(void **state)
761
assert_int_equal(buffer_strlen(instance->labels), 0);
762
}
763
764
+#if ENABLE_PROMETHEUS_REMOTE_WRITE
765
+static void test_init_prometheus_remote_write_instance(void **state)
766
+{
767
+ struct engine *engine = *state;
768
+ struct instance *instance = engine->instance_root;
769
+
770
+ expect_function_call(__wrap_init_write_request);
771
+ will_return(__wrap_init_write_request, 0xff);
772
+
773
+ assert_int_equal(init_prometheus_remote_write_instance(instance), 0);
774
+
775
+ assert_ptr_equal(instance->worker, simple_connector_worker);
776
+ assert_ptr_equal(instance->start_batch_formatting, NULL);
777
+ assert_ptr_equal(instance->start_host_formatting, format_host_prometheus_remote_write);
778
+ assert_ptr_equal(instance->start_chart_formatting, format_chart_prometheus_remote_write);
779
+ assert_ptr_equal(instance->metric_formatting, format_dimension_prometheus_remote_write);
780
+ assert_ptr_equal(instance->end_chart_formatting, NULL);
781
+ assert_ptr_equal(instance->end_host_formatting, NULL);
782
+ assert_ptr_equal(instance->end_batch_formatting, format_batch_prometheus_remote_write);
783
+ assert_ptr_equal(instance->send_header, prometheus_remote_write_send_header);
784
+ assert_ptr_equal(instance->check_response, process_prometheus_remote_write_response);
785
+
786
+ assert_ptr_not_equal(instance->buffer, NULL);
787
+ buffer_free(instance->buffer);
788
+
789
+ struct prometheus_remote_write_specific_data *connector_specific_data =
790
+ (struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
791
+
792
+ assert_ptr_not_equal(instance->connector_specific_data, NULL);
793
+ assert_ptr_equal(connector_specific_data->write_request, 0xff);
794
+ freez(instance->connector_specific_data);
795
+}
796
+
797
+static void test_prometheus_remote_write_send_header(void **state)
798
+{
799
+ struct engine *engine = *state;
800
+ struct instance *instance = engine->instance_root;
801
+ int sock = 1;
802
+
803
+ struct prometheus_remote_write_specific_config *connector_specific_config =
804
+ callocz(1, sizeof(struct prometheus_remote_write_specific_config));
805
+ instance->config.connector_specific_config = connector_specific_config;
806
+ connector_specific_config->remote_write_path = strdupz("/receive");
807
+
808
+ buffer_sprintf(instance->buffer, "test buffer");
809
+
810
+ expect_function_call(__wrap_send);
811
+ expect_value(__wrap_send, sockfd, 1);
812
+ expect_not_value(__wrap_send, buf, NULL);
813
+ expect_string(
814
+ __wrap_send, buf,
815
+ "POST /receive HTTP/1.1\r\n"
816
+ "Host: test-host\r\n"
817
+ "Accept: */*\r\n"
818
+ "Content-Length: 11\r\n"
819
+ "Content-Type: application/x-www-form-urlencoded\r\n\r\n");
820
+ expect_value(__wrap_send, len, 125);
821
+ expect_value(__wrap_send, flags, MSG_NOSIGNAL);
822
+
823
+ assert_int_equal(prometheus_remote_write_send_header(&sock, instance),0);
824
+
825
+ free(connector_specific_config->remote_write_path);
826
+}
827
+
828
+static void test_process_prometheus_remote_write_response(void **state)
829
+{
830
+ (void)state;
831
+ BUFFER *buffer = buffer_create(0);
832
+
833
+ buffer_sprintf(buffer, "HTTP/1.1 200 OK\r\n");
834
+ assert_int_equal(process_prometheus_remote_write_response(buffer, NULL), 0);
835
+
836
+ buffer_free(buffer);
837
+}
838
+
839
+static void test_format_host_prometheus_remote_write(void **state)
840
+{
841
+ struct engine *engine = *state;
842
+ struct instance *instance = engine->instance_root;
843
+
844
+ instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
845
+ instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
846
+
847
+ struct prometheus_remote_write_specific_data *connector_specific_data =
848
+ mallocz(sizeof(struct prometheus_remote_write_specific_data *));
849
+ instance->connector_specific_data = (void *)connector_specific_data;
850
+ connector_specific_data->write_request = (void *)0xff;
851
+
852
+ localhost->program_name = strdupz("test_program");
853
+ localhost->program_version = strdupz("test_version");
854
+
855
+ expect_function_call(__wrap_add_host_info);
856
+ expect_value(__wrap_add_host_info, write_request_p, 0xff);
857
+ expect_string(__wrap_add_host_info, name, "netdata_info");
858
+ expect_string(__wrap_add_host_info, instance, "test-host");
859
+ expect_string(__wrap_add_host_info, application, "test_program");
860
+ expect_string(__wrap_add_host_info, version, "test_version");
861
+ expect_in_range(
862
+ __wrap_add_host_info, timestamp, now_realtime_usec() / USEC_PER_MS - 1000, now_realtime_usec() / USEC_PER_MS);
863
+
864
+ expect_function_call(__wrap_add_label);
865
+ expect_value(__wrap_add_label, write_request_p, 0xff);
866
+ expect_string(__wrap_add_label, key, "key1");
867
+ expect_string(__wrap_add_label, value, "value1");
868
+
869
+ expect_function_call(__wrap_add_label);
870
+ expect_value(__wrap_add_label, write_request_p, 0xff);
871
+ expect_string(__wrap_add_label, key, "key2");
872
+ expect_string(__wrap_add_label, value, "value2");
873
+
874
+ assert_int_equal(format_host_prometheus_remote_write(instance, localhost), 0);
875
+
876
+ freez(connector_specific_data);
877
+ free(localhost->program_name);
878
+ free(localhost->program_version);
879
+}
880
+
881
+static void test_format_dimension_prometheus_remote_write(void **state)
882
+{
883
+ struct engine *engine = *state;
884
+ struct instance *instance = engine->instance_root;
885
+
886
+ struct prometheus_remote_write_specific_data *connector_specific_data =
887
+ mallocz(sizeof(struct prometheus_remote_write_specific_data *));
888
+ instance->connector_specific_data = (void *)connector_specific_data;
889
+ connector_specific_data->write_request = (void *)0xff;
890
+
891
+ RRDDIM *rd = localhost->rrdset_root->dimensions;
892
+
893
+ expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
894
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
895
+
896
+ expect_function_call(__wrap_add_metric);
897
+ expect_value(__wrap_add_metric, write_request_p, 0xff);
898
+ expect_string(__wrap_add_metric, name, "netdata_");
899
+ expect_string(__wrap_add_metric, chart, "");
900
+ expect_string(__wrap_add_metric, family, "");
901
+ expect_string(__wrap_add_metric, dimension, "dimension_name");
902
+ expect_string(__wrap_add_metric, instance, "test-host");
903
+ expect_value(__wrap_add_metric, value, 0x292932e0);
904
+ expect_value(__wrap_add_metric, timestamp, 15052 * MSEC_PER_SEC);
905
+
906
+ assert_int_equal(format_dimension_prometheus_remote_write(instance, rd), 0);
907
+}
908
+
909
+static void test_format_batch_prometheus_remote_write(void **state)
910
+{
911
+ struct engine *engine = *state;
912
+ struct instance *instance = engine->instance_root;
913
+
914
+ struct prometheus_remote_write_specific_data *connector_specific_data =
915
+ mallocz(sizeof(struct prometheus_remote_write_specific_data *));
916
+ instance->connector_specific_data = (void *)connector_specific_data;
917
+ connector_specific_data->write_request = __real_init_write_request();
918
+
919
+ __real_add_host_info(
920
+ connector_specific_data->write_request,
921
+ "test_name", "test_instance", "test_application", "test_version", 15051);
922
+
923
+ __real_add_label(connector_specific_data->write_request, "test_key", "test_value");
924
+
925
+ __real_add_metric(
926
+ connector_specific_data->write_request,
927
+ "test_name", "test chart", "test_family", "test_dimension", "test_instance",
928
+ 123000321, 15052);
929
+
930
+ assert_int_equal(format_batch_prometheus_remote_write(instance), 0);
931
+
932
+ BUFFER *buffer = instance->buffer;
933
+ assert_int_equal(buffer_strlen(buffer), 192);
934
+
935
+ BUFFER *escaped_buffer = buffer_create(850);
936
+ size_t len = buffer_strlen(buffer);
937
+ char *ch = (char *)buffer_tostring(buffer);
938
+ for (; len > 0; ch++, len--)
939
+ buffer_sprintf(escaped_buffer, "\\%03o", (unsigned int)*ch);
940
+ assert_string_equal(
941
+ buffer_tostring(escaped_buffer),
942
+ "\\37777777641\\002\\120\\012\\37777777622\\001\\012\\025\\012\\010\\137\\137\\156\\141\\155\\145\\137\\137"
943
+ "\\022\\011\\164\\145\\163\\164\\005\\015\\064\\012\\031\\012\\010\\151\\156\\163\\164\\141\\156\\143\\145\\022"
944
+ "\\015\\005\\027\\021\\017\\100\\012\\037\\012\\013\\141\\160\\160\\154\\151\\143\\141\\164\\151\\157\\156\\022"
945
+ "\\020\\005\\036\\035\\022\\034\\012\\027\\012\\007\\166\\145\\162\\163\\001\\035\\000\\014\\005\\035\\015\\016"
946
+ "\\014\\012\\026\\012\\010\\005\\020\\020\\153\\145\\171\\022\\012\\005\\012\\040\\166\\141\\154\\165\\145\\022"
947
+ "\\014\\011\\000\\005\\001\\030\\37777777760\\077\\020\\37777777713\\165\\012\\37777777611\\142\\37777777625"
948
+ "\\000\\034\\023\\012\\005\\143\\150\\141\\162\\164\\011\\075\\000\\040\\005\\014\\054\\012\\025\\012\\006\\146"
949
+ "\\141\\155\\151\\154\\171\\022\\013\\005\\123\\011\\015\\040\\012\\033\\012\\011\\144\\151\\155\\145\\156\\005"
950
+ "\\37777777607\\000\\016\\005\\032\\025\\020\\000\\012\\146\\37777777736\\000\\064\\022\\014\\011\\000\\000\\000"
951
+ "\\004\\130\\123\\37777777635\\101\\020\\37777777714\\165");
952
+
953
+ buffer_free(escaped_buffer);
954
+ protocol_buffers_shutdown();
955
+}
956
+#endif // ENABLE_PROMETHEUS_REMOTE_WRITE
957
+
958
#if HAVE_KINESIS
959
static void test_init_aws_kinesis_instance(void **state)
960
{
@@ -851,7 +1045,7 @@ static void test_aws_kinesis_connector_worker(void **state)
1045
expect_string(__wrap_kinesis_put_record, stream_name, "test_stream");
1046
expect_string(__wrap_kinesis_put_record, partition_key, "netdata_0");
1047
expect_value(__wrap_kinesis_put_record, data, buffer_tostring(buffer));
854
- // The buffer is prepated by Graphite exporting connector
1048
+ // The buffer is prepared by Graphite exporting connector
1049
expect_string(
1050
__wrap_kinesis_put_record, data,
1051
"netdata.test-host.chart_name.dimension_name;TAG1=VALUE1 TAG2=VALUE2 123000321 15051\n");
@@ -941,6 +1135,28 @@ int main(void)
1135
cmocka_unit_test_setup_teardown(test_flush_host_labels, setup_initialized_engine, teardown_initialized_engine),
1136
};
1137
1138
+ int test_res = cmocka_run_group_tests_name("exporting_engine", tests, NULL, NULL) +
1139
+ cmocka_run_group_tests_name("labels_in_exporting_engine", label_tests, NULL, NULL);
1140
+
1141
+#if ENABLE_PROMETHEUS_REMOTE_WRITE
1142
+ const struct CMUnitTest prometheus_remote_write_tests[] = {
1143
+ cmocka_unit_test_setup_teardown(
1144
+ test_init_prometheus_remote_write_instance, setup_configured_engine, teardown_configured_engine),
1145
+ cmocka_unit_test_setup_teardown(
1146
+ test_prometheus_remote_write_send_header, setup_initialized_engine, teardown_initialized_engine),
1147
+ cmocka_unit_test(test_process_prometheus_remote_write_response),
1148
+ cmocka_unit_test_setup_teardown(
1149
+ test_format_host_prometheus_remote_write, setup_initialized_engine, teardown_initialized_engine),
1150
+ cmocka_unit_test_setup_teardown(
1151
+ test_format_dimension_prometheus_remote_write, setup_initialized_engine, teardown_initialized_engine),
1152
+ cmocka_unit_test_setup_teardown(
1153
+ test_format_batch_prometheus_remote_write, setup_initialized_engine, teardown_initialized_engine),
1154
+ };
1155
+
1156
+ test_res += cmocka_run_group_tests_name(
1157
+ "prometheus_remote_write_exporting_connector", prometheus_remote_write_tests, NULL, NULL);
1158
+#endif
1159
+
1160
#if HAVE_KINESIS
1161
const struct CMUnitTest kinesis_tests[] = {
1162
cmocka_unit_test_setup_teardown(
@@ -948,12 +1164,7 @@ int main(void)
1164
cmocka_unit_test_setup_teardown(
1165
test_aws_kinesis_connector_worker, setup_initialized_engine, teardown_initialized_engine),
1166
};
951
-#endif
1167
953
- int test_res = cmocka_run_group_tests_name("exporting_engine", tests, NULL, NULL) +
954
- cmocka_run_group_tests_name("labels_in_exporting_engine", label_tests, NULL, NULL);
955
-
956
-#if HAVE_KINESIS
1168
test_res += cmocka_run_group_tests_name("kinesis_exporting_connector", kinesis_tests, NULL, NULL);
1169
#endif
1170
exporting/tests/test_exporting_engine.h
+23
@@ -9,6 +9,7 @@
9
#include "exporting/graphite/graphite.h"
10
#include "exporting/json/json.h"
11
#include "exporting/opentsdb/opentsdb.h"
12
+#include "exporting/prometheus/remote_write/remote_write.h"
13
#include "exporting/aws_kinesis/aws_kinesis.h"
14
15
#include <stdarg.h>
@@ -96,6 +97,28 @@ int __mock_end_chart_formatting(struct instance *instance, RRDSET *st);
97
int __mock_end_host_formatting(struct instance *instance, RRDHOST *host);
98
int __mock_end_batch_formatting(struct instance *instance);
99
100
+void *__real_init_write_request();
101
+void *__wrap_init_write_request();
102
+
103
+void __real_add_host_info(
104
+ void *write_request_p,
105
+ const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp);
106
+void __wrap_add_host_info(
107
+ void *write_request_p,
108
+ const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp);
109
+
110
+void __real_add_label(void *write_request_p, char *key, char *value);
111
+void __wrap_add_label(void *write_request_p, char *key, char *value);
112
+
113
+void __real_add_metric(
114
+ void *write_request_p,
115
+ const char *name, const char *chart, const char *family, const char *dimension,
116
+ const char *instance, const double value, const int64_t timestamp);
117
+void __wrap_add_metric(
118
+ void *write_request_p,
119
+ const char *name, const char *chart, const char *family, const char *dimension,
120
+ const char *instance, const double value, const int64_t timestamp);
121
+
122
void __wrap_aws_sdk_init();
123
void __wrap_kinesis_init(
124
void *kinesis_specific_data_p, const char *region, const char *access_key_id, const char *secret_key,
web/api/exporters/allmetrics.c
+12
-12
@@ -4,24 +4,24 @@
4
5
struct prometheus_output_options {
6
char *name;
7
- PROMETHEUS_OUTPUT_OPTIONS flag;
7
+ BACKENDS_PROMETHEUS_OUTPUT_OPTIONS flag;
8
} prometheus_output_flags_root[] = {
9
- { "help", PROMETHEUS_OUTPUT_HELP },
10
- { "types", PROMETHEUS_OUTPUT_TYPES },
11
- { "names", PROMETHEUS_OUTPUT_NAMES },
12
- { "timestamps", PROMETHEUS_OUTPUT_TIMESTAMPS },
13
- { "variables", PROMETHEUS_OUTPUT_VARIABLES },
14
- { "oldunits", PROMETHEUS_OUTPUT_OLDUNITS },
15
- { "hideunits", PROMETHEUS_OUTPUT_HIDEUNITS },
9
+ { "help", BACKENDS_PROMETHEUS_OUTPUT_HELP },
10
+ { "types", BACKENDS_PROMETHEUS_OUTPUT_TYPES },
11
+ { "names", BACKENDS_PROMETHEUS_OUTPUT_NAMES },
12
+ { "timestamps", BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS },
13
+ { "variables", BACKENDS_PROMETHEUS_OUTPUT_VARIABLES },
14
+ { "oldunits", BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS },
15
+ { "hideunits", BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS },
16
// terminator
17
- { NULL, PROMETHEUS_OUTPUT_NONE },
17
+ { NULL, BACKENDS_PROMETHEUS_OUTPUT_NONE },
18
};
19
20
inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) {
21
int format = ALLMETRICS_SHELL;
22
const char *prometheus_server = w->client_ip;
23
uint32_t prometheus_backend_options = global_backend_options;
24
- PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options = PROMETHEUS_OUTPUT_TIMESTAMPS | ((global_backend_options & BACKEND_OPTION_SEND_NAMES)?PROMETHEUS_OUTPUT_NAMES:0);
24
+ BACKENDS_PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options = BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS | ((global_backend_options & BACKEND_OPTION_SEND_NAMES)?BACKENDS_PROMETHEUS_OUTPUT_NAMES:0);
25
const char *prometheus_prefix = global_backend_prefix;
26
27
while(url) {
@@ -84,7 +84,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
84
85
case ALLMETRICS_PROMETHEUS:
86
w->response.data->contenttype = CT_PROMETHEUS;
87
- rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
87
+ backends_rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
88
host
89
, w->response.data
90
, prometheus_server
@@ -96,7 +96,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
96
97
case ALLMETRICS_PROMETHEUS_ALL_HOSTS:
98
w->response.data->contenttype = CT_PROMETHEUS;
99
- rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
99
+ backends_rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
100
host
101
, w->response.data
102
, prometheus_server