@cryptotaxi247 / netdata-1 / commits / ebbce7c77

Prometheus web api connector (#8540)

* Fix the Prometheus web API code in the exporting engine * Rename connector types * Remove the conditional compilation of the exporting engine * Use labels instead of tags * Fix the exporter configuration * Document functions * Add unit tests

Vladimir Kobal committed Apr 6, 2020 at 09:26 UTC ebbce7c7773a886d222dbaa60d098b6c25150f69
19 files changed +574 -254
Makefile.am
+4 -26
@@ -573,6 +573,7 @@ NETDATA_FILES = \
573 $(LIBNETDATA_FILES) \
574 $(API_PLUGIN_FILES) \
575 $(BACKENDS_PLUGIN_FILES) \
576 + $(EXPORTING_ENGINE_FILES) \
577 $(CHECKS_PLUGIN_FILES) \
578 $(HEALTH_PLUGIN_FILES) \
579 $(IDLEJITTER_PLUGIN_FILES) \
@@ -608,12 +609,6 @@ if LINUX
609
610 endif
611
611 -if ENABLE_EXPORTING
612 - NETDATA_FILES += \
613 - $(EXPORTING_ENGINE_FILES) \
614 - $(NULL)
615 -endif
616 -
612 NETDATA_COMMON_LIBS = \
613 $(OPTIONAL_MATH_LIBS) \
614 $(OPTIONAL_ZLIB_LIBS) \
@@ -745,23 +740,13 @@ if ENABLE_PLUGIN_SLABINFO
740 $(NULL)
741 endif
742
748 -if ENABLE_EXPORTING
749 -if ENABLE_BACKEND_KINESIS
750 - netdata_SOURCES += $(KINESIS_EXPORTING_FILES)
751 - netdata_LDADD += $(OPTIONAL_KINESIS_LIBS)
752 -endif
753 -endif
754 -
743 if ENABLE_BACKEND_KINESIS
756 - netdata_SOURCES += $(KINESIS_BACKEND_FILES)
744 + netdata_SOURCES += $(KINESIS_BACKEND_FILES) $(KINESIS_EXPORTING_FILES)
745 netdata_LDADD += $(OPTIONAL_KINESIS_LIBS)
746 endif
747
748 if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE
761 -if ENABLE_EXPORTING
762 - netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
763 -endif
764 - netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES)
749 + netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES) $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
750 netdata_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS)
751 BUILT_SOURCES = \
752 exporting/prometheus/remote_write/remote_write.pb.cc \
@@ -775,15 +760,8 @@ exporting/prometheus/remote_write/remote_write.pb.h: exporting/prometheus/remote
760
761 endif
762
778 -if ENABLE_EXPORTING
779 -if ENABLE_BACKEND_MONGODB
780 - netdata_SOURCES += $(MONGODB_EXPORTING_FILES)
781 - netdata_LDADD += $(OPTIONAL_MONGOC_LIBS)
782 -endif
783 -endif
784 -
763 if ENABLE_BACKEND_MONGODB
786 - netdata_SOURCES += $(MONGODB_BACKEND_FILES)
764 + netdata_SOURCES += $(MONGODB_BACKEND_FILES) $(MONGODB_EXPORTING_FILES)
765 netdata_LDADD += $(OPTIONAL_MONGOC_LIBS)
766 endif
767
backends/backends.h
-4
@@ -27,10 +27,6 @@ typedef enum backend_types {
27 BACKEND_TYPE_NUM // Number of backend types
28 } BACKEND_TYPE;
29
30 -#ifdef ENABLE_EXPORTING
31 -#include "exporting/exporting_engine.h"
32 -#endif
33 -
30 typedef int (**backend_response_checker_t)(BUFFER *);
31 typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS);
32
backends/prometheus/backend_prometheus.c
+20 -20
@@ -44,7 +44,7 @@ static inline time_t prometheus_server_last_access(const char *server, RRDHOST *
44 return 0;
45 }
46
47 -static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) {
47 +static inline size_t backends_prometheus_name_copy(char *d, const char *s, size_t usable) {
48 size_t n;
49
50 for(n = 0; *s && n < usable ; d++, s++, n++) {
@@ -58,7 +58,7 @@ static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable)
58 return n;
59 }
60
61 -static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable) {
61 +static inline size_t backends_prometheus_label_copy(char *d, const char *s, size_t usable) {
62 size_t n;
63
64 // make sure we can escape one character without overflowing the buffer
@@ -78,7 +78,7 @@ static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable
78 return n;
79 }
80
81 -static inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits) {
81 +static inline char *backends_prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits) {
82 const char *sorig = s;
83 char *ret = d;
84 size_t n;
@@ -194,7 +194,7 @@ static int print_host_variables(RRDVAR *rv, void *data) {
194 label_post = "}";
195 }
196
197 - prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));
197 + backends_prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));
198
199 if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
200 buffer_sprintf(opts->wb
@@ -227,7 +227,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
227 rrdhost_rdlock(host);
228
229 char hostname[PROMETHEUS_ELEMENT_MAX + 1];
230 - prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
230 + backends_prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
231
232 char labels[PROMETHEUS_LABELS_MAX + 1] = "";
233 if(allhosts) {
@@ -299,9 +299,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
299 char family[PROMETHEUS_ELEMENT_MAX + 1];
300 char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
301
302 - prometheus_label_copy(chart, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
303 - prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
304 - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
302 + backends_prometheus_label_copy(chart, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
303 + backends_prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
304 + backends_prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
305
306 if(likely(backends_can_send_rrdset(backend_options, st))) {
307 rrdset_rdlock(st);
@@ -317,7 +317,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
317 }
318 else {
319 if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE && !(output_options & BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS))
320 - prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS);
320 + backends_prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS);
321 }
322
323 if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
@@ -354,7 +354,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
354 // all the dimensions of the chart, has the same algorithm, multiplier and divisor
355 // we add all dimensions as labels
356
357 - prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
357 + backends_prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
358
359 if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
360 buffer_sprintf(wb
@@ -411,7 +411,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
411 // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
412 // we create a metric per dimension
413
414 - prometheus_name_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
414 + backends_prometheus_name_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
415
416 if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
417 buffer_sprintf(wb
@@ -480,7 +480,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
480 else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)
481 suffix = "_sum";
482
483 - prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
483 + backends_prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
484
485 if (unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
486 buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n"
@@ -593,7 +593,7 @@ void backends_rrd_stats_remote_write_allmetrics_prometheus(
593 , size_t *count_dims_skipped
594 ) {
595 char hostname[PROMETHEUS_ELEMENT_MAX + 1];
596 - prometheus_label_copy(hostname, __hostname, PROMETHEUS_ELEMENT_MAX);
596 + backends_prometheus_label_copy(hostname, __hostname, PROMETHEUS_ELEMENT_MAX);
597
598 backends_add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
599
@@ -620,9 +620,9 @@ void backends_rrd_stats_remote_write_allmetrics_prometheus(
620 char family[PROMETHEUS_ELEMENT_MAX + 1];
621 char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
622
623 - prometheus_label_copy(chart, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
624 - prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
625 - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
623 + backends_prometheus_label_copy(chart, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
624 + backends_prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
625 + backends_prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
626
627 if(likely(backends_can_send_rrdset(backend_options, st))) {
628 rrdset_rdlock(st);
@@ -640,7 +640,7 @@ void backends_rrd_stats_remote_write_allmetrics_prometheus(
640 }
641 else {
642 if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)
643 - prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0);
643 + backends_prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0);
644 }
645
646 // for each dimension
@@ -664,7 +664,7 @@ void backends_rrd_stats_remote_write_allmetrics_prometheus(
664 // all the dimensions of the chart, has the same algorithm, multiplier and divisor
665 // we add all dimensions as labels
666
667 - prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
667 + backends_prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
668 snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", prefix, context, suffix);
669
670 backends_add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));
@@ -674,7 +674,7 @@ void backends_rrd_stats_remote_write_allmetrics_prometheus(
674 // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
675 // we create a metric per dimension
676
677 - prometheus_name_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
677 + backends_prometheus_name_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
678 snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", prefix, context, dimension, suffix);
679
680 backends_add_metric(name, chart, family, NULL, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));
@@ -694,7 +694,7 @@ void backends_rrd_stats_remote_write_allmetrics_prometheus(
694 else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)
695 suffix = "_sum";
696
697 - prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
697 + backends_prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
698 snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", prefix, context, units, suffix);
699
700 backends_add_metric(name, chart, family, dimension, hostname, value, last_t * MSEC_PER_SEC);
configure.ac
-14
@@ -433,20 +433,6 @@ fi
433 AC_MSG_RESULT([${enable_https}])
434 AM_CONDITIONAL([ENABLE_HTTPS], [test "${enable_https}" = "yes"])
435
436 -# -----------------------------------------------------------------------------
437 -# Exporting engine
438 -AC_MSG_CHECKING([if netdata exporting engine should be used])
439 -if test "${UV_LIBS}"; then
440 - enable_exporting_engine="yes"
441 - AC_DEFINE([ENABLE_EXPORTING], [1], [netdata exporting engine usability])
442 - OPTIONAL_UV_CFLAGS="${UV_CFLAGS}"
443 - OPTIONAL_UV_LIBS="${UV_LIBS}"
444 -else
445 - enable_exporting_engine="no"
446 -fi
447 -AC_MSG_RESULT([${enable_exporting_engine}])
448 -AM_CONDITIONAL([ENABLE_EXPORTING], [test "${enable_exporting_engine}" = "yes"])
449 -
436 # -----------------------------------------------------------------------------
437 # JSON-C
438 test "${enable_jsonc}" = "yes" -a -z "${JSONC_LIBS}" && \
daemon/common.h
+2
@@ -50,6 +50,8 @@
50
51 // backends for archiving the metrics
52 #include "backends/backends.h"
53 +// the new exporting engine for archiving the metrics
54 +#include "exporting/exporting_engine.h"
55
56 // the netdata API
57 #include "web/api/web_api_v1.h"
daemon/main.c
-2
@@ -80,9 +80,7 @@ struct netdata_static_thread static_threads[] = {
80
81 // common plugins for all systems
82 {"BACKENDS", NULL, NULL, 1, NULL, NULL, backends_main},
83 -#ifdef ENABLE_EXPORTING
83 {"EXPORTING", NULL, NULL, 1, NULL, NULL, exporting_main},
85 -#endif
84 {"WEB_SERVER[static1]", NULL, NULL, 0, NULL, NULL, socket_listen_main_static_threaded},
85 {"STREAM", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread},
86
exporting/exporting_engine.h
+20 -4
@@ -14,10 +14,10 @@
14 extern struct config exporting_config;
15
16 #define EXPORTING_UPDATE_EVERY_OPTION_NAME "update every"
17 -#define EXPORTING_UPDATE_EVERY_DEFAULT 10
17 +#define EXPORTING_UPDATE_EVERY_DEFAULT 10
18
19 typedef enum exporting_options {
20 - EXPORTING_OPTION_NONE = 0,
20 + EXPORTING_OPTION_NON = 0,
21
22 EXPORTING_SOURCE_DATA_AS_COLLECTED = (1 << 0),
23 EXPORTING_SOURCE_DATA_AVERAGE = (1 << 1),
@@ -42,10 +42,22 @@ typedef enum exporting_options {
42 (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS && \
43 label->label_source != LABEL_SOURCE_NETDATA_CONF))
44
45 +typedef enum exporting_connector_types {
46 + EXPORTING_CONNECTOR_TYPE_UNKNOWN, // Invalid type
47 + EXPORTING_CONNECTOR_TYPE_GRAPHITE, // Send plain text to Graphite
48 + EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET, // Send data to OpenTSDB using telnet API
49 + EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP, // Send data to OpenTSDB using HTTP API
50 + EXPORTING_CONNECTOR_TYPE_JSON, // Stores the data using JSON.
51 + EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // The user selected to use Prometheus backend
52 + EXPORTING_CONNECTOR_TYPE_KINESIS, // Send message to AWS Kinesis
53 + EXPORTING_CONNECTOR_TYPE_MONGODB, // Send data to MongoDB collection
54 + EXPORTING_CONNECTOR_TYPE_NUM // Number of backend types
55 +} EXPORTING_CONNECTOR_TYPE;
56 +
57 struct engine;
58
59 struct instance_config {
48 - BACKEND_TYPE type;
60 + EXPORTING_CONNECTOR_TYPE type;
61
62 const char *name;
63 const char *destination;
@@ -150,10 +162,12 @@ struct engine {
162 struct instance *instance_root;
163 };
164
165 +extern struct instance *prometheus_exporter_instance;
166 +
167 void *exporting_main(void *ptr);
168
169 struct engine *read_exporting_config();
156 -BACKEND_TYPE exporting_select_type(const char *type);
170 +EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type);
171
172 int init_connectors(struct engine *engine);
173
@@ -187,4 +201,6 @@ void simple_connector_worker(void *instance_p);
201
202 int send_internal_metrics(struct engine *engine);
203
204 +#include "exporting/prometheus/prometheus.h"
205 +
206 #endif /* NETDATA_EXPORTING_ENGINE_H */
exporting/init_connectors.c
+8 -8
@@ -32,35 +32,35 @@ int init_connectors(struct engine *engine)
32 instance->after = engine->now;
33
34 switch (instance->config.type) {
35 - case BACKEND_TYPE_GRAPHITE:
35 + case EXPORTING_CONNECTOR_TYPE_GRAPHITE:
36 if (init_graphite_instance(instance) != 0)
37 return 1;
38 break;
39 - case BACKEND_TYPE_JSON:
39 + case EXPORTING_CONNECTOR_TYPE_JSON:
40 if (init_json_instance(instance) != 0)
41 return 1;
42 break;
43 - case BACKEND_TYPE_OPENTSDB_USING_TELNET:
43 + case EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET:
44 if (init_opentsdb_telnet_instance(instance) != 0)
45 return 1;
46 break;
47 - case BACKEND_TYPE_OPENTSDB_USING_HTTP:
47 + case EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP:
48 if (init_opentsdb_http_instance(instance) != 0)
49 return 1;
50 break;
51 - case BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE:
51 + case EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE:
52 #if ENABLE_PROMETHEUS_REMOTE_WRITE
53 if (init_prometheus_remote_write_instance(instance) != 0)
54 return 1;
55 #endif
56 break;
57 - case BACKEND_TYPE_KINESIS:
57 + case EXPORTING_CONNECTOR_TYPE_KINESIS:
58 #if HAVE_KINESIS
59 if (init_aws_kinesis_instance(instance) != 0)
60 return 1;
61 #endif
62 break;
63 - case BACKEND_TYPE_MONGODB:
63 + case EXPORTING_CONNECTOR_TYPE_MONGODB:
64 #if HAVE_MONGOC
65 if (init_mongodb_instance(instance) != 0)
66 return 1;
@@ -77,7 +77,7 @@ int init_connectors(struct engine *engine)
77 error("EXPORTING: cannot create tread worker. uv_thread_create(): %s", uv_strerror(error));
78 return 1;
79 }
80 - char threadname[NETDATA_THREAD_NAME_MAX+1];
80 + char threadname[NETDATA_THREAD_NAME_MAX + 1];
81 snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "EXPORTING-%zu", instance->index);
82 uv_thread_set_name_np(instance->thread, threadname);
83 }
exporting/mongodb/mongodb.c
+1 -1
@@ -208,7 +208,7 @@ int format_batch_mongodb(struct instance *instance)
208 insert[documents_inserted] = bson_new_from_json((const uint8_t *)start, -1, &bson_error);
209
210 if (unlikely(!insert[documents_inserted])) {
211 - error("EXPORTING: %s", bson_error.message);
211 + error("EXPORTING: Failed creating a BSON document from a JSON string \"%s\" : %s", start, bson_error.message);
212 free_bson(insert, documents_inserted);
213 return 1;
214 }
exporting/process_data.c
+1 -1
@@ -206,7 +206,7 @@ int start_host_formatting(struct engine *engine, RRDHOST *host)
206 * Start chart formatting for every connector instance's buffer
207 *
208 * @param engine an engine data structure.
209 - * @param a chart.
209 + * @param st a chart.
210 * @return Returns 0 on success, 1 on failure.
211 */
212 int start_chart_formatting(struct engine *engine, RRDSET *st)
exporting/prometheus/prometheus.c
+217 -73
@@ -7,10 +7,16 @@
7 // PROMETHEUS
8 // /api/v1/allmetrics?format=prometheus and /api/v1/allmetrics?format=prometheus_all_hosts
9
10 +/**
11 + * Check if a chart can be sent to an external databese
12 + *
13 + * @param instance an instance data structure.
14 + * @param st a chart.
15 + * @return Returns 1 if the chart can be sent, 0 otherwise.
16 + */
17 inline int can_send_rrdset(struct instance *instance, RRDSET *st)
18 {
19 RRDHOST *host = st->rrdhost;
13 - (void)host;
20
21 if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE)))
22 return 0;
@@ -24,7 +30,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)
30 rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE);
31 debug(
32 D_BACKEND,
27 - "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.",
33 + "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.",
34 st->id,
35 host->hostname);
36 return 0;
@@ -34,7 +40,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)
40 if (unlikely(!rrdset_is_available_for_backends(st))) {
41 debug(
42 D_BACKEND,
37 - "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.",
43 + "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.",
44 st->id,
45 host->hostname);
46 return 0;
@@ -42,10 +48,10 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)
48
49 if (unlikely(
50 st->rrd_memory_mode == RRD_MEMORY_MODE_NONE &&
45 - !(BACKEND_OPTIONS_DATA_SOURCE(instance->config.options) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {
51 + !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
52 debug(
53 D_BACKEND,
48 - "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.",
54 + "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.",
55 st->id,
56 host->hostname,
57 rrd_memory_mode_name(host->rrd_memory_mode));
@@ -63,8 +69,19 @@ static struct prometheus_server {
69 struct prometheus_server *next;
70 } *prometheus_server_root = NULL;
71
72 +/**
73 + * Get the last time when a Prometheus server scraped the Netdata Prometheus exporter.
74 + *
75 + * @param server the name of the Prometheus server.
76 + * @param host a data collecting host.
77 + * @param now actual time.
78 + * @return Returns the last time when the server accessed Netdata, or 0 if it is the first occurrence.
79 + */
80 static inline time_t prometheus_server_last_access(const char *server, RRDHOST *host, time_t now)
81 {
82 +#ifdef UNIT_TESTING
83 + return 0;
84 +#endif
85 static netdata_mutex_t prometheus_server_root_mutex = NETDATA_MUTEX_INITIALIZER;
86
87 uint32_t hash = simple_hash(server);
@@ -93,6 +110,14 @@ static inline time_t prometheus_server_last_access(const char *server, RRDHOST *
110 return 0;
111 }
112
113 +/**
114 + * Copy and sanitize name.
115 + *
116 + * @param d a destination string.
117 + * @param s a source sting.
118 + * @param usable the number of characters to copy.
119 + * @return Returns the length of the copied string.
120 + */
121 inline size_t prometheus_name_copy(char *d, const char *s, size_t usable)
122 {
123 size_t n;
@@ -110,6 +135,14 @@ inline size_t prometheus_name_copy(char *d, const char *s, size_t usable)
135 return n;
136 }
137
138 +/**
139 + * Copy and sanitize label.
140 + *
141 + * @param d a destination string.
142 + * @param s a source sting.
143 + * @param usable the number of characters to copy.
144 + * @return Returns the length of the copied string.
145 + */
146 inline size_t prometheus_label_copy(char *d, const char *s, size_t usable)
147 {
148 size_t n;
@@ -131,6 +164,15 @@ inline size_t prometheus_label_copy(char *d, const char *s, size_t usable)
164 return n;
165 }
166
167 +/**
168 + * Copy and sanitize units.
169 + *
170 + * @param d a destination string.
171 + * @param s a source sting.
172 + * @param usable the number of characters to copy.
173 + * @param showoldunits set this flag to 1 to show old (before v1.12) units.
174 + * @return Returns the destination string.
175 + */
176 inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits)
177 {
178 const char *sorig = s;
@@ -203,6 +245,43 @@ inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int sh
245 return ret;
246 }
247
248 +/**
249 + * Format host labels for the Prometheus exporter
250 + *
251 + * @param instance an instance data structure.
252 + * @param host a data collecting host.
253 + */
254 +void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
255 +{
256 + if (unlikely(!sending_labels_configured(instance)))
257 + return;
258 +
259 + if (!instance->labels)
260 + instance->labels = buffer_create(1024);
261 +
262 + int count = 0;
263 + rrdhost_check_rdlock(host);
264 + netdata_rwlock_rdlock(&host->labels_rwlock);
265 + for (struct label *label = host->labels; label; label = label->next) {
266 + if (!should_send_label(instance, label))
267 + continue;
268 +
269 + char key[PROMETHEUS_ELEMENT_MAX + 1];
270 + char value[PROMETHEUS_ELEMENT_MAX + 1];
271 +
272 + prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
273 + prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
274 +
275 + if (*key && *value) {
276 + if (count > 0)
277 + buffer_strcat(instance->labels, ",");
278 + buffer_sprintf(instance->labels, "%s=\"%s\"", key, value);
279 + count++;
280 + }
281 + }
282 + netdata_rwlock_unlock(&host->labels_rwlock);
283 +}
284 +
285 struct host_variables_callback_options {
286 RRDHOST *host;
287 BUFFER *wb;
@@ -215,6 +294,13 @@ struct host_variables_callback_options {
294 char name[PROMETHEUS_VARIABLE_MAX + 1];
295 };
296
297 +/**
298 + * Print host variables.
299 + *
300 + * @param rv a variable.
301 + * @param data callback options.
302 + * @return Returns 1 if the chart can be sent, 0 otherwise.
303 + */
304 static int print_host_variables(RRDVAR *rv, void *data)
305 {
306 struct host_variables_callback_options *opts = data;
@@ -274,14 +360,23 @@ static int print_host_variables(RRDVAR *rv, void *data)
360 return 0;
361 }
362
363 +/**
364 + * Write metrics in Prometheus format to a buffer.
365 + *
366 + * @param instance an instance data structure.
367 + * @param host a data collecting host.
368 + * @param wb the buffer to fill with metrics.
369 + * @param prefix a prefix for every metric.
370 + * @param exporting_options options to configure what data is exported.
371 + * @param allhosts set to 1 if host instance should be in the output for tags.
372 + * @param output_options options to configure the format of the output.
373 + */
374 static void rrd_stats_api_v1_charts_allmetrics_prometheus(
375 struct instance *instance,
376 RRDHOST *host,
377 BUFFER *wb,
378 const char *prefix,
379 EXPORTING_OPTIONS exporting_options,
283 - time_t after,
284 - time_t before,
380 int allhosts,
381 PROMETHEUS_OUTPUT_OPTIONS output_options)
382 {
@@ -290,31 +385,33 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
385 char hostname[PROMETHEUS_ELEMENT_MAX + 1];
386 prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
387
388 + format_host_labels_prometheus(instance, host);
389 +
390 + if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
391 + buffer_sprintf(
392 + wb,
393 + "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
394 + hostname,
395 + host->program_name,
396 + host->program_version,
397 + now_realtime_usec() / USEC_PER_MS);
398 + else
399 + buffer_sprintf(
400 + wb,
401 + "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
402 + hostname,
403 + host->program_name,
404 + host->program_version);
405 +
406 char labels[PROMETHEUS_LABELS_MAX + 1] = "";
407 if (allhosts) {
295 - if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
296 - buffer_sprintf(
297 - wb,
298 - "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
299 - hostname,
300 - host->program_name,
301 - host->program_version,
302 - now_realtime_usec() / USEC_PER_MS);
303 - else
304 - buffer_sprintf(
305 - wb,
306 - "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
307 - hostname,
308 - host->program_name,
309 - host->program_version);
310 -
311 - if (host->tags && *(host->tags)) {
408 + if (instance->labels && buffer_tostring(instance->labels)) {
409 if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
410 buffer_sprintf(
411 wb,
412 "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n",
413 hostname,
317 - host->tags,
414 + buffer_tostring(instance->labels),
415 now_realtime_usec() / USEC_PER_MS);
416
417 // deprecated, exists only for compatibility with older queries
@@ -322,50 +419,46 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
419 wb,
420 "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n",
421 hostname,
325 - host->tags,
422 + buffer_tostring(instance->labels),
423 now_realtime_usec() / USEC_PER_MS);
424 } else {
328 - buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, host->tags);
425 + buffer_sprintf(
426 + wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
427
428 // deprecated, exists only for compatibility with older queries
331 - buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, host->tags);
429 + buffer_sprintf(
430 + wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
431 }
432 }
433
434 snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
435 } else {
337 - if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
338 - buffer_sprintf(
339 - wb,
340 - "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
341 - hostname,
342 - host->program_name,
343 - host->program_version,
344 - now_realtime_usec() / USEC_PER_MS);
345 - else
346 - buffer_sprintf(
347 - wb,
348 - "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
349 - hostname,
350 - host->program_name,
351 - host->program_version);
352 -
353 - if (host->tags && *(host->tags)) {
436 + if (instance->labels && buffer_tostring(instance->labels)) {
437 if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
438 buffer_sprintf(
356 - wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
439 + wb,
440 + "netdata_host_tags_info{%s} 1 %llu\n",
441 + buffer_tostring(instance->labels),
442 + now_realtime_usec() / USEC_PER_MS);
443
444 // deprecated, exists only for compatibility with older queries
359 - buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
445 + buffer_sprintf(
446 + wb,
447 + "netdata_host_tags{%s} 1 %llu\n",
448 + buffer_tostring(instance->labels),
449 + now_realtime_usec() / USEC_PER_MS);
450 } else {
361 - buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", host->tags);
451 + buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels));
452
453 // deprecated, exists only for compatibility with older queries
364 - buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", host->tags);
454 + buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels));
455 }
456 }
457 }
458
459 + if (instance->labels)
460 + buffer_flush(instance->labels);
461 +
462 // send custom variables set for the host
463 if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {
464 struct host_variables_callback_options opts = { .host = host,
@@ -383,20 +476,20 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
476 RRDSET *st;
477 rrdset_foreach_read(st, host)
478 {
386 - char chart[PROMETHEUS_ELEMENT_MAX + 1];
387 - char context[PROMETHEUS_ELEMENT_MAX + 1];
388 - char family[PROMETHEUS_ELEMENT_MAX + 1];
389 -
390 - prometheus_label_copy(
391 - chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX);
392 - prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
393 - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
479
480 if (likely(can_send_rrdset(instance, st))) {
481 rrdset_rdlock(st);
482
483 + char chart[PROMETHEUS_ELEMENT_MAX + 1];
484 + char context[PROMETHEUS_ELEMENT_MAX + 1];
485 + char family[PROMETHEUS_ELEMENT_MAX + 1];
486 char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
487
488 + prometheus_label_copy(
489 + chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX);
490 + prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
491 + prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
492 +
493 int as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED);
494 int homogeneous = 1;
495 if (as_collected) {
@@ -433,7 +526,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
526 if (as_collected) {
527 // we need as-collected / raw data
528
436 - if (unlikely(rd->last_collected_time.tv_sec < after))
529 + if (unlikely(rd->last_collected_time.tv_sec < instance->after))
530 continue;
531
532 const char *t = "gauge", *h = "gives";
@@ -562,8 +655,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
655 } else {
656 // we need average or sum of the data
657
565 - time_t first_t = after, last_t = before;
566 - calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
658 + time_t first_time = instance->after;
659 + time_t last_time = instance->before;
660 + calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
661
662 if (!isnan(value) && !isinf(value)) {
663 if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
@@ -586,8 +680,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
680 suffix,
681 (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
682 st->units,
589 - (unsigned long long)first_t,
590 - (unsigned long long)last_t);
683 + (unsigned long long)first_time,
684 + (unsigned long long)last_time);
685
686 if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
687 buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n", prefix, context, units, suffix);
@@ -606,7 +700,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
700 dimension,
701 labels,
702 value,
609 - last_t * MSEC_PER_SEC);
703 + last_time * MSEC_PER_SEC);
704 else
705 buffer_sprintf(
706 wb,
@@ -633,6 +727,18 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
727 rrdhost_unlock(host);
728 }
729
730 +/**
731 + * Get the last time time when a server accessed Netdata. Write information about an API request to a buffer.
732 + *
733 + * @param instance an instance data structure.
734 + * @param host a data collecting host.
735 + * @param wb the buffer to write to.
736 + * @param exporting_options options to configure what data is exported.
737 + * @param server the name of a Prometheus server..
738 + * @param now actual time.
739 + * @param output_options options to configure the format of the output.
740 + * @return Returns the last time when the server accessed Netdata.
741 + */
742 static inline time_t prometheus_preparation(
743 struct instance *instance,
744 RRDHOST *host,
@@ -649,13 +755,13 @@ static inline time_t prometheus_preparation(
755
756 int first_seen = 0;
757 if (!after) {
652 - after = now - instance->engine->config.update_every;
758 + after = now - instance->config.update_every;
759 first_seen = 1;
760 }
761
762 if (after > now) {
763 // oops! this should never happen
658 - after = now - instance->engine->config.update_every;
764 + after = now - instance->config.update_every;
765 }
766
767 if (output_options & PROMETHEUS_OUTPUT_HELP) {
@@ -685,8 +791,17 @@ static inline time_t prometheus_preparation(
791 return after;
792 }
793
794 +/**
795 + * Write metrics and auxiliary information for one host to a buffer.
796 + *
797 + * @param host a data collecting host.
798 + * @param wb the buffer to write to.
799 + * @param server the name of a Prometheus server.
800 + * @param prefix a prefix for every metric.
801 + * @param exporting_options options to configure what data is exported.
802 + * @param output_options options to configure the format of the output.
803 + */
804 void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
689 - struct instance *instance,
805 RRDHOST *host,
806 BUFFER *wb,
807 const char *server,
@@ -694,17 +809,36 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
809 EXPORTING_OPTIONS exporting_options,
810 PROMETHEUS_OUTPUT_OPTIONS output_options)
811 {
697 - time_t before = now_realtime_sec();
812 + if (unlikely(!prometheus_exporter_instance))
813 + return;
814 +
815 + prometheus_exporter_instance->before = now_realtime_sec();
816
817 // we start at the point we had stopped before
700 - time_t after = prometheus_preparation(instance, host, wb, exporting_options, server, before, output_options);
818 + prometheus_exporter_instance->after = prometheus_preparation(
819 + prometheus_exporter_instance,
820 + host,
821 + wb,
822 + exporting_options,
823 + server,
824 + prometheus_exporter_instance->before,
825 + output_options);
826
827 rrd_stats_api_v1_charts_allmetrics_prometheus(
703 - instance, host, wb, prefix, exporting_options, after, before, 0, output_options);
828 + prometheus_exporter_instance, host, wb, prefix, exporting_options, 0, output_options);
829 }
830
831 +/**
832 + * Write metrics and auxiliary information for all hosts to a buffer.
833 + *
834 + * @param host a data collecting host.
835 + * @param wb the buffer to write to.
836 + * @param server the name of a Prometheus server.
837 + * @param prefix a prefix for every metric.
838 + * @param exporting_options options to configure what data is exported.
839 + * @param output_options options to configure the format of the output.
840 + */
841 void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
707 - struct instance *instance,
842 RRDHOST *host,
843 BUFFER *wb,
844 const char *server,
@@ -712,16 +846,26 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
846 EXPORTING_OPTIONS exporting_options,
847 PROMETHEUS_OUTPUT_OPTIONS output_options)
848 {
715 - time_t before = now_realtime_sec();
849 + if (unlikely(!prometheus_exporter_instance))
850 + return;
851 +
852 + prometheus_exporter_instance->before = now_realtime_sec();
853
854 // we start at the point we had stopped before
718 - time_t after = prometheus_preparation(instance, host, wb, exporting_options, server, before, output_options);
855 + prometheus_exporter_instance->after = prometheus_preparation(
856 + prometheus_exporter_instance,
857 + host,
858 + wb,
859 + exporting_options,
860 + server,
861 + prometheus_exporter_instance->before,
862 + output_options);
863
864 rrd_rdlock();
865 rrdhost_foreach_read(host)
866 {
867 rrd_stats_api_v1_charts_allmetrics_prometheus(
724 - instance, host, wb, prefix, exporting_options, after, before, 1, output_options);
868 + prometheus_exporter_instance, host, wb, prefix, exporting_options, 1, output_options);
869 }
870 rrd_unlock();
871 }
exporting/prometheus/prometheus.h
+4 -2
@@ -23,10 +23,10 @@ typedef enum prometheus_output_flags {
23 } PROMETHEUS_OUTPUT_OPTIONS;
24
25 extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
26 - struct instance *instance, RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
26 + RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
27 EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
28 extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
29 - struct instance *instance, RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
29 + RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
30 EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
31
32 int can_send_rrdset(struct instance *instance, RRDSET *st);
@@ -34,4 +34,6 @@ size_t prometheus_name_copy(char *d, const char *s, size_t usable);
34 size_t prometheus_label_copy(char *d, const char *s, size_t usable);
35 char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits);
36
37 +void format_host_labels_prometheus(struct instance *instance, RRDHOST *host);
38 +
39 #endif //NETDATA_EXPORTING_PROMETHEUS_H
exporting/read_config.c
+99 -84
@@ -2,18 +2,19 @@
2
3 #include "exporting_engine.h"
4
5 -struct config exporting_config = {.first_section = NULL,
6 - .last_section = NULL,
7 - .mutex = NETDATA_MUTEX_INITIALIZER,
8 - .index = {.avl_tree = {.root = NULL, .compar = appconfig_section_compare},
9 - .rwlock = AVL_LOCK_INITIALIZER}};
5 +struct config exporting_config = { .first_section = NULL,
6 + .last_section = NULL,
7 + .mutex = NETDATA_MUTEX_INITIALIZER,
8 + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
9 + .rwlock = AVL_LOCK_INITIALIZER } };
10
11 +struct instance *prometheus_exporter_instance = NULL;
12
13 static _CONNECTOR_INSTANCE *find_instance(const char *section)
14 {
15 _CONNECTOR_INSTANCE *local_ci;
16
16 - local_ci = add_connector_instance(NULL, NULL); // Get root section
17 + local_ci = add_connector_instance(NULL, NULL); // Get root section
18 if (unlikely(!local_ci))
19 return local_ci;
20
@@ -38,12 +39,10 @@ char *expconfig_get(struct config *root, const char *section, const char *name,
39 local_ci = find_instance(section);
40
41 if (!local_ci)
41 - return NULL; // TODO: Check if it is meaningful to return default_value
42 + return NULL; // TODO: Check if it is meaningful to return default_value
43
44 return appconfig_get(
44 - root,
45 - local_ci->instance_name,
46 - name,
45 + root, local_ci->instance_name, name,
46 appconfig_get(
47 root, local_ci->connector_name, name, appconfig_get(root, CONFIG_SECTION_EXPORTING, name, default_value)));
48 }
@@ -58,16 +57,12 @@ int expconfig_get_boolean(struct config *root, const char *section, const char *
57 local_ci = find_instance(section);
58
59 if (!local_ci)
61 - return 0; // TODO: Check if it is meaningful to return default_value
60 + return 0; // TODO: Check if it is meaningful to return default_value
61
62 return appconfig_get_boolean(
64 - root,
65 - local_ci->instance_name,
66 - name,
63 + root, local_ci->instance_name, name,
64 appconfig_get_boolean(
68 - root,
69 - local_ci->connector_name,
70 - name,
65 + root, local_ci->connector_name, name,
66 appconfig_get_boolean(root, CONFIG_SECTION_EXPORTING, name, default_value)));
67 }
68
@@ -81,16 +76,12 @@ long long expconfig_get_number(struct config *root, const char *section, const c
76 local_ci = find_instance(section);
77
78 if (!local_ci)
84 - return 0; // TODO: Check if it is meaningful to return default_value
79 + return 0; // TODO: Check if it is meaningful to return default_value
80
81 return appconfig_get_number(
87 - root,
88 - local_ci->instance_name,
89 - name,
82 + root, local_ci->instance_name, name,
83 appconfig_get_number(
91 - root,
92 - local_ci->connector_name,
93 - name,
84 + root, local_ci->connector_name, name,
85 appconfig_get_number(root, CONFIG_SECTION_EXPORTING, name, default_value)));
86 }
87
@@ -108,7 +99,7 @@ int get_connector_instance(struct connector_instance *target_ci)
99 static _CONNECTOR_INSTANCE *local_ci = NULL;
100 _CONNECTOR_INSTANCE *global_connector_instance;
101
111 - global_connector_instance = find_instance(NULL); // Fetch head of instances
102 + global_connector_instance = find_instance(NULL); // Fetch head of instances
103
104 if (unlikely(!global_connector_instance))
105 return 0;
@@ -140,40 +131,39 @@ int get_connector_instance(struct connector_instance *target_ci)
131 *
132 * @return It returns the connector id.
133 */
143 -BACKEND_TYPE exporting_select_type(const char *type)
134 +EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type)
135 {
136 if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
146 - return BACKEND_TYPE_GRAPHITE;
137 + return EXPORTING_CONNECTOR_TYPE_GRAPHITE;
138 } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
148 - return BACKEND_TYPE_OPENTSDB_USING_TELNET;
139 + return EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET;
140 } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
150 - return BACKEND_TYPE_OPENTSDB_USING_HTTP;
141 + return EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP;
142 } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
152 - return BACKEND_TYPE_JSON;
143 + return EXPORTING_CONNECTOR_TYPE_JSON;
144 } else if (!strcmp(type, "prometheus_remote_write")) {
154 - return BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE;
145 + return EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE;
146 } else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
156 - return BACKEND_TYPE_KINESIS;
147 + return EXPORTING_CONNECTOR_TYPE_KINESIS;
148 } else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext"))
158 - return BACKEND_TYPE_MONGODB;
149 + return EXPORTING_CONNECTOR_TYPE_MONGODB;
150
160 - return BACKEND_TYPE_UNKNOWN;
151 + return EXPORTING_CONNECTOR_TYPE_UNKNOWN;
152 }
153
163 -EXPORTING_OPTIONS exporting_parse_data_source(const char *data_source, EXPORTING_OPTIONS exporting_options) {
164 - if(!strcmp(data_source, "raw") || !strcmp(data_source, "as collected") || !strcmp(data_source, "as-collected") || !strcmp(data_source, "as_collected") || !strcmp(data_source, "ascollected")) {
154 +EXPORTING_OPTIONS exporting_parse_data_source(const char *data_source, EXPORTING_OPTIONS exporting_options)
155 +{
156 + if (!strcmp(data_source, "raw") || !strcmp(data_source, "as collected") || !strcmp(data_source, "as-collected") ||
157 + !strcmp(data_source, "as_collected") || !strcmp(data_source, "ascollected")) {
158 exporting_options |= EXPORTING_SOURCE_DATA_AS_COLLECTED;
159 exporting_options &= ~(EXPORTING_OPTIONS_SOURCE_BITS ^ EXPORTING_SOURCE_DATA_AS_COLLECTED);
167 - }
168 - else if(!strcmp(data_source, "average")) {
160 + } else if (!strcmp(data_source, "average")) {
161 exporting_options |= EXPORTING_SOURCE_DATA_AVERAGE;
162 exporting_options &= ~(EXPORTING_OPTIONS_SOURCE_BITS ^ EXPORTING_SOURCE_DATA_AVERAGE);
171 - }
172 - else if(!strcmp(data_source, "sum") || !strcmp(data_source, "volume")) {
163 + } else if (!strcmp(data_source, "sum") || !strcmp(data_source, "volume")) {
164 exporting_options |= EXPORTING_SOURCE_DATA_SUM;
165 exporting_options &= ~(EXPORTING_OPTIONS_SOURCE_BITS ^ EXPORTING_SOURCE_DATA_SUM);
175 - }
176 - else {
166 + } else {
167 error("EXPORTING: invalid data data_source method '%s'.", data_source);
168 }
169
@@ -195,7 +185,7 @@ struct engine *read_exporting_config()
185 static struct engine *engine = NULL;
186 struct connector_instance_list {
187 struct connector_instance local_ci;
198 - BACKEND_TYPE backend_type;
188 + EXPORTING_CONNECTOR_TYPE backend_type;
189
190 struct connector_instance_list *next;
191 };
@@ -220,6 +210,46 @@ struct engine *read_exporting_config()
210
211 freez(filename);
212
213 +#define prometheus_config_get(name, value) \
214 + appconfig_get( \
215 + &exporting_config, CONFIG_SECTION_PROMETHEUS, name, \
216 + appconfig_get(&exporting_config, CONFIG_SECTION_EXPORTING, name, value))
217 +#define prometheus_config_get_number(name, value) \
218 + appconfig_get_number( \
219 + &exporting_config, CONFIG_SECTION_PROMETHEUS, name, \
220 + appconfig_get_number(&exporting_config, CONFIG_SECTION_EXPORTING, name, value))
221 +#define prometheus_config_get_boolean(name, value) \
222 + appconfig_get_boolean( \
223 + &exporting_config, CONFIG_SECTION_PROMETHEUS, name, \
224 + appconfig_get_boolean(&exporting_config, CONFIG_SECTION_EXPORTING, name, value))
225 +
226 + if (!prometheus_exporter_instance) {
227 + prometheus_exporter_instance = callocz(1, sizeof(struct instance));
228 +
229 + prometheus_exporter_instance->config.update_every =
230 + prometheus_config_get_number(EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
231 +
232 + if (prometheus_config_get_boolean("send names instead of ids", CONFIG_BOOLEAN_YES))
233 + prometheus_exporter_instance->config.options |= EXPORTING_OPTION_SEND_NAMES;
234 + else
235 + prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;
236 +
237 + if (prometheus_config_get_boolean("send configured labels", CONFIG_BOOLEAN_YES))
238 + prometheus_exporter_instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
239 + else
240 + prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
241 +
242 + if (prometheus_config_get_boolean("send automatic labels", CONFIG_BOOLEAN_NO))
243 + prometheus_exporter_instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
244 + else
245 + prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
246 +
247 + prometheus_exporter_instance->config.charts_pattern =
248 + simple_pattern_create(prometheus_config_get("send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
249 + prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create(
250 + prometheus_config_get("send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
251 + }
252 +
253 // TODO: change BACKEND to EXPORTING
254 while (get_connector_instance(&local_ci)) {
255 info("Processing connector instance (%s)", local_ci.instance_name);
@@ -227,8 +257,7 @@ struct engine *read_exporting_config()
257 if (exporter_get_boolean(local_ci.instance_name, "enabled", 0)) {
258 info(
259 "Instance (%s) on connector (%s) is enabled and scheduled for activation",
230 - local_ci.instance_name,
231 - local_ci.connector_name);
260 + local_ci.instance_name, local_ci.connector_name);
261
262 tmp_ci_list = (struct connector_instance_list *)callocz(1, sizeof(struct connector_instance_list));
263 memcpy(&tmp_ci_list->local_ci, &local_ci, sizeof(local_ci));
@@ -252,8 +281,8 @@ struct engine *read_exporting_config()
281 engine->config.hostname =
282 strdupz(exporter_get(CONFIG_SECTION_EXPORTING, "hostname", netdata_configured_hostname));
283 engine->config.prefix = strdupz(exporter_get(CONFIG_SECTION_EXPORTING, "prefix", "netdata"));
255 - engine->config.update_every =
256 - exporter_get_number(CONFIG_SECTION_EXPORTING, EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
284 + engine->config.update_every = exporter_get_number(
285 + CONFIG_SECTION_EXPORTING, EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
286 }
287
288 while (tmp_ci_list) {
@@ -262,27 +291,27 @@ struct engine *read_exporting_config()
291
292 info("Instance %s on %s", tmp_ci_list->local_ci.instance_name, tmp_ci_list->local_ci.connector_name);
293
265 - if (tmp_ci_list->backend_type == BACKEND_TYPE_UNKNOWN) {
294 + if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_UNKNOWN) {
295 error("Unknown exporting connector type");
296 goto next_connector_instance;
297 }
298
299 #ifndef ENABLE_PROMETHEUS_REMOTE_WRITE
271 - if (tmp_ci_list->backend_type == BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE) {
300 + if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) {
301 error("Prometheus Remote Write support isn't compiled");
302 goto next_connector_instance;
303 }
304 #endif
305
306 #ifndef HAVE_KINESIS
278 - if (tmp_ci_list->backend_type == BACKEND_TYPE_KINESIS) {
307 + if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_KINESIS) {
308 error("AWS Kinesis support isn't compiled");
309 goto next_connector_instance;
310 }
311 #endif
312
313 #ifndef HAVE_MONGOC
285 - if (tmp_ci_list->backend_type == BACKEND_TYPE_MONGODB) {
314 + if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_MONGODB) {
315 error("MongoDB support isn't compiled");
316 goto next_connector_instance;
317 }
@@ -299,41 +328,31 @@ struct engine *read_exporting_config()
328
329 tmp_instance->config.name = strdupz(tmp_ci_list->local_ci.instance_name);
330
302 - tmp_instance->config.destination =
303 - strdupz(exporter_get(instance_name, "destination", "localhost"));
331 + tmp_instance->config.destination = strdupz(exporter_get(instance_name, "destination", "localhost"));
332
333 tmp_instance->config.update_every =
334 exporter_get_number(instance_name, EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
335
308 - tmp_instance->config.buffer_on_failures =
309 - exporter_get_number(instance_name, "buffer on failures", 10);
336 + tmp_instance->config.buffer_on_failures = exporter_get_number(instance_name, "buffer on failures", 10);
337
311 - tmp_instance->config.timeoutms =
312 - exporter_get_number(instance_name, "timeout ms", 10000);
338 + tmp_instance->config.timeoutms = exporter_get_number(instance_name, "timeout ms", 10000);
339
314 - tmp_instance->config.charts_pattern = simple_pattern_create(
315 - exporter_get(instance_name, "send charts matching", "*"),
316 - NULL,
317 - SIMPLE_PATTERN_EXACT);
340 + tmp_instance->config.charts_pattern =
341 + simple_pattern_create(exporter_get(instance_name, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
342
343 tmp_instance->config.hosts_pattern = simple_pattern_create(
320 - exporter_get(instance_name, "send hosts matching", "localhost *"),
321 - NULL,
322 - SIMPLE_PATTERN_EXACT);
344 + exporter_get(instance_name, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
345
324 - char *data_source =
325 - exporter_get(instance_name, "data source", "average");
346 + char *data_source = exporter_get(instance_name, "data source", "average");
347
348 tmp_instance->config.options = exporting_parse_data_source(data_source, tmp_instance->config.options);
349
329 - if (exporter_get_boolean(
330 - instance_name, "send configured labels", CONFIG_BOOLEAN_YES))
350 + if (exporter_get_boolean(instance_name, "send configured labels", CONFIG_BOOLEAN_YES))
351 tmp_instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
352 else
353 tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
354
335 - if (exporter_get_boolean(
336 - instance_name, "send automatic labels", CONFIG_BOOLEAN_NO))
355 + if (exporter_get_boolean(instance_name, "send automatic labels", CONFIG_BOOLEAN_NO))
356 tmp_instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
357 else
358 tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
@@ -343,33 +362,30 @@ struct engine *read_exporting_config()
362 else
363 tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;
364
346 - if (tmp_instance->config.type == BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE) {
365 + if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) {
366 struct prometheus_remote_write_specific_config *connector_specific_config =
367 callocz(1, sizeof(struct prometheus_remote_write_specific_config));
368
369 tmp_instance->config.connector_specific_config = connector_specific_config;
370
352 - connector_specific_config->remote_write_path = strdupz(exporter_get(
353 - instance_name, "remote write URL path", "/receive"));
371 + connector_specific_config->remote_write_path =
372 + strdupz(exporter_get(instance_name, "remote write URL path", "/receive"));
373 }
374
356 - if (tmp_instance->config.type == BACKEND_TYPE_KINESIS) {
375 + if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_KINESIS) {
376 struct aws_kinesis_specific_config *connector_specific_config =
377 callocz(1, sizeof(struct aws_kinesis_specific_config));
378
379 tmp_instance->config.connector_specific_config = connector_specific_config;
380
362 - connector_specific_config->stream_name = strdupz(exporter_get(
363 - instance_name, "stream name", "netdata"));
381 + connector_specific_config->stream_name = strdupz(exporter_get(instance_name, "stream name", "netdata"));
382
365 - connector_specific_config->auth_key_id = strdupz(exporter_get(
366 - instance_name, "aws_access_key_id", ""));
383 + connector_specific_config->auth_key_id = strdupz(exporter_get(instance_name, "aws_access_key_id", ""));
384
368 - connector_specific_config->secure_key = strdupz(exporter_get(
369 - instance_name, "aws_secret_access_key", ""));
385 + connector_specific_config->secure_key = strdupz(exporter_get(instance_name, "aws_secret_access_key", ""));
386 }
387
372 - if (tmp_instance->config.type == BACKEND_TYPE_MONGODB) {
388 + if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_MONGODB) {
389 struct mongodb_specific_config *connector_specific_config =
390 callocz(1, sizeof(struct mongodb_specific_config));
391
@@ -393,14 +409,13 @@ struct engine *read_exporting_config()
409 #endif
410
411 if (unlikely(!exporting_config_exists) && !engine->config.hostname) {
396 - engine->config.hostname =
397 - strdupz(config_get(instance_name, "hostname", netdata_configured_hostname));
412 + engine->config.hostname = strdupz(config_get(instance_name, "hostname", netdata_configured_hostname));
413 engine->config.prefix = strdupz(config_get(instance_name, "prefix", "netdata"));
414 engine->config.update_every =
415 config_get_number(instance_name, EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);
416 }
417
403 -next_connector_instance:
418 + next_connector_instance:
419 tmp_ci_list1 = tmp_ci_list->next;
420 freez(tmp_ci_list);
421 tmp_ci_list = tmp_ci_list1;
exporting/tests/exporting_doubles.c
+1 -1
@@ -20,7 +20,7 @@ struct engine *__mock_read_exporting_config()
20 engine->instance_root = calloc(1, sizeof(struct instance));
21 struct instance *instance = engine->instance_root;
22 instance->engine = engine;
23 - instance->config.type = BACKEND_TYPE_GRAPHITE;
23 + instance->config.type = EXPORTING_CONNECTOR_TYPE_GRAPHITE;
24 instance->config.name = strdupz("instance_name");
25 instance->config.destination = strdupz("localhost");
26 instance->config.update_every = 1;
exporting/tests/exporting_fixtures.c
+32
@@ -127,3 +127,35 @@ int teardown_initialized_engine(void **state)
127
128 return 0;
129 }
130 +
131 +int setup_prometheus(void **state)
132 +{
133 + (void)state;
134 +
135 + prometheus_exporter_instance = calloc(1, sizeof(struct instance));
136 +
137 + setup_rrdhost();
138 +
139 + prometheus_exporter_instance->config.update_every = 10;
140 +
141 + prometheus_exporter_instance->config.options |=
142 + EXPORTING_OPTION_SEND_NAMES | EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
143 +
144 + prometheus_exporter_instance->config.charts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT);
145 + prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT);
146 +
147 + return 0;
148 +}
149 +
150 +int teardown_prometheus(void **state)
151 +{
152 + (void)state;
153 +
154 + teardown_rrdhost();
155 +
156 + simple_pattern_free(prometheus_exporter_instance->config.charts_pattern);
157 + simple_pattern_free(prometheus_exporter_instance->config.hosts_pattern);
158 + free(prometheus_exporter_instance);
159 +
160 + return 0;
161 +}
exporting/tests/test_exporting_engine.c
+148 -1
@@ -87,7 +87,7 @@ static void test_read_exporting_config(void **state)
87 assert_ptr_not_equal(instance, NULL);
88 assert_ptr_equal(instance->next, NULL);
89 assert_ptr_equal(instance->engine, engine);
90 - assert_int_equal(instance->config.type, BACKEND_TYPE_GRAPHITE);
90 + assert_int_equal(instance->config.type, EXPORTING_CONNECTOR_TYPE_GRAPHITE);
91 assert_string_equal(instance->config.destination, "localhost");
92 assert_int_equal(instance->config.update_every, 1);
93 assert_int_equal(instance->config.buffer_on_failures, 10);
@@ -761,6 +761,140 @@ static void test_flush_host_labels(void **state)
761 assert_int_equal(buffer_strlen(instance->labels), 0);
762 }
763
764 +static void test_can_send_rrdset(void **state)
765 +{
766 + (void)*state;
767 +
768 + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 1);
769 +
770 + rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE);
771 + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 0);
772 + rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE);
773 +
774 + // TODO: test with a denying simple pattern
775 +
776 + rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_OBSOLETE);
777 + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 0);
778 + rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_OBSOLETE);
779 +
780 + localhost->rrdset_root->rrd_memory_mode = RRD_MEMORY_MODE_NONE;
781 + prometheus_exporter_instance->config.options |= EXPORTING_SOURCE_DATA_AVERAGE;
782 + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 0);
783 +}
784 +
785 +static void test_prometheus_name_copy(void **state)
786 +{
787 + (void)*state;
788 +
789 + char destination_name[PROMETHEUS_ELEMENT_MAX + 1];
790 + assert_int_equal(prometheus_name_copy(destination_name, "test-name", PROMETHEUS_ELEMENT_MAX), 9);
791 +
792 + assert_string_equal(destination_name, "test_name");
793 +}
794 +
795 +static void test_prometheus_label_copy(void **state)
796 +{
797 + (void)*state;
798 +
799 + char destination_name[PROMETHEUS_ELEMENT_MAX + 1];
800 + assert_int_equal(prometheus_label_copy(destination_name, "test\"\\\nlabel", PROMETHEUS_ELEMENT_MAX), 15);
801 +
802 + assert_string_equal(destination_name, "test\\\"\\\\\\\nlabel");
803 +}
804 +
805 +static void test_prometheus_units_copy(void **state)
806 +{
807 + (void)*state;
808 +
809 + char destination_name[PROMETHEUS_ELEMENT_MAX + 1];
810 + assert_string_equal(prometheus_units_copy(destination_name, "test-units", PROMETHEUS_ELEMENT_MAX, 0), "_test_units");
811 + assert_string_equal(destination_name, "_test_units");
812 +
813 + assert_string_equal(prometheus_units_copy(destination_name, "%", PROMETHEUS_ELEMENT_MAX, 0), "_percent");
814 + assert_string_equal(prometheus_units_copy(destination_name, "test-units/s", PROMETHEUS_ELEMENT_MAX, 0), "_test_units_persec");
815 +
816 + assert_string_equal(prometheus_units_copy(destination_name, "KiB", PROMETHEUS_ELEMENT_MAX, 1), "_KB");
817 +}
818 +
819 +static void test_format_host_labels_prometheus(void **state)
820 +{
821 + struct engine *engine = *state;
822 + struct instance *instance = engine->instance_root;
823 +
824 + instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
825 + instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
826 +
827 + format_host_labels_prometheus(instance, localhost);
828 + assert_string_equal(buffer_tostring(instance->labels), "key1=\"netdata\",key2=\"value2\"");
829 +}
830 +
831 +static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
832 +{
833 + (void)state;
834 +
835 + BUFFER *buffer = buffer_create(0);
836 +
837 + localhost->hostname = strdupz("test_hostname");
838 + localhost->rrdset_root->family = strdupz("test_family");
839 + localhost->rrdset_root->context = strdupz("test_context");
840 +
841 + expect_function_call(__wrap_now_realtime_sec);
842 + will_return(__wrap_now_realtime_sec, 2);
843 +
844 + expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
845 + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
846 +
847 + rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(localhost, buffer, "test_server", "test_prefix", 0, 0);
848 +
849 + assert_string_equal(
850 + buffer_tostring(buffer),
851 + "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n"
852 + "netdata_host_tags_info{key1=\"value1\",key2=\"value2\"} 1\n"
853 + "netdata_host_tags{key1=\"value1\",key2=\"value2\"} 1\n"
854 + "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\"} 690565856.0000000\n");
855 +
856 + buffer_flush(buffer);
857 +
858 + expect_function_call(__wrap_now_realtime_sec);
859 + will_return(__wrap_now_realtime_sec, 2);
860 +
861 + expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
862 + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
863 +
864 + rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
865 + localhost, buffer, "test_server", "test_prefix", 0, PROMETHEUS_OUTPUT_NAMES | PROMETHEUS_OUTPUT_TYPES);
866 +
867 + assert_string_equal(
868 + buffer_tostring(buffer),
869 + "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n"
870 + "netdata_host_tags_info{key1=\"value1\",key2=\"value2\"} 1\n"
871 + "netdata_host_tags{key1=\"value1\",key2=\"value2\"} 1\n"
872 + "# COMMENT TYPE test_prefix_test_context gauge\n"
873 + "test_prefix_test_context{chart=\"chart_name\",family=\"test_family\",dimension=\"dimension_name\"} 690565856.0000000\n");
874 +
875 + buffer_flush(buffer);
876 +
877 + expect_function_call(__wrap_now_realtime_sec);
878 + will_return(__wrap_now_realtime_sec, 2);
879 +
880 + expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
881 + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
882 +
883 + rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(localhost, buffer, "test_server", "test_prefix", 0, 0);
884 +
885 + assert_string_equal(
886 + buffer_tostring(buffer),
887 + "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n"
888 + "netdata_host_tags_info{instance=\"test_hostname\",key1=\"value1\",key2=\"value2\"} 1\n"
889 + "netdata_host_tags{instance=\"test_hostname\",key1=\"value1\",key2=\"value2\"} 1\n"
890 + "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\",instance=\"test_hostname\"} 690565856.0000000\n");
891 +
892 + free(localhost->rrdset_root->context);
893 + free(localhost->rrdset_root->family);
894 + free(localhost->hostname);
895 + buffer_free(buffer);
896 +}
897 +
898 #if ENABLE_PROMETHEUS_REMOTE_WRITE
899 static void test_init_prometheus_remote_write_instance(void **state)
900 {
@@ -1311,6 +1445,19 @@ int main(void)
1445 int test_res = cmocka_run_group_tests_name("exporting_engine", tests, NULL, NULL) +
1446 cmocka_run_group_tests_name("labels_in_exporting_engine", label_tests, NULL, NULL);
1447
1448 + const struct CMUnitTest prometheus_web_api_tests[] = {
1449 + cmocka_unit_test_setup_teardown(test_can_send_rrdset, setup_prometheus, teardown_prometheus),
1450 + cmocka_unit_test_setup_teardown(test_prometheus_name_copy, setup_prometheus, teardown_prometheus),
1451 + cmocka_unit_test_setup_teardown(test_prometheus_label_copy, setup_prometheus, teardown_prometheus),
1452 + cmocka_unit_test_setup_teardown(test_prometheus_units_copy, setup_prometheus, teardown_prometheus),
1453 + cmocka_unit_test_setup_teardown(
1454 + test_format_host_labels_prometheus, setup_configured_engine, teardown_configured_engine),
1455 + cmocka_unit_test_setup_teardown(
1456 + rrd_stats_api_v1_charts_allmetrics_prometheus, setup_prometheus, teardown_prometheus),
1457 + };
1458 +
1459 + test_res += cmocka_run_group_tests_name("prometheus_web_api", prometheus_web_api_tests, NULL, NULL);
1460 +
1461 #if ENABLE_PROMETHEUS_REMOTE_WRITE
1462 const struct CMUnitTest prometheus_remote_write_tests[] = {
1463 cmocka_unit_test_setup_teardown(
exporting/tests/test_exporting_engine.h
+2
@@ -166,6 +166,8 @@ int setup_rrdhost();
166 int teardown_rrdhost();
167 int setup_initialized_engine(void **state);
168 int teardown_initialized_engine(void **state);
169 +int setup_prometheus(void **state);
170 +int teardown_prometheus(void **state);
171
172 void init_connectors_in_tests(struct engine *engine);
173
libnetdata/config/appconfig.c
+2 -1
@@ -547,7 +547,8 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons
547 s++;
548
549 if (is_exporter_config) {
550 - global_exporting_section = !(strcmp(s, CONFIG_SECTION_EXPORTING));
550 + global_exporting_section =
551 + !(strcmp(s, CONFIG_SECTION_EXPORTING)) || !(strcmp(s, CONFIG_SECTION_PROMETHEUS));
552 if (unlikely(!global_exporting_section)) {
553 int rc;
554 rc = is_valid_connector(s, 0);
libnetdata/config/appconfig.h
+13 -12
@@ -82,18 +82,19 @@
82
83 #define CONFIG_FILENAME "netdata.conf"
84
85 -#define CONFIG_SECTION_GLOBAL "global"
86 -#define CONFIG_SECTION_WEB "web"
87 -#define CONFIG_SECTION_STATSD "statsd"
88 -#define CONFIG_SECTION_PLUGINS "plugins"
89 -#define CONFIG_SECTION_CLOUD "cloud"
90 -#define CONFIG_SECTION_REGISTRY "registry"
91 -#define CONFIG_SECTION_HEALTH "health"
92 -#define CONFIG_SECTION_BACKEND "backend"
93 -#define CONFIG_SECTION_STREAM "stream"
94 -#define CONFIG_SECTION_EXPORTING "exporting:global"
95 -#define CONFIG_SECTION_HOST_LABEL "host labels"
96 -#define EXPORTING_CONF "exporting.conf"
85 +#define CONFIG_SECTION_GLOBAL "global"
86 +#define CONFIG_SECTION_WEB "web"
87 +#define CONFIG_SECTION_STATSD "statsd"
88 +#define CONFIG_SECTION_PLUGINS "plugins"
89 +#define CONFIG_SECTION_CLOUD "cloud"
90 +#define CONFIG_SECTION_REGISTRY "registry"
91 +#define CONFIG_SECTION_HEALTH "health"
92 +#define CONFIG_SECTION_BACKEND "backend"
93 +#define CONFIG_SECTION_STREAM "stream"
94 +#define CONFIG_SECTION_EXPORTING "exporting:global"
95 +#define CONFIG_SECTION_PROMETHEUS "prometheus:exporter"
96 +#define CONFIG_SECTION_HOST_LABEL "host labels"
97 +#define EXPORTING_CONF "exporting.conf"
98
99 // these are used to limit the configuration names and values lengths
100 // they are not enforced by config.c functions (they will strdup() all strings, no matter of their length)