@cryptotaxi247 / netdata-1 / commits / d3679c1d7

prometheus support all data sources netdata supports for the other backends: as collected, average, sum. The prometheus metrics format differs between as collected and the other types so that average and sum fixe #1519

Costa Tsaousis (ktsaou) committed Jul 9, 2017 at 20:50 UTC d3679c1d71e256b152b1c17b21bd91ae19ce6451
10 files changed +321 -176
CMakeLists.txt
+1 -1
@@ -152,7 +152,7 @@ set(NETDATA_SOURCE_FILES
152 src/web_client.h
153 src/web_server.c
154 src/web_server.h
155 - src/locks.h src/statsd.c src/statsd.h src/statistical.c src/statistical.h)
155 + src/locks.h src/statsd.c src/statsd.h src/statistical.c src/statistical.h src/backend_prometheus.c src/backend_prometheus.h)
156
157 set(APPS_PLUGIN_SOURCE_FILES
158 src/appconfig.c
src/Makefile.am
+2
@@ -44,6 +44,8 @@ netdata_SOURCES = \
44 appconfig.h \
45 avl.c \
46 avl.h \
47 + backend_prometheus.c \
48 + backend_prometheus.h \
49 backends.c \
50 backends.h \
51 clocks.c \
src/backend_prometheus.c new
+234
@@ -0,0 +1,234 @@
1 +#include "common.h"
2 +
3 +// ----------------------------------------------------------------------------
4 +// PROMETHEUS
5 +// /api/v1/allmetrics?format=prometheus
6 +
7 +static struct prometheus_server {
8 + const char *server;
9 + uint32_t hash;
10 + time_t last_access;
11 + struct prometheus_server *next;
12 +} *prometheus_server_root = NULL;
13 +
14 +static inline time_t prometheus_server_last_access(const char *server, time_t now) {
15 + uint32_t hash = simple_hash(server);
16 +
17 + struct prometheus_server *ps;
18 + for(ps = prometheus_server_root; ps ;ps = ps->next) {
19 + if (hash == ps->hash && !strcmp(server, ps->server)) {
20 + time_t last = ps->last_access;
21 + ps->last_access = now;
22 + return last;
23 + }
24 + }
25 +
26 + ps = callocz(1, sizeof(struct prometheus_server));
27 + ps->server = strdupz(server);
28 + ps->hash = hash;
29 + ps->last_access = now;
30 + ps->next = prometheus_server_root;
31 + prometheus_server_root = ps;
32 +
33 + return 0;
34 +}
35 +
36 +static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) {
37 + size_t n;
38 +
39 + for(n = 0; *s && n < usable ; d++, s++, n++) {
40 + register char c = *s;
41 +
42 + if(!isalnum(c)) *d = '_';
43 + else *d = c;
44 + }
45 + *d = '\0';
46 +
47 + return n;
48 +}
49 +
50 +static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable) {
51 + size_t n;
52 +
53 + // make sure we can escape one character without overflowing the buffer
54 + usable--;
55 +
56 + for(n = 0; *s && n < usable ; d++, s++, n++) {
57 + register char c = *s;
58 +
59 + if(unlikely(c == '"' || c == '\\' || c == '\n')) {
60 + *d++ = '\\';
61 + n++;
62 + }
63 + *d = c;
64 + }
65 + *d = '\0';
66 +
67 + return n;
68 +}
69 +
70 +#define PROMETHEUS_ELEMENT_MAX 256
71 +#define PROMETHEUS_LABELS_MAX 1024
72 +
73 +void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, uint32_t options, time_t after, time_t before, int allhosts, int help, int types, int names) {
74 + rrdhost_rdlock(host);
75 +
76 + char hostname[PROMETHEUS_ELEMENT_MAX + 1];
77 + prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
78 +
79 + char labels[PROMETHEUS_LABELS_MAX + 1] = "";
80 + if(allhosts) {
81 + if(host->tags && *(host->tags))
82 + buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);
83 +
84 + snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
85 + }
86 + else {
87 + if(host->tags && *(host->tags))
88 + buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
89 + }
90 +
91 + // for each chart
92 + RRDSET *st;
93 + rrdset_foreach_read(st, host) {
94 + char chart[PROMETHEUS_ELEMENT_MAX + 1];
95 + char context[PROMETHEUS_ELEMENT_MAX + 1];
96 + char family[PROMETHEUS_ELEMENT_MAX + 1];
97 +
98 + prometheus_label_copy(chart, (names && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
99 + prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
100 + prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
101 +
102 + if(unlikely(help || types))
103 + buffer_strcat(wb, "\n");
104 +
105 + if(rrdset_is_available_for_backends(st)) {
106 + rrdset_rdlock(st);
107 +
108 + // for each dimension
109 + RRDDIM *rd;
110 + rrddim_foreach_read(rd, st) {
111 + if(rd->collections_counter) {
112 + char dimension[PROMETHEUS_ELEMENT_MAX + 1];
113 +
114 + if ((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) {
115 + // we need as-collected / raw data
116 +
117 + prometheus_name_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
118 +
119 + const char *t = "gauge", *h = "gives";
120 + if (rd->algorithm == RRD_ALGORITHM_INCREMENTAL ||
121 + rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) {
122 + t = "counter";
123 + h = "delta gives";
124 + }
125 +
126 + if (unlikely(help))
127 + buffer_sprintf(wb, "# HELP %s_%s netdata chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n",
128 + context, dimension, (names && st->name) ? st->name : st->id, st->context,
129 + st->family, (names && rd->name) ? rd->name : rd->id, rd->multiplier,
130 + rd->divisor, h, st->units, t
131 + );
132 +
133 + if (unlikely(types))
134 + buffer_sprintf(wb, "# TYPE %s_%s %s\n", context, dimension, t);
135 +
136 + buffer_sprintf(wb, "%s_%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n",
137 + context, dimension, chart, family, labels, rd->last_collected_value,
138 + timeval_msec(&rd->last_collected_time)
139 + );
140 + }
141 + else {
142 + // we need average or sum of the data
143 +
144 + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options);
145 +
146 + if(!isnan(value) && !isinf(value)) {
147 + prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
148 +
149 + if (unlikely(help))
150 + buffer_sprintf(wb, "# HELP %s netdata chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value gives %s (gauge)\n",
151 + context, (names && st->name) ? st->name : st->id, st->context,
152 + st->family, (names && rd->name) ? rd->name : rd->id,
153 + st->units
154 + );
155 +
156 + if (unlikely(types))
157 + buffer_sprintf(wb, "# TYPE %s gauge\n", context);
158 +
159 + buffer_sprintf(wb, "%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n",
160 + context, chart, family, dimension, labels, value,
161 + timeval_msec(&rd->last_collected_time)
162 + );
163 + }
164 + }
165 + }
166 + }
167 +
168 + rrdset_unlock(st);
169 + }
170 + }
171 +
172 + rrdhost_unlock(host);
173 +}
174 +
175 +static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, uint32_t options, const char *server, time_t now, int help) {
176 + if(!server || !*server) server = "default";
177 +
178 + time_t after = prometheus_server_last_access(server, now);
179 +
180 + int first_seen = 0;
181 + if(!after) {
182 + after = now - backend_update_every;
183 + first_seen = 1;
184 + }
185 +
186 + if(help) {
187 + int show_range = 1;
188 + char *mode;
189 + if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) {
190 + mode = "as collected";
191 + show_range = 0;
192 + }
193 + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE)
194 + mode = "average";
195 + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)
196 + mode = "sum";
197 + else
198 + mode = "unknown";
199 +
200 + buffer_sprintf(wb, "# HELP netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s\n"
201 + , host->hostname
202 + , (first_seen)?"FIRST SEEN ":""
203 + , server
204 + , mode
205 + , (unsigned long)((first_seen)?0:(now - after))
206 + , (first_seen)?"never":"seconds ago"
207 + );
208 +
209 + if(show_range)
210 + buffer_sprintf(wb, "# HELP netdata values for time range %lu to %lu\n", (unsigned long)after, (unsigned long)now);
211 +
212 + buffer_strcat(wb, "\n");
213 + }
214 +
215 + return after;
216 +}
217 +
218 +void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, uint32_t options, int help, int types, int names) {
219 + time_t before = now_realtime_sec();
220 + time_t after = prometheus_preparation(host, wb, options, server, before, help);
221 +
222 + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, options, after, before, 0, help, types, names);
223 +}
224 +
225 +void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, uint32_t options, int help, int types, int names) {
226 + time_t before = now_realtime_sec();
227 + time_t after = prometheus_preparation(host, wb, options, server, before, help);
228 +
229 + rrd_rdlock();
230 + rrdhost_foreach_read(host) {
231 + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, options, after, before, 1, help, types, names);
232 + }
233 + rrd_unlock();
234 +}
src/backend_prometheus.h new
+11
@@ -0,0 +1,11 @@
1 +//
2 +// Created by costa on 09/07/17.
3 +//
4 +
5 +#ifndef NETDATA_BACKEND_PROMETHEUS_H
6 +#define NETDATA_BACKEND_PROMETHEUS_H
7 +
8 +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, uint32_t options, int help, int types, int names);
9 +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, uint32_t options, int help, int types, int names);
10 +
11 +#endif //NETDATA_BACKEND_PROMETHEUS_H
src/backends.c
+44 -40
@@ -22,11 +22,9 @@
22 // 5. repeats the above forever.
23 //
24
25 -#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001
26 -#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002
27 -#define BACKEND_SOURCE_DATA_SUM 0x00000004
28 -
25 int backend_send_names = 1;
26 +int backend_update_every = 10;
27 +uint32_t backend_options = BACKEND_SOURCE_DATA_AVERAGE;
28
29 // ----------------------------------------------------------------------------
30 // helper functions for backends
@@ -48,7 +46,7 @@ static inline size_t backend_name_copy(char *d, const char *s, size_t usable) {
46 // calculate the SUM or AVERAGE of a dimension, for any timeframe
47 // may return NAN if the database does not have any value in the give timeframe
48
51 -static inline calculated_number backend_calculate_value_from_stored_data(
49 +inline calculated_number backend_calculate_value_from_stored_data(
50 RRDSET *st // the chart
51 , RRDDIM *rd // the dimension
52 , time_t after // the start timestamp
@@ -124,7 +122,7 @@ static inline calculated_number backend_calculate_value_from_stored_data(
122 return NAN;
123 }
124
127 - if(unlikely(options & BACKEND_SOURCE_DATA_SUM))
125 + if(unlikely((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM))
126 return sum;
127
128 return sum / (calculated_number)counter;
@@ -457,7 +455,7 @@ static inline int backends_can_send_rrdset(uint32_t options, RRDSET *st) {
455 return 0;
456 }
457
460 - if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(options & BACKEND_SOURCE_DATA_AS_COLLECTED))) {
458 + if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {
459 debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, st->rrdhost->hostname, rrd_memory_mode_name(st->rrdhost->rrd_memory_mode));
460 return 0;
461 }
@@ -465,6 +463,26 @@ static inline int backends_can_send_rrdset(uint32_t options, RRDSET *st) {
463 return 1;
464 }
465
466 +inline uint32_t backend_parse_data_source(const char *source, uint32_t mode) {
467 + if(!strcmp(source, "raw") || !strcmp(source, "as collected") || !strcmp(source, "as-collected") || !strcmp(source, "as_collected") || !strcmp(source, "ascollected")) {
468 + mode |= BACKEND_SOURCE_DATA_AS_COLLECTED;
469 + mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AS_COLLECTED);
470 + }
471 + else if(!strcmp(source, "average")) {
472 + mode |= BACKEND_SOURCE_DATA_AVERAGE;
473 + mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AVERAGE);
474 + }
475 + else if(!strcmp(source, "sum") || !strcmp(source, "volume")) {
476 + mode |= BACKEND_SOURCE_DATA_SUM;
477 + mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_SUM);
478 + }
479 + else {
480 + error("BACKEND: invalid data source method '%s'.", source);
481 + }
482 +
483 + return mode;
484 +}
485 +
486 void *backends_main(void *ptr) {
487 int default_port = 0;
488 int sock = -1;
@@ -489,16 +507,15 @@ void *backends_main(void *ptr) {
507 .tv_sec = 0,
508 .tv_usec = 0
509 };
492 - uint32_t options = 0x00000000;
510 int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0);
511 const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average");
512 const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");
513 const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost");
514 const char *prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");
515 const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname);
499 - int frequency = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", 10);
516 + backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", backend_update_every);
517 int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10);
501 - long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", frequency * 2 * 1000);
518 + long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", backend_update_every * 2 * 1000);
519 backend_send_names = config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", backend_send_names);
520
521 charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), SIMPLE_PATTERN_EXACT);
@@ -508,30 +525,17 @@ void *backends_main(void *ptr) {
525 // validate configuration options
526 // and prepare for sending data to our backend
527
511 - if(!enabled || frequency < 1)
512 - goto cleanup;
513 -
514 - if(!strcmp(source, "as collected")) {
515 - options |= BACKEND_SOURCE_DATA_AS_COLLECTED;
516 - }
517 - else if(!strcmp(source, "average")) {
518 - options |= BACKEND_SOURCE_DATA_AVERAGE;
519 - }
520 - else if(!strcmp(source, "sum") || !strcmp(source, "volume")) {
521 - options |= BACKEND_SOURCE_DATA_SUM;
522 - }
523 - else {
524 - error("BACKEND: invalid data source method '%s' for backend given. Disabling backed.", source);
525 - goto cleanup;
526 - }
528 + backend_options = backend_parse_data_source(source, backend_options);
529
530 if(timeoutms < 1) {
529 - error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, frequency * 2 * 1000);
530 - timeoutms = frequency * 2 * 1000;
531 + error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, backend_update_every * 2 * 1000);
532 + timeoutms = backend_update_every * 2 * 1000;
533 }
534 timeout.tv_sec = (timeoutms * 1000) / 1000000;
535 timeout.tv_usec = (timeoutms * 1000) % 1000000;
536
537 + if(!enabled || backend_update_every < 1)
538 + goto cleanup;
539
540 // ------------------------------------------------------------------------
541 // select the backend type
@@ -541,7 +545,7 @@ void *backends_main(void *ptr) {
545 default_port = 2003;
546 backend_response_checker = process_graphite_response;
547
544 - if(options & BACKEND_SOURCE_DATA_AS_COLLECTED)
548 + if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
549 backend_request_formatter = format_dimension_collected_graphite_plaintext;
550 else
551 backend_request_formatter = format_dimension_stored_graphite_plaintext;
@@ -552,7 +556,7 @@ void *backends_main(void *ptr) {
556 default_port = 4242;
557 backend_response_checker = process_opentsdb_response;
558
555 - if(options & BACKEND_SOURCE_DATA_AS_COLLECTED)
559 + if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
560 backend_request_formatter = format_dimension_collected_opentsdb_telnet;
561 else
562 backend_request_formatter = format_dimension_stored_opentsdb_telnet;
@@ -563,7 +567,7 @@ void *backends_main(void *ptr) {
567 default_port = 5448;
568 backend_response_checker = process_json_response;
569
566 - if (options & BACKEND_SOURCE_DATA_AS_COLLECTED)
570 + if ((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
571 backend_request_formatter = format_dimension_collected_json_plaintext;
572 else
573 backend_request_formatter = format_dimension_stored_json_plaintext;
@@ -600,18 +604,18 @@ void *backends_main(void *ptr) {
604 chart_backend_reconnects = 0,
605 chart_backend_latency = 0;
606
603 - RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, frequency, RRDSET_TYPE_LINE);
607 + RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, backend_update_every, RRDSET_TYPE_LINE);
608 rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
609 rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
610 rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
611
608 - RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, frequency, RRDSET_TYPE_AREA);
612 + RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, backend_update_every, RRDSET_TYPE_AREA);
613 rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
614 rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
615 rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
616 rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
617
614 - RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, frequency, RRDSET_TYPE_LINE);
618 + RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, backend_update_every, RRDSET_TYPE_LINE);
619 rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
620 rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
621 rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -625,11 +629,11 @@ void *backends_main(void *ptr) {
629 *
630 * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
631 *
628 - RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, frequency, RRDSET_TYPE_AREA);
632 + RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, backend_update_every, RRDSET_TYPE_AREA);
633 rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
634 */
635
632 - RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, frequency, RRDSET_TYPE_STACKED);
636 + RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, backend_update_every, RRDSET_TYPE_STACKED);
637 rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
638 rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
639
@@ -637,9 +641,9 @@ void *backends_main(void *ptr) {
641 // ------------------------------------------------------------------------
642 // prepare the backend main loop
643
640 - info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, frequency, hostname, prefix);
644 + info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, backend_update_every, hostname, prefix);
645
642 - usec_t step_ut = frequency * USEC_PER_SEC;
646 + usec_t step_ut = backend_update_every * USEC_PER_SEC;
647 time_t after = now_realtime_sec();
648 int failures = 0;
649 heartbeat_t hb;
@@ -680,7 +684,7 @@ void *backends_main(void *ptr) {
684
685 RRDSET *st;
686 rrdset_foreach_read(st, host) {
683 - if(likely(backends_can_send_rrdset(options, st))) {
687 + if(likely(backends_can_send_rrdset(backend_options, st))) {
688 rrdset_rdlock(st);
689
690 count_charts++;
@@ -688,7 +692,7 @@ void *backends_main(void *ptr) {
692 RRDDIM *rd;
693 rrddim_foreach_read(rd, st) {
694 if (likely(rd->last_collected_time.tv_sec >= after)) {
691 - chart_buffered_metrics += backend_request_formatter(b, prefix, host, __hostname, st, rd, after, before, options);
695 + chart_buffered_metrics += backend_request_formatter(b, prefix, host, __hostname, st, rd, after, before, backend_options);
696 count_dims++;
697 }
698 else {
src/backends.h
+18
@@ -1,7 +1,25 @@
1 #ifndef NETDATA_BACKENDS_H
2 #define NETDATA_BACKENDS_H 1
3
4 +#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001
5 +#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002
6 +#define BACKEND_SOURCE_DATA_SUM 0x00000004
7 +
8 +#define BACKEND_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM)
9 +
10 extern int backend_send_names;
11 +extern int backend_update_every;
12 +extern uint32_t backend_options;
13 extern void *backends_main(void *ptr);
14
15 +extern uint32_t backend_parse_data_source(const char *source, uint32_t mode);
16 +
17 +extern calculated_number backend_calculate_value_from_stored_data(
18 + RRDSET *st // the chart
19 + , RRDDIM *rd // the dimension
20 + , time_t after // the start timestamp
21 + , time_t before // the end timestamp
22 + , uint32_t options // BACKEND_SOURCE_* bitmap
23 +);
24 +
25 #endif /* NETDATA_BACKENDS_H */
src/common.h
+1
@@ -220,6 +220,7 @@
220 #include "unit_test.h"
221 #include "ipc.h"
222 #include "backends.h"
223 +#include "backend_prometheus.h"
224 #include "inlined.h"
225 #include "adaptive_resortable_list.h"
226 #include "rrdpush.h"
src/rrd2json.c
-131
@@ -166,137 +166,6 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) {
166 buffer_sprintf(wb, "\n\t]\n}\n");
167 }
168
169 -// ----------------------------------------------------------------------------
170 -// PROMETHEUS
171 -// /api/v1/allmetrics?format=prometheus
172 -
173 -static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) {
174 - size_t n;
175 -
176 - for(n = 0; *s && n < usable ; d++, s++, n++) {
177 - register char c = *s;
178 -
179 - if(!isalnum(c)) *d = '_';
180 - else *d = c;
181 - }
182 - *d = '\0';
183 -
184 - return n;
185 -}
186 -
187 -static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable) {
188 - size_t n;
189 -
190 - // make sure we can escape one character without overflowing the buffer
191 - usable--;
192 -
193 - for(n = 0; *s && n < usable ; d++, s++, n++) {
194 - register char c = *s;
195 -
196 - if(unlikely(c == '"' || c == '\\' || c == '\n')) {
197 - *d++ = '\\';
198 - n++;
199 - }
200 - *d = c;
201 - }
202 - *d = '\0';
203 -
204 - return n;
205 -}
206 -
207 -#define PROMETHEUS_ELEMENT_MAX 256
208 -#define PROMETHEUS_LABELS_MAX 1024
209 -
210 -void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, int allhosts, int help, int types, int names) {
211 - rrdhost_rdlock(host);
212 -
213 - char hostname[PROMETHEUS_ELEMENT_MAX + 1];
214 - prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
215 -
216 - char labels[PROMETHEUS_LABELS_MAX + 1] = "";
217 - if(allhosts) {
218 - if(host->tags && *(host->tags))
219 - buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);
220 -
221 - snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
222 - }
223 - else {
224 - if(host->tags && *(host->tags))
225 - buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
226 - }
227 -
228 - // for each chart
229 - RRDSET *st;
230 - rrdset_foreach_read(st, host) {
231 - char chart[PROMETHEUS_ELEMENT_MAX + 1];
232 - char context[PROMETHEUS_ELEMENT_MAX + 1];
233 - char family[PROMETHEUS_ELEMENT_MAX + 1];
234 -
235 - prometheus_label_copy(chart, (names && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
236 - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
237 - prometheus_name_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
238 -
239 - if(unlikely(help || types))
240 - buffer_strcat(wb, "\n");
241 -
242 - if(rrdset_is_available_for_backends(st)) {
243 - rrdset_rdlock(st);
244 -
245 - // for each dimension
246 - RRDDIM *rd;
247 - rrddim_foreach_read(rd, st) {
248 - if(rd->collections_counter) {
249 - char dimension[PROMETHEUS_ELEMENT_MAX + 1];
250 - prometheus_name_copy(dimension, (names && rd->name)?rd->name:rd->id, PROMETHEUS_ELEMENT_MAX);
251 -
252 - const char *t = "gauge", *h = "gives";
253 - if(rd->algorithm == RRD_ALGORITHM_INCREMENTAL || rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) {
254 - t = "counter";
255 - h = "delta gives";
256 - }
257 -
258 - if(unlikely(help))
259 - buffer_sprintf(wb, "# HELP %s_%s_%s netdata chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"
260 - , context, family, dimension
261 - , (names && st->name)?st->name:st->id
262 - , st->context
263 - , st->family
264 - , (names && rd->name)?rd->name:rd->id
265 - , rd->multiplier
266 - , rd->divisor
267 - , h
268 - , st->units
269 - , t
270 - );
271 -
272 - if(unlikely(types))
273 - buffer_sprintf(wb, "# TYPE %s_%s_%s %s\n", context, family, dimension, t);
274 -
275 - // calculated_number n = (calculated_number)rd->last_collected_value * (calculated_number)(abs(rd->multiplier)) / (calculated_number)(abs(rd->divisor));
276 - // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n, timeval_msec(&rd->last_collected_time));
277 -
278 - buffer_sprintf(wb, "%s_%s_%s{chart=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n",
279 - context, family, dimension, chart, labels, rd->last_collected_value, timeval_msec(&rd->last_collected_time)
280 - );
281 - }
282 - }
283 -
284 - rrdset_unlock(st);
285 - }
286 - }
287 -
288 - rrdhost_unlock(host);
289 -}
290 -
291 -void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(BUFFER *wb, int help, int types, int names) {
292 - RRDHOST *host;
293 - rrd_rdlock();
294 - rrdhost_foreach_read(host) {
295 - rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, 1, help, types, names);
296 - }
297 - rrd_unlock();
298 -}
299 -
169 // ----------------------------------------------------------------------------
170 // BASH
171 // /api/v1/allmetrics?format=bash
src/rrd2json.h
-2
@@ -67,8 +67,6 @@ extern void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb);
67
68 extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb);
69 extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb);
70 -extern void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, int allhosts, int help, int types, int names);
71 -extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(BUFFER *wb, int help, int types, int names);
70
71 extern int rrdset2anything_api_v1(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points
72 , long long after, long long before, int group_method, uint32_t options
src/web_api_v1.c
+10 -2
@@ -209,6 +209,8 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,
209 inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) {
210 int format = ALLMETRICS_SHELL;
211 int help = 0, types = 0, names = backend_send_names; // prometheus options
212 + const char *prometheus_server = w->client_ip;
213 + uint32_t prometheus_options = backend_options;
214
215 while(url) {
216 char *value = mystrsep(&url, "?&");
@@ -248,6 +250,12 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
250 else
251 names = 0;
252 }
253 + else if(!strcmp(name, "server")) {
254 + prometheus_server = value;
255 + }
256 + else if(!strcmp(name, "source")) {
257 + prometheus_options = backend_parse_data_source(value, prometheus_options);
258 + }
259 }
260
261 buffer_flush(w->response.data);
@@ -266,12 +274,12 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
274
275 case ALLMETRICS_PROMETHEUS:
276 w->response.data->contenttype = CT_PROMETHEUS;
269 - rrd_stats_api_v1_charts_allmetrics_prometheus(host, w->response.data, 0, help, types, names);
277 + rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(host, w->response.data, prometheus_server, prometheus_options, help, types, names);
278 return 200;
279
280 case ALLMETRICS_PROMETHEUS_ALL_HOSTS:
281 w->response.data->contenttype = CT_PROMETHEUS;
274 - rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(w->response.data, help, types, names);
282 + rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(host, w->response.data, prometheus_server, prometheus_options, help, types, names);
283 return 200;
284
285 default: