@cryptotaxi247 / netdata-1 / commits / 0fba85e2c

Send host labels via exporting connectors (#7554)

* Add labels to the JSON exporting connector * Add labels to the Graphite exporting connector * Add labels to the OpenTSDB telnet exporting connector * Add labels to the OpenTSDB HTTP exporting connector * Replace control characters in JSON strings * Add unit tests

Vladimir Kobal committed Jan 9, 2020 at 12:51 UTC 0fba85e2c20add69546cefbf37bb2033d2d1e052
14 files changed +449 -53
database/rrdset.c
+1
@@ -603,6 +603,7 @@ RRDSET *rrdset_create_custom(
603 st->variables = NULL;
604 st->alarms = NULL;
605 st->flags = 0x00000000;
606 + st->exporting_flags = NULL;
607
608 if(memory_mode == RRD_MEMORY_MODE_RAM) {
609 memset(st, 0, size);
exporting/exporting_engine.h
+42 -21
@@ -13,44 +13,62 @@
13
14 extern struct config exporting_config;
15
16 -#define EXPORTER_DATA_SOURCE "data source"
17 -#define EXPORTER_DATA_SOURCE_DEFAULT "average"
16 +#define EXPORTER_DATA_SOURCE "data source"
17 +#define EXPORTER_DATA_SOURCE_DEFAULT "average"
18
19 -#define EXPORTER_DESTINATION "destination"
20 -#define EXPORTER_DESTINATION_DEFAULT "localhost"
19 +#define EXPORTER_DESTINATION "destination"
20 +#define EXPORTER_DESTINATION_DEFAULT "localhost"
21
22 -#define EXPORTER_UPDATE_EVERY "update every"
23 -#define EXPORTER_UPDATE_EVERY_DEFAULT 10
22 +#define EXPORTER_UPDATE_EVERY "update every"
23 +#define EXPORTER_UPDATE_EVERY_DEFAULT 10
24
25 -#define EXPORTER_BUF_ONFAIL "buffer on failures"
26 -#define EXPORTER_BUF_ONFAIL_DEFAULT 10
25 +#define EXPORTER_BUF_ONFAIL "buffer on failures"
26 +#define EXPORTER_BUF_ONFAIL_DEFAULT 10
27
28 -#define EXPORTER_TIMEOUT_MS "timeout ms"
29 -#define EXPORTER_TIMEOUT_MS_DEFAULT 10000
28 +#define EXPORTER_TIMEOUT_MS "timeout ms"
29 +#define EXPORTER_TIMEOUT_MS_DEFAULT 10000
30
31 -#define EXPORTER_SEND_CHART_MATCH "send charts matching"
32 -#define EXPORTER_SEND_CHART_MATCH_DEFAULT "*"
31 +#define EXPORTER_SEND_CHART_MATCH "send charts matching"
32 +#define EXPORTER_SEND_CHART_MATCH_DEFAULT "*"
33
34 -#define EXPORTER_SEND_HOST_MATCH "send hosts matching"
35 -#define EXPORTER_SEND_HOST_MATCH_DEFAULT "localhost *"
34 +#define EXPORTER_SEND_HOST_MATCH "send hosts matching"
35 +#define EXPORTER_SEND_HOST_MATCH_DEFAULT "localhost *"
36
37 -#define EXPORTER_SEND_NAMES "send names instead of ids"
38 -#define EXPORTER_SEND_NAMES_DEFAULT CONFIG_BOOLEAN_YES
37 +#define EXPORTER_SEND_CONFIGURED_LABELS "send configured labels"
38 +#define EXPORTER_SEND_CONFIGURED_LABELS_DEFAULT CONFIG_BOOLEAN_YES
39 +
40 +#define EXPORTER_SEND_AUTOMATIC_LABELS "send automatic labels"
41 +#define EXPORTER_SEND_AUTOMATIC_LABELS_DEFAULT CONFIG_BOOLEAN_NO
42 +
43 +#define EXPORTER_SEND_NAMES "send names instead of ids"
44 +#define EXPORTER_SEND_NAMES_DEFAULT CONFIG_BOOLEAN_YES
45
46 typedef enum exporting_options {
41 - EXPORTING_OPTION_NONE = 0,
47 + EXPORTING_OPTION_NONE = 0,
48 +
49 + EXPORTING_SOURCE_DATA_AS_COLLECTED = (1 << 0),
50 + EXPORTING_SOURCE_DATA_AVERAGE = (1 << 1),
51 + EXPORTING_SOURCE_DATA_SUM = (1 << 2),
52
43 - EXPORTING_SOURCE_DATA_AS_COLLECTED = (1 << 0),
44 - EXPORTING_SOURCE_DATA_AVERAGE = (1 << 1),
45 - EXPORTING_SOURCE_DATA_SUM = (1 << 2),
53 + EXPORTING_OPTION_SEND_CONFIGURED_LABELS = (1 << 3),
54 + EXPORTING_OPTION_SEND_AUTOMATIC_LABELS = (1 << 4),
55
47 - EXPORTING_OPTION_SEND_NAMES = (1 << 16)
56 + EXPORTING_OPTION_SEND_NAMES = (1 << 16)
57 } EXPORTING_OPTIONS;
58
59 #define EXPORTING_OPTIONS_SOURCE_BITS \
60 (EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_SOURCE_DATA_SUM)
61 #define EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) (exporting_options & EXPORTING_OPTIONS_SOURCE_BITS)
62
63 +#define sending_labels_configured(instance) \
64 + (instance->config.options & (EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS))
65 +
66 +#define should_send_label(instance, label) \
67 + ((instance->config.options & EXPORTING_OPTION_SEND_CONFIGURED_LABELS && \
68 + label->label_source == LABEL_SOURCE_NETDATA_CONF) || \
69 + (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS && \
70 + label->label_source != LABEL_SOURCE_NETDATA_CONF))
71 +
72 struct engine;
73
74 struct instance_config {
@@ -107,6 +125,8 @@ struct instance {
125 int skip_host;
126 int skip_chart;
127
128 + BUFFER *labels;
129 +
130 time_t after;
131 time_t before;
132
@@ -174,6 +194,7 @@ int metric_formatting(struct engine *engine, RRDDIM *rd);
194 int end_chart_formatting(struct engine *engine, RRDSET *st);
195 int end_host_formatting(struct engine *engine, RRDHOST *host);
196 int end_batch_formatting(struct engine *engine);
197 +int flush_host_labels(struct instance *instance, RRDHOST *host);
198
199 int exporting_discard_response(BUFFER *buffer, struct instance *instance);
200 void simple_connector_receive_response(int *sock, struct instance *instance);
exporting/graphite/graphite.c
+60 -4
@@ -28,7 +28,7 @@ int init_graphite_connector(struct connector *connector)
28 int init_graphite_instance(struct instance *instance)
29 {
30 instance->start_batch_formatting = NULL;
31 - instance->start_host_formatting = NULL;
31 + instance->start_host_formatting = format_host_labels_graphite_plaintext;
32 instance->start_chart_formatting = NULL;
33
34 if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
@@ -37,7 +37,7 @@ int init_graphite_instance(struct instance *instance)
37 instance->metric_formatting = format_dimension_stored_graphite_plaintext;
38
39 instance->end_chart_formatting = NULL;
40 - instance->end_host_formatting = NULL;
40 + instance->end_host_formatting = flush_host_labels;
41 instance->end_batch_formatting = NULL;
42
43 instance->buffer = (void *)buffer_create(0);
@@ -51,6 +51,60 @@ int init_graphite_instance(struct instance *instance)
51 return 0;
52 }
53
54 +/**
55 + * Copy a label value and substitute underscores in place of charachters which can't be used in Graphite output
56 + *
57 + * @param dst a destination string.
58 + * @param src a source string.
59 + * @param len the maximum number of characters copied.
60 + */
61 +
62 +void sanitize_graphite_label_value(char *dst, char *src, size_t len)
63 +{
64 + while (*src != '\0' && len) {
65 + if (isspace(*src) || *src == ';' || *src == '~')
66 + *dst++ = '_';
67 + else
68 + *dst++ = *src;
69 + src++;
70 + len--;
71 + }
72 + *dst = '\0';
73 +}
74 +
75 +/**
76 + * Format host labels for JSON connector
77 + *
78 + * @param instance an instance data structure.
79 + * @param host a data collecting host.
80 + * @return Always returns 0.
81 + */
82 +int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host)
83 +{
84 + if (!instance->labels)
85 + instance->labels = buffer_create(1024);
86 +
87 + if (unlikely(!sending_labels_configured(instance)))
88 + return 0;
89 +
90 + netdata_rwlock_rdlock(&host->labels_rwlock);
91 + for (struct label *label = host->labels; label; label = label->next) {
92 + if (!should_send_label(instance, label))
93 + continue;
94 +
95 + char value[CONFIG_MAX_VALUE + 1];
96 + sanitize_graphite_label_value(value, label->value, CONFIG_MAX_VALUE);
97 +
98 + if (*value) {
99 + buffer_strcat(instance->labels, ";");
100 + buffer_sprintf(instance->labels, "%s=%s", label->key, value);
101 + }
102 + }
103 + netdata_rwlock_unlock(&host->labels_rwlock);
104 +
105 + return 0;
106 +}
107 +
108 /**
109 * Format dimension using collected data for Graphite connector
110 *
@@ -78,13 +132,14 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
132
133 buffer_sprintf(
134 instance->buffer,
81 - "%s.%s.%s.%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n",
135 + "%s.%s.%s.%s%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n",
136 engine->config.prefix,
137 engine->config.hostname,
138 chart_name,
139 dimension_name,
140 (host->tags) ? ";" : "",
141 (host->tags) ? host->tags : "",
142 + (instance->labels) ? buffer_tostring(instance->labels) : "",
143 rd->last_collected_value,
144 (unsigned long long)rd->last_collected_time.tv_sec);
145
@@ -124,13 +179,14 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
179
180 buffer_sprintf(
181 instance->buffer,
127 - "%s.%s.%s.%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
182 + "%s.%s.%s.%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
183 engine->config.prefix,
184 engine->config.hostname,
185 chart_name,
186 dimension_name,
187 (host->tags) ? ";" : "",
188 (host->tags) ? host->tags : "",
189 + (instance->labels) ? buffer_tostring(instance->labels) : "",
190 value,
191 (unsigned long long)last_t);
192
exporting/graphite/graphite.h
+4
@@ -7,6 +7,10 @@
7
8 int init_graphite_connector(struct connector *connector);
9 int init_graphite_instance(struct instance *instance);
10 +
11 +void sanitize_graphite_label_value(char *dst, char *src, size_t len);
12 +int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host);
13 +
14 int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd);
15 int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM *rd);
16
exporting/json/json.c
+50 -5
@@ -28,7 +28,7 @@ int init_json_connector(struct connector *connector)
28 int init_json_instance(struct instance *instance)
29 {
30 instance->start_batch_formatting = NULL;
31 - instance->start_host_formatting = NULL;
31 + instance->start_host_formatting = format_host_labels_json_plaintext;
32 instance->start_chart_formatting = NULL;
33
34 if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
@@ -37,7 +37,7 @@ int init_json_instance(struct instance *instance)
37 instance->metric_formatting = format_dimension_stored_json_plaintext;
38
39 instance->end_chart_formatting = NULL;
40 - instance->end_host_formatting = NULL;
40 + instance->end_host_formatting = flush_host_labels;
41 instance->end_batch_formatting = NULL;
42
43 instance->buffer = (void *)buffer_create(0);
@@ -51,6 +51,44 @@ int init_json_instance(struct instance *instance)
51 return 0;
52 }
53
54 +/**
55 + * Format host labels for JSON connector
56 + *
57 + * @param instance an instance data structure.
58 + * @param host a data collecting host.
59 + * @return Always returns 0.
60 + */
61 +int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host)
62 +{
63 + if (!instance->labels)
64 + instance->labels = buffer_create(1024);
65 +
66 + if (unlikely(!sending_labels_configured(instance)))
67 + return 0;
68 +
69 + buffer_strcat(instance->labels, "\"labels\":{");
70 +
71 + int count = 0;
72 + netdata_rwlock_rdlock(&host->labels_rwlock);
73 + for (struct label *label = host->labels; label; label = label->next) {
74 + if (!should_send_label(instance, label))
75 + continue;
76 +
77 + char value[CONFIG_MAX_VALUE * 2 + 1];
78 + sanitize_json_string(value, label->value, CONFIG_MAX_VALUE);
79 + if (count > 0)
80 + buffer_strcat(instance->labels, ",");
81 + buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
82 +
83 + count++;
84 + }
85 + netdata_rwlock_unlock(&host->labels_rwlock);
86 +
87 + buffer_strcat(instance->labels, "},");
88 +
89 + return 0;
90 +}
91 +
92 /**
93 * Format dimension using collected data for JSON connector
94 *
@@ -80,28 +118,32 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
118
119 buffer_sprintf(
120 instance->buffer,
121 +
122 "{"
123 "\"prefix\":\"%s\","
124 "\"hostname\":\"%s\","
125 "%s%s%s"
126 + "%s"
127
128 "\"chart_id\":\"%s\","
129 "\"chart_name\":\"%s\","
130 "\"chart_family\":\"%s\","
91 - "\"chart_context\": \"%s\","
131 + "\"chart_context\":\"%s\","
132 "\"chart_type\":\"%s\","
93 - "\"units\": \"%s\","
133 + "\"units\":\"%s\","
134
135 "\"id\":\"%s\","
136 "\"name\":\"%s\","
137 "\"value\":" COLLECTED_NUMBER_FORMAT ","
138
99 - "\"timestamp\": %llu}\n",
139 + "\"timestamp\":%llu}\n",
140 +
141 engine->config.prefix,
142 engine->config.hostname,
143 tags_pre,
144 tags,
145 tags_post,
146 + instance->labels ? buffer_tostring(instance->labels) : "",
147
148 st->id,
149 st->name,
@@ -158,6 +200,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
200 "\"prefix\":\"%s\","
201 "\"hostname\":\"%s\","
202 "%s%s%s"
203 + "%s"
204
205 "\"chart_id\":\"%s\","
206 "\"chart_name\":\"%s\","
@@ -171,11 +214,13 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
214 "\"value\":" CALCULATED_NUMBER_FORMAT ","
215
216 "\"timestamp\": %llu}\n",
217 +
218 engine->config.prefix,
219 engine->config.hostname,
220 tags_pre,
221 tags,
222 tags_post,
223 + instance->labels ? buffer_tostring(instance->labels) : "",
224
225 st->id,
226 st->name,
exporting/json/json.h
+3
@@ -7,6 +7,9 @@
7
8 int init_json_connector(struct connector *connector);
9 int init_json_instance(struct instance *instance);
10 +
11 +int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host);
12 +
13 int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM *rd);
14 int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd);
15
exporting/opentsdb/opentsdb.c
+104 -12
@@ -28,7 +28,7 @@ int init_opentsdb_connector(struct connector *connector)
28 int init_opentsdb_telnet_instance(struct instance *instance)
29 {
30 instance->start_batch_formatting = NULL;
31 - instance->start_host_formatting = NULL;
31 + instance->start_host_formatting = format_host_labels_opentsdb_telnet;
32 instance->start_chart_formatting = NULL;
33
34 if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
@@ -37,7 +37,7 @@ int init_opentsdb_telnet_instance(struct instance *instance)
37 instance->metric_formatting = format_dimension_stored_opentsdb_telnet;
38
39 instance->end_chart_formatting = NULL;
40 - instance->end_host_formatting = NULL;
40 + instance->end_host_formatting = flush_host_labels;
41 instance->end_batch_formatting = NULL;
42
43 instance->buffer = (void *)buffer_create(0);
@@ -60,7 +60,7 @@ int init_opentsdb_telnet_instance(struct instance *instance)
60 int init_opentsdb_http_instance(struct instance *instance)
61 {
62 instance->start_batch_formatting = NULL;
63 - instance->start_host_formatting = NULL;
63 + instance->start_host_formatting = format_host_labels_opentsdb_http;
64 instance->start_chart_formatting = NULL;
65
66 if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
@@ -69,7 +69,7 @@ int init_opentsdb_http_instance(struct instance *instance)
69 instance->metric_formatting = format_dimension_stored_opentsdb_http;
70
71 instance->end_chart_formatting = NULL;
72 - instance->end_host_formatting = NULL;
72 + instance->end_host_formatting = flush_host_labels;
73 instance->end_batch_formatting = NULL;
74
75 instance->buffer = (void *)buffer_create(0);
@@ -83,6 +83,58 @@ int init_opentsdb_http_instance(struct instance *instance)
83 return 0;
84 }
85
86 +/**
87 + * Copy a label value and substitute underscores in place of charachters which can't be used in OpenTSDB output
88 + *
89 + * @param dst a destination string.
90 + * @param src a source string.
91 + * @param len the maximum number of characters copied.
92 + */
93 +
94 +void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
95 +{
96 + while (*src != '\0' && len) {
97 + if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
98 + *dst++ = *src;
99 + else
100 + *dst++ = '_';
101 + src++;
102 + len--;
103 + }
104 + *dst = '\0';
105 +}
106 +
107 +/**
108 + * Format host labels for JSON connector
109 + *
110 + * @param instance an instance data structure.
111 + * @param host a data collecting host.
112 + * @return Always returns 0.
113 + */
114 +int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host)
115 +{
116 + if (!instance->labels)
117 + instance->labels = buffer_create(1024);
118 +
119 + if (unlikely(!sending_labels_configured(instance)))
120 + return 0;
121 +
122 + netdata_rwlock_rdlock(&host->labels_rwlock);
123 + for (struct label *label = host->labels; label; label = label->next) {
124 + if (!should_send_label(instance, label))
125 + continue;
126 +
127 + char value[CONFIG_MAX_VALUE + 1];
128 + sanitize_opentsdb_label_value(value, label->value, CONFIG_MAX_VALUE);
129 +
130 + if (*value)
131 + buffer_sprintf(instance->labels, " %s=%s", label->key, value);
132 + }
133 + netdata_rwlock_unlock(&host->labels_rwlock);
134 +
135 + return 0;
136 +}
137 +
138 /**
139 * Format dimension using collected data for OpenTSDB telnet connector
140 *
@@ -110,7 +162,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
162
163 buffer_sprintf(
164 instance->buffer,
113 - "put %s.%s.%s %llu " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n",
165 + "put %s.%s.%s %llu " COLLECTED_NUMBER_FORMAT " host=%s%s%s%s\n",
166 engine->config.prefix,
167 chart_name,
168 dimension_name,
@@ -118,7 +170,8 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
170 rd->last_collected_value,
171 engine->config.hostname,
172 (host->tags) ? " " : "",
121 - (host->tags) ? host->tags : "");
173 + (host->tags) ? host->tags : "",
174 + (instance->labels) ? buffer_tostring(instance->labels) : "");
175
176 return 0;
177 }
@@ -156,7 +209,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
209
210 buffer_sprintf(
211 instance->buffer,
159 - "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n",
212 + "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s%s\n",
213 engine->config.prefix,
214 chart_name,
215 dimension_name,
@@ -164,7 +217,8 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
217 value,
218 engine->config.hostname,
219 (host->tags) ? " " : "",
167 - (host->tags) ? host->tags : "");
220 + (host->tags) ? host->tags : "",
221 + (instance->labels) ? buffer_tostring(instance->labels) : "");
222
223 return 0;
224 }
@@ -192,6 +246,42 @@ static inline void opentsdb_build_message(BUFFER *buffer, char *message, const c
246 message);
247 }
248
249 +/**
250 + * Format host labels for OpenTSDB HTTP connector
251 + *
252 + * @param instance an instance data structure.
253 + * @param host a data collecting host.
254 + * @return Always returns 0.
255 + */
256 +int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host)
257 +{
258 + if (!instance->labels)
259 + instance->labels = buffer_create(1024);
260 +
261 + if (unlikely(!sending_labels_configured(instance)))
262 + return 0;
263 +
264 + netdata_rwlock_rdlock(&host->labels_rwlock);
265 + for (struct label *label = host->labels; label; label = label->next) {
266 + if (!should_send_label(instance, label))
267 + continue;
268 +
269 + char escaped_value[CONFIG_MAX_VALUE * 2 + 1];
270 + sanitize_json_string(escaped_value, label->value, CONFIG_MAX_VALUE);
271 +
272 + char value[CONFIG_MAX_VALUE + 1];
273 + sanitize_opentsdb_label_value(value, escaped_value, CONFIG_MAX_VALUE);
274 +
275 + if (*value) {
276 + buffer_strcat(instance->labels, ",");
277 + buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
278 + }
279 + }
280 + netdata_rwlock_unlock(&host->labels_rwlock);
281 +
282 + return 0;
283 +}
284 +
285 /**
286 * Format dimension using collected data for OpenTSDB HTTP connector
287 *
@@ -226,7 +316,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
316 " \"timestamp\": %llu,"
317 " \"value\": " COLLECTED_NUMBER_FORMAT ","
318 " \"tags\": {"
229 - " \"host\": \"%s%s%s\""
319 + " \"host\": \"%s%s%s\"%s"
320 " }"
321 "}",
322 engine->config.prefix,
@@ -236,7 +326,8 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
326 rd->last_collected_value,
327 engine->config.hostname,
328 (host->tags) ? " " : "",
239 - (host->tags) ? host->tags : "");
329 + (host->tags) ? host->tags : "",
330 + instance->labels ? buffer_tostring(instance->labels) : "");
331
332 if (length > 0) {
333 opentsdb_build_message(instance->buffer, message, engine->config.hostname, length);
@@ -285,7 +376,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
376 " \"timestamp\": %llu,"
377 " \"value\": " CALCULATED_NUMBER_FORMAT ","
378 " \"tags\": {"
288 - " \"host\": \"%s%s%s\""
379 + " \"host\": \"%s%s%s\"%s"
380 " }"
381 "}",
382 engine->config.prefix,
@@ -295,7 +386,8 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
386 value,
387 engine->config.hostname,
388 (host->tags) ? " " : "",
298 - (host->tags) ? host->tags : "");
389 + (host->tags) ? host->tags : "",
390 + instance->labels ? buffer_tostring(instance->labels) : "");
391
392 if (length > 0) {
393 opentsdb_build_message(instance->buffer, message, engine->config.hostname, length);
exporting/opentsdb/opentsdb.h
+4
@@ -9,6 +9,10 @@ int init_opentsdb_connector(struct connector *connector);
9 int init_opentsdb_telnet_instance(struct instance *instance);
10 int init_opentsdb_http_instance(struct instance *instance);
11
12 +void sanitize_opentsdb_label_value(char *dst, char *src, size_t len);
13 +int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host);
14 +int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host);
15 +
16 int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM *rd);
17 int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *rd);
18
exporting/process_data.c
+17
@@ -388,6 +388,23 @@ int prepare_buffers(struct engine *engine)
388 return 0;
389 }
390
391 +/**
392 + * Flush a buffer with host labels
393 + *
394 + * @param instance an instance data structure.
395 + * @param host a data collecting host.
396 + * @return Always returns 0.
397 + */
398 +int flush_host_labels(struct instance *instance, RRDHOST *host)
399 +{
400 + (void)host;
401 +
402 + if (instance->labels)
403 + buffer_flush(instance->labels);
404 +
405 + return 0;
406 +}
407 +
408 /**
409 * Notify workers
410 *
exporting/read_config.c
+12
@@ -318,6 +318,18 @@ struct engine *read_exporting_config()
318
319 tmp_instance->config.options = exporting_parse_data_source(data_source, tmp_instance->config.options);
320
321 + if (exporter_get_boolean(
322 + instance_name, EXPORTER_SEND_CONFIGURED_LABELS, EXPORTER_SEND_CONFIGURED_LABELS_DEFAULT))
323 + tmp_instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
324 + else
325 + tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
326 +
327 + if (exporter_get_boolean(
328 + instance_name, EXPORTER_SEND_AUTOMATIC_LABELS, EXPORTER_SEND_AUTOMATIC_LABELS_DEFAULT))
329 + tmp_instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
330 + else
331 + tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
332 +
333 if (exporter_get_boolean(instance_name, EXPORTER_SEND_NAMES, EXPORTER_SEND_NAMES_DEFAULT))
334 tmp_instance->config.options |= EXPORTING_OPTION_SEND_NAMES;
335 else
exporting/tests/exporting_fixtures.c
+20
@@ -39,6 +39,18 @@ int setup_rrdhost()
39
40 localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2");
41
42 + struct label *label = calloc(1, sizeof(struct label));
43 + label->key = strdupz("key1");
44 + label->value = strdupz("value1");
45 + label->label_source = LABEL_SOURCE_NETDATA_CONF;
46 + localhost->labels = label;
47 +
48 + label = calloc(1, sizeof(struct label));
49 + label->key = strdupz("key2");
50 + label->value = strdupz("value2");
51 + label->label_source = LABEL_SOURCE_AUTO;
52 + localhost->labels->next = label;
53 +
54 localhost->rrdset_root = calloc(1, sizeof(RRDSET));
55 RRDSET *st = localhost->rrdset_root;
56 st->rrdhost = localhost;
@@ -80,6 +92,13 @@ int teardown_rrdhost()
92 free((void *)st->name);
93 free(st);
94
95 + free(localhost->labels->next->key);
96 + free(localhost->labels->next->value);
97 + free(localhost->labels->next);
98 + free(localhost->labels->key);
99 + free(localhost->labels->value);
100 + free(localhost->labels);
101 +
102 free((void *)localhost->tags);
103 free(localhost);
104
@@ -103,6 +122,7 @@ int teardown_initialized_engine(void **state)
122 struct engine *engine = *state;
123
124 teardown_rrdhost();
125 + buffer_free(engine->connector_root->instance_root->labels);
126 buffer_free(engine->connector_root->instance_root->buffer);
127 teardown_configured_engine(state);
128
exporting/tests/test_exporting_engine.c
+121 -6
@@ -61,7 +61,9 @@ static void test_exporting_engine(void **state)
61 expect_memory(__wrap_send_internal_metrics, engine, engine, sizeof(struct engine));
62 will_return(__wrap_send_internal_metrics, 0);
63
64 - void *ptr = malloc(sizeof(int));
64 + expect_function_call(__wrap_info_int);
65 +
66 + void *ptr = malloc(sizeof(struct netdata_static_thread));
67 assert_ptr_equal(exporting_main(ptr), NULL);
68 assert_int_equal(engine->now, 2);
69 free(ptr);
@@ -118,11 +120,11 @@ static void test_init_connectors(void **state)
120 assert_ptr_equal(instance->next, NULL);
121 assert_int_equal(instance->index, 0);
122 assert_ptr_equal(instance->start_batch_formatting, NULL);
121 - assert_ptr_equal(instance->start_host_formatting, NULL);
123 + assert_ptr_equal(instance->start_host_formatting, format_host_labels_graphite_plaintext);
124 assert_ptr_equal(instance->start_chart_formatting, NULL);
125 assert_ptr_equal(instance->metric_formatting, format_dimension_collected_graphite_plaintext);
126 assert_ptr_equal(instance->end_chart_formatting, NULL);
125 - assert_ptr_equal(instance->end_host_formatting, NULL);
127 + assert_ptr_equal(instance->end_host_formatting, flush_host_labels);
128 assert_ptr_equal(instance->end_batch_formatting, NULL);
129
130 BUFFER *buffer = instance->buffer;
@@ -460,8 +462,8 @@ static void test_format_dimension_collected_json_plaintext(void **state)
462 buffer_tostring(engine->connector_root->instance_root->buffer),
463 "{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\","
464 "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\","
463 - "\"chart_context\": \"(null)\",\"chart_type\":\"(null)\",\"units\": \"(null)\",\"id\":\"dimension_id\","
464 - "\"name\":\"dimension_name\",\"value\":123000321,\"timestamp\": 15051}\n");
465 + "\"chart_context\":\"(null)\",\"chart_type\":\"(null)\",\"units\":\"(null)\",\"id\":\"dimension_id\","
466 + "\"name\":\"dimension_name\",\"value\":123000321,\"timestamp\":15051}\n");
467 }
468
469 static void test_format_dimension_stored_json_plaintext(void **state)
@@ -678,6 +680,103 @@ static void test_simple_connector_worker(void **state)
680 simple_connector_worker(instance);
681 }
682
683 +static void test_sanitize_json_string(void **state)
684 +{
685 + (void)state;
686 +
687 + char *src = "check \t\\\" string";
688 + char dst[19 + 1];
689 +
690 + sanitize_json_string(dst, src, 19);
691 +
692 + assert_string_equal(dst, "check _\\\\\\\" string");
693 +}
694 +
695 +static void test_sanitize_graphite_label_value(void **state)
696 +{
697 + (void)state;
698 +
699 + char *src = "check ;~ string";
700 + char dst[15 + 1];
701 +
702 + sanitize_graphite_label_value(dst, src, 15);
703 +
704 + assert_string_equal(dst, "check____string");
705 +}
706 +
707 +static void test_sanitize_opentsdb_label_value(void **state)
708 +{
709 + (void)state;
710 +
711 + char *src = "check \t\\\" #&$? -_./ string";
712 + char dst[26 + 1];
713 +
714 + sanitize_opentsdb_label_value(dst, src, 26);
715 +
716 + assert_string_equal(dst, "check__________-_./_string");
717 +}
718 +
719 +static void test_format_host_labels_json_plaintext(void **state)
720 +{
721 + struct engine *engine = *state;
722 + struct instance *instance = engine->connector_root->instance_root;
723 +
724 + instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
725 + instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
726 +
727 + assert_int_equal(format_host_labels_json_plaintext(instance, localhost), 0);
728 + assert_string_equal(buffer_tostring(instance->labels), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
729 +}
730 +
731 +static void test_format_host_labels_graphite_plaintext(void **state)
732 +{
733 + struct engine *engine = *state;
734 + struct instance *instance = engine->connector_root->instance_root;
735 +
736 + instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
737 + instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
738 +
739 + assert_int_equal(format_host_labels_graphite_plaintext(instance, localhost), 0);
740 + assert_string_equal(buffer_tostring(instance->labels), ";key1=value1;key2=value2");
741 +}
742 +
743 +static void test_format_host_labels_opentsdb_telnet(void **state)
744 +{
745 + struct engine *engine = *state;
746 + struct instance *instance = engine->connector_root->instance_root;
747 +
748 + instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
749 + instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
750 +
751 + assert_int_equal(format_host_labels_opentsdb_telnet(instance, localhost), 0);
752 + assert_string_equal(buffer_tostring(instance->labels), " key1=value1 key2=value2");
753 +}
754 +
755 +static void test_format_host_labels_opentsdb_http(void **state)
756 +{
757 + struct engine *engine = *state;
758 + struct instance *instance = engine->connector_root->instance_root;
759 +
760 + instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
761 + instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
762 +
763 + assert_int_equal(format_host_labels_opentsdb_http(instance, localhost), 0);
764 + assert_string_equal(buffer_tostring(instance->labels), ",\"key1\":\"value1\",\"key2\":\"value2\"");
765 +}
766 +
767 +static void test_flush_host_labels(void **state)
768 +{
769 + struct engine *engine = *state;
770 + struct instance *instance = engine->connector_root->instance_root;
771 +
772 + instance->labels = buffer_create(12);
773 + buffer_strcat(instance->labels, "check string");
774 + assert_int_equal(buffer_strlen(instance->labels), 12);
775 +
776 + assert_int_equal(flush_host_labels(instance, localhost), 0);
777 + assert_int_equal(buffer_strlen(instance->labels), 0);
778 +}
779 +
780 int main(void)
781 {
782 const struct CMUnitTest tests[] = {
@@ -734,5 +833,21 @@ int main(void)
833 test_simple_connector_worker, setup_initialized_engine, teardown_initialized_engine),
834 };
835
737 - return cmocka_run_group_tests_name("exporting_engine", tests, NULL, NULL);
836 + const struct CMUnitTest label_tests[] = {
837 + cmocka_unit_test(test_sanitize_json_string),
838 + cmocka_unit_test(test_sanitize_graphite_label_value),
839 + cmocka_unit_test(test_sanitize_opentsdb_label_value),
840 + cmocka_unit_test_setup_teardown(
841 + test_format_host_labels_json_plaintext, setup_initialized_engine, teardown_initialized_engine),
842 + cmocka_unit_test_setup_teardown(
843 + test_format_host_labels_graphite_plaintext, setup_initialized_engine, teardown_initialized_engine),
844 + cmocka_unit_test_setup_teardown(
845 + test_format_host_labels_opentsdb_telnet, setup_initialized_engine, teardown_initialized_engine),
846 + cmocka_unit_test_setup_teardown(
847 + test_format_host_labels_opentsdb_http, setup_initialized_engine, teardown_initialized_engine),
848 + cmocka_unit_test_setup_teardown(test_flush_host_labels, setup_initialized_engine, teardown_initialized_engine),
849 + };
850 +
851 + return cmocka_run_group_tests_name("exporting_engine", tests, NULL, NULL) +
852 + cmocka_run_group_tests_name("labels_in_exporting_engine", label_tests, NULL, NULL);
853 }
libnetdata/inlined.h
+10 -4
@@ -246,12 +246,18 @@ static inline char *strncpyz(char *dst, const char *src, size_t n) {
246 return p;
247 }
248
249 -static inline void escape_json_string(char *dst, char *src, size_t len) {
249 +static inline void sanitize_json_string(char *dst, char *src, size_t len) {
250 while (*src != '\0' && len > 1) {
251 if (*src == '\\' || *src == '\"' || *src < 0x1F) {
252 - *dst++ = '\\';
253 - *dst++ = *src++;
254 - len -= 2;
252 + if (*src < 0x1F) {
253 + *dst++ = '_';
254 + src++;
255 + len--;
256 + } else {
257 + *dst++ = '\\';
258 + *dst++ = *src++;
259 + len -= 2;
260 + }
261 } else {
262 *dst++ = *src++;
263 len--;
web/api/web_api_v1.c
+1 -1
@@ -780,7 +780,7 @@ inline void host_labels2json(RRDHOST *host, BUFFER *wb, size_t indentation) {
780 buffer_strcat(wb, tabs);
781
782 char value[CONFIG_MAX_VALUE * 2 + 1];
783 - escape_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
783 + sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
784 buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value);
785
786 count++;