@cryptotaxi247 / netdata-1 / commits / 3071aa055

Add additional metadata to the data response (#13036)

* Consolidate query params * Add new option to show full dimensions in the json header (this will include dimensions, charts and chart labels) * Group and pass parameters with query_params

Stelios Fragkakis committed May 31, 2022 at 21:46 UTC 3071aa055ccf9629278bd79aa2005291f5ad01c4
5 files changed +132 -37
web/api/formatters/json_wrapper.c
+73 -1
@@ -2,9 +2,26 @@
2
3 #include "json_wrapper.h"
4
5 +struct value_output {
6 + int c;
7 + BUFFER *wb;
8 +};
9 +
10 +static int value_list_output(void *entry, void *data) {
11 + struct value_output *ap = (struct value_output *)data;
12 + BUFFER *wb = ap->wb;
13 + char *output = (char *) entry;
14 + if(ap->c) buffer_strcat(wb, ",");
15 + buffer_strcat(wb, output);
16 + (ap->c)++;
17 + return 0;
18 +}
19 +
20 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
6 - struct context_param *context_param_list, char *chart_label_key)
21 + QUERY_PARAMS *rrdset_query_data)
22 {
23 + struct context_param *context_param_list = rrdset_query_data->context_param_list;
24 + char *chart_label_key = rrdset_query_data->chart_label_key;
25
26 RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
27 int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
@@ -98,6 +115,61 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
115 }
116 buffer_strcat(wb, "],\n");
117
118 + if (rrdset_query_data->show_dimensions) {
119 + buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq);
120 +
121 + char name[RRD_ID_LENGTH_MAX * 2 + 2];
122 + char output[RRD_ID_LENGTH_MAX * 2 + 8];
123 + char value[RRD_ID_LENGTH_MAX * 2 + 1];
124 +
125 + struct value_output co = {.c = 0, .wb = wb};
126 +
127 + DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
128 + for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
129 + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name);
130 + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name);
131 + dictionary_set(dict, name, output, len+1);
132 + }
133 + dictionary_get_all(dict, value_list_output, &co);
134 + dictionary_destroy(dict);
135 +
136 + co.c = 0;
137 + buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq);
138 + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
139 + for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
140 + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name);
141 + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name);
142 + dictionary_set(dict, name, output, len + 1);
143 + }
144 +
145 + dictionary_get_all(dict, value_list_output, &co);
146 + dictionary_destroy(dict);
147 +
148 + RRDSET *st;
149 + co.c = 0;
150 + buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq);
151 + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
152 + for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
153 + st = rd->rrdset;
154 + if (likely(st->state)) {
155 + struct label_index *labels = &st->state->labels;
156 + if (labels->head) {
157 + netdata_rwlock_rdlock(&labels->labels_rwlock);
158 + for (struct label *label = labels->head; label; label = label->next) {
159 + sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2);
160 + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value);
161 + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value);
162 + dictionary_set(dict, name, output, len + 1);
163 + }
164 + netdata_rwlock_unlock(&labels->labels_rwlock);
165 + }
166 + }
167 + }
168 + dictionary_get_all(dict, value_list_output, &co);
169 + dictionary_destroy(dict);
170 + buffer_strcat(wb, "],\n");
171 + }
172 +
173 // Composite charts
174 if (context_mode && temp_rd) {
175 buffer_sprintf(
web/api/formatters/json_wrapper.h
+2 -1
@@ -5,7 +5,8 @@
5
6 #include "rrd2json.h"
7
8 -extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key);
8 +extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
9 + QUERY_PARAMS *query_params);
10 extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
11
12 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
web/api/formatters/rrd2json.c
+33 -25
@@ -211,7 +211,7 @@ cleanup:
211 int rrdset2anything_api_v1(
212 ONEWAYALLOC *owa
213 , RRDSET *st
214 - , BUFFER *wb
214 + , QUERY_PARAMS *query_params
215 , BUFFER *dimensions
216 , uint32_t format
217 , long points
@@ -221,16 +221,24 @@ int rrdset2anything_api_v1(
221 , long group_time
222 , uint32_t options
223 , time_t *latest_timestamp
224 - , struct context_param *context_param_list
225 - , char *chart_label_key
226 - , int max_anomaly_rates
227 - , int timeout
224 )
225 {
230 - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
226 + BUFFER *wb = query_params->wb;
227 + if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
228 st->last_accessed_time = now_realtime_sec();
229
233 - RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
230 + RRDR *r = rrd2rrdr(
231 + owa,
232 + st,
233 + points,
234 + after,
235 + before,
236 + group_method,
237 + group_time,
238 + options,
239 + dimensions ? buffer_tostring(dimensions) : NULL,
240 + query_params->context_param_list,
241 + query_params->timeout);
242 if(!r) {
243 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
244 return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -242,9 +250,9 @@ int rrdset2anything_api_v1(
250 }
251
252 if (st && st->state && st->state->is_ar_chart)
245 - ml_process_rrdr(r, max_anomaly_rates);
253 + ml_process_rrdr(r, query_params->max_anomaly_rates);
254
247 - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
255 + RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;
256
257 if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
258 buffer_no_cacheable(wb);
@@ -258,7 +266,7 @@ int rrdset2anything_api_v1(
266 case DATASOURCE_SSV:
267 if(options & RRDR_OPTION_JSON_WRAP) {
268 wb->contenttype = CT_APPLICATION_JSON;
261 - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
269 + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
270 rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
271 rrdr_json_wrapper_end(r, wb, format, options, 1);
272 }
@@ -271,7 +279,7 @@ int rrdset2anything_api_v1(
279 case DATASOURCE_SSV_COMMA:
280 if(options & RRDR_OPTION_JSON_WRAP) {
281 wb->contenttype = CT_APPLICATION_JSON;
274 - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
282 + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
283 rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
284 rrdr_json_wrapper_end(r, wb, format, options, 1);
285 }
@@ -284,7 +292,7 @@ int rrdset2anything_api_v1(
292 case DATASOURCE_JS_ARRAY:
293 if(options & RRDR_OPTION_JSON_WRAP) {
294 wb->contenttype = CT_APPLICATION_JSON;
287 - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
295 + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
296 rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
297 rrdr_json_wrapper_end(r, wb, format, options, 0);
298 }
@@ -297,7 +305,7 @@ int rrdset2anything_api_v1(
305 case DATASOURCE_CSV:
306 if(options & RRDR_OPTION_JSON_WRAP) {
307 wb->contenttype = CT_APPLICATION_JSON;
300 - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
308 + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
309 rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
310 rrdr_json_wrapper_end(r, wb, format, options, 1);
311 }
@@ -310,7 +318,7 @@ int rrdset2anything_api_v1(
318 case DATASOURCE_CSV_MARKDOWN:
319 if(options & RRDR_OPTION_JSON_WRAP) {
320 wb->contenttype = CT_APPLICATION_JSON;
313 - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
321 + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
322 rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
323 rrdr_json_wrapper_end(r, wb, format, options, 1);
324 }
@@ -323,7 +331,7 @@ int rrdset2anything_api_v1(
331 case DATASOURCE_CSV_JSON_ARRAY:
332 wb->contenttype = CT_APPLICATION_JSON;
333 if(options & RRDR_OPTION_JSON_WRAP) {
326 - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
334 + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
335 buffer_strcat(wb, "[\n");
336 rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
337 buffer_strcat(wb, "\n]");
@@ -340,7 +348,7 @@ int rrdset2anything_api_v1(
348 case DATASOURCE_TSV:
349 if(options & RRDR_OPTION_JSON_WRAP) {
350 wb->contenttype = CT_APPLICATION_JSON;
343 - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
351 + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
352 rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
353 rrdr_json_wrapper_end(r, wb, format, options, 1);
354 }
@@ -353,7 +361,7 @@ int rrdset2anything_api_v1(
361 case DATASOURCE_HTML:
362 if(options & RRDR_OPTION_JSON_WRAP) {
363 wb->contenttype = CT_APPLICATION_JSON;
356 - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
364 + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
365 buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
366 rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
367 buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
@@ -371,9 +379,9 @@ int rrdset2anything_api_v1(
379 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
380
381 if(options & RRDR_OPTION_JSON_WRAP)
374 - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
382 + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
383
376 - rrdr2json(r, wb, options, 1, context_param_list);
384 + rrdr2json(r, wb, options, 1, query_params->context_param_list);
385
386 if(options & RRDR_OPTION_JSON_WRAP)
387 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -383,9 +391,9 @@ int rrdset2anything_api_v1(
391 wb->contenttype = CT_APPLICATION_JSON;
392
393 if(options & RRDR_OPTION_JSON_WRAP)
386 - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
394 + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
395
388 - rrdr2json(r, wb, options, 1, context_param_list);
396 + rrdr2json(r, wb, options, 1, query_params->context_param_list);
397
398 if(options & RRDR_OPTION_JSON_WRAP)
399 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -394,9 +402,9 @@ int rrdset2anything_api_v1(
402 case DATASOURCE_JSONP:
403 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
404 if(options & RRDR_OPTION_JSON_WRAP)
397 - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
405 + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
406
399 - rrdr2json(r, wb, options, 0, context_param_list);
407 + rrdr2json(r, wb, options, 0, query_params->context_param_list);
408
409 if(options & RRDR_OPTION_JSON_WRAP)
410 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -407,9 +415,9 @@ int rrdset2anything_api_v1(
415 wb->contenttype = CT_APPLICATION_JSON;
416
417 if(options & RRDR_OPTION_JSON_WRAP)
410 - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
418 + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
419
412 - rrdr2json(r, wb, options, 0, context_param_list);
420 + rrdr2json(r, wb, options, 0, query_params->context_param_list);
421
422 if(options & RRDR_OPTION_JSON_WRAP)
423 rrdr_json_wrapper_end(r, wb, format, options, 0);
web/api/formatters/rrd2json.h
+13 -6
@@ -4,6 +4,17 @@
4 #define NETDATA_RRD2JSON_H 1
5
6 #include "web/api/web_api_v1.h"
7 +
8 +typedef struct query_params {
9 + struct context_param *context_param_list;
10 + BUFFER *wb;
11 + char *chart_label_key;
12 + int max_anomaly_rates;
13 + int timeout;
14 + int show_dimensions;
15 +} QUERY_PARAMS;
16 +
17 +
18 #include "web/api/exporters/allmetrics.h"
19 #include "web/api/queries/rrdr.h"
20
@@ -56,8 +67,8 @@ extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
67 extern int rrdset2anything_api_v1(
68 ONEWAYALLOC *owa
69 , RRDSET *st
59 - , BUFFER *wb
60 - , BUFFER *dimensions
70 + ,
71 + QUERY_PARAMS *query_params, BUFFER *dimensions
72 , uint32_t format
73 , long points
74 , long long after
@@ -66,10 +77,6 @@ extern int rrdset2anything_api_v1(
77 , long group_time
78 , uint32_t options
79 , time_t *latest_timestamp
69 - , struct context_param *context_param_list
70 - , char *chart_label_key
71 - , int max_anomaly_rates
72 - , int timeout
80 );
81
82 extern int rrdset2value_api_v1(
web/api/web_api_v1.c
+11 -4
@@ -422,6 +422,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
422 char *chart_labels_filter = NULL;
423
424 int group = RRDR_GROUPING_AVERAGE;
425 + int show_dimensions = 0;
426 uint32_t format = DATASOURCE_JSON;
427 uint32_t options = 0x00000000;
428
@@ -447,6 +448,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
448 buffer_strcat(dimensions, "|");
449 buffer_strcat(dimensions, value);
450 }
451 + else if(!strcmp(name, "show_dimensions")) show_dimensions = 1;
452 else if(!strcmp(name, "after")) after_str = value;
453 else if(!strcmp(name, "before")) before_str = value;
454 else if(!strcmp(name, "points")) points_str = value;
@@ -645,10 +647,15 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
647 buffer_strcat(w->response.data, "(");
648 }
649
648 - ret = rrdset2anything_api_v1(owa, st, w->response.data, dimensions, format,
649 - points, after, before, group, group_time,
650 - options, &last_timestamp_in_data, context_param_list,
651 - chart_label_key, max_anomaly_rates, timeout);
650 + QUERY_PARAMS query_params = {
651 + .context_param_list = context_param_list,
652 + .timeout = timeout,
653 + .max_anomaly_rates = max_anomaly_rates,
654 + .show_dimensions = show_dimensions,
655 + .wb = w->response.data};
656 +
657 + ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,
658 + points, after, before, group, group_time, options, &last_timestamp_in_data);
659
660 free_context_param_list(owa, &context_param_list);
661