/api/v2/data - multi-host/context/instance/dimension/label queries (#14564)
* fundamentals for having /api/v2/ working * use an atomic to prevent writing to internal pipe too much * first attempt of multi-node, multi-context, multi-chart, multi-dimension queries * v2 jsonwrap * first attempt for group by * cleaned up RRDR and fixed group by * improvements to /api/v2/api * query instance may be realloced, so pointers to it get invalid; solved memory leaks * count of quried metrics in summary information * provide detailed information about selected, excluded, queried and failed metrics for each entity * select instances by fqdn too * add timing information to json output * link charts to rrdcontexts, if a query comes in and it is found unlinked * calculate min, max, sum, average, volume, count per metric * api v2 parameters naming * renders alerts and units * render machine_guid and node_id in all sections it is relevant * unified keys * group by now takes into account units and when there are multiple units involved, it creates a dimension per unit * request and detailed are hidden behind an option * summary includes only a flattened list of alerts * alert counts per host and instance * count of grouped metrics per dimension * added contexts to summary * added chart title * added dimension priorities and chart type * support for multiple group by at the same time * minor fixes * labels are now a tree * keys uniformity * filtering by alerts, both having a specific alert and having a specific alert in a specific status * added scope of hosts and contexts * count of instances on contexts and hosts * make the api return valid responses even when the response contains no data * calculate average and contribution % for every item in the summary * fix compilation warnings * fix compilation warnings - again
Costa Tsaousis committed
Feb 22, 2023 at 22:30 UTC
1cfad181a8cdcafc04d6f2d52aa0ffb5e57e182a
43 files changed
+2983
-802
CMakeLists.txt
+4
@@ -796,8 +796,12 @@ set(WEB_PLUGIN_FILES
796
)
797
798
set(API_PLUGIN_FILES
799
+ web/api/web_api.c
800
+ web/api/web_api.h
801
web/api/web_api_v1.c
802
web/api/web_api_v1.h
803
+ web/api/web_api_v2.c
804
+ web/api/web_api_v2.h
805
web/api/badges/web_buffer_svg.c
806
web/api/badges/web_buffer_svg.h
807
web/api/exporters/allmetrics.c
Makefile.am
+4
@@ -628,8 +628,12 @@ API_PLUGIN_FILES = \
628
web/api/formatters/rrdset2json.h \
629
web/api/health/health_cmdapi.c \
630
web/api/health/health_cmdapi.h \
631
+ web/api/web_api.c \
632
+ web/api/web_api.h \
633
web/api/web_api_v1.c \
634
web/api/web_api_v1.h \
635
+ web/api/web_api_v2.c \
636
+ web/api/web_api_v2.h \
637
$(NULL)
638
639
STREAMING_PLUGIN_FILES = \
daemon/common.h
+1
-1
@@ -58,7 +58,7 @@
58
#include "exporting/exporting_engine.h"
59
60
// the netdata API
61
-#include "web/api/web_api_v1.h"
61
+#include "web/server/web_client.h"
62
63
// all data collection plugins
64
#include "collectors/all.h"
database/rrd.h
+19
-17
@@ -132,6 +132,22 @@ typedef struct storage_point {
132
SN_FLAGS flags; // flags stored with the point
133
} STORAGE_POINT;
134
135
+// ----------------------------------------------------------------------------
136
+// chart types
137
+
138
+typedef enum __attribute__ ((__packed__)) rrdset_type {
139
+ RRDSET_TYPE_LINE = 0,
140
+ RRDSET_TYPE_AREA = 1,
141
+ RRDSET_TYPE_STACKED = 2,
142
+} RRDSET_TYPE;
143
+
144
+#define RRDSET_TYPE_LINE_NAME "line"
145
+#define RRDSET_TYPE_AREA_NAME "area"
146
+#define RRDSET_TYPE_STACKED_NAME "stacked"
147
+
148
+RRDSET_TYPE rrdset_type_id(const char *name);
149
+const char *rrdset_type_name(RRDSET_TYPE chart_type);
150
+
151
#include "rrdcontext.h"
152
153
extern bool unittest_running;
@@ -177,23 +193,6 @@ extern bool ieee754_doubles;
193
typedef long long total_number;
194
#define TOTAL_NUMBER_FORMAT "%lld"
195
180
-// ----------------------------------------------------------------------------
181
-// chart types
182
-
183
-typedef enum __attribute__ ((__packed__)) rrdset_type {
184
- RRDSET_TYPE_LINE = 0,
185
- RRDSET_TYPE_AREA = 1,
186
- RRDSET_TYPE_STACKED = 2,
187
-} RRDSET_TYPE;
188
-
189
-#define RRDSET_TYPE_LINE_NAME "line"
190
-#define RRDSET_TYPE_AREA_NAME "area"
191
-#define RRDSET_TYPE_STACKED_NAME "stacked"
192
-
193
-RRDSET_TYPE rrdset_type_id(const char *name);
194
-const char *rrdset_type_name(RRDSET_TYPE chart_type);
195
-
196
-
196
// ----------------------------------------------------------------------------
197
// algorithms types
198
@@ -285,6 +284,9 @@ void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
284
void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
285
void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key);
286
void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const char *key);
287
+void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key);
288
+STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key);
289
+STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset);
290
void rrdlabels_flush(DICTIONARY *labels_dict);
291
292
void rrdlabels_unmark_all(DICTIONARY *labels);
database/rrdcontext.c
+494
-172
@@ -325,6 +325,16 @@ const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma) {
325
return string2str(rm->name);
326
}
327
328
+STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma) {
329
+ RRDMETRIC *rm = rrdmetric_acquired_value(rma);
330
+ return string_dup(rm->id);
331
+}
332
+
333
+STRING *rrdmetric_acquired_name_dup(RRDMETRIC_ACQUIRED *rma) {
334
+ RRDMETRIC *rm = rrdmetric_acquired_value(rma);
335
+ return string_dup(rm->name);
336
+}
337
+
338
NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma) {
339
RRDMETRIC *rm = rrdmetric_acquired_value(rma);
340
@@ -361,6 +371,16 @@ const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria) {
371
return string2str(ri->name);
372
}
373
374
+const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria) {
375
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
376
+ return string2str(ri->units);
377
+}
378
+
379
+STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria) {
380
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
381
+ return string_dup(ri->units);
382
+}
383
+
384
DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) {
385
RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
386
return ri->rrdlabels;
@@ -372,6 +392,11 @@ DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria) {
392
return ri->rrdset->functions_view;
393
}
394
395
+RRDHOST *rrdinstance_acquired_rrdhost(RRDINSTANCE_ACQUIRED *ria) {
396
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
397
+ return ri->rc->rrdhost;
398
+}
399
+
400
// ----------------------------------------------------------------------------
401
// helper one-liners for RRDCONTEXT
402
@@ -384,6 +409,40 @@ const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca) {
409
return string2str(rc->id);
410
}
411
412
+bool rrdcontext_acquired_belongs_to_host(RRDCONTEXT_ACQUIRED *rca, RRDHOST *host) {
413
+ RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
414
+ return rc->rrdhost == host;
415
+}
416
+
417
+bool rrdinstance_acquired_belongs_to_context(RRDINSTANCE_ACQUIRED *ria, RRDCONTEXT_ACQUIRED *rca) {
418
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
419
+ RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
420
+ return ri->rc == rc;
421
+}
422
+
423
+bool rrdmetric_acquired_belongs_to_instance(RRDMETRIC_ACQUIRED *rma, RRDINSTANCE_ACQUIRED *ria) {
424
+ RRDMETRIC *rm = rrdmetric_acquired_value(rma);
425
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
426
+ return rm->ri == ri;
427
+}
428
+
429
+time_t rrdinstance_acquired_update_every(RRDINSTANCE_ACQUIRED *ria) {
430
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
431
+ return ri->update_every_s;
432
+}
433
+time_t rrdmetric_acquired_first_entry(RRDMETRIC_ACQUIRED *rma) {
434
+ RRDMETRIC *rm = rrdmetric_acquired_value(rma);
435
+ return rm->first_time_s;
436
+}
437
+time_t rrdmetric_acquired_last_entry(RRDMETRIC_ACQUIRED *rma) {
438
+ RRDMETRIC *rm = rrdmetric_acquired_value(rma);
439
+
440
+ if(rrd_flag_check(rm, RRD_FLAG_COLLECTED))
441
+ return 0;
442
+
443
+ return rm->last_time_s;
444
+}
445
+
446
static inline RRDCONTEXT_ACQUIRED *rrdcontext_acquired_dup(RRDCONTEXT_ACQUIRED *rca) {
447
RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
448
return (RRDCONTEXT_ACQUIRED *)dictionary_acquired_item_dup((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca);
@@ -2203,33 +2262,38 @@ typedef struct query_target_locals {
2262
2263
RRDSET *st;
2264
2265
+ const char *scope_hosts;
2266
+ const char *scope_contexts;
2267
+
2268
const char *hosts;
2269
const char *contexts;
2270
const char *charts;
2271
const char *dimensions;
2272
const char *chart_label_key;
2211
- const char *charts_labels_filter;
2273
+ const char *labels;
2274
+ const char *alerts;
2275
2276
long long after;
2277
long long before;
2278
bool match_ids;
2279
bool match_names;
2280
2218
- RRDHOST *host;
2219
- RRDCONTEXT_ACQUIRED *rca;
2220
- RRDINSTANCE_ACQUIRED *ria;
2221
-
2281
size_t metrics_skipped_due_to_not_matching_timeframe;
2282
} QUERY_TARGET_LOCALS;
2283
2284
static __thread QUERY_TARGET thread_query_target = {};
2285
void query_target_release(QUERY_TARGET *qt) {
2227
- if(unlikely(!qt)) return;
2228
- if(unlikely(!qt->used)) return;
2286
+ if(unlikely(!qt || !qt->used)) return;
2287
+
2288
+ simple_pattern_free(qt->hosts.scope_pattern);
2289
+ qt->hosts.scope_pattern = NULL;
2290
2291
simple_pattern_free(qt->hosts.pattern);
2292
qt->hosts.pattern = NULL;
2293
2294
+ simple_pattern_free(qt->contexts.scope_pattern);
2295
+ qt->contexts.scope_pattern = NULL;
2296
+
2297
simple_pattern_free(qt->contexts.pattern);
2298
qt->contexts.pattern = NULL;
2299
@@ -2239,82 +2303,83 @@ void query_target_release(QUERY_TARGET *qt) {
2303
simple_pattern_free(qt->instances.chart_label_key_pattern);
2304
qt->instances.chart_label_key_pattern = NULL;
2305
2242
- simple_pattern_free(qt->instances.charts_labels_filter_pattern);
2243
- qt->instances.charts_labels_filter_pattern = NULL;
2306
+ simple_pattern_free(qt->instances.labels_pattern);
2307
+ qt->instances.labels_pattern = NULL;
2308
2309
simple_pattern_free(qt->query.pattern);
2310
qt->query.pattern = NULL;
2311
2312
// release the query
2313
for(size_t i = 0, used = qt->query.used; i < used ;i++) {
2250
- string_freez(qt->query.array[i].dimension.id);
2251
- qt->query.array[i].dimension.id = NULL;
2252
-
2253
- string_freez(qt->query.array[i].dimension.name);
2254
- qt->query.array[i].dimension.name = NULL;
2255
-
2256
- string_freez(qt->query.array[i].chart.id);
2257
- qt->query.array[i].chart.id = NULL;
2258
-
2259
- string_freez(qt->query.array[i].chart.name);
2260
- qt->query.array[i].chart.name = NULL;
2314
+ QUERY_METRIC *qm = query_metric(qt, i);
2315
2316
// reset the plans
2263
- for(size_t p = 0; p < qt->query.array[i].plan.used; p++) {
2264
- internal_fatal(qt->query.array[i].plan.array[p].initialized &&
2265
- !qt->query.array[i].plan.array[p].finalized,
2317
+ for(size_t p = 0; p < qm->plan.used; p++) {
2318
+ internal_fatal(qm->plan.array[p].initialized &&
2319
+ !qm->plan.array[p].finalized,
2320
"QUERY: left-over initialized plan");
2321
2268
- qt->query.array[i].plan.array[p].initialized = false;
2269
- qt->query.array[i].plan.array[p].finalized = false;
2322
+ qm->plan.array[p].initialized = false;
2323
+ qm->plan.array[p].finalized = false;
2324
}
2271
- qt->query.array[i].plan.used = 0;
2325
+ qm->plan.used = 0;
2326
2327
// reset the tiers
2328
for(size_t tier = 0; tier < storage_tiers ;tier++) {
2275
- if(qt->query.array[i].tiers[tier].db_metric_handle) {
2276
- STORAGE_ENGINE *eng = qt->query.array[i].tiers[tier].eng;
2277
- eng->api.metric_release(qt->query.array[i].tiers[tier].db_metric_handle);
2278
- qt->query.array[i].tiers[tier].db_metric_handle = NULL;
2279
- qt->query.array[i].tiers[tier].weight = 0;
2280
- qt->query.array[i].tiers[tier].eng = NULL;
2329
+ if(qm->tiers[tier].db_metric_handle) {
2330
+ STORAGE_ENGINE *eng = qm->tiers[tier].eng;
2331
+ eng->api.metric_release(qm->tiers[tier].db_metric_handle);
2332
+ qm->tiers[tier].db_metric_handle = NULL;
2333
+ qm->tiers[tier].eng = NULL;
2334
}
2335
}
2336
}
2337
+ qt->query.used = 0;
2338
2285
- // release the metrics
2286
- for(size_t i = 0, used = qt->metrics.used; i < used ;i++) {
2287
- rrdmetric_release(qt->metrics.array[i]);
2288
- qt->metrics.array[i] = NULL;
2339
+ // release the dimensions
2340
+ for(size_t i = 0, used = qt->dimensions.used; i < used ; i++) {
2341
+ QUERY_DIMENSION *qd = query_dimension(qt, i);
2342
+ rrdmetric_release(qd->rma);
2343
+ qd->rma = NULL;
2344
}
2345
+ qt->dimensions.used = 0;
2346
2347
// release the instances
2348
for(size_t i = 0, used = qt->instances.used; i < used ;i++) {
2293
- rrdinstance_release(qt->instances.array[i]);
2294
- qt->instances.array[i] = NULL;
2349
+ QUERY_INSTANCE *qi = &qt->instances.array[i];
2350
+
2351
+ rrdinstance_release(qi->ria);
2352
+ qi->ria = NULL;
2353
+
2354
+ string_freez(qi->id_fqdn);
2355
+ qi->id_fqdn = NULL;
2356
+
2357
+ string_freez(qi->name_fqdn);
2358
+ qi->name_fqdn = NULL;
2359
}
2360
+ qt->instances.used = 0;
2361
2362
// release the contexts
2363
for(size_t i = 0, used = qt->contexts.used; i < used ;i++) {
2299
- rrdcontext_release(qt->contexts.array[i]);
2300
- qt->contexts.array[i] = NULL;
2364
+ QUERY_CONTEXT *qc = query_context(qt, i);
2365
+ rrdcontext_release(qc->rca);
2366
+ qc->rca = NULL;
2367
}
2368
+ qt->contexts.used = 0;
2369
2370
// release the hosts
2371
for(size_t i = 0, used = qt->hosts.used; i < used ;i++) {
2305
- qt->hosts.array[i] = NULL;
2372
+ QUERY_HOST *qh = query_host(qt, i);
2373
+ qh->host = NULL;
2374
}
2307
-
2308
- qt->query.used = 0;
2309
- qt->metrics.used = 0;
2310
- qt->instances.used = 0;
2311
- qt->contexts.used = 0;
2375
qt->hosts.used = 0;
2376
2377
qt->db.minimum_latest_update_every_s = 0;
2378
qt->db.first_time_s = 0;
2379
qt->db.last_time_s = 0;
2380
2381
+ qt->group_by.used = 0;
2382
+
2383
qt->id[0] = '\0';
2384
2385
qt->used = false;
@@ -2330,10 +2395,10 @@ void query_target_free(void) {
2395
qt->query.array = NULL;
2396
qt->query.size = 0;
2397
2333
- __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->metrics.size * sizeof(RRDMETRIC_ACQUIRED *), __ATOMIC_RELAXED);
2334
- freez(qt->metrics.array);
2335
- qt->metrics.array = NULL;
2336
- qt->metrics.size = 0;
2398
+ __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->dimensions.size * sizeof(RRDMETRIC_ACQUIRED *), __ATOMIC_RELAXED);
2399
+ freez(qt->dimensions.array);
2400
+ qt->dimensions.array = NULL;
2401
+ qt->dimensions.size = 0;
2402
2403
__atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->instances.size * sizeof(RRDINSTANCE_ACQUIRED *), __ATOMIC_RELAXED);
2404
freez(qt->instances.array);
@@ -2351,31 +2416,17 @@ void query_target_free(void) {
2416
qt->hosts.size = 0;
2417
}
2418
2354
-static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED *rma, RRDINSTANCE *ri,
2355
- bool queryable_instance) {
2419
+static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, QUERY_CONTEXT *qc,
2420
+ QUERY_INSTANCE *qi, QUERY_DIMENSION *qd) {
2421
QUERY_TARGET *qt = qtl->qt;
2357
-
2358
- RRDMETRIC *rm = rrdmetric_acquired_value(rma);
2359
- if(rrd_flag_is_deleted(rm))
2360
- return;
2361
-
2362
- if(qt->metrics.used == qt->metrics.size) {
2363
- size_t old_mem = qt->metrics.size * sizeof(RRDMETRIC_ACQUIRED *);
2364
- qt->metrics.size = (qt->metrics.size) ? qt->metrics.size * 2 : 1;
2365
- size_t new_mem = qt->metrics.size * sizeof(RRDMETRIC_ACQUIRED *);
2366
- qt->metrics.array = reallocz(qt->metrics.array, new_mem);
2367
-
2368
- __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2369
- }
2370
- qt->metrics.array[qt->metrics.used++] = rrdmetric_acquired_dup(rma);
2371
-
2372
- if(!queryable_instance)
2373
- return;
2422
+ RRDMETRIC *rm = rrdmetric_acquired_value(qd->rma);
2423
+ RRDINSTANCE *ri = rm->ri;
2424
2425
time_t common_first_time_s = 0;
2426
time_t common_last_time_s = 0;
2427
time_t common_update_every_s = 0;
2428
size_t tiers_added = 0;
2429
+
2430
struct {
2431
STORAGE_ENGINE *eng;
2432
STORAGE_METRIC_HANDLE *db_metric_handle;
@@ -2385,14 +2436,14 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2436
} tier_retention[storage_tiers];
2437
2438
for (size_t tier = 0; tier < storage_tiers; tier++) {
2388
- STORAGE_ENGINE *eng = qtl->host->db[tier].eng;
2439
+ STORAGE_ENGINE *eng = qh->host->db[tier].eng;
2440
tier_retention[tier].eng = eng;
2390
- tier_retention[tier].db_update_every_s = (time_t) (qtl->host->db[tier].tier_grouping * ri->update_every_s);
2441
+ tier_retention[tier].db_update_every_s = (time_t) (qh->host->db[tier].tier_grouping * ri->update_every_s);
2442
2443
if(rm->rrddim && rm->rrddim->tiers[tier].db_metric_handle)
2444
tier_retention[tier].db_metric_handle = eng->api.metric_dup(rm->rrddim->tiers[tier].db_metric_handle);
2445
else
2395
- tier_retention[tier].db_metric_handle = eng->api.metric_get(qtl->host->db[tier].instance, &rm->uuid);
2446
+ tier_retention[tier].db_metric_handle = eng->api.metric_get(qh->host->db[tier].instance, &rm->uuid);
2447
2448
if(tier_retention[tier].db_metric_handle) {
2449
tier_retention[tier].db_first_time_s = tier_retention[tier].eng->api.query_ops.oldest_time_s(tier_retention[tier].db_metric_handle);
@@ -2425,8 +2476,8 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2476
bool release_retention = true;
2477
bool timeframe_matches =
2478
(tiers_added
2428
- && (common_first_time_s - common_update_every_s * 2) <= qt->window.before
2429
- && (common_last_time_s + common_update_every_s * 2) >= qt->window.after
2479
+ && (common_first_time_s - common_update_every_s * 2) <= qt->window.before
2480
+ && (common_last_time_s + common_update_every_s * 2) >= qt->window.after
2481
) ? true : false;
2482
2483
if(timeframe_matches) {
@@ -2435,7 +2486,7 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2486
if (rrd_flag_check(rm, RRD_FLAG_HIDDEN)
2487
|| (rm->rrddim && rrddim_option_check(rm->rrddim, RRDDIM_OPTION_HIDDEN))) {
2488
options |= RRDR_DIMENSION_HIDDEN;
2438
- options &= ~RRDR_DIMENSION_QUERIED;
2489
+ options &= ~RRDR_DIMENSION_SELECTED;
2490
}
2491
2492
if (qt->query.pattern) {
@@ -2443,16 +2494,16 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2494
// lets see if this dimension is selected
2495
2496
if ((qtl->match_ids && simple_pattern_matches(qt->query.pattern, string2str(rm->id)))
2446
- || (qtl->match_names && simple_pattern_matches(qt->query.pattern, string2str(rm->name)))
2497
+ || (qtl->match_names && simple_pattern_matches(qt->query.pattern, string2str(rm->name)))
2498
) {
2499
// it matches the pattern
2449
- options |= (RRDR_DIMENSION_QUERIED | RRDR_DIMENSION_NONZERO);
2500
+ options |= (RRDR_DIMENSION_SELECTED | RRDR_DIMENSION_NONZERO);
2501
options &= ~RRDR_DIMENSION_HIDDEN;
2502
}
2503
else {
2504
// it does not match the pattern
2505
options |= RRDR_DIMENSION_HIDDEN;
2455
- options &= ~RRDR_DIMENSION_QUERIED;
2506
+ options &= ~RRDR_DIMENSION_SELECTED;
2507
}
2508
}
2509
else {
@@ -2460,10 +2511,10 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2511
// so this is a selected dimension
2512
// if it is not hidden
2513
if(!(options & RRDR_DIMENSION_HIDDEN))
2463
- options |= RRDR_DIMENSION_QUERIED;
2514
+ options |= RRDR_DIMENSION_SELECTED;
2515
}
2516
2466
- if((options & RRDR_DIMENSION_HIDDEN) && (options & RRDR_DIMENSION_QUERIED))
2517
+ if((options & RRDR_DIMENSION_HIDDEN) && (options & RRDR_DIMENSION_SELECTED))
2518
options &= ~RRDR_DIMENSION_HIDDEN;
2519
2520
if(!(options & RRDR_DIMENSION_HIDDEN) || (qt->request.options & RRDR_OPTION_PERCENTAGE)) {
@@ -2474,28 +2525,22 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2525
ri->rrdset->last_accessed_time_s = qtl->start_s;
2526
2527
if (qt->query.used == qt->query.size) {
2477
- size_t old_mem = qt->query.size * sizeof(QUERY_METRIC);
2528
+ size_t old_mem = qt->query.size * sizeof(*qt->query.array);
2529
qt->query.size = (qt->query.size) ? qt->query.size * 2 : 1;
2479
- size_t new_mem = qt->query.size * sizeof(QUERY_METRIC);
2530
+ size_t new_mem = qt->query.size * sizeof(*qt->query.array);
2531
qt->query.array = reallocz(qt->query.array, new_mem);
2532
2533
__atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2534
}
2535
QUERY_METRIC *qm = &qt->query.array[qt->query.used++];
2536
+ memset(qm, 0, sizeof(*qm));
2537
2486
- qm->plan.used = 0;
2487
- qm->dimension.options = options;
2538
+ qm->status = options;
2539
2489
- qm->link.host = qtl->host;
2490
- qm->link.rca = qtl->rca;
2491
- qm->link.ria = qtl->ria;
2492
- qm->link.rma = rma;
2493
-
2494
- qm->chart.id = string_dup(ri->id);
2495
- qm->chart.name = string_dup(ri->name);
2496
-
2497
- qm->dimension.id = string_dup(rm->id);
2498
- qm->dimension.name = string_dup(rm->name);
2540
+ qm->link.query_host_id = qh->slot;
2541
+ qm->link.query_context_id = qc->slot;
2542
+ qm->link.query_instance_id = qi->slot;
2543
+ qm->link.query_dimension_id = qd->slot;
2544
2545
if (!qt->db.first_time_s || common_first_time_s < qt->db.first_time_s)
2546
qt->db.first_time_s = common_first_time_s;
@@ -2510,11 +2555,29 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2555
qm->tiers[tier].db_last_time_s = tier_retention[tier].db_last_time_s;
2556
qm->tiers[tier].db_update_every_s = tier_retention[tier].db_update_every_s;
2557
}
2558
+
2559
release_retention = false;
2560
+
2561
+ qi->metrics.selected++;
2562
+ qc->metrics.selected++;
2563
+ qh->metrics.selected++;
2564
+ }
2565
+ else {
2566
+ qi->metrics.excluded++;
2567
+ qc->metrics.excluded++;
2568
+ qh->metrics.excluded++;
2569
+
2570
+ qd->status |= QUERY_STATUS_DIMENSION_HIDDEN;
2571
}
2572
}
2516
- else
2573
+ else {
2574
+ qi->metrics.excluded++;
2575
+ qc->metrics.excluded++;
2576
+ qh->metrics.excluded++;
2577
+
2578
+ qd->status |= QUERY_STATUS_DIMENSION_NODATA;
2579
qtl->metrics_skipped_due_to_not_matching_timeframe++;
2580
+ }
2581
2582
if(release_retention) {
2583
// cleanup anything we allocated to the retention we will not use
@@ -2525,7 +2588,180 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
2588
}
2589
}
2590
2528
-static void query_target_add_instance(QUERY_TARGET_LOCALS *qtl, RRDINSTANCE_ACQUIRED *ria, bool queryable_instance) {
2591
+static void query_target_add_dimension(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi,
2592
+ RRDMETRIC_ACQUIRED *rma, bool queryable_instance) {
2593
+ QUERY_TARGET *qt = qtl->qt;
2594
+
2595
+ RRDMETRIC *rm = rrdmetric_acquired_value(rma);
2596
+ if(rrd_flag_is_deleted(rm))
2597
+ return;
2598
+
2599
+ if(qt->dimensions.used == qt->dimensions.size) {
2600
+ size_t old_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
2601
+ qt->dimensions.size = (qt->dimensions.size) ? qt->dimensions.size * 2 : 1;
2602
+ size_t new_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
2603
+ qt->dimensions.array = reallocz(qt->dimensions.array, new_mem);
2604
+
2605
+ __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2606
+ }
2607
+ QUERY_DIMENSION *qd = &qt->dimensions.array[qt->dimensions.used];
2608
+ memset(qd, 0, sizeof(*qd));
2609
+
2610
+ qd->slot = qt->dimensions.used++;
2611
+ qd->rma = rrdmetric_acquired_dup(rma);
2612
+ qd->status = QUERY_STATUS_NONE;
2613
+
2614
+ if(!queryable_instance) {
2615
+ qi->metrics.excluded++;
2616
+ qc->metrics.excluded++;
2617
+ qh->metrics.excluded++;
2618
+ qd->status |= QUERY_STATUS_EXCLUDED;
2619
+ return;
2620
+ }
2621
+
2622
+ query_target_add_metric(qtl, qh, qc, qi, qd);
2623
+}
2624
+
2625
+static inline STRING *rrdinstance_id_fqdn_v1(RRDINSTANCE_ACQUIRED *ria) {
2626
+ if(unlikely(!ria))
2627
+ return NULL;
2628
+
2629
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2630
+ return string_dup(ri->id);
2631
+}
2632
+
2633
+static inline STRING *rrdinstance_name_fqdn_v1(RRDINSTANCE_ACQUIRED *ria) {
2634
+ if(unlikely(!ria))
2635
+ return NULL;
2636
+
2637
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2638
+ return string_dup(ri->name);
2639
+}
2640
+
2641
+static inline STRING *rrdinstance_id_fqdn_v2(RRDINSTANCE_ACQUIRED *ria) {
2642
+ if(unlikely(!ria))
2643
+ return NULL;
2644
+
2645
+ char buffer[RRD_ID_LENGTH_MAX + 1];
2646
+
2647
+ RRDHOST *host = rrdinstance_acquired_rrdhost(ria);
2648
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s@%s", rrdinstance_acquired_id(ria), host->machine_guid);
2649
+ return string_strdupz(buffer);
2650
+}
2651
+
2652
+static inline STRING *rrdinstance_name_fqdn_v2(RRDINSTANCE_ACQUIRED *ria) {
2653
+ if(unlikely(!ria))
2654
+ return NULL;
2655
+
2656
+ char buffer[RRD_ID_LENGTH_MAX + 1];
2657
+
2658
+ RRDHOST *host = rrdinstance_acquired_rrdhost(ria);
2659
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s@%s", rrdinstance_acquired_name(ria), rrdhost_hostname(host));
2660
+ return string_strdupz(buffer);
2661
+}
2662
+
2663
+RRDSET *rrdinstance_acquired_rrdset(RRDINSTANCE_ACQUIRED *ria) {
2664
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2665
+ return ri->rrdset;
2666
+}
2667
+
2668
+const char *rrdcontext_acquired_units(RRDCONTEXT_ACQUIRED *rca) {
2669
+ RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2670
+ return string2str(rc->units);
2671
+}
2672
+
2673
+RRDSET_TYPE rrdcontext_acquired_chart_type(RRDCONTEXT_ACQUIRED *rca) {
2674
+ RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2675
+ return rc->chart_type;
2676
+}
2677
+
2678
+const char *rrdcontext_acquired_title(RRDCONTEXT_ACQUIRED *rca) {
2679
+ RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2680
+ return string2str(rc->title);
2681
+}
2682
+
2683
+static void query_target_eval_instance_rrdcalc(QUERY_TARGET_LOCALS *qtl __maybe_unused,
2684
+ QUERY_HOST *qh, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi) {
2685
+ RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
2686
+ if (st) {
2687
+ netdata_rwlock_rdlock(&st->alerts.rwlock);
2688
+ if (st->alerts.base) {
2689
+ for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
2690
+ switch(rc->status) {
2691
+ case RRDCALC_STATUS_CLEAR:
2692
+ qi->alerts.clear++;
2693
+ qc->alerts.clear++;
2694
+ qh->alerts.clear++;
2695
+ break;
2696
+
2697
+ case RRDCALC_STATUS_WARNING:
2698
+ qi->alerts.warning++;
2699
+ qc->alerts.warning++;
2700
+ qh->alerts.warning++;
2701
+ break;
2702
+
2703
+ case RRDCALC_STATUS_CRITICAL:
2704
+ qi->alerts.critical++;
2705
+ qc->alerts.critical++;
2706
+ qh->alerts.critical++;
2707
+ break;
2708
+
2709
+ default:
2710
+ case RRDCALC_STATUS_UNINITIALIZED:
2711
+ case RRDCALC_STATUS_UNDEFINED:
2712
+ case RRDCALC_STATUS_REMOVED:
2713
+ qi->alerts.other++;
2714
+ qc->alerts.other++;
2715
+ qh->alerts.other++;
2716
+ break;
2717
+ }
2718
+ }
2719
+ }
2720
+ netdata_rwlock_unlock(&st->alerts.rwlock);
2721
+ }
2722
+}
2723
+
2724
+static bool query_target_match_alert_pattern(QUERY_INSTANCE *qi, SIMPLE_PATTERN *pattern) {
2725
+ if(!pattern)
2726
+ return true;
2727
+
2728
+ RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
2729
+ if (!st)
2730
+ return false;
2731
+
2732
+ BUFFER *wb = NULL;
2733
+ bool matched = false;
2734
+ netdata_rwlock_rdlock(&st->alerts.rwlock);
2735
+ if (st->alerts.base) {
2736
+ for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
2737
+ if(simple_pattern_matches(pattern, string2str(rc->name))) {
2738
+ matched = true;
2739
+ break;
2740
+ }
2741
+
2742
+ if(!wb)
2743
+ wb = buffer_create(0, NULL);
2744
+ else
2745
+ buffer_flush(wb);
2746
+
2747
+ buffer_fast_strcat(wb, string2str(rc->name), string_strlen(rc->name));
2748
+ buffer_fast_strcat(wb, ":", 1);
2749
+ buffer_strcat(wb, rrdcalc_status2string(rc->status));
2750
+
2751
+ if(simple_pattern_matches(pattern, buffer_tostring(wb))) {
2752
+ matched = true;
2753
+ break;
2754
+ }
2755
+ }
2756
+ }
2757
+ netdata_rwlock_unlock(&st->alerts.rwlock);
2758
+
2759
+ buffer_free(wb);
2760
+ return matched;
2761
+}
2762
+
2763
+static void query_target_add_instance(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, QUERY_CONTEXT *qc,
2764
+ RRDINSTANCE_ACQUIRED *ria, bool queryable_instance, bool filter_instances) {
2765
QUERY_TARGET *qt = qtl->qt;
2766
2767
RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
@@ -2533,47 +2769,94 @@ static void query_target_add_instance(QUERY_TARGET_LOCALS *qtl, RRDINSTANCE_ACQU
2769
return;
2770
2771
if(qt->instances.used == qt->instances.size) {
2536
- size_t old_mem = qt->instances.size * sizeof(RRDINSTANCE_ACQUIRED *);
2772
+ size_t old_mem = qt->instances.size * sizeof(*qt->instances.array);
2773
qt->instances.size = (qt->instances.size) ? qt->instances.size * 2 : 1;
2538
- size_t new_mem = qt->instances.size * sizeof(RRDINSTANCE_ACQUIRED *);
2774
+ size_t new_mem = qt->instances.size * sizeof(*qt->instances.array);
2775
qt->instances.array = reallocz(qt->instances.array, new_mem);
2776
2777
__atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2778
}
2779
+ QUERY_INSTANCE *qi = &qt->instances.array[qt->instances.used];
2780
+ memset(qi, 0, sizeof(*qi));
2781
2544
- qtl->ria = qt->instances.array[qt->instances.used++] = rrdinstance_acquired_dup(ria);
2782
+ qi->slot = qt->instances.used;
2783
+ qt->instances.used++;
2784
+ qi->ria = rrdinstance_acquired_dup(ria);
2785
+ qi->query_host_id = qh->slot;
2786
+
2787
+ if(qt->request.version <= 1) {
2788
+ qi->id_fqdn = rrdinstance_id_fqdn_v1(ria);
2789
+ qi->name_fqdn = rrdinstance_name_fqdn_v1(ria);
2790
+ }
2791
+ else {
2792
+ qi->id_fqdn = rrdinstance_id_fqdn_v2(ria);
2793
+ qi->name_fqdn = rrdinstance_name_fqdn_v2(ria);
2794
+ }
2795
2796
if(qt->db.minimum_latest_update_every_s == 0 || ri->update_every_s < qt->db.minimum_latest_update_every_s)
2797
qt->db.minimum_latest_update_every_s = ri->update_every_s;
2798
2799
+ if(queryable_instance && filter_instances) {
2800
+ queryable_instance = false;
2801
+ if(!qt->instances.pattern
2802
+ || (qtl->match_ids && simple_pattern_matches(qt->instances.pattern, string2str(ri->id)))
2803
+ || (qtl->match_ids && simple_pattern_matches(qt->instances.pattern, string2str(qi->id_fqdn)))
2804
+ || (qtl->match_names && simple_pattern_matches(qt->instances.pattern, string2str(ri->name)))
2805
+ || (qtl->match_names && simple_pattern_matches(qt->instances.pattern, string2str(qi->name_fqdn)))
2806
+ )
2807
+ queryable_instance = true;
2808
+ }
2809
+
2810
if(queryable_instance) {
2811
if ((qt->instances.chart_label_key_pattern && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, qt->instances.chart_label_key_pattern, ':')) ||
2551
- (qt->instances.charts_labels_filter_pattern && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, qt->instances.charts_labels_filter_pattern, ':')))
2812
+ (qt->instances.labels_pattern && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, qt->instances.labels_pattern, ':')))
2813
queryable_instance = false;
2814
}
2815
2816
+ if(queryable_instance) {
2817
+ if(qt->instances.alerts_pattern && !query_target_match_alert_pattern(qi, qt->instances.alerts_pattern))
2818
+ queryable_instance = false;
2819
+ }
2820
+
2821
+ if(queryable_instance && qt->request.version >= 2)
2822
+ query_target_eval_instance_rrdcalc(qtl, qh, qc, qi);
2823
+
2824
size_t added = 0;
2825
2826
if(unlikely(qt->request.rma)) {
2558
- query_target_add_metric(qtl, qt->request.rma, ri, queryable_instance);
2827
+ query_target_add_dimension(qtl, qh, qc, qi, qt->request.rma, queryable_instance);
2828
added++;
2829
}
2830
else {
2831
RRDMETRIC *rm;
2832
dfe_start_read(ri->rrdmetrics, rm) {
2564
- query_target_add_metric(qtl, (RRDMETRIC_ACQUIRED *) rm_dfe.item, ri, queryable_instance);
2833
+ query_target_add_dimension(qtl, qh, qc, qi, (RRDMETRIC_ACQUIRED *) rm_dfe.item, queryable_instance);
2834
added++;
2835
}
2836
dfe_done(rm);
2837
}
2838
2839
if(!added) {
2840
+ qc->instances.excluded++;
2841
+ qh->instances.excluded++;
2842
+
2843
qt->instances.used--;
2844
rrdinstance_release(ria);
2845
+ qi->ria = NULL;
2846
+
2847
+ string_freez(qi->id_fqdn);
2848
+ qi->id_fqdn = NULL;
2849
+
2850
+ string_freez(qi->name_fqdn);
2851
+ qi->name_fqdn = NULL;
2852
+ }
2853
+ else {
2854
+ qc->instances.selected++;
2855
+ qh->instances.selected++;
2856
}
2857
}
2858
2576
-static void query_target_add_context(QUERY_TARGET_LOCALS *qtl, RRDCONTEXT_ACQUIRED *rca) {
2859
+static void query_target_add_context(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, RRDCONTEXT_ACQUIRED *rca, bool queryable_context) {
2860
QUERY_TARGET *qt = qtl->qt;
2861
2862
RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
@@ -2581,35 +2864,31 @@ static void query_target_add_context(QUERY_TARGET_LOCALS *qtl, RRDCONTEXT_ACQUIR
2864
return;
2865
2866
if(qt->contexts.used == qt->contexts.size) {
2584
- size_t old_mem = qt->contexts.size * sizeof(RRDCONTEXT_ACQUIRED *);
2867
+ size_t old_mem = qt->contexts.size * sizeof(*qt->contexts.array);
2868
qt->contexts.size = (qt->contexts.size) ? qt->contexts.size * 2 : 1;
2586
- size_t new_mem = qt->contexts.size * sizeof(RRDCONTEXT_ACQUIRED *);
2869
+ size_t new_mem = qt->contexts.size * sizeof(*qt->contexts.array);
2870
qt->contexts.array = reallocz(qt->contexts.array, new_mem);
2871
2872
__atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2873
}
2591
- qtl->rca = qt->contexts.array[qt->contexts.used++] = rrdcontext_acquired_dup(rca);
2874
+ QUERY_CONTEXT *qc = &qt->contexts.array[qt->contexts.used];
2875
+ memset(qc, 0, sizeof(*qc));
2876
+ qc->slot = qt->contexts.used++;
2877
+ qc->rca = rrdcontext_acquired_dup(rca);
2878
2879
size_t added = 0;
2880
if(unlikely(qt->request.ria)) {
2595
- query_target_add_instance(qtl, qt->request.ria, true);
2881
+ query_target_add_instance(qtl, qh, qc, qt->request.ria, queryable_context, false);
2882
added++;
2883
}
2884
else if(unlikely(qtl->st && qtl->st->rrdcontext == rca && qtl->st->rrdinstance)) {
2599
- query_target_add_instance(qtl, qtl->st->rrdinstance, true);
2885
+ query_target_add_instance(qtl, qh, qc, qtl->st->rrdinstance, queryable_context, false);
2886
added++;
2887
}
2888
else {
2889
RRDINSTANCE *ri;
2890
dfe_start_read(rc->rrdinstances, ri) {
2605
- bool queryable_instance = false;
2606
- if(!qt->instances.pattern
2607
- || (qtl->match_ids && simple_pattern_matches(qt->instances.pattern, string2str(ri->id)))
2608
- || (qtl->match_names && simple_pattern_matches(qt->instances.pattern, string2str(ri->name)))
2609
- )
2610
- queryable_instance = true;
2611
-
2612
- query_target_add_instance(qtl, (RRDINSTANCE_ACQUIRED *)ri_dfe.item, queryable_instance);
2891
+ query_target_add_instance(qtl, qh, qc, (RRDINSTANCE_ACQUIRED *)ri_dfe.item, queryable_context, true);
2892
added++;
2893
}
2894
dfe_done(ri);
@@ -2621,56 +2900,79 @@ static void query_target_add_context(QUERY_TARGET_LOCALS *qtl, RRDCONTEXT_ACQUIR
2900
}
2901
}
2902
2624
-static void query_target_add_host(QUERY_TARGET_LOCALS *qtl, RRDHOST *host) {
2903
+static void query_target_add_host(QUERY_TARGET_LOCALS *qtl, RRDHOST *host, bool queryable_host) {
2904
QUERY_TARGET *qt = qtl->qt;
2905
2906
if(qt->hosts.used == qt->hosts.size) {
2628
- size_t old_mem = qt->hosts.size * sizeof(RRDHOST *);
2907
+ size_t old_mem = qt->hosts.size * sizeof(*qt->hosts.array);
2908
qt->hosts.size = (qt->hosts.size) ? qt->hosts.size * 2 : 1;
2630
- size_t new_mem = qt->hosts.size * sizeof(RRDHOST *);
2909
+ size_t new_mem = qt->hosts.size * sizeof(*qt->hosts.array);
2910
qt->hosts.array = reallocz(qt->hosts.array, new_mem);
2911
2912
__atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2913
}
2635
- qtl->host = qt->hosts.array[qt->hosts.used++] = host;
2914
+ QUERY_HOST *qh = &qt->hosts.array[qt->hosts.used];
2915
+ memset(qh, 0, sizeof(*qh));
2916
+ qh->slot = qt->hosts.used++;
2917
+ qh->host = host;
2918
+
2919
+ if(host->node_id)
2920
+ uuid_unparse_lower(*host->node_id, qh->node_id);
2921
+ else
2922
+ qh->node_id[0] = '\0';
2923
2924
// is the chart given valid?
2925
if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
2639
- error("QUERY TARGET: RRDSET '%s' given, because it is not linked to rrdcontext structures. Switching to context query.", rrdset_name(qtl->st));
2926
+ error("QUERY TARGET: RRDSET '%s' given, but it is not linked to rrdcontext structures. Linking it now.", rrdset_name(qtl->st));
2927
+ rrdinstance_from_rrdset(qtl->st);
2928
2641
- if(!is_valid_sp(qtl->charts))
2642
- qtl->charts = rrdset_name(qtl->st);
2929
+ if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
2930
+ error("QUERY TARGET: RRDSET '%s' given, but failed to be linked to rrdcontext structures. Switching to context query.",
2931
+ rrdset_name(qtl->st));
2932
2644
- qtl->st = NULL;
2933
+ if (!is_valid_sp(qtl->charts))
2934
+ qtl->charts = rrdset_name(qtl->st);
2935
+
2936
+ qtl->st = NULL;
2937
+ }
2938
}
2939
2940
size_t added = 0;
2941
if(unlikely(qt->request.rca)) {
2649
- query_target_add_context(qtl, qt->request.rca);
2942
+ query_target_add_context(qtl, qh, qt->request.rca, true);
2943
added++;
2944
}
2945
else if(unlikely(qtl->st)) {
2946
// single chart data queries
2654
- query_target_add_context(qtl, qtl->st->rrdcontext);
2947
+ query_target_add_context(qtl, qh, qtl->st->rrdcontext, true);
2948
added++;
2949
}
2950
else {
2951
// context pattern queries
2659
- RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)qtl->host->rrdctx, qtl->contexts);
2952
+ RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)host->rrdctx, qtl->scope_contexts);
2953
if(likely(rca)) {
2954
// we found it!
2662
- query_target_add_context(qtl, rca);
2955
+
2956
+ bool queryable_context = queryable_host;
2957
+ if(queryable_context && qt->contexts.pattern && !simple_pattern_matches(qt->contexts.pattern, rrdcontext_acquired_id(rca)))
2958
+ queryable_context = false;
2959
+
2960
+ query_target_add_context(qtl, qh, rca, queryable_context);
2961
rrdcontext_release(rca);
2962
added++;
2963
}
2964
else {
2965
// Probably it is a pattern, we need to search for it...
2966
RRDCONTEXT *rc;
2669
- dfe_start_read((DICTIONARY *)qtl->host->rrdctx, rc) {
2670
- if(qt->contexts.pattern && !simple_pattern_matches(qt->contexts.pattern, string2str(rc->id)))
2967
+ dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
2968
+ if(qt->contexts.scope_pattern && !simple_pattern_matches(qt->contexts.scope_pattern, string2str(rc->id)))
2969
continue;
2970
2673
- query_target_add_context(qtl, (RRDCONTEXT_ACQUIRED *)rc_dfe.item);
2971
+ bool queryable_context = queryable_host;
2972
+ if(queryable_context && qt->contexts.pattern && !simple_pattern_matches(qt->contexts.pattern, string2str(rc->id)))
2973
+ queryable_context = false;
2974
+
2975
+ query_target_add_context(qtl, qh, (RRDCONTEXT_ACQUIRED *)rc_dfe.item, queryable_context);
2976
added++;
2977
}
2978
dfe_done(rc);
@@ -2695,7 +2997,7 @@ void query_target_generate_name(QUERY_TARGET *qt) {
2997
snprintfz(tier_buffer, 20, "/tier:%zu", qt->request.tier);
2998
2999
if(qt->request.st)
2698
- snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "chart://host:%s/instance:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3000
+ snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "chart://hosts:%s/instance:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3001
, rrdhost_hostname(qt->request.st->rrdhost)
3002
, rrdset_name(qt->request.st)
3003
, (qt->request.dimensions) ? qt->request.dimensions : "*"
@@ -2709,7 +3011,7 @@ void query_target_generate_name(QUERY_TARGET *qt) {
3011
, tier_buffer
3012
);
3013
else if(qt->request.host && qt->request.rca && qt->request.ria && qt->request.rma)
2712
- snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "metric://host:%s/context:%s/instance:%s/dimension:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3014
+ snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "metric://hosts:%s/context:%s/instance:%s/dimension:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3015
, rrdhost_hostname(qt->request.host)
3016
, rrdcontext_acquired_id(qt->request.rca)
3017
, rrdinstance_acquired_id(qt->request.ria)
@@ -2724,7 +3026,7 @@ void query_target_generate_name(QUERY_TARGET *qt) {
3026
, tier_buffer
3027
);
3028
else
2727
- snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "context://host:%s/contexts:%s/instances:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3029
+ snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "context://hosts:%s/contexts:%s/instances:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3030
, (qt->request.host) ? rrdhost_hostname(qt->request.host) : ((qt->request.hosts) ? qt->request.hosts : "*")
3031
, (qt->request.contexts) ? qt->request.contexts : "*"
3032
, (qt->request.charts) ? qt->request.charts : "*"
@@ -2754,6 +3056,19 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
3056
qt->used = true;
3057
qt->queries++;
3058
3059
+ if(!qtr->received_ut)
3060
+ qtr->received_ut = now_monotonic_usec();
3061
+
3062
+ qt->timings.received_ut = qtr->received_ut;
3063
+
3064
+ if(qtr->hosts && !qtr->scope_hosts)
3065
+ qtr->scope_hosts = qtr->hosts;
3066
+
3067
+ if(qtr->contexts && !qtr->scope_contexts)
3068
+ qtr->scope_contexts = qtr->contexts;
3069
+
3070
+ memset(&qt->query_stats, 0, sizeof(qt->query_stats));
3071
+
3072
// copy the request into query_thread_target
3073
qt->request = *qtr;
3074
@@ -2766,25 +3081,34 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
3081
QUERY_TARGET_LOCALS qtl = {
3082
.qt = qt,
3083
.start_s = now_realtime_sec(),
2769
- .host = qt->request.host,
3084
.st = qt->request.st,
3085
+ .scope_hosts = qt->request.scope_hosts,
3086
+ .scope_contexts = qt->request.scope_contexts,
3087
.hosts = qt->request.hosts,
3088
.contexts = qt->request.contexts,
3089
.charts = qt->request.charts,
3090
.dimensions = qt->request.dimensions,
3091
.chart_label_key = qt->request.chart_label_key,
2776
- .charts_labels_filter = qt->request.charts_labels_filter,
3092
+ .labels = qt->request.labels,
3093
+ .alerts = qt->request.alerts,
3094
};
3095
3096
+ RRDHOST *host = qt->request.host;
3097
+
3098
qt->db.minimum_latest_update_every_s = 0; // it will be updated by query_target_add_query()
3099
3100
// prepare all the patterns
3101
qt->hosts.pattern = is_valid_sp(qtl.hosts) ? simple_pattern_create(qtl.hosts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3102
+ qt->hosts.scope_pattern = is_valid_sp(qtl.scope_hosts) ? simple_pattern_create(qtl.scope_hosts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3103
+
3104
qt->contexts.pattern = is_valid_sp(qtl.contexts) ? simple_pattern_create(qtl.contexts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3105
+ qt->contexts.scope_pattern = is_valid_sp(qtl.scope_contexts) ? simple_pattern_create(qtl.scope_contexts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3106
+
3107
qt->instances.pattern = is_valid_sp(qtl.charts) ? simple_pattern_create(qtl.charts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3108
qt->query.pattern = is_valid_sp(qtl.dimensions) ? simple_pattern_create(qtl.dimensions, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3109
qt->instances.chart_label_key_pattern = is_valid_sp(qtl.chart_label_key) ? simple_pattern_create(qtl.chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
2787
- qt->instances.charts_labels_filter_pattern = is_valid_sp(qtl.charts_labels_filter) ? simple_pattern_create(qtl.charts_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3110
+ qt->instances.labels_pattern = is_valid_sp(qtl.labels) ? simple_pattern_create(qtl.labels, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3111
+ qt->instances.alerts_pattern = is_valid_sp(qtl.alerts) ? simple_pattern_create(qtl.alerts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3112
3113
qtl.match_ids = qt->request.options & RRDR_OPTION_MATCH_IDS;
3114
qtl.match_names = qt->request.options & RRDR_OPTION_MATCH_NAMES;
@@ -2793,58 +3117,56 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
3117
3118
// verify that the chart belongs to the host we are interested
3119
if(qtl.st) {
2796
- if (!qtl.host) {
3120
+ if (!host) {
3121
// It is NULL, set it ourselves.
2798
- qtl.host = qtl.st->rrdhost;
3122
+ host = qtl.st->rrdhost;
3123
}
2800
- else if (unlikely(qtl.host != qtl.st->rrdhost)) {
3124
+ else if (unlikely(host != qtl.st->rrdhost)) {
3125
// Oops! A different host!
3126
error("QUERY TARGET: RRDSET '%s' given does not belong to host '%s'. Switching query host to '%s'",
2803
- rrdset_name(qtl.st), rrdhost_hostname(qtl.host), rrdhost_hostname(qtl.st->rrdhost));
2804
- qtl.host = qtl.st->rrdhost;
3127
+ rrdset_name(qtl.st), rrdhost_hostname(host), rrdhost_hostname(qtl.st->rrdhost));
3128
+ host = qtl.st->rrdhost;
3129
}
3130
}
3131
2808
- if(qtl.host) {
3132
+ if(host) {
3133
// single host query
2810
- query_target_add_host(&qtl, qtl.host);
2811
- qtl.hosts = rrdhost_hostname(qtl.host);
3134
+ query_target_add_host(&qtl, host, true);
3135
+ qtl.hosts = rrdhost_hostname(host);
3136
}
3137
else {
3138
+ char uuid[UUID_STR_LEN];
3139
+
3140
// multi host query
3141
rrd_rdlock();
2816
- rrdhost_foreach_read(qtl.host) {
2817
- if(!qt->hosts.pattern || simple_pattern_matches(qt->hosts.pattern, rrdhost_hostname(qtl.host)))
2818
- query_target_add_host(&qtl, qtl.host);
3142
+ rrdhost_foreach_read(host) {
3143
+
3144
+ if(!host->node_id)
3145
+ uuid_unparse_lower(*host->node_id, uuid);
3146
+ else
3147
+ uuid[0] = '\0';
3148
+
3149
+ if(!qt->hosts.scope_pattern ||
3150
+ simple_pattern_matches(qt->hosts.scope_pattern, rrdhost_hostname(host)) ||
3151
+ simple_pattern_matches(qt->hosts.scope_pattern, host->machine_guid) ||
3152
+ (*uuid && simple_pattern_matches(qt->hosts.scope_pattern, uuid))) {
3153
+
3154
+ bool queryable_host = false;
3155
+ if(!qt->hosts.pattern ||
3156
+ simple_pattern_matches(qt->hosts.pattern, rrdhost_hostname(host)) ||
3157
+ simple_pattern_matches(qt->hosts.pattern, host->machine_guid) ||
3158
+ simple_pattern_matches(qt->hosts.pattern, uuid))
3159
+ queryable_host = true;
3160
+
3161
+ query_target_add_host(&qtl, host, queryable_host);
3162
+ }
3163
}
3164
rrd_unlock();
3165
}
3166
2823
- // make sure everything is good
2824
- if(!qt->query.used || !qt->metrics.used || !qt->instances.used || !qt->contexts.used || !qt->hosts.used) {
2825
- internal_error(
2826
- true
2827
- , "QUERY TARGET: query '%s' does not have all the data required. "
2828
- "Matched %u hosts, %u contexts, %u instances, %u dimensions, %u metrics to query, "
2829
- "%zu metrics skipped because they don't have data in the desired time-frame. "
2830
- "Aborting it."
2831
- , qt->id
2832
- , qt->hosts.used
2833
- , qt->contexts.used
2834
- , qt->instances.used
2835
- , qt->metrics.used
2836
- , qt->query.used
2837
- , qtl.metrics_skipped_due_to_not_matching_timeframe
2838
- );
2839
-
2840
- query_target_release(qt);
2841
- return NULL;
2842
- }
3167
+ query_target_calculate_window(qt);
3168
2844
- if(!query_target_calculate_window(qt)) {
2845
- query_target_release(qt);
2846
- return NULL;
2847
- }
3169
+ qt->timings.preprocessed_ut = now_monotonic_usec();
3170
3171
return qt;
3172
}
database/rrdcontext.h
+174
-19
@@ -25,12 +25,30 @@ typedef struct rrdcontext_acquired RRDCONTEXT_ACQUIRED;
25
26
const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma);
27
const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma);
28
+
29
+STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma);
30
+STRING *rrdmetric_acquired_name_dup(RRDMETRIC_ACQUIRED *rma);
31
+
32
NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma);
33
+time_t rrdmetric_acquired_first_entry(RRDMETRIC_ACQUIRED *rma);
34
+time_t rrdmetric_acquired_last_entry(RRDMETRIC_ACQUIRED *rma);
35
+bool rrdmetric_acquired_belongs_to_instance(RRDMETRIC_ACQUIRED *rma, RRDINSTANCE_ACQUIRED *ria);
36
37
const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria);
38
const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria);
39
+const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria);
40
+STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria);
41
DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria);
42
DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria);
43
+RRDHOST *rrdinstance_acquired_rrdhost(RRDINSTANCE_ACQUIRED *ria);
44
+RRDSET *rrdinstance_acquired_rrdset(RRDINSTANCE_ACQUIRED *ria);
45
+
46
+bool rrdinstance_acquired_belongs_to_context(RRDINSTANCE_ACQUIRED *ria, RRDCONTEXT_ACQUIRED *rca);
47
+time_t rrdinstance_acquired_update_every(RRDINSTANCE_ACQUIRED *ria);
48
+
49
+const char *rrdcontext_acquired_units(RRDCONTEXT_ACQUIRED *rca);
50
+const char *rrdcontext_acquired_title(RRDCONTEXT_ACQUIRED *rca);
51
+RRDSET_TYPE rrdcontext_acquired_chart_type(RRDCONTEXT_ACQUIRED *rca);
52
53
// ----------------------------------------------------------------------------
54
// public API for rrdhost
@@ -67,6 +85,7 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
85
// public API for rrdcontexts
86
87
const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca);
88
+bool rrdcontext_acquired_belongs_to_host(RRDCONTEXT_ACQUIRED *rca, RRDHOST *host);
89
90
// ----------------------------------------------------------------------------
91
// public API for rrddims
@@ -118,6 +137,15 @@ DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contex
137
// ----------------------------------------------------------------------------
138
// public API for queries
139
140
+typedef enum __attribute__ ((__packed__)) {
141
+ QUERY_STATUS_NONE = 0,
142
+ QUERY_STATUS_QUERIED = (1 << 0),
143
+ QUERY_STATUS_DIMENSION_NODATA = (1 << 1),
144
+ QUERY_STATUS_DIMENSION_HIDDEN = (1 << 2),
145
+ QUERY_STATUS_EXCLUDED = (1 << 3),
146
+ QUERY_STATUS_FAILED = (1 << 4),
147
+} QUERY_STATUS;
148
+
149
typedef struct query_plan_entry {
150
size_t tier;
151
time_t after;
@@ -134,7 +162,75 @@ typedef struct query_plan_entry {
162
163
#define QUERY_PLANS_MAX (RRD_STORAGE_TIERS * 2)
164
165
+struct query_instances_counts {
166
+ size_t selected;
167
+ size_t excluded;
168
+};
169
+
170
+struct query_metrics_counts {
171
+ size_t selected;
172
+ size_t excluded;
173
+ size_t queried;
174
+ size_t failed;
175
+};
176
+
177
+struct query_alerts_counts {
178
+ size_t clear;
179
+ size_t warning;
180
+ size_t critical;
181
+ size_t other;
182
+};
183
+
184
+struct query_data_statistics {
185
+ size_t count;
186
+ NETDATA_DOUBLE min;
187
+ NETDATA_DOUBLE max;
188
+ NETDATA_DOUBLE sum;
189
+ NETDATA_DOUBLE volume;
190
+};
191
+
192
+typedef struct query_host {
193
+ uint32_t slot;
194
+ RRDHOST *host;
195
+ char node_id[UUID_STR_LEN];
196
+
197
+ struct query_data_statistics query_stats;
198
+ struct query_instances_counts instances;
199
+ struct query_metrics_counts metrics;
200
+ struct query_alerts_counts alerts;
201
+} QUERY_HOST;
202
+
203
+typedef struct query_context {
204
+ uint32_t slot;
205
+ RRDCONTEXT_ACQUIRED *rca;
206
+
207
+ struct query_data_statistics query_stats;
208
+ struct query_instances_counts instances;
209
+ struct query_metrics_counts metrics;
210
+ struct query_alerts_counts alerts;
211
+} QUERY_CONTEXT;
212
+
213
+typedef struct query_instance {
214
+ uint32_t slot;
215
+ uint32_t query_host_id;
216
+ RRDINSTANCE_ACQUIRED *ria;
217
+ STRING *id_fqdn;
218
+ STRING *name_fqdn;
219
+
220
+ struct query_data_statistics query_stats;
221
+ struct query_metrics_counts metrics;
222
+ struct query_alerts_counts alerts;
223
+} QUERY_INSTANCE;
224
+
225
+typedef struct query_dimension {
226
+ uint32_t slot;
227
+ RRDMETRIC_ACQUIRED *rma;
228
+ QUERY_STATUS status;
229
+} QUERY_DIMENSION;
230
+
231
typedef struct query_metric {
232
+ RRDR_DIMENSION_FLAGS status;
233
+
234
struct query_metric_tier {
235
struct storage_engine *eng;
236
STORAGE_METRIC_HANDLE *db_metric_handle;
@@ -150,28 +246,31 @@ typedef struct query_metric {
246
} plan;
247
248
struct {
153
- RRDHOST *host;
154
- RRDCONTEXT_ACQUIRED *rca;
155
- RRDINSTANCE_ACQUIRED *ria;
156
- RRDMETRIC_ACQUIRED *rma;
249
+ uint32_t query_host_id;
250
+ uint32_t query_context_id;
251
+ uint32_t query_instance_id;
252
+ uint32_t query_dimension_id;
253
} link;
254
159
- struct {
160
- STRING *id;
161
- STRING *name;
162
- RRDR_DIMENSION_FLAGS options;
163
- } dimension;
255
+ struct query_data_statistics query_stats;
256
257
struct {
258
+ size_t slot;
259
STRING *id;
260
STRING *name;
168
- } chart;
261
+ STRING *units;
262
+ } grouped_as;
263
264
} QUERY_METRIC;
265
266
#define MAX_QUERY_TARGET_ID_LENGTH 255
267
268
typedef struct query_target_request {
269
+ size_t version;
270
+
271
+ const char *scope_hosts;
272
+ const char *scope_contexts;
273
+
274
// selecting / filtering metrics to be queried
275
RRDHOST *host; // the host to be queried (can be NULL, hosts will be used)
276
RRDCONTEXT_ACQUIRED *rca; // the context to be queried (can be NULL)
@@ -183,7 +282,8 @@ typedef struct query_target_request {
282
const char *charts; // charts simple pattern (for context queries)
283
const char *dimensions; // dimensions simple pattern
284
const char *chart_label_key; // select only the chart having this label key
186
- const char *charts_labels_filter; // select only the charts having this combo of label key:value
285
+ const char *labels; // select only the charts having this combo of label key:value
286
+ const char *alerts; // select only the charts having this combo of alert name:status
287
288
time_t after; // the requested timeframe
289
time_t before; // the requested timeframe
@@ -206,11 +306,14 @@ typedef struct query_target_request {
306
307
// group by across multiple time-series
308
RRDR_GROUP_BY group_by;
209
- const char *group_by_key;
210
- RRDR_GROUP_BY_FUNCTION group_by_function;
309
+ char *group_by_label;
310
+ RRDR_GROUP_BY_FUNCTION group_by_aggregate_function;
311
312
+ usec_t received_ut;
313
} QUERY_TARGET_REQUEST;
314
315
+#define GROUP_BY_MAX_LABEL_KEYS 10
316
+
317
typedef struct query_target {
318
char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging)
319
QUERY_TARGET_REQUEST request;
@@ -248,36 +351,88 @@ typedef struct query_target {
351
} query;
352
353
struct {
251
- RRDMETRIC_ACQUIRED **array;
354
+ QUERY_DIMENSION *array;
355
uint32_t used; // how many items of the array are used
356
uint32_t size; // the size of the array
254
- } metrics;
357
+ } dimensions;
358
359
struct {
257
- RRDINSTANCE_ACQUIRED **array;
360
+ QUERY_INSTANCE *array;
361
uint32_t used; // how many items of the array are used
362
uint32_t size; // the size of the array
363
SIMPLE_PATTERN *pattern;
364
+ SIMPLE_PATTERN *labels_pattern;
365
+ SIMPLE_PATTERN *alerts_pattern;
366
SIMPLE_PATTERN *chart_label_key_pattern;
262
- SIMPLE_PATTERN *charts_labels_filter_pattern;
367
} instances;
368
369
struct {
266
- RRDCONTEXT_ACQUIRED **array;
370
+ QUERY_CONTEXT *array;
371
uint32_t used; // how many items of the array are used
372
uint32_t size; // the size of the array
373
SIMPLE_PATTERN *pattern;
374
+ SIMPLE_PATTERN *scope_pattern;
375
} contexts;
376
377
struct {
273
- RRDHOST **array;
378
+ QUERY_HOST *array;
379
uint32_t used; // how many items of the array are used
380
uint32_t size; // the size of the array
381
SIMPLE_PATTERN *pattern;
382
+ SIMPLE_PATTERN *scope_pattern;
383
} hosts;
384
385
+ struct {
386
+ size_t used;
387
+ char *label_keys[GROUP_BY_MAX_LABEL_KEYS];
388
+ } group_by;
389
+
390
+ struct query_data_statistics query_stats;
391
+
392
+ struct {
393
+ usec_t received_ut;
394
+ usec_t preprocessed_ut;
395
+ usec_t executed_ut;
396
+ usec_t group_by_ut;
397
+ usec_t finished_ut;
398
+ } timings;
399
} QUERY_TARGET;
400
401
+static inline NEVERNULL QUERY_HOST *query_host(QUERY_TARGET *qt, size_t id) {
402
+ internal_fatal(id >= qt->hosts.used, "QUERY: invalid query host id");
403
+ return &qt->hosts.array[id];
404
+}
405
+
406
+static inline NEVERNULL QUERY_CONTEXT *query_context(QUERY_TARGET *qt, size_t query_context_id) {
407
+ internal_fatal(query_context_id >= qt->contexts.used, "QUERY: invalid query context id");
408
+ return &qt->contexts.array[query_context_id];
409
+}
410
+
411
+static inline NEVERNULL QUERY_INSTANCE *query_instance(QUERY_TARGET *qt, size_t query_instance_id) {
412
+ internal_fatal(query_instance_id >= qt->instances.used, "QUERY: invalid query instance id");
413
+ return &qt->instances.array[query_instance_id];
414
+}
415
+
416
+static inline NEVERNULL QUERY_DIMENSION *query_dimension(QUERY_TARGET *qt, size_t query_dimension_id) {
417
+ internal_fatal(query_dimension_id >= qt->dimensions.used, "QUERY: invalid query dimension id");
418
+ return &qt->dimensions.array[query_dimension_id];
419
+}
420
+
421
+static inline NEVERNULL QUERY_METRIC *query_metric(QUERY_TARGET *qt, size_t id) {
422
+ internal_fatal(id >= qt->query.used, "QUERY: invalid query metric id");
423
+ return &qt->query.array[id];
424
+}
425
+
426
+static inline const char *query_metric_id(QUERY_TARGET *qt, QUERY_METRIC *qm) {
427
+ QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
428
+ return rrdmetric_acquired_id(qd->rma);
429
+}
430
+
431
+static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm) {
432
+ QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
433
+ return rrdmetric_acquired_name(qd->rma);
434
+}
435
+
436
void query_target_free(void);
437
void query_target_release(QUERY_TARGET *qt);
438
database/rrdlabels.c
+40
@@ -677,6 +677,46 @@ void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const
677
dictionary_acquired_item_release(labels, acquired_item);
678
}
679
680
+void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key) {
681
+ const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
682
+ RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
683
+
684
+ if(lb && lb->label_value)
685
+ strncpyz(dst, string2str(lb->label_value), dst_len);
686
+ else
687
+ dst[0] = '\0';
688
+
689
+ dictionary_acquired_item_release(labels, acquired_item);
690
+}
691
+
692
+STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key) {
693
+ const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
694
+ RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
695
+
696
+ STRING *ret = NULL;
697
+ if(lb && lb->label_value)
698
+ ret = string_dup(lb->label_value);
699
+
700
+ dictionary_acquired_item_release(labels, acquired_item);
701
+
702
+ return ret;
703
+}
704
+
705
+STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset) {
706
+ const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
707
+ RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
708
+
709
+ STRING *ret = NULL;
710
+ if(lb && lb->label_value)
711
+ buffer_strcat(wb, string2str(lb->label_value));
712
+ else
713
+ buffer_strcat(wb, unset);
714
+
715
+ dictionary_acquired_item_release(labels, acquired_item);
716
+
717
+ return ret;
718
+}
719
+
720
// ----------------------------------------------------------------------------
721
// rrdlabels_unmark_all()
722
// remove labels RRDLABEL_FLAG_OLD and RRDLABEL_FLAG_NEW from all dictionary items
streaming/replication.c
+67
-49
@@ -1626,67 +1626,85 @@ static void replication_initialize_workers(bool master) {
1626
#define REQUEST_QUEUE_EMPTY (-1)
1627
#define REQUEST_CHART_NOT_FOUND (-2)
1628
1629
-static int replication_execute_next_pending_request(bool cancel) {
1630
- static __thread int max_requests_ahead = 0;
1631
- static __thread struct replication_request *rqs = NULL;
1632
- static __thread int rqs_last_executed = 0, rqs_last_prepared = 0;
1633
- static __thread size_t queue_rounds = 0; (void)queue_rounds;
1629
+static __thread struct replication_thread_pipeline {
1630
+ int max_requests_ahead;
1631
+ struct replication_request *rqs;
1632
+ int rqs_last_executed, rqs_last_prepared;
1633
+ size_t queue_rounds;
1634
+} rtp = {
1635
+ .max_requests_ahead = 0,
1636
+ .rqs = NULL,
1637
+ .rqs_last_executed = 0,
1638
+ .rqs_last_prepared = 0,
1639
+ .queue_rounds = 0,
1640
+};
1641
+
1642
+static void replication_pipeline_cancel_and_cleanup(void) {
1643
+ if(!rtp.rqs)
1644
+ return;
1645
+
1646
struct replication_request *rq;
1647
+ size_t cancelled = 0;
1648
1636
- if(unlikely(cancel)) {
1637
- if(rqs) {
1638
- size_t cancelled = 0;
1639
- do {
1640
- if (++rqs_last_executed >= max_requests_ahead)
1641
- rqs_last_executed = 0;
1649
+ do {
1650
+ if (++rtp.rqs_last_executed >= rtp.max_requests_ahead)
1651
+ rtp.rqs_last_executed = 0;
1652
1643
- rq = &rqs[rqs_last_executed];
1653
+ rq = &rtp.rqs[rtp.rqs_last_executed];
1654
1645
- if (rq->q) {
1646
- internal_fatal(rq->executed, "REPLAY FATAL: query has already been executed!");
1647
- internal_fatal(!rq->found, "REPLAY FATAL: orphan q in rq");
1655
+ if (rq->q) {
1656
+ internal_fatal(rq->executed, "REPLAY FATAL: query has already been executed!");
1657
+ internal_fatal(!rq->found, "REPLAY FATAL: orphan q in rq");
1658
1649
- replication_response_cancel_and_finalize(rq->q);
1650
- rq->q = NULL;
1651
- cancelled++;
1652
- }
1659
+ replication_response_cancel_and_finalize(rq->q);
1660
+ rq->q = NULL;
1661
+ cancelled++;
1662
+ }
1663
1654
- rq->executed = true;
1655
- rq->found = false;
1664
+ rq->executed = true;
1665
+ rq->found = false;
1666
1657
- } while (rqs_last_executed != rqs_last_prepared);
1667
+ } while (rtp.rqs_last_executed != rtp.rqs_last_prepared);
1668
1659
- internal_error(true, "REPLICATION: cancelled %zu inflight queries", cancelled);
1660
- }
1661
- return REQUEST_QUEUE_EMPTY;
1662
- }
1669
+ internal_error(true, "REPLICATION: cancelled %zu inflight queries", cancelled);
1670
+
1671
+ freez(rtp.rqs);
1672
+ rtp.rqs = NULL;
1673
+ rtp.max_requests_ahead = 0;
1674
+ rtp.rqs_last_executed = 0;
1675
+ rtp.rqs_last_prepared = 0;
1676
+ rtp.queue_rounds = 0;
1677
+}
1678
+
1679
+static int replication_pipeline_execute_next(void) {
1680
+ struct replication_request *rq;
1681
1664
- if(unlikely(!rqs)) {
1665
- max_requests_ahead = get_netdata_cpus() / 2;
1682
+ if(unlikely(!rtp.rqs)) {
1683
+ rtp.max_requests_ahead = (int)get_netdata_cpus() / 2;
1684
1667
- if(max_requests_ahead > libuv_worker_threads * 2)
1668
- max_requests_ahead = libuv_worker_threads * 2;
1685
+ if(rtp.max_requests_ahead > libuv_worker_threads * 2)
1686
+ rtp.max_requests_ahead = libuv_worker_threads * 2;
1687
1670
- if(max_requests_ahead < 2)
1671
- max_requests_ahead = 2;
1688
+ if(rtp.max_requests_ahead < 2)
1689
+ rtp.max_requests_ahead = 2;
1690
1673
- rqs = callocz(max_requests_ahead, sizeof(struct replication_request));
1674
- __atomic_add_fetch(&replication_buffers_allocated, max_requests_ahead * sizeof(struct replication_request), __ATOMIC_RELAXED);
1691
+ rtp.rqs = callocz(rtp.max_requests_ahead, sizeof(struct replication_request));
1692
+ __atomic_add_fetch(&replication_buffers_allocated, rtp.max_requests_ahead * sizeof(struct replication_request), __ATOMIC_RELAXED);
1693
}
1694
1695
// fill the queue
1696
do {
1679
- if(++rqs_last_prepared >= max_requests_ahead) {
1680
- rqs_last_prepared = 0;
1681
- queue_rounds++;
1697
+ if(++rtp.rqs_last_prepared >= rtp.max_requests_ahead) {
1698
+ rtp.rqs_last_prepared = 0;
1699
+ rtp.queue_rounds++;
1700
}
1701
1684
- internal_fatal(rqs[rqs_last_prepared].q,
1702
+ internal_fatal(rtp.rqs[rtp.rqs_last_prepared].q,
1703
"REPLAY FATAL: slot is used by query that has not been executed!");
1704
1705
worker_is_busy(WORKER_JOB_FIND_NEXT);
1688
- rqs[rqs_last_prepared] = replication_request_get_first_available();
1689
- rq = &rqs[rqs_last_prepared];
1706
+ rtp.rqs[rtp.rqs_last_prepared] = replication_request_get_first_available();
1707
+ rq = &rtp.rqs[rtp.rqs_last_prepared];
1708
1709
if(rq->found) {
1710
if (!rq->st) {
@@ -1707,14 +1725,14 @@ static int replication_execute_next_pending_request(bool cancel) {
1725
rq->executed = false;
1726
}
1727
1710
- } while(rq->found && rqs_last_prepared != rqs_last_executed);
1728
+ } while(rq->found && rtp.rqs_last_prepared != rtp.rqs_last_executed);
1729
1730
// pick the first usable
1731
do {
1714
- if (++rqs_last_executed >= max_requests_ahead)
1715
- rqs_last_executed = 0;
1732
+ if (++rtp.rqs_last_executed >= rtp.max_requests_ahead)
1733
+ rtp.rqs_last_executed = 0;
1734
1717
- rq = &rqs[rqs_last_executed];
1735
+ rq = &rtp.rqs[rtp.rqs_last_executed];
1736
1737
if(rq->found) {
1738
internal_fatal(rq->executed, "REPLAY FATAL: query has already been executed!");
@@ -1747,7 +1765,7 @@ static int replication_execute_next_pending_request(bool cancel) {
1765
else
1766
internal_fatal(rq->q, "REPLAY FATAL: slot status says slot is empty, but it has a pending query!");
1767
1750
- } while(!rq->found && rqs_last_executed != rqs_last_prepared);
1768
+ } while(!rq->found && rtp.rqs_last_executed != rtp.rqs_last_prepared);
1769
1770
if(unlikely(!rq->found)) {
1771
worker_is_idle();
@@ -1771,7 +1789,7 @@ static int replication_execute_next_pending_request(bool cancel) {
1789
}
1790
1791
static void replication_worker_cleanup(void *ptr __maybe_unused) {
1774
- replication_execute_next_pending_request(true);
1792
+ replication_pipeline_cancel_and_cleanup();
1793
worker_unregister();
1794
}
1795
@@ -1781,7 +1799,7 @@ static void *replication_worker_thread(void *ptr) {
1799
netdata_thread_cleanup_push(replication_worker_cleanup, ptr);
1800
1801
while(service_running(SERVICE_REPLICATION)) {
1784
- if(unlikely(replication_execute_next_pending_request(false) == REQUEST_QUEUE_EMPTY)) {
1802
+ if(unlikely(replication_pipeline_execute_next() == REQUEST_QUEUE_EMPTY)) {
1803
sender_thread_buffer_free();
1804
worker_is_busy(WORKER_JOB_WAIT);
1805
worker_is_idle();
@@ -1797,7 +1815,7 @@ static void replication_main_cleanup(void *ptr) {
1815
struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
1816
static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
1817
1800
- replication_execute_next_pending_request(true);
1818
+ replication_pipeline_cancel_and_cleanup();
1819
1820
int threads = (int)replication_globals.main_thread.threads;
1821
for(int i = 0; i < threads ;i++) {
@@ -1914,7 +1932,7 @@ void *replication_thread_main(void *ptr __maybe_unused) {
1932
worker_is_idle();
1933
}
1934
1917
- if(unlikely(replication_execute_next_pending_request(false) == REQUEST_QUEUE_EMPTY)) {
1935
+ if(unlikely(replication_pipeline_execute_next() == REQUEST_QUEUE_EMPTY)) {
1936
1937
worker_is_busy(WORKER_JOB_WAIT);
1938
replication_recursive_lock();
streaming/rrdpush.h
+5
@@ -186,12 +186,17 @@ struct sender_state {
186
} replication;
187
188
struct {
189
+ bool pending_data;
190
size_t buffer_used_percentage; // the current utilization of the sending buffer
191
usec_t last_flush_time_ut; // the last time the sender flushed the sending buffer in USEC
192
time_t last_buffer_recreate_s; // true when the sender buffer should be re-created
193
} atomic;
194
};
195
196
+#define rrdpush_sender_pipe_has_pending_data(sender) __atomic_load_n(&(sender)->atomic.pending_data, __ATOMIC_RELAXED)
197
+#define rrdpush_sender_pipe_set_pending_data(sender) __atomic_store_n(&(sender)->atomic.pending_data, true, __ATOMIC_RELAXED)
198
+#define rrdpush_sender_pipe_clear_pending_data(sender) __atomic_store_n(&(sender)->atomic.pending_data, false, __ATOMIC_RELAXED)
199
+
200
#define rrdpush_sender_last_buffer_recreate_get(sender) __atomic_load_n(&(sender)->atomic.last_buffer_recreate_s, __ATOMIC_RELAXED)
201
#define rrdpush_sender_last_buffer_recreate_set(sender, value) __atomic_store_n(&(sender)->atomic.last_buffer_recreate_s, value, __ATOMIC_RELAXED)
202
streaming/sender.c
+12
-4
@@ -178,8 +178,16 @@ void sender_commit(struct sender_state *s, BUFFER *wb) {
178
179
replication_recalculate_buffer_used_ratio_unsafe(s);
180
181
+ bool signal_sender = false;
182
+ if(!rrdpush_sender_pipe_has_pending_data(s)) {
183
+ rrdpush_sender_pipe_set_pending_data(s);
184
+ signal_sender = true;
185
+ }
186
+
187
netdata_mutex_unlock(&s->mutex);
182
- rrdpush_signal_sender_to_wake_up(s);
188
+
189
+ if(signal_sender)
190
+ rrdpush_signal_sender_to_wake_up(s);
191
}
192
193
static inline void rrdpush_sender_add_host_variable_to_buffer(BUFFER *wb, const RRDVAR_ACQUIRED *rva) {
@@ -1016,7 +1024,6 @@ void execute_commands(struct sender_state *s) {
1024
}
1025
1026
struct rrdpush_sender_thread_data {
1019
- struct sender_state *sender_state;
1027
RRDHOST *host;
1028
char *pipe_buffer;
1029
};
@@ -1249,7 +1256,6 @@ void *rrdpush_sender_thread(void *ptr) {
1256
1257
struct rrdpush_sender_thread_data *thread_data = callocz(1, sizeof(struct rrdpush_sender_thread_data));
1258
thread_data->pipe_buffer = mallocz(pipe_buffer_size);
1252
- thread_data->sender_state = s;
1259
thread_data->host = s->host;
1260
1261
netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, thread_data);
@@ -1305,8 +1311,10 @@ void *rrdpush_sender_thread(void *ptr) {
1311
netdata_mutex_lock(&s->mutex);
1312
size_t outstanding = cbuffer_next_unsafe(s->buffer, NULL);
1313
size_t available = cbuffer_available_size_unsafe(s->buffer);
1308
- if (unlikely(!outstanding))
1314
+ if (unlikely(!outstanding)) {
1315
+ rrdpush_sender_pipe_clear_pending_data(s);
1316
rrdpush_sender_cbuffer_recreate_timed(s, now_s, true, false);
1317
+ }
1318
netdata_mutex_unlock(&s->mutex);
1319
1320
worker_set_metric(WORKER_SENDER_JOB_BUFFER_RATIO, (NETDATA_DOUBLE)(s->buffer->max_size - available) * 100.0 / (NETDATA_DOUBLE)s->buffer->max_size);
web/api/formatters/csv/csv.c
+5
-6
@@ -5,9 +5,8 @@
5
6
void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) {
7
//info("RRD2CSV(): %s: BEGIN", r->st->id);
8
- QUERY_TARGET *qt = r->internal.qt;
8
long c, i;
10
- const long used = qt->query.used;
9
+ const long used = (long)r->d;
10
11
// print the csv header
12
for(c = 0, i = 0; c < used ; c++) {
@@ -22,7 +21,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
21
}
22
buffer_strcat(wb, separator);
23
if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
25
- buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
24
+ buffer_strcat(wb, string2str(r->dn[c]));
25
if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
26
i++;
27
}
@@ -126,12 +125,12 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
125
n = n * 100 / total;
126
127
if(unlikely(set_min_max)) {
129
- r->min = r->max = n;
128
+ r->view.min = r->view.max = n;
129
set_min_max = 0;
130
}
131
133
- if(n < r->min) r->min = n;
134
- if(n > r->max) r->max = n;
132
+ if(n < r->view.min) r->view.min = n;
133
+ if(n > r->view.max) r->view.max = n;
134
}
135
136
buffer_print_netdata_double(wb, n);
web/api/formatters/json/json.c
+20
-16
@@ -42,7 +42,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
42
strcpy(post_value, "}");
43
strcpy(post_line, "]}");
44
snprintfz(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq);
45
- strcpy(finish, "\n ]\n }");
45
+ strcpy(finish, "\n ]\n }");
46
47
snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
48
snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
@@ -69,9 +69,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
69
dates_with_new = 0;
70
}
71
if( options & RRDR_OPTION_OBJECTSROWS )
72
- strcpy(pre_date, " {");
72
+ strcpy(pre_date, " {");
73
else
74
- strcpy(pre_date, " [");
74
+ strcpy(pre_date, " [");
75
strcpy(pre_label, ",\"");
76
strcpy(post_label, "\"");
77
strcpy(pre_value, ",");
@@ -79,10 +79,10 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
79
strcpy(post_line, "}");
80
else
81
strcpy(post_line, "]");
82
- snprintfz(data_begin, 100, "],\n %sdata%s:[\n", kq, kq);
83
- strcpy(finish, "\n ]\n }");
82
+ snprintfz(data_begin, 100, "],\n %sdata%s:[\n", kq, kq);
83
+ strcpy(finish, "\n ]\n }");
84
85
- buffer_sprintf(wb, "{\n %slabels%s:[", kq, kq);
85
+ buffer_sprintf(wb, "{\n %slabels%s:[", kq, kq);
86
buffer_sprintf(wb, "%stime%s", sq, sq);
87
88
if( options & RRDR_OPTION_OBJECTSROWS )
@@ -104,9 +104,8 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
104
// -------------------------------------------------------------------------
105
// print the JSON header
106
107
- QUERY_TARGET *qt = r->internal.qt;
107
long c, i;
109
- const long used = qt->query.used;
108
+ const long used = (long)r->d;
109
110
// print the header lines
111
for(c = 0, i = 0; c < used ; c++) {
@@ -114,7 +113,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
113
continue;
114
115
buffer_fast_strcat(wb, pre_label, pre_label_len);
117
- buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
116
+ buffer_strcat(wb, string2str(r->dn[c]));
117
buffer_fast_strcat(wb, post_label, post_label_len);
118
i++;
119
}
@@ -155,6 +154,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
154
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
155
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
156
NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
157
+ uint32_t *gbc = &r->gbc [ i * r->d ];
158
159
time_t now = r->t[i];
160
@@ -218,7 +218,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
218
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
219
220
NETDATA_DOUBLE n;
221
- if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
221
+ if(unlikely(options & RRDR_OPTION_INTERNAL_GBC))
222
+ n = gbc[c];
223
+ else if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
224
n = ar[c];
225
else
226
n = cn[c];
@@ -239,7 +241,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
241
continue;
242
243
NETDATA_DOUBLE n;
242
- if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
244
+ if(unlikely(options & RRDR_OPTION_INTERNAL_GBC))
245
+ n = gbc[c];
246
+ else if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
247
n = ar[c];
248
else
249
n = cn[c];
@@ -247,9 +251,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
251
buffer_fast_strcat(wb, pre_value, pre_value_len);
252
253
if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
250
- buffer_sprintf(wb, "%s%s%s: ", kq, string2str(qt->query.array[c].dimension.name), kq);
254
+ buffer_sprintf(wb, "%s%s%s: ", kq, string2str(r->dn[c]), kq);
255
252
- if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) {
256
+ if(co[c] & RRDR_VALUE_EMPTY && !(options & (RRDR_OPTION_INTERNAL_AR | RRDR_OPTION_INTERNAL_GBC))) {
257
if(unlikely(options & RRDR_OPTION_NULL2ZERO))
258
buffer_fast_strcat(wb, "0", 1);
259
else
@@ -263,12 +267,12 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
267
n = n * 100 / total;
268
269
if(unlikely(set_min_max)) {
266
- r->min = r->max = n;
270
+ r->view.min = r->view.max = n;
271
set_min_max = 0;
272
}
273
270
- if(n < r->min) r->min = n;
271
- if(n > r->max) r->max = n;
274
+ if(n < r->view.min) r->view.min = n;
275
+ if(n > r->view.max) r->view.max = n;
276
}
277
278
buffer_print_netdata_double(wb, n);
web/api/formatters/json_wrapper.c
+872
-133
@@ -2,54 +2,54 @@
2
3
#include "json_wrapper.h"
4
5
+static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) {
6
+ buffer_json_member_add_array(wb, "plans");
7
+ for (size_t p = 0; p < qm->plan.used; p++) {
8
+ QUERY_PLAN_ENTRY *qp = &qm->plan.array[p];
9
+
10
+ buffer_json_add_array_item_object(wb);
11
+ buffer_json_member_add_uint64(wb, "tr", qp->tier);
12
+ buffer_json_member_add_time_t(wb, "af", qp->after);
13
+ buffer_json_member_add_time_t(wb, "bf", qp->before);
14
+ buffer_json_object_close(wb);
15
+ }
16
+ buffer_json_array_close(wb);
17
+
18
+ buffer_json_member_add_array(wb, "tiers");
19
+ for (size_t tier = 0; tier < storage_tiers; tier++) {
20
+ buffer_json_add_array_item_object(wb);
21
+ buffer_json_member_add_uint64(wb, "tr", tier);
22
+ buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s);
23
+ buffer_json_member_add_time_t(wb, "le", qm->tiers[tier].db_last_time_s);
24
+ buffer_json_member_add_int64(wb, "wg", qm->tiers[tier].weight);
25
+ buffer_json_object_close(wb);
26
+ }
27
+ buffer_json_array_close(wb);
28
+}
29
+
30
void jsonwrap_query_plan(RRDR *r, BUFFER *wb) {
31
QUERY_TARGET *qt = r->internal.qt;
32
33
buffer_json_member_add_object(wb, "query_plan");
34
for(size_t m = 0; m < qt->query.used; m++) {
10
- QUERY_METRIC *qm = &qt->query.array[m];
11
-
12
- buffer_json_member_add_object(wb, string2str(qm->dimension.id));
13
- {
14
- buffer_json_member_add_array(wb, "plans");
15
- for (size_t p = 0; p < qm->plan.used; p++) {
16
- QUERY_PLAN_ENTRY *qp = &qm->plan.array[p];
17
-
18
- buffer_json_add_array_item_object(wb);
19
- buffer_json_member_add_uint64(wb, "tier", qp->tier);
20
- buffer_json_member_add_time_t(wb, "after", qp->after);
21
- buffer_json_member_add_time_t(wb, "before", qp->before);
22
- buffer_json_object_close(wb);
23
- }
24
- buffer_json_array_close(wb);
25
-
26
- buffer_json_member_add_array(wb, "tiers");
27
- for (size_t tier = 0; tier < storage_tiers; tier++) {
28
- buffer_json_add_array_item_object(wb);
29
- buffer_json_member_add_uint64(wb, "tier", tier);
30
- buffer_json_member_add_time_t(wb, "db_first_time", qm->tiers[tier].db_first_time_s);
31
- buffer_json_member_add_time_t(wb, "db_last_time", qm->tiers[tier].db_last_time_s);
32
- buffer_json_member_add_int64(wb, "weight", qm->tiers[tier].weight);
33
- buffer_json_object_close(wb);
34
- }
35
- buffer_json_array_close(wb);
36
- }
35
+ QUERY_METRIC *qm = query_metric(qt, m);
36
+ buffer_json_member_add_object(wb, query_metric_id(qt, qm));
37
+ jsonwrap_query_metric_plan(wb, qm);
38
buffer_json_object_close(wb);
39
}
40
buffer_json_object_close(wb);
41
}
42
42
-static inline long jsonwrap_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
43
- QUERY_TARGET *qt = r->internal.qt;
44
- const long query_used = qt->query.used;
45
- long c, i;
43
+static inline size_t rrdr_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
44
+ const size_t dimensions = r->d;
45
+ size_t c, i;
46
47
buffer_json_member_add_array(wb, key);
48
- for(c = 0, i = 0; c < query_used ; c++) {
48
+ for(c = 0, i = 0; c < dimensions ; c++) {
49
if(!rrdr_dimension_should_be_exposed(r->od[c], options))
50
continue;
51
52
- buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.name));
52
+ buffer_json_add_array_item_string(wb, string2str(r->dn[c]));
53
i++;
54
}
55
buffer_json_array_close(wb);
@@ -57,17 +57,16 @@ static inline long jsonwrap_dimension_names(BUFFER *wb, const char *key, RRDR *r
57
return i;
58
}
59
60
-static inline long jsonwrap_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
61
- QUERY_TARGET *qt = r->internal.qt;
62
- const long query_used = qt->query.used;
63
- long c, i;
60
+static inline size_t rrdr_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
61
+ const size_t dimensions = r->d;
62
+ size_t c, i;
63
64
buffer_json_member_add_array(wb, key);
66
- for(c = 0, i = 0; c < query_used ; c++) {
65
+ for(c = 0, i = 0; c < dimensions ; c++) {
66
if(!rrdr_dimension_should_be_exposed(r->od[c], options))
67
continue;
68
70
- buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.id));
69
+ buffer_json_add_array_item_string(wb, string2str(r->di[c]));
70
i++;
71
}
72
buffer_json_array_close(wb);
@@ -75,7 +74,7 @@ static inline long jsonwrap_dimension_ids(BUFFER *wb, const char *key, RRDR *r,
74
return i;
75
}
76
78
-static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
77
+static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
78
QUERY_TARGET *qt = r->internal.qt;
79
const long query_used = qt->query.used;
80
long c, i;
@@ -85,8 +84,9 @@ static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR
84
if(!rrdr_dimension_should_be_exposed(r->od[c], options))
85
continue;
86
88
- QUERY_METRIC *qm = &qt->query.array[c];
89
- buffer_json_add_array_item_string(wb, string2str(qm->chart.id));
87
+ QUERY_METRIC *qm = query_metric(qt, c);
88
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
89
+ buffer_json_add_array_item_string(wb, string2str(qi->id_fqdn));
90
i++;
91
}
92
buffer_json_array_close(wb);
@@ -94,106 +94,401 @@ static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR
94
return i;
95
}
96
97
-struct rrdlabels_formatting_v2 {
98
- BUFFER *wb;
99
- DICTIONARY *dict;
100
-};
97
+static inline void query_target_metric_counts(BUFFER *wb, struct query_metrics_counts *metrics) {
98
+ buffer_json_member_add_object(wb, "ds");
99
+ buffer_json_member_add_uint64(wb, "sl", metrics->selected);
100
+ buffer_json_member_add_uint64(wb, "ex", metrics->excluded);
101
+ buffer_json_member_add_uint64(wb, "qr", metrics->queried);
102
+ buffer_json_member_add_uint64(wb, "fl", metrics->failed);
103
+ buffer_json_object_close(wb);
104
+}
105
102
-static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
103
- struct rrdlabels_formatting_v2 *t = data;
104
- BUFFER *wb = t->wb;
105
- DICTIONARY *dict = t->dict;
106
+static inline void query_target_instance_counts(BUFFER *wb, struct query_instances_counts *instances) {
107
+ buffer_json_member_add_object(wb, "is");
108
+ buffer_json_member_add_uint64(wb, "sl", instances->selected);
109
+ buffer_json_member_add_uint64(wb, "ex", instances->excluded);
110
+ buffer_json_object_close(wb);
111
+}
112
107
- char n[RRD_ID_LENGTH_MAX * 2 + 2];
108
- snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value);
113
+static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_counts *alerts, const char *name, bool array) {
114
+ if(array)
115
+ buffer_json_add_array_item_object(wb);
116
+ else
117
+ buffer_json_member_add_object(wb, "al");
118
+
119
+ if(name)
120
+ buffer_json_member_add_string(wb, "nm", name);
121
+ buffer_json_member_add_uint64(wb, "cl", alerts->clear);
122
+ buffer_json_member_add_uint64(wb, "wr", alerts->warning);
123
+ buffer_json_member_add_uint64(wb, "cr", alerts->critical);
124
+ buffer_json_member_add_uint64(wb, "ot", alerts->other);
125
+ buffer_json_object_close(wb);
126
+}
127
110
- bool existing = 0;
111
- bool *set = dictionary_set(dict, n, &existing, sizeof(bool));
112
- if(!*set) {
113
- *set = true;
114
- buffer_json_add_array_item_array(wb);
115
- buffer_json_add_array_item_string(wb, name);
116
- buffer_json_add_array_item_string(wb, value);
117
- buffer_json_array_close(wb);
118
- }
128
+static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, struct query_data_statistics *d) {
129
+ buffer_json_member_add_object(wb, "sts");
130
+// buffer_json_member_add_double(wb, "min", d->min);
131
+// buffer_json_member_add_double(wb, "max", d->max);
132
+// buffer_json_member_add_double(wb, "sum", d->sum);
133
+ buffer_json_member_add_double(wb, "avg", (d->count) ? d->sum / (NETDATA_DOUBLE)d->count : 0.0);
134
+// buffer_json_member_add_double(wb, "vol", d->volume);
135
+ buffer_json_member_add_double(wb, "con", (qt->query_stats.volume > 0) ? d->volume * 100.0 / qt->query_stats.volume : 0.0);
136
+ buffer_json_object_close(wb);
137
+}
138
120
- return 1;
139
+static void query_target_summary_hosts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
140
+ buffer_json_member_add_array(wb, key);
141
+ for (long c = 0; c < (long) qt->hosts.used; c++) {
142
+ QUERY_HOST *qh = query_host(qt, c);
143
+ RRDHOST *host = qh->host;
144
+ buffer_json_add_array_item_object(wb);
145
+ buffer_json_member_add_string(wb, "mg", host->machine_guid);
146
+ if(qh->node_id[0])
147
+ buffer_json_member_add_string(wb, "nd", qh->node_id);
148
+ buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
149
+ query_target_instance_counts(wb, &qh->instances);
150
+ query_target_metric_counts(wb, &qh->metrics);
151
+ query_target_alerts_counts(wb, &qh->alerts, NULL, false);
152
+ query_target_data_statistics(wb, qt, &qh->query_stats);
153
+ buffer_json_object_close(wb);
154
+ }
155
+ buffer_json_array_close(wb);
156
}
157
123
-static inline void jsonwrap_full_dimension_list(BUFFER *wb, RRDR *r) {
124
- QUERY_TARGET *qt = r->internal.qt;
158
+static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
159
+ buffer_json_member_add_array(wb, key);
160
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
161
+
162
+ struct {
163
+ struct query_data_statistics query_stats;
164
+ struct query_instances_counts instances;
165
+ struct query_metrics_counts metrics;
166
+ struct query_alerts_counts alerts;
167
+ } x = {
168
+ .instances = (struct query_instances_counts){ 0 },
169
+ .metrics = (struct query_metrics_counts){ 0 },
170
+ .alerts = (struct query_alerts_counts){ 0 },
171
+ }, *z;
172
+
173
+ for (long c = 0; c < (long) qt->contexts.used; c++) {
174
+ QUERY_CONTEXT *qc = query_context(qt, c);
175
+
176
+ z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), &x, sizeof(x));
177
+
178
+ z->instances.selected += qc->instances.selected;
179
+ z->instances.excluded += qc->instances.selected;
180
181
+ z->metrics.selected += qc->metrics.selected;
182
+ z->metrics.excluded += qc->metrics.excluded;
183
+ z->metrics.queried += qc->metrics.queried;
184
+ z->metrics.failed += qc->metrics.failed;
185
+
186
+ z->alerts.clear += qc->alerts.clear;
187
+ z->alerts.warning += qc->alerts.warning;
188
+ z->alerts.critical += qc->alerts.critical;
189
+
190
+ query_target_merge_data_statistics(&z->query_stats, &qc->query_stats);
191
+ }
192
+
193
+ size_t unique_contexts = dictionary_entries(dict);
194
+ dfe_start_read(dict, z) {
195
+ buffer_json_add_array_item_object(wb);
196
+ buffer_json_member_add_string(wb, "id", z_dfe.name);
197
+ query_target_instance_counts(wb, &z->instances);
198
+ query_target_metric_counts(wb, &z->metrics);
199
+ query_target_alerts_counts(wb, &z->alerts, NULL, false);
200
+ query_target_data_statistics(wb, qt, &z->query_stats);
201
+ buffer_json_object_close(wb);
202
+ }
203
+ dfe_done(z);
204
+ buffer_json_array_close(wb);
205
+ dictionary_destroy(dict);
206
+
207
+ return unique_contexts;
208
+}
209
+
210
+static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
211
char name[RRD_ID_LENGTH_MAX * 2 + 2];
212
128
- buffer_json_member_add_array(wb, "full_dimension_list");
129
- DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
130
- for (long c = 0; c < (long)qt->metrics.used ;c++) {
131
- RRDMETRIC_ACQUIRED *rma = qt->metrics.array[c];
213
+ buffer_json_member_add_array(wb, key);
214
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
215
+ for (long c = 0; c < (long) qt->instances.used; c++) {
216
+ QUERY_INSTANCE *qi = query_instance(qt, c);
217
218
snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
134
- rrdmetric_acquired_id(rma),
135
- rrdmetric_acquired_name(rma));
219
+ rrdinstance_acquired_id(qi->ria),
220
+ rrdinstance_acquired_name(qi->ria));
221
222
bool existing = 0;
223
bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
139
- if(!*set) {
224
+ if (!*set) {
225
*set = true;
226
buffer_json_add_array_item_array(wb);
142
- buffer_json_add_array_item_string(wb, rrdmetric_acquired_id(rma));
143
- buffer_json_add_array_item_string(wb, rrdmetric_acquired_name(rma));
227
+ buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria));
228
+ buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(qi->ria));
229
buffer_json_array_close(wb);
230
}
231
}
232
dictionary_destroy(dict);
233
buffer_json_array_close(wb);
234
+}
235
150
- buffer_json_member_add_array(wb, "full_chart_list");
151
- dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
152
- for (long c = 0; c < (long)qt->instances.used ; c++) {
153
- RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
236
+static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
237
+ buffer_json_member_add_array(wb, key);
238
+ for (long c = 0; c < (long) qt->instances.used; c++) {
239
+ QUERY_INSTANCE *qi = query_instance(qt, c);
240
+ QUERY_HOST *qh = query_host(qt, qi->query_host_id);
241
+
242
+ buffer_json_add_array_item_object(wb);
243
+ buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn));
244
+ buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn));
245
+ buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria));
246
+ buffer_json_member_add_string(wb, "mg", qh->host->machine_guid);
247
+ if(qh->node_id[0])
248
+ buffer_json_member_add_string(wb, "nd", qh->node_id);
249
+ query_target_metric_counts(wb, &qi->metrics);
250
+ query_target_alerts_counts(wb, &qi->alerts, NULL, false);
251
+ query_target_data_statistics(wb, qt, &qi->query_stats);
252
+ buffer_json_object_close(wb);
253
+ }
254
+ buffer_json_array_close(wb);
255
+}
256
+
257
+static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2) {
258
+ char name[RRD_ID_LENGTH_MAX * 2 + 2];
259
+
260
+ buffer_json_member_add_array(wb, key);
261
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
262
+ struct {
263
+ const char *id;
264
+ const char *name;
265
+ struct query_data_statistics query_stats;
266
+ struct query_metrics_counts metrics;
267
+ } x, *z;
268
+ size_t q = 0;
269
+ for (long c = 0; c < (long) qt->dimensions.used; c++) {
270
+ QUERY_DIMENSION * qd = query_dimension(qt, c);
271
+ RRDMETRIC_ACQUIRED *rma = qd->rma;
272
+
273
+ QUERY_METRIC *qm = NULL;
274
+ for( ; q < qt->query.used ;q++) {
275
+ QUERY_METRIC *tqm = query_metric(qt, q);
276
+ QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id);
277
+ if(tqd->rma != rma) break;
278
+ qm = tqm;
279
+ }
280
281
snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
156
- rrdinstance_acquired_id(ria),
157
- rrdinstance_acquired_name(ria));
282
+ rrdmetric_acquired_id(rma),
283
+ rrdmetric_acquired_name(rma));
284
159
- bool existing = 0;
160
- bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
161
- if(!*set) {
162
- *set = true;
163
- buffer_json_add_array_item_array(wb);
164
- buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(ria));
165
- buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(ria));
166
- buffer_json_array_close(wb);
285
+ x.id = rrdmetric_acquired_id(rma);
286
+ x.name = rrdmetric_acquired_name(rma);
287
+ x.metrics = (struct query_metrics_counts){ 0 };
288
+
289
+ z = dictionary_set(dict, name, &x, sizeof(x));
290
+
291
+ if(qm) {
292
+ z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0;
293
+ z->metrics.failed += (qm->status & RRDR_DIMENSION_FAILED) ? 1 : 0;
294
+
295
+ if(qm->status & RRDR_DIMENSION_QUERIED) {
296
+ z->metrics.queried++;
297
+ query_target_merge_data_statistics(&z->query_stats, &qm->query_stats);
298
+ }
299
}
300
+ else
301
+ z->metrics.excluded++;
302
}
303
+ dfe_start_read(dict, z) {
304
+ if(v2) {
305
+ buffer_json_add_array_item_object(wb);
306
+ buffer_json_member_add_string(wb, "id", z->id);
307
+ buffer_json_member_add_string(wb, "nm", z->name);
308
+ query_target_metric_counts(wb, &z->metrics);
309
+ query_target_data_statistics(wb, qt, &z->query_stats);
310
+ buffer_json_object_close(wb);
311
+ }
312
+ else {
313
+ buffer_json_add_array_item_array(wb);
314
+ buffer_json_add_array_item_string(wb, z->id);
315
+ buffer_json_add_array_item_string(wb, z->name);
316
+ buffer_json_array_close(wb);
317
+ }
318
+ }
319
+ dfe_done(z);
320
dictionary_destroy(dict);
321
buffer_json_array_close(wb);
322
+}
323
+
324
+struct rrdlabels_formatting_v2 {
325
+ DICTIONARY *keys;
326
+ QUERY_INSTANCE *qi;
327
+ bool v2;
328
+};
329
+
330
+struct rrdlabels_keys_dict_entry {
331
+ const char *name;
332
+ DICTIONARY *values;
333
+ struct query_data_statistics query_stats;
334
+ struct query_metrics_counts metrics;
335
+};
336
172
- buffer_json_member_add_array(wb, "full_chart_labels");
337
+struct rrdlabels_key_value_dict_entry {
338
+ const char *key;
339
+ const char *value;
340
+ struct query_data_statistics query_stats;
341
+ struct query_metrics_counts metrics;
342
+};
343
+
344
+static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
345
+ struct rrdlabels_formatting_v2 *t = data;
346
+
347
+ struct rrdlabels_keys_dict_entry k = {
348
+ .name = name,
349
+ .values = NULL,
350
+ .metrics = (struct query_metrics_counts){ 0 },
351
+ }, *d = dictionary_set(t->keys, name, &k, sizeof(k));
352
+
353
+ if(!d->values)
354
+ d->values = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
355
+
356
+ char n[RRD_ID_LENGTH_MAX * 2 + 2];
357
+ snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value);
358
+
359
+ struct rrdlabels_key_value_dict_entry x = {
360
+ .key = name,
361
+ .value = value,
362
+ .metrics = (struct query_metrics_counts){ 0 },
363
+ }, *z = dictionary_set(d->values, n, &x, sizeof(x));
364
+
365
+ if(t->v2) {
366
+ QUERY_INSTANCE *qi = t->qi;
367
+
368
+ z->metrics.selected += qi->metrics.selected;
369
+ z->metrics.excluded += qi->metrics.excluded;
370
+ z->metrics.queried += qi->metrics.queried;
371
+ z->metrics.failed += qi->metrics.failed;
372
+
373
+ d->metrics.selected += qi->metrics.selected;
374
+ d->metrics.excluded += qi->metrics.excluded;
375
+ d->metrics.queried += qi->metrics.queried;
376
+ d->metrics.failed += qi->metrics.failed;
377
+
378
+ query_target_merge_data_statistics(&z->query_stats, &qi->query_stats);
379
+ query_target_merge_data_statistics(&d->query_stats, &qi->query_stats);
380
+ }
381
+
382
+ return 1;
383
+}
384
+
385
+static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2) {
386
+ buffer_json_member_add_array(wb, key);
387
struct rrdlabels_formatting_v2 t = {
174
- .wb = wb,
175
- .dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE),
388
+ .keys = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE),
389
+ .v2 = v2,
390
};
177
- for (long c = 0; c < (long)qt->instances.used ; c++) {
178
- RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
391
+ for (long c = 0; c < (long) qt->instances.used; c++) {
392
+ QUERY_INSTANCE *qi = query_instance(qt, c);
393
+ RRDINSTANCE_ACQUIRED *ria = qi->ria;
394
+ t.qi = qi;
395
rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), rrdlabels_formatting_v2, &t);
396
}
181
- dictionary_destroy(t.dict);
397
+ struct rrdlabels_keys_dict_entry *d;
398
+ dfe_start_read(t.keys, d) {
399
+ if(v2) {
400
+ buffer_json_add_array_item_object(wb);
401
+ buffer_json_member_add_string(wb, "id", d_dfe.name);
402
+ query_target_metric_counts(wb, &d->metrics);
403
+ query_target_data_statistics(wb, qt, &d->query_stats);
404
+ buffer_json_member_add_array(wb, "vl");
405
+ }
406
+ struct rrdlabels_key_value_dict_entry *z;
407
+ dfe_start_read(d->values, z){
408
+ if (v2) {
409
+ buffer_json_add_array_item_object(wb);
410
+ buffer_json_member_add_string(wb, "id", z->value);
411
+ query_target_metric_counts(wb, &z->metrics);
412
+ query_target_data_statistics(wb, qt, &z->query_stats);
413
+ buffer_json_object_close(wb);
414
+ } else {
415
+ buffer_json_add_array_item_array(wb);
416
+ buffer_json_add_array_item_string(wb, z->key);
417
+ buffer_json_add_array_item_string(wb, z->value);
418
+ buffer_json_array_close(wb);
419
+ }
420
+ }
421
+ dfe_done(z);
422
+ dictionary_destroy(d->values);
423
+ if(v2) {
424
+ buffer_json_array_close(wb);
425
+ buffer_json_object_close(wb);
426
+ }
427
+ }
428
+ dfe_done(d);
429
+ dictionary_destroy(t.keys);
430
buffer_json_array_close(wb);
431
}
432
185
-static inline void jsonwrap_functions(BUFFER *wb, const char *key, RRDR *r) {
433
+static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
434
+ buffer_json_member_add_array(wb, key);
435
+ struct query_alerts_counts x = { 0 }, *z;
436
+
437
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
438
+ for (long c = 0; c < (long) qt->instances.used; c++) {
439
+ QUERY_INSTANCE *qi = query_instance(qt, c);
440
+ RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
441
+ if (st) {
442
+ netdata_rwlock_rdlock(&st->alerts.rwlock);
443
+ if (st->alerts.base) {
444
+ for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
445
+ z = dictionary_set(dict, string2str(rc->name), &x, sizeof(x));
446
+
447
+ switch(rc->status) {
448
+ case RRDCALC_STATUS_CLEAR:
449
+ z->clear++;
450
+ break;
451
+
452
+ case RRDCALC_STATUS_WARNING:
453
+ z->warning++;
454
+ break;
455
+
456
+ case RRDCALC_STATUS_CRITICAL:
457
+ z->critical++;
458
+ break;
459
+
460
+ default:
461
+ case RRDCALC_STATUS_UNINITIALIZED:
462
+ case RRDCALC_STATUS_UNDEFINED:
463
+ case RRDCALC_STATUS_REMOVED:
464
+ z->other++;
465
+ break;
466
+ }
467
+ }
468
+ }
469
+ netdata_rwlock_unlock(&st->alerts.rwlock);
470
+ }
471
+ }
472
+ dfe_start_read(dict, z)
473
+ query_target_alerts_counts(wb, z, z_dfe.name, true);
474
+ dfe_done(z);
475
+ dictionary_destroy(dict);
476
+ buffer_json_array_close(wb); // alerts
477
+}
478
+
479
+static inline void query_target_functions(BUFFER *wb, const char *key, RRDR *r) {
480
QUERY_TARGET *qt = r->internal.qt;
481
const long query_used = qt->query.used;
482
483
DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
484
RRDINSTANCE_ACQUIRED *ria = NULL;
485
for (long c = 0; c < query_used ; c++) {
192
- QUERY_METRIC *qm = &qt->query.array[c];
193
- if(qm->link.ria == ria)
486
+ QUERY_METRIC *qm = query_metric(qt, c);
487
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
488
+ if(qi->ria == ria)
489
continue;
490
196
- ria = qm->link.ria;
491
+ ria = qi->ria;
492
chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs);
493
}
494
@@ -206,7 +501,7 @@ static inline void jsonwrap_functions(BUFFER *wb, const char *key, RRDR *r) {
501
buffer_json_array_close(wb);
502
}
503
209
-static inline long jsonwrap_chart_labels_filter(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
504
+static inline long query_target_chart_labels_filter(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
505
QUERY_TARGET *qt = r->internal.qt;
506
const long query_used = qt->query.used;
507
long c, i = 0;
@@ -222,8 +517,9 @@ static inline long jsonwrap_chart_labels_filter(BUFFER *wb, const char *key, RRD
517
if(!rrdr_dimension_should_be_exposed(r->od[c], options))
518
continue;
519
225
- QUERY_METRIC *qm = &qt->query.array[c];
226
- rrdlabels_value_to_buffer_array_item_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key);
520
+ QUERY_METRIC *qm = query_metric(qt, c);
521
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
522
+ rrdlabels_value_to_buffer_array_item_or_null(rrdinstance_acquired_labels(qi->ria), wb, label_key);
523
i++;
524
}
525
buffer_json_array_close(wb);
@@ -234,7 +530,7 @@ static inline long jsonwrap_chart_labels_filter(BUFFER *wb, const char *key, RRD
530
return i;
531
}
532
237
-static inline long jsonwrap_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
533
+static inline long query_target_metrics_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
534
QUERY_TARGET *qt = r->internal.qt;
535
const long query_used = qt->query.used;
536
long c, i;
@@ -245,8 +541,9 @@ static inline long jsonwrap_latest_values(BUFFER *wb, const char *key, RRDR *r,
541
if(!rrdr_dimension_should_be_exposed(r->od[c], options))
542
continue;
543
248
- QUERY_METRIC *qm = &qt->query.array[c];
249
- buffer_json_add_array_item_double(wb, rrdmetric_acquired_last_stored_value(qm->link.rma));
544
+ QUERY_METRIC *qm = query_metric(qt, c);
545
+ QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
546
+ buffer_json_add_array_item_double(wb, rrdmetric_acquired_last_stored_value(qd->rma));
547
i++;
548
}
549
@@ -255,10 +552,8 @@ static inline long jsonwrap_latest_values(BUFFER *wb, const char *key, RRDR *r,
552
return i;
553
}
554
258
-static inline long jsonwrap_view_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
259
- QUERY_TARGET *qt = r->internal.qt;
260
- const long query_used = qt->query.used;
261
- long c, i;
555
+static inline size_t rrdr_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
556
+ size_t c, i;
557
558
buffer_json_member_add_array(wb, key);
559
@@ -266,7 +561,7 @@ static inline long jsonwrap_view_latest_values(BUFFER *wb, const char *key, RRDR
561
562
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
563
total = 0;
269
- for(c = 0; c < query_used ;c++) {
564
+ for(c = 0; c < r->d ; c++) {
565
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
566
567
NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
@@ -281,7 +576,7 @@ static inline long jsonwrap_view_latest_values(BUFFER *wb, const char *key, RRDR
576
if(total == 0) total = 1;
577
}
578
284
- for(c = 0, i = 0; c < query_used ;c++) {
579
+ for(c = 0, i = 0; c < r->d ; c++) {
580
if(!rrdr_dimension_should_be_exposed(r->od[c], options))
581
continue;
582
@@ -313,7 +608,7 @@ static inline long jsonwrap_view_latest_values(BUFFER *wb, const char *key, RRDR
608
return i;
609
}
610
316
-void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
611
+void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value,
612
RRDR_TIME_GROUPING group_method)
613
{
614
QUERY_TARGET *qt = r->internal.qt;
@@ -337,36 +632,39 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
632
buffer_json_member_add_uint64(wb, "api", 1);
633
buffer_json_member_add_string(wb, "id", qt->id);
634
buffer_json_member_add_string(wb, "name", qt->id);
340
- buffer_json_member_add_time_t(wb, "view_update_every", r->update_every);
635
+ buffer_json_member_add_time_t(wb, "view_update_every", r->view.update_every);
636
buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
637
buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
638
buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
344
- buffer_json_member_add_time_t(wb, "after", r->after);
345
- buffer_json_member_add_time_t(wb, "before", r->before);
639
+ buffer_json_member_add_time_t(wb, "after", r->view.after);
640
+ buffer_json_member_add_time_t(wb, "before", r->view.before);
641
buffer_json_member_add_string(wb, "group", time_grouping_tostring(group_method));
347
- web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->internal.query_options);
642
+ web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->view.options);
643
349
- if(!jsonwrap_dimension_names(wb, "dimension_names", r, options))
644
+ if(!rrdr_dimension_names(wb, "dimension_names", r, options))
645
rows = 0;
646
352
- if(!jsonwrap_dimension_ids(wb, "dimension_ids", r, options))
647
+ if(!rrdr_dimension_ids(wb, "dimension_ids", r, options))
648
rows = 0;
649
355
- if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS)
356
- jsonwrap_full_dimension_list(wb, r);
650
+ if (r->view.options & RRDR_OPTION_ALL_DIMENSIONS) {
651
+ query_target_summary_instances_v1(wb, qt, "full_chart_list");
652
+ query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false);
653
+ query_target_summary_labels_v12(wb, qt, "full_chart_labels", false);
654
+ }
655
358
- jsonwrap_functions(wb, "functions", r);
656
+ query_target_functions(wb, "functions", r);
657
360
- if (!qt->request.st && !jsonwrap_chart_ids(wb, "chart_ids", r, options))
658
+ if (!qt->request.st && !jsonwrap_v1_chart_ids(wb, "chart_ids", r, options))
659
rows = 0;
660
363
- if (qt->instances.chart_label_key_pattern && !jsonwrap_chart_labels_filter(wb, "chart_labels", r, options))
661
+ if (qt->instances.chart_label_key_pattern && !query_target_chart_labels_filter(wb, "chart_labels", r, options))
662
rows = 0;
663
366
- if(!jsonwrap_latest_values(wb, "latest_values", r, options))
664
+ if(!query_target_metrics_latest_values(wb, "latest_values", r, options))
665
rows = 0;
666
369
- long dimensions = jsonwrap_view_latest_values(wb, "view_latest_values", r, options);
667
+ size_t dimensions = rrdr_latest_values(wb, "view_latest_values", r, options);
668
if(!dimensions)
669
rows = 0;
670
@@ -376,19 +674,431 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
674
675
buffer_json_member_add_array(wb, "db_points_per_tier");
676
for(size_t tier = 0; tier < storage_tiers ; tier++)
379
- buffer_json_add_array_item_uint64(wb, r->internal.tier_points_read[tier]);
677
+ buffer_json_add_array_item_uint64(wb, r->stats.tier_points_read[tier]);
678
buffer_json_array_close(wb);
679
680
if(options & RRDR_OPTION_SHOW_PLAN)
681
jsonwrap_query_plan(r, wb);
682
385
- buffer_sprintf(wb, ",\n %sresult%s:", kq, kq);
683
+ buffer_sprintf(wb, ",\n %sresult%s:", kq, kq);
684
if(string_value) buffer_strcat(wb, sq);
685
}
686
389
-void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) {
390
- (void)r;
391
- (void)format;
687
+static void rrdset_rrdcalc_entries(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) {
688
+ RRDSET *st = rrdinstance_acquired_rrdset(ria);
689
+ if(st) {
690
+ netdata_rwlock_rdlock(&st->alerts.rwlock);
691
+ if(st->alerts.base) {
692
+ buffer_json_member_add_object(wb, "alerts");
693
+ for(RRDCALC *rc = st->alerts.base; rc ;rc = rc->next) {
694
+ if(rc->status < RRDCALC_STATUS_CLEAR)
695
+ continue;
696
+
697
+ buffer_json_member_add_object(wb, string2str(rc->name));
698
+ buffer_json_member_add_string(wb, "st", rrdcalc_status2string(rc->status));
699
+ buffer_json_member_add_double(wb, "vl", rc->value);
700
+ buffer_json_member_add_string(wb, "un", string2str(rc->units));
701
+ buffer_json_object_close(wb);
702
+ }
703
+ buffer_json_object_close(wb);
704
+ }
705
+ netdata_rwlock_unlock(&st->alerts.rwlock);
706
+ }
707
+}
708
+
709
+static void query_target_combined_units(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
710
+ if(contexts == 1) {
711
+ buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca));
712
+ }
713
+ else if(contexts > 1) {
714
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
715
+ for(size_t c = 0; c < qt->contexts.used ;c++)
716
+ dictionary_set(dict, rrdcontext_acquired_units(qt->contexts.array[c].rca), NULL, 0);
717
+
718
+ if(dictionary_entries(dict) == 1)
719
+ buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca));
720
+ else {
721
+ buffer_json_member_add_array(wb, "units");
722
+ const char *s;
723
+ dfe_start_read(dict, s)
724
+ buffer_json_add_array_item_string(wb, s_dfe.name);
725
+ dfe_done(s);
726
+ buffer_json_array_close(wb);
727
+ }
728
+ dictionary_destroy(dict);
729
+ }
730
+}
731
+
732
+static void query_target_combined_chart_type(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
733
+ if(contexts >= 1)
734
+ buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rrdcontext_acquired_chart_type(qt->contexts.array[0].rca)));
735
+}
736
+
737
+static void rrdr_dimension_units_array(BUFFER *wb, RRDR *r) {
738
+ if(!r->du)
739
+ return;
740
+
741
+ buffer_json_member_add_array(wb, "units");
742
+ for(size_t c = 0; c < r->d ; c++)
743
+ buffer_json_add_array_item_string(wb, string2str(r->du[c]));
744
+ buffer_json_array_close(wb);
745
+}
746
+
747
+static void rrdr_dimension_priority_array(BUFFER *wb, RRDR *r) {
748
+ if(!r->dp)
749
+ return;
750
+
751
+ buffer_json_member_add_array(wb, "priorities");
752
+ for(size_t c = 0; c < r->d ; c++)
753
+ buffer_json_add_array_item_uint64(wb, r->dp[c]);
754
+ buffer_json_array_close(wb);
755
+}
756
+
757
+static void rrdr_dimension_grouped_array(BUFFER *wb, RRDR *r) {
758
+ if(!r->dgbc)
759
+ return;
760
+
761
+ buffer_json_member_add_array(wb, "grouped");
762
+ for(size_t c = 0; c < r->d ;c++)
763
+ buffer_json_add_array_item_uint64(wb, r->dgbc[c]);
764
+ buffer_json_array_close(wb);
765
+}
766
+
767
+static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
768
+ if(contexts == 1) {
769
+ buffer_json_member_add_string(wb, "title", rrdcontext_acquired_title(qt->contexts.array[0].rca));
770
+ }
771
+ else if(contexts > 1) {
772
+ BUFFER *t = buffer_create(0, NULL);
773
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
774
+
775
+ buffer_strcat(t, "Chart for contexts: ");
776
+
777
+ size_t added = 0;
778
+ for(size_t c = 0; c < qt->contexts.used ;c++) {
779
+ bool old = false;
780
+ bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), &old, sizeof(old));
781
+ if(!*set) {
782
+ *set = true;
783
+ if(added)
784
+ buffer_fast_strcat(t, ", ", 2);
785
+
786
+ buffer_strcat(t, rrdcontext_acquired_id(qt->contexts.array[c].rca));
787
+ added++;
788
+ }
789
+ }
790
+ buffer_json_member_add_string(wb, "title", buffer_tostring(t));
791
+ dictionary_destroy(dict);
792
+ buffer_free(t);
793
+ }
794
+}
795
+
796
+static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) {
797
+ QUERY_TARGET *qt = r->internal.qt;
798
+ buffer_json_member_add_object(wb, "hosts");
799
+
800
+ time_t now_s = now_realtime_sec();
801
+ RRDHOST *last_host = NULL;
802
+ RRDCONTEXT_ACQUIRED *last_rca = NULL;
803
+ RRDINSTANCE_ACQUIRED *last_ria = NULL;
804
+
805
+ size_t h = 0, c = 0, i = 0, m = 0, q = 0;
806
+ for(; h < qt->hosts.used ; h++) {
807
+ QUERY_HOST *qh = query_host(qt, h);
808
+ RRDHOST *host = qh->host;
809
+
810
+ for( ;c < qt->contexts.used ;c++) {
811
+ QUERY_CONTEXT *qc = query_context(qt, c);
812
+ RRDCONTEXT_ACQUIRED *rca = qc->rca;
813
+ if(!rrdcontext_acquired_belongs_to_host(rca, host)) break;
814
+
815
+ for( ;i < qt->instances.used ;i++) {
816
+ QUERY_INSTANCE *qi = query_instance(qt, i);
817
+ RRDINSTANCE_ACQUIRED *ria = qi->ria;
818
+ if(!rrdinstance_acquired_belongs_to_context(ria, rca)) break;
819
+
820
+ for( ; m < qt->dimensions.used ; m++) {
821
+ QUERY_DIMENSION *qd = query_dimension(qt, m);
822
+ RRDMETRIC_ACQUIRED *rma = qd->rma;
823
+ if(!rrdmetric_acquired_belongs_to_instance(rma, ria)) break;
824
+
825
+ QUERY_METRIC *qm = NULL;
826
+ bool queried = false;
827
+ for( ; q < qt->query.used ;q++) {
828
+ QUERY_METRIC *tqm = query_metric(qt, q);
829
+ QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id);
830
+ if(tqd->rma != rma) break;
831
+
832
+ queried = tqm->status & RRDR_DIMENSION_QUERIED;
833
+ qm = tqm;
834
+ }
835
+
836
+ if(!queried & !(options & RRDR_OPTION_ALL_DIMENSIONS))
837
+ continue;
838
+
839
+ if(host != last_host) {
840
+ if(last_host) {
841
+ if(last_rca) {
842
+ if(last_ria) {
843
+ buffer_json_object_close(wb); // dimensions
844
+ buffer_json_object_close(wb); // instance
845
+ last_ria = NULL;
846
+ }
847
+ buffer_json_object_close(wb); // instances
848
+ buffer_json_object_close(wb); // context
849
+ last_rca = NULL;
850
+ }
851
+ buffer_json_object_close(wb); // contexts
852
+ buffer_json_object_close(wb); // host
853
+ last_host = NULL;
854
+ }
855
+
856
+ buffer_json_member_add_object(wb, host->machine_guid);
857
+ if(qh->node_id[0])
858
+ buffer_json_member_add_string(wb, "nd", qh->node_id);
859
+ buffer_json_member_add_uint64(wb, "idx", qh->slot);
860
+ buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
861
+ buffer_json_member_add_object(wb, "contexts");
862
+
863
+ last_host = host;
864
+ }
865
+
866
+ if(rca != last_rca) {
867
+ if(last_rca) {
868
+ if(last_ria) {
869
+ buffer_json_object_close(wb); // dimensions
870
+ buffer_json_object_close(wb); // instance
871
+ last_ria = NULL;
872
+ }
873
+ buffer_json_object_close(wb); // instances
874
+ buffer_json_object_close(wb); // context
875
+ last_rca = NULL;
876
+ }
877
+
878
+ buffer_json_member_add_object(wb, rrdcontext_acquired_id(rca));
879
+ buffer_json_member_add_object(wb, "instances");
880
+
881
+ last_rca = rca;
882
+ }
883
+
884
+ if(ria != last_ria) {
885
+ if(last_ria) {
886
+ buffer_json_object_close(wb); // dimensions
887
+ buffer_json_object_close(wb); // instance
888
+ last_ria = NULL;
889
+ }
890
+
891
+ buffer_json_member_add_object(wb, rrdinstance_acquired_id(ria));
892
+ buffer_json_member_add_uint64(wb, "idx", qi->slot);
893
+ buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(ria));
894
+ buffer_json_member_add_time_t(wb, "ue", rrdinstance_acquired_update_every(ria));
895
+ DICTIONARY *labels = rrdinstance_acquired_labels(ria);
896
+ if(labels) {
897
+ buffer_json_member_add_object(wb, "labels");
898
+ rrdlabels_to_buffer_json_members(labels, wb);
899
+ buffer_json_object_close(wb);
900
+ }
901
+ rrdset_rrdcalc_entries(wb, ria);
902
+ buffer_json_member_add_object(wb, "dimensions");
903
+
904
+ last_ria = ria;
905
+ }
906
+
907
+ buffer_json_member_add_object(wb, rrdmetric_acquired_id(rma));
908
+ {
909
+ buffer_json_member_add_string(wb, "nm", rrdmetric_acquired_name(rma));
910
+ buffer_json_member_add_uint64(wb, "qr", queried ? 1 : 0);
911
+ time_t first_entry_s = rrdmetric_acquired_first_entry(rma);
912
+ time_t last_entry_s = rrdmetric_acquired_last_entry(rma);
913
+ buffer_json_member_add_time_t(wb, "fe", first_entry_s);
914
+ buffer_json_member_add_time_t(wb, "le", last_entry_s ? last_entry_s : now_s);
915
+
916
+ if(qm) {
917
+ if(qm->status & RRDR_DIMENSION_GROUPED) {
918
+ // buffer_json_member_add_string(wb, "grouped_as_id", string2str(qm->grouped_as.id));
919
+ buffer_json_member_add_string(wb, "as", string2str(qm->grouped_as.name));
920
+ }
921
+
922
+ query_target_data_statistics(wb, qt, &qm->query_stats);
923
+
924
+ if(options & RRDR_OPTION_SHOW_PLAN)
925
+ jsonwrap_query_metric_plan(wb, qm);
926
+ }
927
+ }
928
+ buffer_json_object_close(wb); // metric
929
+ }
930
+ }
931
+ }
932
+ }
933
+
934
+ if(last_host) {
935
+ if(last_rca) {
936
+ if(last_ria) {
937
+ buffer_json_object_close(wb); // dimensions
938
+ buffer_json_object_close(wb); // instance
939
+ last_ria = NULL;
940
+ }
941
+ buffer_json_object_close(wb); // instances
942
+ buffer_json_object_close(wb); // context
943
+ last_rca = NULL;
944
+ }
945
+ buffer_json_object_close(wb); // contexts
946
+ buffer_json_object_close(wb); // host
947
+ last_host = NULL;
948
+ }
949
+ buffer_json_object_close(wb); // hosts
950
+}
951
+
952
+void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value,
953
+ RRDR_TIME_GROUPING group_method)
954
+{
955
+ QUERY_TARGET *qt = r->internal.qt;
956
+
957
+ long rows = rrdr_rows(r);
958
+
959
+ char kq[2] = "\"", // key quote
960
+ sq[2] = "\""; // string quote
961
+
962
+ if(unlikely(options & RRDR_OPTION_GOOGLE_JSON)) {
963
+ kq[0] = '\0';
964
+ sq[0] = '\'';
965
+ }
966
+
967
+ buffer_json_initialize(wb, kq, sq, 0, true);
968
+
969
+ buffer_json_member_add_uint64(wb, "api", 2);
970
+
971
+ if(options & RRDR_OPTION_DEBUG) {
972
+ buffer_json_member_add_string(wb, "id", qt->id);
973
+ buffer_json_member_add_object(wb, "request");
974
+ {
975
+ buffer_json_member_add_string(wb, "format", rrdr_format_to_string(qt->request.format));
976
+ web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", qt->request.options);
977
+
978
+ buffer_json_member_add_object(wb, "scope");
979
+ buffer_json_member_add_string(wb, "scope_hosts", qt->request.scope_hosts);
980
+ buffer_json_member_add_string(wb, "scope_contexts", qt->request.scope_contexts);
981
+ buffer_json_object_close(wb); // scope
982
+
983
+ buffer_json_member_add_object(wb, "selectors");
984
+ if (qt->request.host)
985
+ buffer_json_member_add_string(wb, "host", rrdhost_hostname(qt->request.host));
986
+ else
987
+ buffer_json_member_add_string(wb, "hosts", qt->request.hosts);
988
+ buffer_json_member_add_string(wb, "contexts", qt->request.contexts);
989
+ buffer_json_member_add_string(wb, "instances", qt->request.charts);
990
+ buffer_json_member_add_string(wb, "dimensions", qt->request.dimensions);
991
+ buffer_json_member_add_string(wb, "labels", qt->request.labels);
992
+ buffer_json_member_add_string(wb, "alerts", qt->request.alerts);
993
+ buffer_json_object_close(wb); // selectors
994
+
995
+ buffer_json_member_add_object(wb, "window");
996
+ buffer_json_member_add_time_t(wb, "after", qt->request.after);
997
+ buffer_json_member_add_time_t(wb, "before", qt->request.before);
998
+ buffer_json_member_add_uint64(wb, "points", qt->request.points);
999
+ if (qt->request.options & RRDR_OPTION_SELECTED_TIER)
1000
+ buffer_json_member_add_uint64(wb, "tier", qt->request.tier);
1001
+ else
1002
+ buffer_json_member_add_string(wb, "tier", NULL);
1003
+ buffer_json_object_close(wb); // window
1004
+
1005
+ buffer_json_member_add_object(wb, "aggregations");
1006
+ {
1007
+ buffer_json_member_add_object(wb, "time");
1008
+ buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method));
1009
+ buffer_json_member_add_string(wb, "time_group_options", qt->request.time_group_options);
1010
+ if (qt->request.resampling_time > 0)
1011
+ buffer_json_member_add_time_t(wb, "time_resampling", qt->request.resampling_time);
1012
+ else
1013
+ buffer_json_member_add_string(wb, "time_resampling", NULL);
1014
+ buffer_json_object_close(wb); // time
1015
+
1016
+ buffer_json_member_add_object(wb, "metrics");
1017
+
1018
+ buffer_json_member_add_array(wb, "group_by");
1019
+ buffer_json_group_by_to_array(wb, qt->request.group_by);
1020
+ buffer_json_array_close(wb);
1021
+
1022
+ buffer_json_member_add_array(wb, "group_by_label");
1023
+ for(size_t l = 0; l < qt->group_by.used ;l++)
1024
+ buffer_json_add_array_item_string(wb, qt->group_by.label_keys[l]);
1025
+ buffer_json_array_close(wb);
1026
+
1027
+ buffer_json_member_add_string(wb, "group_by_aggregate",
1028
+ group_by_aggregate_function_to_string(
1029
+ qt->request.group_by_aggregate_function));
1030
+ buffer_json_object_close(wb); // dimensions
1031
+ }
1032
+ buffer_json_object_close(wb); // aggregations
1033
+
1034
+ buffer_json_member_add_uint64(wb, "timeout", qt->request.timeout);
1035
+ }
1036
+ buffer_json_object_close(wb); // request
1037
+ }
1038
+
1039
+ size_t contexts;
1040
+ buffer_json_member_add_object(wb, "summary");
1041
+ {
1042
+ query_target_summary_hosts_v2(wb, qt, "hosts");
1043
+ contexts = query_target_summary_contexts_v2(wb, qt, "contexts");
1044
+ query_target_summary_instances_v2(wb, qt, "instances");
1045
+ query_target_summary_dimensions_v12(wb, qt, "dimensions", true);
1046
+ query_target_summary_labels_v12(wb, qt, "labels", true);
1047
+ query_target_summary_alerts_v2(wb, qt, "alerts");
1048
+ }
1049
+ buffer_json_object_close(wb); // summary
1050
+
1051
+ if(options & RRDR_OPTION_SHOW_DETAILS) {
1052
+ buffer_json_member_add_object(wb, "detailed");
1053
+ query_target_detailed_objects_tree(wb, r, options);
1054
+ buffer_json_object_close(wb); // detailed
1055
+ }
1056
+
1057
+ query_target_functions(wb, "functions", r);
1058
+
1059
+ buffer_json_member_add_object(wb, "db");
1060
+ {
1061
+ buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
1062
+ buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
1063
+ buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
1064
+ buffer_json_member_add_array(wb, "points_per_tier");
1065
+ for(size_t tier = 0; tier < storage_tiers ; tier++)
1066
+ buffer_json_add_array_item_uint64(wb, r->stats.tier_points_read[tier]);
1067
+ buffer_json_array_close(wb);
1068
+ }
1069
+ buffer_json_object_close(wb);
1070
+
1071
+ buffer_json_member_add_object(wb, "view");
1072
+ {
1073
+ query_target_title(wb, qt, contexts);
1074
+ buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
1075
+ web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->view.options);
1076
+ buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(group_method));
1077
+ buffer_json_member_add_time_t(wb, "update_every", r->view.update_every);
1078
+ buffer_json_member_add_time_t(wb, "after", r->view.after);
1079
+ buffer_json_member_add_time_t(wb, "before", r->view.before);
1080
+ buffer_json_member_add_uint64(wb, "points", rows);
1081
+ query_target_combined_units(wb, qt, contexts);
1082
+ query_target_combined_chart_type(wb, qt, contexts);
1083
+ buffer_json_member_add_object(wb, "dimensions");
1084
+ {
1085
+ rrdr_dimension_ids(wb, "ids", r, options);
1086
+ rrdr_dimension_names(wb, "names", r, options);
1087
+ rrdr_dimension_units_array(wb, r);
1088
+ rrdr_dimension_priority_array(wb, r);
1089
+ rrdr_dimension_grouped_array(wb, r);
1090
+ size_t dims = rrdr_latest_values(wb, "view_latest_values", r, options);
1091
+ buffer_json_member_add_uint64(wb, "count", dims);
1092
+ }
1093
+ buffer_json_object_close(wb);
1094
+ }
1095
+ buffer_json_object_close(wb);
1096
+
1097
+ buffer_sprintf(wb, ",\n %sresult%s:", kq, kq);
1098
+ if(string_value) buffer_strcat(wb, sq);
1099
+}
1100
+
1101
+void rrdr_json_wrapper_anomaly_rates(RRDR *r __maybe_unused, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, bool string_value) {
1102
1103
char kq[2] = "", // key quote
1104
sq[2] = ""; // string quote
@@ -404,11 +1114,30 @@ void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint3
1114
1115
if(string_value) buffer_strcat(wb, sq);
1116
407
- buffer_sprintf(wb, ",\n %sanomaly_rates%s: ", kq, kq);
1117
+ buffer_sprintf(wb, ",\n %sanomaly_rates%s: ", kq, kq);
1118
}
1119
410
-void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) {
411
- (void)format;
1120
+void rrdr_json_wrapper_group_by_count(RRDR *r __maybe_unused, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, bool string_value) {
1121
+
1122
+ char kq[2] = "", // key quote
1123
+ sq[2] = ""; // string quote
1124
+
1125
+ if( options & RRDR_OPTION_GOOGLE_JSON ) {
1126
+ kq[0] = '\0';
1127
+ sq[0] = '\'';
1128
+ }
1129
+ else {
1130
+ kq[0] = '"';
1131
+ sq[0] = '"';
1132
+ }
1133
+
1134
+ if(string_value) buffer_strcat(wb, sq);
1135
+
1136
+ buffer_sprintf(wb, ",\n %sgroup_by_count%s: ", kq, kq);
1137
+}
1138
+
1139
+void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, bool string_value) {
1140
+ QUERY_TARGET *qt = r->internal.qt;
1141
1142
char sq[2] = ""; // string quote
1143
@@ -421,7 +1150,17 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t option
1150
1151
if(string_value) buffer_strcat(wb, sq);
1152
424
- buffer_json_member_add_double(wb, "min", r->min);
425
- buffer_json_member_add_double(wb, "max", r->max);
1153
+ buffer_json_member_add_double(wb, "min", r->view.min);
1154
+ buffer_json_member_add_double(wb, "max", r->view.max);
1155
+
1156
+ qt->timings.finished_ut = now_monotonic_usec();
1157
+ buffer_json_member_add_object(wb, "timings");
1158
+ buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(qt->timings.preprocessed_ut - qt->timings.received_ut) / USEC_PER_MS);
1159
+ buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(qt->timings.executed_ut - qt->timings.preprocessed_ut) / USEC_PER_MS);
1160
+ buffer_json_member_add_double(wb, "group_by_ms", (NETDATA_DOUBLE)(qt->timings.group_by_ut - qt->timings.executed_ut) / USEC_PER_MS);
1161
+ buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.group_by_ut) / USEC_PER_MS);
1162
+ buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.received_ut) / USEC_PER_MS);
1163
+ buffer_json_object_close(wb);
1164
+
1165
buffer_json_finalize(wb);
1166
}
web/api/formatters/json_wrapper.h
+8
-4
@@ -6,10 +6,14 @@
6
#include "rrd2json.h"
7
#include "web/api/queries/query.h"
8
9
+typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value, RRDR_TIME_GROUPING group_method);
10
+typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
11
10
-void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
11
- RRDR_TIME_GROUPING group_method);
12
-void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
13
-void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
12
+void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value, RRDR_TIME_GROUPING group_method);
13
+void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
14
+void rrdr_json_wrapper_group_by_count(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
15
+void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
16
+
17
+void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value, RRDR_TIME_GROUPING group_method);
18
19
#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
web/api/formatters/rrd2json.c
+422
-41
@@ -7,7 +7,7 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
7
rrdset2json(st, wb, NULL, NULL, 0);
8
}
9
10
-const char *rrdr_format_to_string(uint32_t format) {
10
+const char *rrdr_format_to_string(DATASOURCE_FORMAT format) {
11
switch(format) {
12
case DATASOURCE_JSON:
13
return DATASOURCE_FORMAT_JSON;
@@ -105,15 +105,15 @@ int rrdset2value_api_v1(
105
}
106
107
if(db_points_read)
108
- *db_points_read += r->internal.db_points_read;
108
+ *db_points_read += r->stats.db_points_read;
109
110
if(db_points_per_tier) {
111
for(size_t t = 0; t < storage_tiers ;t++)
112
- db_points_per_tier[t] += r->internal.tier_points_read[t];
112
+ db_points_per_tier[t] += r->stats.tier_points_read[t];
113
}
114
115
if(result_points_generated)
116
- *result_points_generated += r->internal.result_points_generated;
116
+ *result_points_generated += r->stats.result_points_generated;
117
118
if(rrdr_rows(r) == 0) {
119
if(db_after) *db_after = 0;
@@ -125,14 +125,14 @@ int rrdset2value_api_v1(
125
}
126
127
if(wb) {
128
- if (r->result_options & RRDR_RESULT_OPTION_RELATIVE)
128
+ if (r->view.flags & RRDR_RESULT_FLAG_RELATIVE)
129
buffer_no_cacheable(wb);
130
- else if (r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
130
+ else if (r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE)
131
buffer_cacheable(wb);
132
}
133
134
- if(db_after) *db_after = r->after;
135
- if(db_before) *db_before = r->before;
134
+ if(db_after) *db_after = r->view.after;
135
+ if(db_before) *db_before = r->view.before;
136
137
long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
138
*n = rrdr2value(r, i, options, value_is_null, anomaly_rate);
@@ -144,38 +144,412 @@ cleanup:
144
return ret;
145
}
146
147
+struct group_by_entry {
148
+ size_t priority;
149
+ size_t count;
150
+ STRING *id;
151
+ STRING *name;
152
+ STRING *units;
153
+};
154
+
155
+static int group_by_label_is_space(char c) {
156
+ if(c == ',' || c == '|')
157
+ return 1;
158
+
159
+ return 0;
160
+}
161
+
162
+RRDR *data_query_group_by(RRDR *r) {
163
+ QUERY_TARGET *qt = r->internal.qt;
164
+ RRDR_OPTIONS options = qt->request.options;
165
+ size_t rows = rrdr_rows(r);
166
+
167
+ if(qt->request.group_by == RRDR_GROUP_BY_NONE || !rows)
168
+ return r;
169
+
170
+ struct group_by_entry *entries = onewayalloc_callocz(r->internal.owa, qt->query.used, sizeof(struct group_by_entry));
171
+ DICTIONARY *groups = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
172
+
173
+ if(qt->request.group_by & RRDR_GROUP_BY_LABEL && qt->request.group_by_label && *qt->request.group_by_label)
174
+ qt->group_by.used = quoted_strings_splitter(qt->request.group_by_label, qt->group_by.label_keys, GROUP_BY_MAX_LABEL_KEYS, group_by_label_is_space);
175
+
176
+ if(!qt->group_by.used)
177
+ qt->request.group_by &= ~RRDR_GROUP_BY_LABEL;
178
+
179
+ if(!(qt->request.group_by & (RRDR_GROUP_BY_NODE | RRDR_GROUP_BY_INSTANCE | RRDR_GROUP_BY_DIMENSION | RRDR_GROUP_BY_LABEL)))
180
+ qt->request.group_by = RRDR_GROUP_BY_DIMENSION;
181
+
182
+ int added = 0;
183
+ BUFFER *key = buffer_create(0, NULL);
184
+ QUERY_INSTANCE *last_qi = NULL;
185
+ size_t priority = 0;
186
+ for(size_t c = 0; c < qt->query.used ;c++) {
187
+ if(!rrdr_dimension_should_be_exposed(r->od[c], options))
188
+ continue;
189
+
190
+ QUERY_METRIC *qm = query_metric(qt, c);
191
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
192
+ QUERY_HOST *qh = query_host(qt, qm->link.query_host_id);
193
+
194
+ if(qi != last_qi) {
195
+ priority = 0;
196
+ last_qi = qi;
197
+ }
198
+ else
199
+ priority++;
200
+
201
+ // generate the group by key
202
+
203
+ buffer_flush(key);
204
+ if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
205
+ buffer_fast_strcat(key, "|", 1);
206
+ buffer_strcat(key, query_metric_id(qt, qm));
207
+ }
208
+ if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
209
+ buffer_fast_strcat(key, "|", 1);
210
+ buffer_strcat(key, string2str(qi->id_fqdn));
211
+ }
212
+ if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
213
+ DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
214
+ for(size_t l = 0; l < qt->group_by.used ;l++) {
215
+ buffer_fast_strcat(key, "|", 1);
216
+ rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
217
+ }
218
+ }
219
+ if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
220
+ buffer_fast_strcat(key, "|", 1);
221
+ buffer_strcat(key, qh->host->machine_guid);
222
+ }
223
+ buffer_fast_strcat(key, "|", 1);
224
+ buffer_strcat(key, rrdinstance_acquired_units(qi->ria));
225
+
226
+ // lookup the key in the dictionary
227
+
228
+ int pos = -1;
229
+ int *set = dictionary_set(groups, buffer_tostring(key), &pos, sizeof(pos));
230
+ if(*set == -1) {
231
+ // the key just added to the dictionary
232
+
233
+ *set = pos = added++;
234
+
235
+ // generate the dimension id
236
+
237
+ buffer_flush(key);
238
+ if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
239
+ buffer_strcat(key, query_metric_id(qt, qm));
240
+ }
241
+ if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
242
+ if(buffer_strlen(key) != 0)
243
+ buffer_fast_strcat(key, ",", 1);
244
+
245
+ if(qt->request.group_by & RRDR_GROUP_BY_NODE)
246
+ buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
247
+ else
248
+ buffer_strcat(key, string2str(qi->id_fqdn));
249
+ }
250
+ if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
251
+ DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
252
+ for(size_t l = 0; l < qt->group_by.used ;l++) {
253
+ if(buffer_strlen(key) != 0)
254
+ buffer_fast_strcat(key, ",", 1);
255
+ rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
256
+ }
257
+ }
258
+ if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
259
+ if(buffer_strlen(key) != 0)
260
+ buffer_fast_strcat(key, ",", 1);
261
+
262
+ buffer_strcat(key, qh->host->machine_guid);
263
+ }
264
+ entries[pos].id = string_strdupz(buffer_tostring(key));
265
+
266
+ // generate the dimension name
267
+
268
+ buffer_flush(key);
269
+ if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
270
+ buffer_strcat(key, query_metric_name(qt, qm));
271
+ }
272
+ if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
273
+ if(buffer_strlen(key) != 0)
274
+ buffer_fast_strcat(key, ",", 1);
275
+
276
+ if(qt->request.group_by & RRDR_GROUP_BY_NODE)
277
+ buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
278
+ else
279
+ buffer_strcat(key, string2str(qi->name_fqdn));
280
+ }
281
+ if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
282
+ DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
283
+ for(size_t l = 0; l < qt->group_by.used ;l++) {
284
+ if(buffer_strlen(key) != 0)
285
+ buffer_fast_strcat(key, ",", 1);
286
+ rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
287
+ }
288
+ }
289
+ if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
290
+ if(buffer_strlen(key) != 0)
291
+ buffer_fast_strcat(key, ",", 1);
292
+
293
+ buffer_strcat(key, rrdhost_hostname(qh->host));
294
+ }
295
+ entries[pos].name = string_strdupz(buffer_tostring(key));
296
+
297
+ // add the rest of the info
298
+ entries[pos].units = rrdinstance_acquired_units_dup(qi->ria);
299
+ entries[pos].priority = priority;
300
+ }
301
+ else {
302
+ // the key found in the dictionary
303
+ pos = *set;
304
+ }
305
+
306
+ entries[pos].count++;
307
+
308
+ if(unlikely(priority < entries[pos].priority))
309
+ entries[pos].priority = priority;
310
+
311
+ qm->grouped_as.slot = pos;
312
+ qm->grouped_as.id = entries[pos].id;
313
+ qm->grouped_as.name = entries[pos].name;
314
+ qm->grouped_as.units = entries[pos].units;
315
+ qm->status |= RRDR_DIMENSION_GROUPED;
316
+ }
317
+
318
+ // check if we have multiple units
319
+ bool multiple_units = false;
320
+ for(int i = 1; i < added ; i++) {
321
+ if(entries[i].units != entries[0].units) {
322
+ multiple_units = true;
323
+ break;
324
+ }
325
+ }
326
+
327
+ if(multiple_units) {
328
+ // include the units into the id and name of the dimensions
329
+ for(int i = 0; i < added ; i++) {
330
+ buffer_flush(key);
331
+ buffer_strcat(key, string2str(entries[i].id));
332
+ buffer_fast_strcat(key, ",", 1);
333
+ buffer_strcat(key, string2str(entries[i].units));
334
+ STRING *u = string_strdupz(buffer_tostring(key));
335
+ string_freez(entries[i].id);
336
+ entries[i].id = u;
337
+ }
338
+ }
339
+
340
+ RRDR *r2 = rrdr_create(r->internal.owa, qt, added, rows);
341
+ if(!r2)
342
+ goto cleanup;
343
+
344
+ r2->dp = onewayalloc_callocz(r2->internal.owa, r2->d, sizeof(*r2->dp));
345
+ r2->dgbc = onewayalloc_callocz(r2->internal.owa, r2->d, sizeof(*r2->dgbc));
346
+ r2->gbc = onewayalloc_callocz(r2->internal.owa, r2->n * r2->d, sizeof(*r2->gbc));
347
+
348
+ // copy from previous rrdr
349
+ r2->view = r->view;
350
+ r2->stats = r->stats;
351
+ r2->rows = rows;
352
+ r2->stats.result_points_generated = r2->d * r2->n;
353
+
354
+ // initialize r2 (dimension options, names, and ids)
355
+ for(size_t c2 = 0; c2 < r2->d ; c2++) {
356
+ r2->od[c2] = RRDR_DIMENSION_QUERIED;
357
+ r2->di[c2] = entries[c2].id;
358
+ r2->dn[c2] = entries[c2].name;
359
+ r2->du[c2] = entries[c2].units;
360
+ r2->dp[c2] = entries[c2].priority;
361
+ r2->dgbc[c2] = entries[c2].count;
362
+ }
363
+
364
+ // initialize r2 (timestamps and value flags)
365
+ for(size_t i = 0; i != rows ;i++) {
366
+ // copy the timestamp
367
+ r2->t[i] = r->t[i];
368
+
369
+ // make all values empty
370
+ NETDATA_DOUBLE *cn2 = &r2->v[ i * r2->d ];
371
+ RRDR_VALUE_FLAGS *co2 = &r2->o[ i * r2->d ];
372
+ NETDATA_DOUBLE *ar2 = &r2->ar[ i * r2->d ];
373
+ for (size_t c2 = 0; c2 < r2->d; c2++) {
374
+ cn2[c2] = 0.0;
375
+ ar2[c2] = 0.0;
376
+ co2[c2] = RRDR_VALUE_EMPTY;
377
+ }
378
+ }
379
+
380
+ // do the group_by
381
+ for(size_t i = 0; i != rows ;i++) {
382
+ size_t idx = i * r->d;
383
+ size_t idx2 = i * r2->d;
384
+
385
+ NETDATA_DOUBLE *cn = &r->v[ idx ];
386
+ RRDR_VALUE_FLAGS *co = &r->o[ idx ];
387
+ NETDATA_DOUBLE *ar = &r->ar[ idx ];
388
+
389
+ NETDATA_DOUBLE *cn2 = &r2->v[ idx2 ];
390
+ RRDR_VALUE_FLAGS *co2 = &r2->o[ idx2 ];
391
+ NETDATA_DOUBLE *ar2 = &r2->ar[ idx2 ];
392
+ uint32_t *gbc2 = &r2->gbc[ idx2 ];
393
+
394
+ for(size_t c = 0; c < r->d ;c++) {
395
+ if (!rrdr_dimension_should_be_exposed(r->od[c], options))
396
+ continue;
397
+
398
+ NETDATA_DOUBLE n = cn[c];
399
+ RRDR_VALUE_FLAGS o = co[c];
400
+
401
+ if(o & RRDR_VALUE_EMPTY) {
402
+ if(options & RRDR_OPTION_NULL2ZERO)
403
+ n = 0.0;
404
+ else
405
+ continue;
406
+ }
407
+
408
+ if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
409
+ n = -n;
410
+
411
+ QUERY_METRIC *qm = query_metric(qt, c);
412
+ size_t c2 = qm->grouped_as.slot;
413
+
414
+ switch(qt->request.group_by_aggregate_function) {
415
+ default:
416
+ case RRDR_GROUP_BY_FUNCTION_AVERAGE:
417
+ case RRDR_GROUP_BY_FUNCTION_SUM:
418
+ case RRDR_GROUP_BY_FUNCTION_SUM_COUNT:
419
+ cn2[c2] += n;
420
+ break;
421
+
422
+ case RRDR_GROUP_BY_FUNCTION_MIN:
423
+ if(n < cn2[c2])
424
+ cn2[c2] = n;
425
+ break;
426
+
427
+ case RRDR_GROUP_BY_FUNCTION_MAX:
428
+ if(n > cn2[c2])
429
+ cn2[c2] = n;
430
+ break;
431
+ }
432
+
433
+ if(o & RRDR_VALUE_RESET)
434
+ co2[c2] |= RRDR_VALUE_RESET;
435
+
436
+ ar2[c2] += ar[c];
437
+ gbc2[c2]++;
438
+ }
439
+ }
440
+
441
+ // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
442
+ size_t values = 0;
443
+ NETDATA_DOUBLE min = NAN, max = NAN;
444
+ for (size_t c2 = 0; c2 < r2->d; c2++) {
445
+ size_t non_zero = 0;
446
+
447
+ for(size_t i = 0; i != rows ;i++) {
448
+ size_t idx2 = i * r2->d + c2;
449
+
450
+ NETDATA_DOUBLE *cn2 = &r2->v[ idx2 ];
451
+ RRDR_VALUE_FLAGS *co2 = &r2->o[ idx2 ];
452
+ NETDATA_DOUBLE *ar2 = &r2->ar[ idx2 ];
453
+ uint32_t *gbc2 = &r2->gbc[ idx2 ];
454
+
455
+ if(likely(*gbc2)) {
456
+ *co2 &= ~RRDR_VALUE_EMPTY;
457
+
458
+ *ar2 /= *gbc2;
459
+
460
+ NETDATA_DOUBLE n;
461
+
462
+ if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
463
+ n = *cn2 / *gbc2;
464
+ }
465
+ else if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_AVERAGE) {
466
+ n = *cn2 / *gbc2;
467
+ *cn2 = n;
468
+ }
469
+ else
470
+ n = *cn2;
471
+
472
+ if(islessgreater(n, 0.0))
473
+ non_zero++;
474
+
475
+ if(unlikely(!values++)) {
476
+ min = n;
477
+ max = n;
478
+ }
479
+ else {
480
+ if(n < min)
481
+ min = n;
482
+
483
+ if(n > max)
484
+ max = n;
485
+ }
486
+ }
487
+ }
488
+
489
+ if(non_zero)
490
+ r2->od[c2] |= RRDR_DIMENSION_NONZERO;
491
+ }
492
+
493
+ r2->view.min = min;
494
+ r2->view.max = max;
495
+
496
+cleanup:
497
+ buffer_free(key);
498
+
499
+ if(!r2 && entries && added) {
500
+ for(long c = 0; c < added ;c++) {
501
+ string_freez(entries[c].id);
502
+ string_freez(entries[c].name);
503
+ }
504
+ }
505
+ onewayalloc_freez(r->internal.owa, entries);
506
+ dictionary_destroy(groups);
507
+
508
+ return r2;
509
+}
510
+
511
int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) {
512
+ wrapper_begin_t wrapper_begin = rrdr_json_wrapper_begin;
513
+ wrapper_end_t wrapper_end = rrdr_json_wrapper_end;
514
149
- RRDR *r = rrd2rrdr(owa, qt);
150
- if(!r) {
515
+ if(qt->request.version == 2)
516
+ wrapper_begin = rrdr_json_wrapper_begin2;
517
+
518
+ RRDR *r1 = rrd2rrdr(owa, qt);
519
+ qt->timings.executed_ut = now_monotonic_usec();
520
+
521
+ if(!r1) {
522
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
523
return HTTP_RESP_INTERNAL_SERVER_ERROR;
524
}
525
155
- if (r->result_options & RRDR_RESULT_OPTION_CANCEL) {
156
- rrdr_free(owa, r);
526
+ if (r1->view.flags & RRDR_RESULT_FLAG_CANCEL) {
527
+ rrdr_free(owa, r1);
528
return HTTP_RESP_BACKEND_FETCH_FAILED;
529
}
530
160
- if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
531
+ if(r1->view.flags & RRDR_RESULT_FLAG_RELATIVE)
532
buffer_no_cacheable(wb);
162
- else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
533
+ else if(r1->view.flags & RRDR_RESULT_FLAG_ABSOLUTE)
534
buffer_cacheable(wb);
535
165
- if(latest_timestamp && rrdr_rows(r) > 0)
166
- *latest_timestamp = r->before;
536
+ if(latest_timestamp && rrdr_rows(r1) > 0)
537
+ *latest_timestamp = r1->view.before;
538
539
DATASOURCE_FORMAT format = qt->request.format;
540
RRDR_OPTIONS options = qt->request.options;
541
RRDR_TIME_GROUPING group_method = qt->request.time_group_method;
542
543
+ RRDR *r = data_query_group_by(r1);
544
+ qt->timings.group_by_ut = now_monotonic_usec();
545
+
546
switch(format) {
547
case DATASOURCE_SSV:
548
if(options & RRDR_OPTION_JSON_WRAP) {
549
wb->content_type = CT_APPLICATION_JSON;
176
- rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
550
+ wrapper_begin(r, wb, format, options, true, group_method);
551
rrdr2ssv(r, wb, options, "", " ", "");
178
- rrdr_json_wrapper_end(r, wb, format, options, 1);
552
+ wrapper_end(r, wb, format, options, true);
553
}
554
else {
555
wb->content_type = CT_TEXT_PLAIN;
@@ -186,9 +560,9 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
560
case DATASOURCE_SSV_COMMA:
561
if(options & RRDR_OPTION_JSON_WRAP) {
562
wb->content_type = CT_APPLICATION_JSON;
189
- rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
563
+ wrapper_begin(r, wb, format, options, true, group_method);
564
rrdr2ssv(r, wb, options, "", ",", "");
191
- rrdr_json_wrapper_end(r, wb, format, options, 1);
565
+ wrapper_end(r, wb, format, options, true);
566
}
567
else {
568
wb->content_type = CT_TEXT_PLAIN;
@@ -199,9 +573,9 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
573
case DATASOURCE_JS_ARRAY:
574
if(options & RRDR_OPTION_JSON_WRAP) {
575
wb->content_type = CT_APPLICATION_JSON;
202
- rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
576
+ wrapper_begin(r, wb, format, options, false, group_method);
577
rrdr2ssv(r, wb, options, "[", ",", "]");
204
- rrdr_json_wrapper_end(r, wb, format, options, 0);
578
+ wrapper_end(r, wb, format, options, false);
579
}
580
else {
581
wb->content_type = CT_APPLICATION_JSON;
@@ -212,9 +586,9 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
586
case DATASOURCE_CSV:
587
if(options & RRDR_OPTION_JSON_WRAP) {
588
wb->content_type = CT_APPLICATION_JSON;
215
- rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
589
+ wrapper_begin(r, wb, format, options, true, group_method);
590
rrdr2csv(r, wb, format, options, "", ",", "\\n", "");
217
- rrdr_json_wrapper_end(r, wb, format, options, 1);
591
+ wrapper_end(r, wb, format, options, true);
592
}
593
else {
594
wb->content_type = CT_TEXT_PLAIN;
@@ -225,9 +599,9 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
599
case DATASOURCE_CSV_MARKDOWN:
600
if(options & RRDR_OPTION_JSON_WRAP) {
601
wb->content_type = CT_APPLICATION_JSON;
228
- rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
602
+ wrapper_begin(r, wb, format, options, true, group_method);
603
rrdr2csv(r, wb, format, options, "", "|", "\\n", "");
230
- rrdr_json_wrapper_end(r, wb, format, options, 1);
604
+ wrapper_end(r, wb, format, options, true);
605
}
606
else {
607
wb->content_type = CT_TEXT_PLAIN;
@@ -238,11 +612,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
612
case DATASOURCE_CSV_JSON_ARRAY:
613
wb->content_type = CT_APPLICATION_JSON;
614
if(options & RRDR_OPTION_JSON_WRAP) {
241
- rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
615
+ wrapper_begin(r, wb, format, options, false, group_method);
616
buffer_strcat(wb, "[\n");
617
rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
618
buffer_strcat(wb, "\n]");
245
- rrdr_json_wrapper_end(r, wb, format, options, 0);
619
+ wrapper_end(r, wb, format, options, false);
620
}
621
else {
622
wb->content_type = CT_APPLICATION_JSON;
@@ -255,9 +629,9 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
629
case DATASOURCE_TSV:
630
if(options & RRDR_OPTION_JSON_WRAP) {
631
wb->content_type = CT_APPLICATION_JSON;
258
- rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
632
+ wrapper_begin(r, wb, format, options, true, group_method);
633
rrdr2csv(r, wb, format, options, "", "\t", "\\n", "");
260
- rrdr_json_wrapper_end(r, wb, format, options, 1);
634
+ wrapper_end(r, wb, format, options, true);
635
}
636
else {
637
wb->content_type = CT_TEXT_PLAIN;
@@ -268,11 +642,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
642
case DATASOURCE_HTML:
643
if(options & RRDR_OPTION_JSON_WRAP) {
644
wb->content_type = CT_APPLICATION_JSON;
271
- rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
645
+ wrapper_begin(r, wb, format, options, true, group_method);
646
buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
647
rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
648
buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
275
- rrdr_json_wrapper_end(r, wb, format, options, 1);
649
+ wrapper_end(r, wb, format, options, true);
650
}
651
else {
652
wb->content_type = CT_TEXT_HTML;
@@ -286,35 +660,35 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
660
wb->content_type = CT_APPLICATION_X_JAVASCRIPT;
661
662
if(options & RRDR_OPTION_JSON_WRAP)
289
- rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
663
+ wrapper_begin(r, wb, format, options, false, group_method);
664
665
rrdr2json(r, wb, options, 1);
666
667
if(options & RRDR_OPTION_JSON_WRAP)
294
- rrdr_json_wrapper_end(r, wb, format, options, 0);
668
+ wrapper_end(r, wb, format, options, false);
669
break;
670
671
case DATASOURCE_DATATABLE_JSON:
672
wb->content_type = CT_APPLICATION_JSON;
673
674
if(options & RRDR_OPTION_JSON_WRAP)
301
- rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
675
+ wrapper_begin(r, wb, format, options, false, group_method);
676
677
rrdr2json(r, wb, options, 1);
678
679
if(options & RRDR_OPTION_JSON_WRAP)
306
- rrdr_json_wrapper_end(r, wb, format, options, 0);
680
+ wrapper_end(r, wb, format, options, false);
681
break;
682
683
case DATASOURCE_JSONP:
684
wb->content_type = CT_APPLICATION_X_JAVASCRIPT;
685
if(options & RRDR_OPTION_JSON_WRAP)
312
- rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
686
+ wrapper_begin(r, wb, format, options, false, group_method);
687
688
rrdr2json(r, wb, options, 0);
689
690
if(options & RRDR_OPTION_JSON_WRAP)
317
- rrdr_json_wrapper_end(r, wb, format, options, 0);
691
+ wrapper_end(r, wb, format, options, false);
692
break;
693
694
case DATASOURCE_JSON:
@@ -322,20 +696,27 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
696
wb->content_type = CT_APPLICATION_JSON;
697
698
if(options & RRDR_OPTION_JSON_WRAP)
325
- rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
699
+ wrapper_begin(r, wb, format, options, false, group_method);
700
701
rrdr2json(r, wb, options, 0);
702
703
if(options & RRDR_OPTION_JSON_WRAP) {
704
+ if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
705
+ rrdr_json_wrapper_group_by_count(r, wb, format, options, 0);
706
+ rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_GBC, 0);
707
+ }
708
if(options & RRDR_OPTION_RETURN_JWAR) {
709
rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0);
710
rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0);
711
}
334
- rrdr_json_wrapper_end(r, wb, format, options, 0);
712
+ wrapper_end(r, wb, format, options, false);
713
}
714
break;
715
}
716
339
- rrdr_free(owa, r);
717
+ if(r != r1)
718
+ rrdr_free(owa, r);
719
+
720
+ rrdr_free(owa, r1);
721
return HTTP_RESP_OK;
722
}
web/api/formatters/rrd2json.h
+17
-17
@@ -3,6 +3,22 @@
3
#ifndef NETDATA_RRD2JSON_H
4
#define NETDATA_RRD2JSON_H 1
5
6
+// type of JSON generations
7
+typedef enum {
8
+ DATASOURCE_JSON = 0,
9
+ DATASOURCE_DATATABLE_JSON = 1,
10
+ DATASOURCE_DATATABLE_JSONP = 2,
11
+ DATASOURCE_SSV = 3,
12
+ DATASOURCE_CSV = 4,
13
+ DATASOURCE_JSONP = 5,
14
+ DATASOURCE_TSV = 6,
15
+ DATASOURCE_HTML = 7,
16
+ DATASOURCE_JS_ARRAY = 8,
17
+ DATASOURCE_SSV_COMMA = 9,
18
+ DATASOURCE_CSV_JSON_ARRAY = 10,
19
+ DATASOURCE_CSV_MARKDOWN = 11,
20
+} DATASOURCE_FORMAT;
21
+
22
#include "web/api/web_api_v1.h"
23
24
#include "web/api/exporters/allmetrics.h"
@@ -23,22 +39,6 @@
39
40
#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
41
26
-// type of JSON generations
27
-typedef enum {
28
- DATASOURCE_JSON = 0,
29
- DATASOURCE_DATATABLE_JSON = 1,
30
- DATASOURCE_DATATABLE_JSONP = 2,
31
- DATASOURCE_SSV = 3,
32
- DATASOURCE_CSV = 4,
33
- DATASOURCE_JSONP = 5,
34
- DATASOURCE_TSV = 6,
35
- DATASOURCE_HTML = 7,
36
- DATASOURCE_JS_ARRAY = 8,
37
- DATASOURCE_SSV_COMMA = 9,
38
- DATASOURCE_CSV_JSON_ARRAY = 10,
39
- DATASOURCE_CSV_MARKDOWN = 11,
40
-} DATASOURCE_FORMAT;
41
-
42
#define DATASOURCE_FORMAT_JSON "json"
43
#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
44
#define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource"
@@ -53,7 +53,7 @@ typedef enum {
53
#define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown"
54
55
void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
56
-const char *rrdr_format_to_string(uint32_t format);
56
+const char *rrdr_format_to_string(DATASOURCE_FORMAT format);
57
58
int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp);
59
web/api/formatters/ssv/ssv.c
+4
-4
@@ -20,12 +20,12 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con
20
NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL);
21
22
if(likely(i != start)) {
23
- if(r->min > v) r->min = v;
24
- if(r->max < v) r->max = v;
23
+ if(r->view.min > v) r->view.min = v;
24
+ if(r->view.max < v) r->view.max = v;
25
}
26
else {
27
- r->min = v;
28
- r->max = v;
27
+ r->view.min = v;
28
+ r->view.max = v;
29
}
30
31
if(likely(i != start))
web/api/formatters/value/value.c
+9
-8
@@ -49,12 +49,12 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
49
n = n * 100 / total;
50
51
if(unlikely(set_min_max)) {
52
- r->min = r->max = n;
52
+ r->view.min = r->view.max = n;
53
set_min_max = 0;
54
}
55
56
- if(n < r->min) r->min = n;
57
- if(n > r->max) r->max = n;
56
+ if(n < r->view.min) r->view.min = n;
57
+ if(n > r->view.max) r->view.max = n;
58
}
59
60
if(unlikely(init)) {
@@ -110,6 +110,7 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
110
size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority
111
) {
112
QUERY_TARGET_REQUEST qtr = {
113
+ .version = 1,
114
.host = host,
115
.rca = rca,
116
.ria = ria,
@@ -139,14 +140,14 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
140
}
141
else {
142
qv = (QUERY_VALUE) {
142
- .after = r->after,
143
- .before = r->before,
144
- .points_read = r->internal.db_points_read,
145
- .result_points = r->internal.result_points_generated,
143
+ .after = r->view.after,
144
+ .before = r->view.before,
145
+ .points_read = r->stats.db_points_read,
146
+ .result_points = r->stats.result_points_generated,
147
};
148
149
for(size_t t = 0; t < storage_tiers ;t++)
149
- qv.storage_points_per_tier[t] = r->internal.tier_points_read[t];
150
+ qv.storage_points_per_tier[t] = r->stats.tier_points_read[t];
151
152
long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
153
int all_values_are_null = 0;
web/api/queries/average/average.c
+9
-12
@@ -11,30 +11,30 @@ struct grouping_average {
11
};
12
13
void grouping_create_average(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_average));
14
+ r->grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_average));
15
}
16
17
// resets when switches dimensions
18
// so, clear everything to restart
19
void grouping_reset_average(RRDR *r) {
20
- struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
20
+ struct grouping_average *g = (struct grouping_average *)r->grouping.data;
21
g->sum = 0;
22
g->count = 0;
23
}
24
25
void grouping_free_average(RRDR *r) {
26
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
- r->internal.grouping_data = NULL;
26
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
27
+ r->grouping.data = NULL;
28
}
29
30
void grouping_add_average(RRDR *r, NETDATA_DOUBLE value) {
31
- struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
31
+ struct grouping_average *g = (struct grouping_average *)r->grouping.data;
32
g->sum += value;
33
g->count++;
34
}
35
36
NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
37
- struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
37
+ struct grouping_average *g = (struct grouping_average *)r->grouping.data;
38
39
NETDATA_DOUBLE value;
40
@@ -43,12 +43,9 @@ NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_opt
43
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
44
}
45
else {
46
- if(unlikely(r->internal.resampling_group != 1)) {
47
- if (unlikely(r->result_options & RRDR_RESULT_OPTION_VARIABLE_STEP))
48
- value = g->sum / g->count / r->internal.resampling_divisor;
49
- else
50
- value = g->sum / r->internal.resampling_divisor;
51
- } else
46
+ if(unlikely(r->grouping.resampling_group != 1))
47
+ value = g->sum / r->grouping.resampling_divisor;
48
+ else
49
value = g->sum / g->count;
50
}
51
web/api/queries/countif/countif.c
+6
-6
@@ -38,7 +38,7 @@ static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
38
39
void grouping_create_countif(RRDR *r, const char *options __maybe_unused) {
40
struct grouping_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_countif));
41
- r->internal.grouping_data = g;
41
+ r->grouping.data = g;
42
43
if(options && *options) {
44
// skip any leading spaces
@@ -100,24 +100,24 @@ void grouping_create_countif(RRDR *r, const char *options __maybe_unused) {
100
// resets when switches dimensions
101
// so, clear everything to restart
102
void grouping_reset_countif(RRDR *r) {
103
- struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;
103
+ struct grouping_countif *g = (struct grouping_countif *)r->grouping.data;
104
g->matched = 0;
105
g->count = 0;
106
}
107
108
void grouping_free_countif(RRDR *r) {
109
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
110
- r->internal.grouping_data = NULL;
109
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
110
+ r->grouping.data = NULL;
111
}
112
113
void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value) {
114
- struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;
114
+ struct grouping_countif *g = (struct grouping_countif *)r->grouping.data;
115
g->matched += g->comparison(value, g->target);
116
g->count++;
117
}
118
119
NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
120
- struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;
120
+ struct grouping_countif *g = (struct grouping_countif *)r->grouping.data;
121
122
NETDATA_DOUBLE value;
123
web/api/queries/des/des.c
+9
-9
@@ -35,13 +35,13 @@ static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) {
35
(void)g;
36
37
NETDATA_DOUBLE points;
38
- if(r->group == 1) {
38
+ if(r->view.group == 1) {
39
// provide a running DES
40
- points = (NETDATA_DOUBLE)r->internal.points_wanted;
40
+ points = (NETDATA_DOUBLE)r->grouping.points_wanted;
41
}
42
else {
43
// provide a SES with flush points
44
- points = (NETDATA_DOUBLE)r->group;
44
+ points = (NETDATA_DOUBLE)r->view.group;
45
}
46
47
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
@@ -76,13 +76,13 @@ void grouping_create_des(RRDR *r, const char *options __maybe_unused) {
76
g->level = 0.0;
77
g->trend = 0.0;
78
g->count = 0;
79
- r->internal.grouping_data = g;
79
+ r->grouping.data = g;
80
}
81
82
// resets when switches dimensions
83
// so, clear everything to restart
84
void grouping_reset_des(RRDR *r) {
85
- struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
85
+ struct grouping_des *g = (struct grouping_des *)r->grouping.data;
86
g->level = 0.0;
87
g->trend = 0.0;
88
g->count = 0;
@@ -92,12 +92,12 @@ void grouping_reset_des(RRDR *r) {
92
}
93
94
void grouping_free_des(RRDR *r) {
95
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
96
- r->internal.grouping_data = NULL;
95
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
96
+ r->grouping.data = NULL;
97
}
98
99
void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) {
100
- struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
100
+ struct grouping_des *g = (struct grouping_des *)r->grouping.data;
101
102
if(likely(g->count > 0)) {
103
// we have at least a number so far
@@ -124,7 +124,7 @@ void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) {
124
}
125
126
NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
127
- struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
127
+ struct grouping_des *g = (struct grouping_des *)r->grouping.data;
128
129
if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
130
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
web/api/queries/incremental_sum/incremental_sum.c
+6
-6
@@ -12,25 +12,25 @@ struct grouping_incremental_sum {
12
};
13
14
void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused) {
15
- r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_incremental_sum));
15
+ r->grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_incremental_sum));
16
}
17
18
// resets when switches dimensions
19
// so, clear everything to restart
20
void grouping_reset_incremental_sum(RRDR *r) {
21
- struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
21
+ struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->grouping.data;
22
g->first = 0;
23
g->last = 0;
24
g->count = 0;
25
}
26
27
void grouping_free_incremental_sum(RRDR *r) {
28
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
29
- r->internal.grouping_data = NULL;
28
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
29
+ r->grouping.data = NULL;
30
}
31
32
void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) {
33
- struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
33
+ struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->grouping.data;
34
35
if(unlikely(!g->count)) {
36
g->first = value;
@@ -43,7 +43,7 @@ void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) {
43
}
44
45
NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
46
- struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
46
+ struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->grouping.data;
47
48
NETDATA_DOUBLE value;
49
web/api/queries/max/max.c
+6
-6
@@ -11,24 +11,24 @@ struct grouping_max {
11
};
12
13
void grouping_create_max(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_max));
14
+ r->grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_max));
15
}
16
17
// resets when switches dimensions
18
// so, clear everything to restart
19
void grouping_reset_max(RRDR *r) {
20
- struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
20
+ struct grouping_max *g = (struct grouping_max *)r->grouping.data;
21
g->max = 0;
22
g->count = 0;
23
}
24
25
void grouping_free_max(RRDR *r) {
26
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
- r->internal.grouping_data = NULL;
26
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
27
+ r->grouping.data = NULL;
28
}
29
30
void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) {
31
- struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
31
+ struct grouping_max *g = (struct grouping_max *)r->grouping.data;
32
33
if(!g->count || fabsndd(value) > fabsndd(g->max)) {
34
g->max = value;
@@ -37,7 +37,7 @@ void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) {
37
}
38
39
NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
40
- struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
40
+ struct grouping_max *g = (struct grouping_max *)r->grouping.data;
41
42
NETDATA_DOUBLE value;
43
web/api/queries/median/median.c
+8
-8
@@ -14,7 +14,7 @@ struct grouping_median {
14
};
15
16
void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17
- long entries = r->group;
17
+ long entries = r->view.group;
18
if(entries < 10) entries = 10;
19
20
struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median));
@@ -30,7 +30,7 @@ void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBL
30
}
31
32
g->percent = g->percent / 100.0;
33
- r->internal.grouping_data = g;
33
+ r->grouping.data = g;
34
}
35
36
void grouping_create_median(RRDR *r, const char *options) {
@@ -64,20 +64,20 @@ void grouping_create_trimmed_median25(RRDR *r, const char *options) {
64
// resets when switches dimensions
65
// so, clear everything to restart
66
void grouping_reset_median(RRDR *r) {
67
- struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
67
+ struct grouping_median *g = (struct grouping_median *)r->grouping.data;
68
g->next_pos = 0;
69
}
70
71
void grouping_free_median(RRDR *r) {
72
- struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
72
+ struct grouping_median *g = (struct grouping_median *)r->grouping.data;
73
if(g) onewayalloc_freez(r->internal.owa, g->series);
74
75
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
76
- r->internal.grouping_data = NULL;
75
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
76
+ r->grouping.data = NULL;
77
}
78
79
void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) {
80
- struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
80
+ struct grouping_median *g = (struct grouping_median *)r->grouping.data;
81
82
if(unlikely(g->next_pos >= g->series_size)) {
83
g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
@@ -88,7 +88,7 @@ void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) {
88
}
89
90
NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
91
- struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
91
+ struct grouping_median *g = (struct grouping_median *)r->grouping.data;
92
93
size_t available_slots = g->next_pos;
94
NETDATA_DOUBLE value;
web/api/queries/min/min.c
+6
-6
@@ -11,24 +11,24 @@ struct grouping_min {
11
};
12
13
void grouping_create_min(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_min));
14
+ r->grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_min));
15
}
16
17
// resets when switches dimensions
18
// so, clear everything to restart
19
void grouping_reset_min(RRDR *r) {
20
- struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data;
20
+ struct grouping_min *g = (struct grouping_min *)r->grouping.data;
21
g->min = 0;
22
g->count = 0;
23
}
24
25
void grouping_free_min(RRDR *r) {
26
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
- r->internal.grouping_data = NULL;
26
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
27
+ r->grouping.data = NULL;
28
}
29
30
void grouping_add_min(RRDR *r, NETDATA_DOUBLE value) {
31
- struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data;
31
+ struct grouping_min *g = (struct grouping_min *)r->grouping.data;
32
33
if(!g->count || fabsndd(value) < fabsndd(g->min)) {
34
g->min = value;
@@ -37,7 +37,7 @@ void grouping_add_min(RRDR *r, NETDATA_DOUBLE value) {
37
}
38
39
NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
40
- struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data;
40
+ struct grouping_min *g = (struct grouping_min *)r->grouping.data;
41
42
NETDATA_DOUBLE value;
43
web/api/queries/percentile/percentile.c
+8
-8
@@ -14,7 +14,7 @@ struct grouping_percentile {
14
};
15
16
static void grouping_create_percentile_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17
- long entries = r->group;
17
+ long entries = r->view.group;
18
if(entries < 10) entries = 10;
19
20
struct grouping_percentile *g = (struct grouping_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_percentile));
@@ -30,7 +30,7 @@ static void grouping_create_percentile_internal(RRDR *r, const char *options, NE
30
}
31
32
g->percent = g->percent / 100.0;
33
- r->internal.grouping_data = g;
33
+ r->grouping.data = g;
34
}
35
36
void grouping_create_percentile25(RRDR *r, const char *options) {
@@ -64,20 +64,20 @@ void grouping_create_percentile99(RRDR *r, const char *options) {
64
// resets when switches dimensions
65
// so, clear everything to restart
66
void grouping_reset_percentile(RRDR *r) {
67
- struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
67
+ struct grouping_percentile *g = (struct grouping_percentile *)r->grouping.data;
68
g->next_pos = 0;
69
}
70
71
void grouping_free_percentile(RRDR *r) {
72
- struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
72
+ struct grouping_percentile *g = (struct grouping_percentile *)r->grouping.data;
73
if(g) onewayalloc_freez(r->internal.owa, g->series);
74
75
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
76
- r->internal.grouping_data = NULL;
75
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
76
+ r->grouping.data = NULL;
77
}
78
79
void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value) {
80
- struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
80
+ struct grouping_percentile *g = (struct grouping_percentile *)r->grouping.data;
81
82
if(unlikely(g->next_pos >= g->series_size)) {
83
g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
@@ -88,7 +88,7 @@ void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value) {
88
}
89
90
NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
91
- struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
91
+ struct grouping_percentile *g = (struct grouping_percentile *)r->grouping.data;
92
93
NETDATA_DOUBLE value;
94
size_t available_slots = g->next_pos;
web/api/queries/query.c
+234
-90
@@ -638,24 +638,103 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_TIME_GROUPING group_method)
638
int i, found = 0;
639
for(i = 0; !found && api_v1_data_groups[i].name ;i++) {
640
if(api_v1_data_groups[i].value == group_method) {
641
- r->internal.grouping_create = api_v1_data_groups[i].create;
642
- r->internal.grouping_reset = api_v1_data_groups[i].reset;
643
- r->internal.grouping_free = api_v1_data_groups[i].free;
644
- r->internal.grouping_add = api_v1_data_groups[i].add;
645
- r->internal.grouping_flush = api_v1_data_groups[i].flush;
646
- r->internal.tier_query_fetch = api_v1_data_groups[i].tier_query_fetch;
641
+ r->grouping.create = api_v1_data_groups[i].create;
642
+ r->grouping.reset = api_v1_data_groups[i].reset;
643
+ r->grouping.free = api_v1_data_groups[i].free;
644
+ r->grouping.add = api_v1_data_groups[i].add;
645
+ r->grouping.flush = api_v1_data_groups[i].flush;
646
+ r->grouping.tier_query_fetch = api_v1_data_groups[i].tier_query_fetch;
647
found = 1;
648
}
649
}
650
if(!found) {
651
errno = 0;
652
internal_error(true, "QUERY: grouping method %u not found. Using 'average'", (unsigned int)group_method);
653
- r->internal.grouping_create = grouping_create_average;
654
- r->internal.grouping_reset = grouping_reset_average;
655
- r->internal.grouping_free = grouping_free_average;
656
- r->internal.grouping_add = grouping_add_average;
657
- r->internal.grouping_flush = grouping_flush_average;
658
- r->internal.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE;
653
+ r->grouping.create = grouping_create_average;
654
+ r->grouping.reset = grouping_reset_average;
655
+ r->grouping.free = grouping_free_average;
656
+ r->grouping.add = grouping_add_average;
657
+ r->grouping.flush = grouping_flush_average;
658
+ r->grouping.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE;
659
+ }
660
+}
661
+
662
+RRDR_GROUP_BY group_by_parse(char *s) {
663
+ RRDR_GROUP_BY group_by = RRDR_GROUP_BY_NONE;
664
+
665
+ while(s) {
666
+ char *key = mystrsep(&s, ",| ");
667
+ if (!key || !*key) continue;
668
+
669
+ if (strcmp(key, "dimension") == 0)
670
+ group_by |= RRDR_GROUP_BY_DIMENSION;
671
+
672
+ if (strcmp(key, "node") == 0)
673
+ group_by |= RRDR_GROUP_BY_NODE;
674
+
675
+ if (strcmp(key, "instance") == 0)
676
+ group_by |= RRDR_GROUP_BY_INSTANCE;
677
+
678
+ if (strcmp(key, "label") == 0)
679
+ group_by |= RRDR_GROUP_BY_LABEL;
680
+ }
681
+
682
+ if(group_by == RRDR_GROUP_BY_NONE)
683
+ group_by = RRDR_GROUP_BY_DIMENSION;
684
+
685
+ return group_by;
686
+}
687
+
688
+void buffer_json_group_by_to_array(BUFFER *wb, RRDR_GROUP_BY group_by) {
689
+ if(group_by & RRDR_GROUP_BY_DIMENSION)
690
+ buffer_json_add_array_item_string(wb, "dimension");
691
+
692
+ if(group_by & RRDR_GROUP_BY_NODE)
693
+ buffer_json_add_array_item_string(wb, "node");
694
+
695
+ if(group_by & RRDR_GROUP_BY_INSTANCE)
696
+ buffer_json_add_array_item_string(wb, "instance");
697
+
698
+ if(group_by & RRDR_GROUP_BY_LABEL)
699
+ buffer_json_add_array_item_string(wb, "label");
700
+}
701
+
702
+RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s) {
703
+ if(strcmp(s, "sum-count") == 0)
704
+ return RRDR_GROUP_BY_FUNCTION_SUM_COUNT;
705
+
706
+ if(strcmp(s, "average") == 0)
707
+ return RRDR_GROUP_BY_FUNCTION_AVERAGE;
708
+
709
+ if(strcmp(s, "min") == 0)
710
+ return RRDR_GROUP_BY_FUNCTION_MIN;
711
+
712
+ if(strcmp(s, "max") == 0)
713
+ return RRDR_GROUP_BY_FUNCTION_MAX;
714
+
715
+ if(strcmp(s, "sum") == 0)
716
+ return RRDR_GROUP_BY_FUNCTION_SUM;
717
+
718
+ return RRDR_GROUP_BY_FUNCTION_AVERAGE;
719
+}
720
+
721
+const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_by_function) {
722
+ switch(group_by_function) {
723
+ default:
724
+ case RRDR_GROUP_BY_FUNCTION_AVERAGE:
725
+ return "average";
726
+
727
+ case RRDR_GROUP_BY_FUNCTION_SUM_COUNT:
728
+ return "sum-count";
729
+
730
+ case RRDR_GROUP_BY_FUNCTION_MIN:
731
+ return "min";
732
+
733
+ case RRDR_GROUP_BY_FUNCTION_MAX:
734
+ return "max";
735
+
736
+ case RRDR_GROUP_BY_FUNCTION_SUM:
737
+ return "sum";
738
}
739
}
740
@@ -822,7 +901,7 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
901
902
// find the db time-range for this tier for all metrics
903
for(size_t i = 0, used = qt->query.used; i < used ; i++) {
825
- QUERY_METRIC *qm = &qt->query.array[i];
904
+ QUERY_METRIC *qm = query_metric(qt, i);
905
906
time_t first_time_s = qm->tiers[tier].db_first_time_s;
907
time_t last_time_s = qm->tiers[tier].db_last_time_s;
@@ -872,7 +951,7 @@ static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, t
951
// find the db minimum update every for this tier for all metrics
952
time_t common_update_every_s = default_rrd_update_every;
953
for(size_t i = 0, used = qt->query.used; i < used ; i++) {
875
- QUERY_METRIC *qm = &qt->query.array[i];
954
+ QUERY_METRIC *qm = query_metric(qt, i);
955
956
time_t update_every_s = qm->tiers[best_tier].db_update_every_s;
957
@@ -1118,7 +1197,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1197
// put our selected tier as the first plan
1198
size_t selected_tier;
1199
1121
- if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER
1200
+ if(ops->r->view.options & RRDR_OPTION_SELECTED_TIER
1201
&& ops->r->internal.qt->window.tier < storage_tiers
1202
&& query_metric_is_valid_tier(qm, ops->r->internal.qt->window.tier)) {
1203
selected_tier = ops->r->internal.qt->window.tier;
@@ -1126,8 +1205,8 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1205
else {
1206
selected_tier = query_metric_best_tier_for_timeframe(qm, after_wanted, before_wanted, points_wanted);
1207
1129
- if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)
1130
- ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER;
1208
+ if(ops->r->view.options & RRDR_OPTION_SELECTED_TIER)
1209
+ ops->r->view.options &= ~RRDR_OPTION_SELECTED_TIER;
1210
1211
if(!query_metric_is_valid_tier(qm, selected_tier))
1212
return false;
@@ -1142,7 +1221,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1221
qm->plan.array[0].after = (qm->tiers[selected_tier].db_first_time_s < after_wanted) ? after_wanted : qm->tiers[selected_tier].db_first_time_s;
1222
qm->plan.array[0].before = (qm->tiers[selected_tier].db_last_time_s > before_wanted) ? before_wanted : qm->tiers[selected_tier].db_last_time_s;
1223
1145
- if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) {
1224
+ if(!(ops->r->view.options & RRDR_OPTION_SELECTED_TIER)) {
1225
// the selected tier
1226
time_t selected_tier_first_time_s = qm->plan.array[0].after;
1227
time_t selected_tier_last_time_s = qm->plan.array[0].before;
@@ -1280,12 +1359,12 @@ static QUERY_ENGINE_OPS *rrd2rrdr_query_prep(RRDR *r, size_t dim_id_in_rrdr) {
1359
QUERY_ENGINE_OPS *ops = onewayalloc_mallocz(r->internal.owa, sizeof(QUERY_ENGINE_OPS));
1360
*ops = (QUERY_ENGINE_OPS) {
1361
.r = r,
1283
- .qm = &qt->query.array[dim_id_in_rrdr],
1284
- .grouping_add = r->internal.grouping_add,
1285
- .grouping_flush = r->internal.grouping_flush,
1286
- .tier_query_fetch = r->internal.tier_query_fetch,
1287
- .view_update_every = r->update_every,
1288
- .query_granularity = (time_t)(r->update_every / r->group),
1362
+ .qm = query_metric(qt, dim_id_in_rrdr),
1363
+ .grouping_add = r->grouping.add,
1364
+ .grouping_flush = r->grouping.flush,
1365
+ .tier_query_fetch = r->grouping.tier_query_fetch,
1366
+ .view_update_every = r->view.update_every,
1367
+ .query_granularity = (time_t)(r->view.update_every / r->view.group),
1368
.group_value_flags = RRDR_VALUE_NOTHING,
1369
};
1370
@@ -1297,7 +1376,8 @@ static QUERY_ENGINE_OPS *rrd2rrdr_query_prep(RRDR *r, size_t dim_id_in_rrdr) {
1376
1377
static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_OPS *ops) {
1378
QUERY_TARGET *qt = r->internal.qt;
1300
- QUERY_METRIC *qm = &qt->query.array[dim_id_in_rrdr]; (void)qm;
1379
+ QUERY_METRIC *qm = query_metric(qt, dim_id_in_rrdr);
1380
+
1381
size_t points_wanted = qt->window.points;
1382
time_t after_wanted = qt->window.after;
1383
time_t before_wanted = qt->window.before; (void)before_wanted;
@@ -1312,9 +1392,9 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1392
size_t points_added = 0;
1393
1394
long rrdr_line = -1;
1315
- bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
1395
+ bool use_anomaly_bit_as_value = (r->view.options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
1396
1317
- NETDATA_DOUBLE min = r->min, max = r->max;
1397
+ NETDATA_DOUBLE min = r->view.min, max = r->view.max;
1398
1399
QUERY_POINT last2_point = QUERY_POINT_EMPTY;
1400
QUERY_POINT last1_point = QUERY_POINT_EMPTY;
@@ -1421,7 +1501,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1501
switch (ops->tier_query_fetch) {
1502
default:
1503
case TIER_QUERY_FETCH_AVERAGE:
1424
- new_point.value = sp.sum / sp.count;
1504
+ new_point.value = sp.sum / (NETDATA_DOUBLE)sp.count;
1505
break;
1506
1507
case TIER_QUERY_FETCH_MIN:
@@ -1450,7 +1530,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1530
1531
internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned "
1532
"point %zu from %ld to %ld, that are both equal",
1453
- qt->id, string2str(qm->dimension.id),
1533
+ qt->id, query_metric_id(qt, qm),
1534
new_point.id, new_point.start_time, new_point.end_time);
1535
1536
new_point.start_time = new_point.end_time - ops->tier_ptr->db_update_every_s;
@@ -1465,7 +1545,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1545
"point %zu from %ld to %ld, before the "
1546
"last point %zu from %ld to %ld, "
1547
"now is %ld to %ld",
1468
- qt->id, string2str(qm->dimension.id),
1548
+ qt->id, query_metric_id(qt, qm),
1549
new_point.id, new_point.start_time, new_point.end_time,
1550
last1_point.id, last1_point.start_time, last1_point.end_time,
1551
now_start_time, now_end_time);
@@ -1499,7 +1579,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1579
"returned point %zu from %ld time %ld, "
1580
"which is entirely before our current timeframe %ld to %ld "
1581
"(and before the entire query, after %ld, before %ld)",
1502
- qt->id, string2str(qm->dimension.id),
1582
+ qt->id, query_metric_id(qt, qm),
1583
new_point.id, new_point.start_time, new_point.end_time,
1584
now_start_time, now_end_time,
1585
ops->plan_expanded_after, ops->plan_expanded_before);
@@ -1518,7 +1598,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1598
"QUERY: '%s', dimension '%s', the database does not advance the query,"
1599
" it returned an end time less or equal to the end time of the last "
1600
"point we got %ld, %zu times",
1521
- qt->id, string2str(qm->dimension.id),
1601
+ qt->id, query_metric_id(qt, qm),
1602
last1_point.end_time, count_same_end_time);
1603
1604
if(unlikely(new_point.end_time <= last1_point.end_time))
@@ -1626,6 +1706,24 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1706
min = max = group_value;
1707
}
1708
1709
+ NETDATA_DOUBLE stats_value = group_value < 0 ? -group_value : group_value;
1710
+
1711
+ if(unlikely(!points_added)) {
1712
+ qm->query_stats.min = stats_value;
1713
+ qm->query_stats.max = stats_value;
1714
+ }
1715
+ else {
1716
+ if(stats_value < qm->query_stats.min)
1717
+ qm->query_stats.min = stats_value;
1718
+
1719
+ if(stats_value > qm->query_stats.max)
1720
+ qm->query_stats.max = stats_value;
1721
+ }
1722
+
1723
+ qm->query_stats.sum += stats_value;
1724
+ qm->query_stats.volume += stats_value * (NETDATA_DOUBLE)ops->view_update_every;
1725
+ qm->query_stats.count++;
1726
+
1727
points_added++;
1728
ops->group_points_added = 0;
1729
ops->group_value_flags = RRDR_VALUE_NOTHING;
@@ -1640,20 +1738,20 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1738
}
1739
query_planer_finalize_remaining_plans(ops);
1740
1643
- r->internal.result_points_generated += points_added;
1644
- r->internal.db_points_read += ops->db_total_points_read;
1741
+ r->stats.result_points_generated += points_added;
1742
+ r->stats.db_points_read += ops->db_total_points_read;
1743
for(size_t tr = 0; tr < storage_tiers ; tr++)
1646
- r->internal.tier_points_read[tr] += ops->db_points_read_per_tier[tr];
1744
+ r->stats.tier_points_read[tr] += ops->db_points_read_per_tier[tr];
1745
1648
- r->min = min;
1649
- r->max = max;
1650
- r->before = max_date;
1651
- r->after = min_date - ops->view_update_every + ops->query_granularity;
1746
+ r->view.min = min;
1747
+ r->view.max = max;
1748
+ r->view.before = max_date;
1749
+ r->view.after = min_date - ops->view_update_every + ops->query_granularity;
1750
rrdr_done(r, rrdr_line);
1751
1752
internal_error(points_added != points_wanted,
1753
"QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).",
1656
- qt->id, string2str(qm->dimension.id),
1754
+ qt->id, query_metric_id(qt, qm),
1755
(size_t)points_wanted, (size_t)points_added, ops->db_total_points_read);
1756
}
1757
@@ -1759,19 +1857,19 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1857
, resampling_group
1858
1859
// after
1762
- , r->after
1860
+ , r->view.after
1861
, after_wanted
1862
, after_requested
1863
, first_entry_s
1864
1865
// before
1768
- , r->before
1866
+ , r->view.before
1867
, before_wanted
1868
, before_requested
1869
, last_entry_s
1870
1871
// duration
1774
- , (long)(r->before - r->after + r->internal.qt->window.query_granularity)
1872
+ , (long)(r->view.before - r->view.after + r->internal.qt->window.query_granularity)
1873
, (long)(before_wanted - after_wanted + r->internal.qt->window.query_granularity)
1874
, (long)before_requested - after_requested
1875
, (long)((last_entry_s - first_entry_s) + r->internal.qt->window.query_granularity)
@@ -2165,6 +2263,7 @@ RRDR *rrd2rrdr_legacy(
2263
STORAGE_PRIORITY priority) {
2264
2265
QUERY_TARGET_REQUEST qtr = {
2266
+ .version = 1,
2267
.st = st,
2268
.points = points,
2269
.after = after,
@@ -2183,6 +2282,22 @@ RRDR *rrd2rrdr_legacy(
2282
return rrd2rrdr(owa, query_target_create(&qtr));
2283
}
2284
2285
+void query_target_merge_data_statistics(struct query_data_statistics *d, struct query_data_statistics *s) {
2286
+ if(!d->count)
2287
+ *d = *s;
2288
+ else {
2289
+ d->count += s->count;
2290
+ d->sum += s->sum;
2291
+ d->volume += s->volume;
2292
+
2293
+ if(s->min < d->min)
2294
+ d->min = s->min;
2295
+
2296
+ if(s->max > d->max)
2297
+ d->max = s->max;
2298
+ }
2299
+}
2300
+
2301
RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2302
if(!qt)
2303
return NULL;
@@ -2195,42 +2310,37 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2310
// qt.window members are the WANTED ones.
2311
// qt.request members are the REQUESTED ones.
2312
2198
- RRDR *r = rrdr_create(owa, qt);
2313
+ RRDR *r = rrdr_create(owa, qt, qt->query.used, qt->window.points);
2314
if(unlikely(!r)) {
2315
internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, points=%zu",
2316
qt->id, qt->window.after, qt->window.before, qt->window.points);
2317
+ query_target_release(qt);
2318
return NULL;
2319
}
2320
2205
- if(unlikely(!r->d || !qt->window.points)) {
2206
- internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%ld, before=%ld, points=%zu",
2207
- qt->id, qt->window.after, qt->window.before, qt->window.points);
2208
- return r;
2209
- }
2210
-
2321
if(qt->window.relative)
2212
- r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
2322
+ r->view.flags |= RRDR_RESULT_FLAG_RELATIVE;
2323
else
2214
- r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
2324
+ r->view.flags |= RRDR_RESULT_FLAG_ABSOLUTE;
2325
2326
// -------------------------------------------------------------------------
2327
// initialize RRDR
2328
2219
- r->group = qt->window.group;
2220
- r->update_every = (int) (qt->window.group * qt->window.query_granularity);
2221
- r->before = qt->window.before;
2222
- r->after = qt->window.after;
2223
- r->internal.points_wanted = qt->window.points;
2224
- r->internal.resampling_group = qt->window.resampling_group;
2225
- r->internal.resampling_divisor = qt->window.resampling_divisor;
2226
- r->internal.query_options = qt->window.options;
2329
+ r->view.group = qt->window.group;
2330
+ r->view.update_every = (int) (qt->window.group * qt->window.query_granularity);
2331
+ r->view.before = qt->window.before;
2332
+ r->view.after = qt->window.after;
2333
+ r->grouping.points_wanted = qt->window.points;
2334
+ r->grouping.resampling_group = qt->window.resampling_group;
2335
+ r->grouping.resampling_divisor = qt->window.resampling_divisor;
2336
+ r->view.options = qt->window.options;
2337
2338
// -------------------------------------------------------------------------
2339
// assign the processor functions
2340
rrdr_set_grouping_function(r, qt->window.group_method);
2341
2342
// allocate any memory required by the grouping method
2233
- r->internal.grouping_create(r, qt->window.group_options);
2343
+ r->grouping.create(r, qt->window.group_options);
2344
2345
// -------------------------------------------------------------------------
2346
// do the work for each dimension
@@ -2247,7 +2357,9 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2357
size_t last_db_points_read = 0;
2358
size_t last_result_points_generated = 0;
2359
2250
- QUERY_ENGINE_OPS **ops = onewayalloc_callocz(r->internal.owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
2360
+ QUERY_ENGINE_OPS **ops = NULL;
2361
+ if(qt->query.used)
2362
+ ops = onewayalloc_callocz(r->internal.owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
2363
2364
size_t capacity = libuv_worker_threads * 2;
2365
size_t max_queries_to_prepare = (qt->query.used > (capacity - 1)) ? (capacity - 1) : qt->query.used;
@@ -2259,6 +2371,11 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2371
}
2372
2373
for(size_t c = 0, max = qt->query.used; c < max ; c++) {
2374
+ QUERY_METRIC *qm = query_metric(qt, c);
2375
+ QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
2376
+ QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
2377
+ QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id);
2378
+ QUERY_HOST *qh = query_host(qt, qm->link.query_host_id);
2379
2380
if(queries_prepared < max) {
2381
// preload another query
@@ -2267,26 +2384,51 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2384
}
2385
2386
// set the query target dimension options to rrdr
2270
- r->od[c] = qt->query.array[c].dimension.options;
2387
+ r->od[c] = qm->status;
2388
2389
// reset the grouping for the new dimension
2273
- r->internal.grouping_reset(r);
2390
+ r->grouping.reset(r);
2391
2392
if(ops[c]) {
2393
r->od[c] |= RRDR_DIMENSION_QUERIED;
2394
+ r->di[c] = rrdmetric_acquired_id_dup(qd->rma);
2395
+ r->dn[c] = rrdmetric_acquired_name_dup(qd->rma);
2396
+
2397
+ qi->metrics.queried++;
2398
+ qc->metrics.queried++;
2399
+ qh->metrics.queried++;
2400
+
2401
+ qd->status |= QUERY_STATUS_QUERIED;
2402
+ qm->status |= RRDR_DIMENSION_QUERIED;
2403
+
2404
rrd2rrdr_query_execute(r, c, ops[c]);
2405
+
2406
+ if(qt->request.version >= 2) {
2407
+ query_target_merge_data_statistics(&qi->query_stats, &qm->query_stats);
2408
+ query_target_merge_data_statistics(&qc->query_stats, &qm->query_stats);
2409
+ query_target_merge_data_statistics(&qh->query_stats, &qm->query_stats);
2410
+ query_target_merge_data_statistics(&qt->query_stats, &qm->query_stats);
2411
+ }
2412
}
2279
- else
2413
+ else {
2414
+ qi->metrics.failed++;
2415
+ qc->metrics.failed++;
2416
+ qh->metrics.failed++;
2417
+
2418
+ qd->status |= QUERY_STATUS_FAILED;
2419
+ qm->status |= RRDR_DIMENSION_FAILED;
2420
+
2421
continue;
2422
+ }
2423
2424
global_statistics_rrdr_query_completed(
2425
1,
2284
- r->internal.db_points_read - last_db_points_read,
2285
- r->internal.result_points_generated - last_result_points_generated,
2426
+ r->stats.db_points_read - last_db_points_read,
2427
+ r->stats.result_points_generated - last_result_points_generated,
2428
qt->request.query_source);
2429
2288
- last_db_points_read = r->internal.db_points_read;
2289
- last_result_points_generated = r->internal.result_points_generated;
2430
+ last_db_points_read = r->stats.db_points_read;
2431
+ last_result_points_generated = r->stats.result_points_generated;
2432
2433
if (qt->request.timeout)
2434
now_realtime_timeval(&query_current_time);
@@ -2296,28 +2438,28 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2438
2439
// verify all dimensions are aligned
2440
if(unlikely(!dimensions_used)) {
2299
- min_before = r->before;
2300
- max_after = r->after;
2441
+ min_before = r->view.before;
2442
+ max_after = r->view.after;
2443
max_rows = r->rows;
2444
}
2445
else {
2304
- if(r->after != max_after) {
2446
+ if(r->view.after != max_after) {
2447
internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
2306
- string2str(qt->query.array[c].dimension.id), (size_t)max_after, string2str(qt->query.array[c].dimension.name), (size_t)r->after);
2448
+ rrdinstance_acquired_id(qi->ria), (size_t)max_after, rrdmetric_acquired_id(qd->rma), (size_t)r->view.after);
2449
2308
- r->after = (r->after > max_after) ? r->after : max_after;
2450
+ r->view.after = (r->view.after > max_after) ? r->view.after : max_after;
2451
}
2452
2311
- if(r->before != min_before) {
2453
+ if(r->view.before != min_before) {
2454
internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
2313
- string2str(qt->query.array[c].dimension.id), (size_t)min_before, string2str(qt->query.array[c].dimension.name), (size_t)r->before);
2455
+ rrdinstance_acquired_id(qi->ria), (size_t)min_before, rrdmetric_acquired_id(qd->rma), (size_t)r->view.before);
2456
2315
- r->before = (r->before < min_before) ? r->before : min_before;
2457
+ r->view.before = (r->view.before < min_before) ? r->view.before : min_before;
2458
}
2459
2460
if(r->rows != max_rows) {
2461
internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
2320
- string2str(qt->query.array[c].dimension.id), (size_t)max_rows, string2str(qt->query.array[c].dimension.name), (size_t)r->rows);
2462
+ rrdinstance_acquired_id(qi->ria), (size_t)max_rows, rrdmetric_acquired_id(qd->rma), (size_t)r->rows);
2463
2464
r->rows = (r->rows > max_rows) ? r->rows : max_rows;
2465
}
@@ -2327,7 +2469,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2469
if (qt->request.timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout) {
2470
log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)",
2471
(NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout);
2330
- r->result_options |= RRDR_RESULT_OPTION_CANCEL;
2472
+ r->view.flags |= RRDR_RESULT_FLAG_CANCEL;
2473
2474
for(size_t i = c + 1; i < queries_prepared ; i++) {
2475
if(ops[i])
@@ -2352,7 +2494,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2494
qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2495
"got 'points' is not wanted 'points'");
2496
2355
- if(qt->window.aligned && (r->before % (qt->window.group * qt->window.query_granularity)) != 0)
2497
+ if(qt->window.aligned && (r->view.before % (qt->window.group * qt->window.query_granularity)) != 0)
2498
rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2499
qt->window.after, qt->request.after, qt->window.before,qt->request.before,
2500
qt->request.points, qt->window.points, /*after_slot, before_slot,*/
@@ -2362,20 +2504,20 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2504
//if(qt->window.aligned && (r->after % group) != 0)
2505
// rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
2506
2365
- if(r->before != qt->window.before)
2507
+ if(r->view.before != qt->window.before)
2508
rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2509
qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2510
qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2511
"chart is not aligned to requested 'before'");
2512
2371
- if(r->before != qt->window.before)
2513
+ if(r->view.before != qt->window.before)
2514
rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2515
qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2516
qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2517
"got 'before' is not wanted 'before'");
2518
2519
// reported 'after' varies, depending on group
2378
- if(r->after != qt->window.after)
2520
+ if(r->view.after != qt->window.after)
2521
rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2522
qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2523
qt->request.points, qt->window.points, /*after_slot, before_slot,*/
@@ -2384,13 +2526,19 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2526
}
2527
#endif
2528
2529
+ // free the query pipelining ops
2530
+ for(size_t c = 0; c < qt->query.used ;c++)
2531
+ onewayalloc_freez(owa, ops[c]);
2532
+
2533
+ onewayalloc_freez(owa, ops);
2534
+
2535
// free all resources used by the grouping method
2388
- r->internal.grouping_free(r);
2536
+ r->grouping.free(r);
2537
2538
if(likely(dimensions_used)) {
2539
// when all the dimensions are zero, we should return all of them
2540
if (unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero &&
2393
- !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
2541
+ !(r->view.flags & RRDR_RESULT_FLAG_CANCEL))) {
2542
// all the dimensions are zero
2543
// mark them as NONZERO to send them all
2544
for (size_t c = 0, max = qt->query.used; c < max; c++) {
@@ -2399,11 +2547,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2547
r->od[c] |= RRDR_DIMENSION_NONZERO;
2548
}
2549
}
2402
-
2403
- return r;
2550
}
2551
2406
- // we couldn't query any dimension
2407
- rrdr_free(owa, r);
2408
- return NULL;
2552
+ return r;
2553
}
web/api/queries/query.h
+16
-4
@@ -53,12 +53,18 @@ RRDR_TIME_GROUPING time_grouping_parse(const char *name, RRDR_TIME_GROUPING def)
53
const char *time_grouping_tostring(RRDR_TIME_GROUPING group);
54
55
typedef enum rrdr_group_by {
56
- RRDR_GROUP_BY_UNDEFINED = 0,
57
- RRDR_GROUP_BY_NODE,
58
- RRDR_GROUP_BY_INSTANCE,
59
- RRDR_GROUP_BY_LABEL,
56
+ RRDR_GROUP_BY_NONE = 0,
57
+ RRDR_GROUP_BY_DIMENSION = (1 << 0),
58
+ RRDR_GROUP_BY_NODE = (1 << 1),
59
+ RRDR_GROUP_BY_INSTANCE = (1 << 2),
60
+ RRDR_GROUP_BY_LABEL = (1 << 3),
61
} RRDR_GROUP_BY;
62
63
+struct web_buffer;
64
+
65
+RRDR_GROUP_BY group_by_parse(char *s);
66
+void buffer_json_group_by_to_array(struct web_buffer *wb, RRDR_GROUP_BY group_by);
67
+
68
typedef enum rrdr_group_by_function {
69
RRDR_GROUP_BY_FUNCTION_AVERAGE,
70
RRDR_GROUP_BY_FUNCTION_MIN,
@@ -67,6 +73,12 @@ typedef enum rrdr_group_by_function {
73
RRDR_GROUP_BY_FUNCTION_SUM_COUNT,
74
} RRDR_GROUP_BY_FUNCTION;
75
76
+RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s);
77
+const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_by_function);
78
+
79
+struct query_data_statistics;
80
+void query_target_merge_data_statistics(struct query_data_statistics *d, struct query_data_statistics *s);
81
+
82
#ifdef __cplusplus
83
}
84
#endif
web/api/queries/rrdr.c
+36
-15
@@ -61,41 +61,62 @@ static void rrdr_dump(RRDR *r)
61
inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
62
if(unlikely(!r)) return;
63
64
+ for(size_t d = 0; d < r->d ;d++) {
65
+ string_freez(r->di[d]);
66
+ string_freez(r->dn[d]);
67
+ string_freez(r->du[d]);
68
+ }
69
+
70
query_target_release(r->internal.qt);
71
+
72
onewayalloc_freez(owa, r->t);
73
onewayalloc_freez(owa, r->v);
74
onewayalloc_freez(owa, r->o);
75
onewayalloc_freez(owa, r->od);
76
+ onewayalloc_freez(owa, r->di);
77
+ onewayalloc_freez(owa, r->dn);
78
+ onewayalloc_freez(owa, r->du);
79
+ onewayalloc_freez(owa, r->dp);
80
onewayalloc_freez(owa, r->ar);
81
+ onewayalloc_freez(owa, r->gbc);
82
+ onewayalloc_freez(owa, r->dgbc);
83
onewayalloc_freez(owa, r);
84
}
85
73
-RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
74
- if(unlikely(!qt || !qt->query.used || !qt->window.points))
86
+RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt, size_t dimensions, size_t points) {
87
+ if(unlikely(!qt))
88
return NULL;
89
77
- size_t dimensions = qt->query.used;
78
- size_t points = qt->window.points;
79
-
90
// create the rrdr
91
RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
92
r->internal.owa = owa;
93
r->internal.qt = qt;
94
85
- r->before = qt->window.before;
86
- r->after = qt->window.after;
87
- r->internal.points_wanted = qt->window.points;
95
+ r->view.before = qt->window.before;
96
+ r->view.after = qt->window.after;
97
+ r->grouping.points_wanted = points;
98
r->d = (int)dimensions;
99
r->n = (int)points;
100
91
- r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
92
- r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
93
- r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS));
94
- r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
95
- r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS));
101
+ if(points && dimensions) {
102
+ r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
103
+ r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS));
104
+ r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
105
+ }
106
+
107
+ if(points) {
108
+ r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
109
+ }
110
+
111
+ if(dimensions) {
112
+ r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS));
113
+ r->di = onewayalloc_callocz(owa, dimensions, sizeof(STRING *));
114
+ r->dn = onewayalloc_callocz(owa, dimensions, sizeof(STRING *));
115
+ r->du = onewayalloc_callocz(owa, dimensions, sizeof(STRING *));
116
+ }
117
97
- r->group = 1;
98
- r->update_every = 1;
118
+ r->view.group = 1;
119
+ r->view.update_every = 1;
120
121
return r;
122
}
web/api/queries/rrdr.h
+60
-47
@@ -42,87 +42,100 @@ typedef enum rrdr_options {
42
RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query
43
RRDR_OPTION_ALL_DIMENSIONS = 0x00800000, // Return the full dimensions list
44
RRDR_OPTION_SHOW_PLAN = 0x01000000, // Return the query plan in jsonwrap
45
+ RRDR_OPTION_SHOW_DETAILS = 0x02000000, // v2 returns detailed object tree
46
+ RRDR_OPTION_DEBUG = 0x04000000, // v2 returns request description
47
48
// internal ones - not to be exposed to the API
49
RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
50
+ RRDR_OPTION_INTERNAL_GBC = 0x20000000, // internal use only, to let the formatters we want to render the group by count
51
RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
52
} RRDR_OPTIONS;
53
51
-typedef enum rrdr_value_flag {
52
- RRDR_VALUE_NOTHING = 0x00, // no flag set (a good default)
53
- RRDR_VALUE_EMPTY = 0x01, // the database value is empty
54
- RRDR_VALUE_RESET = 0x02, // the database value is marked as reset (overflown)
54
+typedef enum __attribute__ ((__packed__)) rrdr_value_flag {
55
+ RRDR_VALUE_NOTHING = 0, // no flag set (a good default)
56
+ RRDR_VALUE_EMPTY = (1 << 0), // the database value is empty
57
+ RRDR_VALUE_RESET = (1 << 1), // the database value is marked as reset (overflown)
58
} RRDR_VALUE_FLAGS;
59
57
-typedef enum rrdr_dimension_flag {
58
- RRDR_DIMENSION_DEFAULT = 0x00,
59
- RRDR_DIMENSION_HIDDEN = 0x04, // the dimension is hidden (not to be presented to callers)
60
- RRDR_DIMENSION_NONZERO = 0x08, // the dimension is non zero (contains non-zero values)
61
- RRDR_DIMENSION_QUERIED = 0x10, // the dimension is selected for evaluation in this RRDR
60
+typedef enum __attribute__ ((__packed__)) rrdr_dimension_flag {
61
+ RRDR_DIMENSION_DEFAULT = 0,
62
+ RRDR_DIMENSION_HIDDEN = (1 << 0), // the dimension is hidden (not to be presented to callers)
63
+ RRDR_DIMENSION_NONZERO = (1 << 1), // the dimension is non zero (contains non-zero values)
64
+ RRDR_DIMENSION_SELECTED = (1 << 2), // the dimension has been selected for query
65
+ RRDR_DIMENSION_QUERIED = (1 << 3), // the dimension has been queried
66
+ RRDR_DIMENSION_FAILED = (1 << 4), // the dimension failed to be queried
67
+ RRDR_DIMENSION_GROUPED = (1 << 5), // the dimension has been grouped in this RRDR
68
} RRDR_DIMENSION_FLAGS;
69
70
// RRDR result options
65
-typedef enum rrdr_result_flags {
66
- RRDR_RESULT_OPTION_ABSOLUTE = 0x00000001, // the query uses absolute time-frames
67
- // (can be cached by browsers and proxies)
68
- RRDR_RESULT_OPTION_RELATIVE = 0x00000002, // the query uses relative time-frames
69
- // (should not to be cached by browsers and proxies)
70
- RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames
71
- RRDR_RESULT_OPTION_CANCEL = 0x00000008, // the query needs to be cancelled
72
-} RRDR_RESULT_OPTIONS;
71
+typedef enum __attribute__ ((__packed__)) rrdr_result_flags {
72
+ RRDR_RESULT_FLAG_ABSOLUTE = (1 << 0), // the query uses absolute time-frames
73
+ // (can be cached by browsers and proxies)
74
+ RRDR_RESULT_FLAG_RELATIVE = (1 << 1), // the query uses relative time-frames
75
+ // (should not to be cached by browsers and proxies)
76
+ RRDR_RESULT_FLAG_CANCEL = (1 << 2), // the query needs to be cancelled
77
+} RRDR_RESULT_FLAGS;
78
79
typedef struct rrdresult {
75
- RRDR_RESULT_OPTIONS result_options; // RRDR_RESULT_OPTION_*
76
-
80
size_t d; // the number of dimensions
78
- size_t n; // the number of values in the arrays
79
- size_t rows; // the number of rows used
81
+ size_t n; // the number of values in the arrays (number of points per dimension)
82
+ size_t rows; // the number of actual rows used
83
84
RRDR_DIMENSION_FLAGS *od; // the options for the dimensions
85
86
+ STRING **di; // array of d dimension ids
87
+ STRING **dn; // array of d dimension names
88
+ STRING **du; // array of d dimension units
89
+ uint32_t *dgbc; // array of d dimension units - NOT ALLOCATED when RRDR is created
90
+ uint32_t *dp; // array of d dimension units - NOT ALLOCATED when RRDR is created
91
+
92
time_t *t; // array of n timestamps
93
NETDATA_DOUBLE *v; // array n x d values
94
RRDR_VALUE_FLAGS *o; // array n x d options for each value returned
95
NETDATA_DOUBLE *ar; // array n x d of anomaly rates (0 - 100)
96
+ uint32_t *gbc; // array n x d of group by count - NOT ALLOCATED when RRDR is created
97
88
- size_t group; // how many collected values were grouped for each row
89
- time_t update_every; // what is the suggested update frequency in seconds
90
-
91
- NETDATA_DOUBLE min;
92
- NETDATA_DOUBLE max;
98
+ struct {
99
+ size_t group; // how many collected values were grouped for each row - NEEDED BY GROUPING FUNCTIONS
100
+ time_t after;
101
+ time_t before;
102
+ time_t update_every; // what is the suggested update frequency in seconds
103
+ NETDATA_DOUBLE min;
104
+ NETDATA_DOUBLE max;
105
+ RRDR_RESULT_FLAGS flags; // RRDR_RESULT_FLAG_*
106
+ RRDR_OPTIONS options; // RRDR_OPTION_* (as run by the query)
107
+ } view;
108
94
- time_t before;
95
- time_t after;
109
+ struct {
110
+ size_t db_points_read;
111
+ size_t result_points_generated;
112
+ size_t tier_points_read[RRD_STORAGE_TIERS];
113
+ } stats;
114
97
- // internal rrd2rrdr() members below this point
115
struct {
99
- ONEWAYALLOC *owa; // the allocator used
100
- struct query_target *qt; // the QUERY_TARGET
116
+ void *data; // the internal data of the grouping function
117
102
- RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query)
118
+ // grouping function pointers
119
+ void (*create)(struct rrdresult *r, const char *options);
120
+ void (*reset)(struct rrdresult *r);
121
+ void (*free)(struct rrdresult *r);
122
+ void (*add)(struct rrdresult *r, NETDATA_DOUBLE value);
123
+ NETDATA_DOUBLE (*flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
124
+
125
+ TIER_QUERY_FETCH tier_query_fetch; // which value to use from STORAGE_POINT
126
127
size_t points_wanted; // used by SES and DES
128
size_t resampling_group; // used by AVERAGE
129
NETDATA_DOUBLE resampling_divisor; // used by AVERAGE
130
+ } grouping;
131
108
- // grouping function pointers
109
- void (*grouping_create)(struct rrdresult *r, const char *options);
110
- void (*grouping_reset)(struct rrdresult *r);
111
- void (*grouping_free)(struct rrdresult *r);
112
- void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
113
- NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
114
-
115
- TIER_QUERY_FETCH tier_query_fetch; // which value to use from STORAGE_POINT
116
- void *grouping_data; // the internal data of the grouping function
132
+ struct {
133
+ ONEWAYALLOC *owa; // the allocator used
134
+ struct query_target *qt; // the QUERY_TARGET
135
136
#ifdef NETDATA_INTERNAL_CHECKS
137
const char *log;
138
#endif
121
-
122
- // statistics
123
- size_t db_points_read;
124
- size_t result_points_generated;
125
- size_t tier_points_read[RRD_STORAGE_TIERS];
139
} internal;
140
} RRDR;
141
@@ -130,7 +143,7 @@ typedef struct rrdresult {
143
144
#include "database/rrd.h"
145
void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
133
-RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt);
146
+RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt, size_t dimensions, size_t points);
147
148
#include "../web_api_v1.h"
149
#include "web/api/queries/query.h"
web/api/queries/ses/ses.c
+9
-9
@@ -29,13 +29,13 @@ static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_ses *g) {
29
(void)g;
30
31
NETDATA_DOUBLE points;
32
- if(r->group == 1) {
32
+ if(r->view.group == 1) {
33
// provide a running DES
34
- points = (NETDATA_DOUBLE)r->internal.points_wanted;
34
+ points = (NETDATA_DOUBLE)r->grouping.points_wanted;
35
}
36
else {
37
// provide a SES with flush points
38
- points = (NETDATA_DOUBLE)r->group;
38
+ points = (NETDATA_DOUBLE)r->view.group;
39
}
40
41
return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points;
@@ -52,24 +52,24 @@ void grouping_create_ses(RRDR *r, const char *options __maybe_unused) {
52
struct grouping_ses *g = (struct grouping_ses *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_ses));
53
set_alpha(r, g);
54
g->level = 0.0;
55
- r->internal.grouping_data = g;
55
+ r->grouping.data = g;
56
}
57
58
// resets when switches dimensions
59
// so, clear everything to restart
60
void grouping_reset_ses(RRDR *r) {
61
- struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data;
61
+ struct grouping_ses *g = (struct grouping_ses *)r->grouping.data;
62
g->level = 0.0;
63
g->count = 0;
64
}
65
66
void grouping_free_ses(RRDR *r) {
67
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
68
- r->internal.grouping_data = NULL;
67
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
68
+ r->grouping.data = NULL;
69
}
70
71
void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value) {
72
- struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data;
72
+ struct grouping_ses *g = (struct grouping_ses *)r->grouping.data;
73
74
if(unlikely(!g->count))
75
g->level = value;
@@ -79,7 +79,7 @@ void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value) {
79
}
80
81
NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
82
- struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data;
82
+ struct grouping_ses *g = (struct grouping_ses *)r->grouping.data;
83
84
if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
85
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
web/api/queries/stddev/stddev.c
+9
-9
@@ -15,23 +15,23 @@ struct grouping_stddev {
15
};
16
17
void grouping_create_stddev(RRDR *r, const char *options __maybe_unused) {
18
- r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_stddev));
18
+ r->grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_stddev));
19
}
20
21
// resets when switches dimensions
22
// so, clear everything to restart
23
void grouping_reset_stddev(RRDR *r) {
24
- struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
24
+ struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.data;
25
g->count = 0;
26
}
27
28
void grouping_free_stddev(RRDR *r) {
29
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
30
- r->internal.grouping_data = NULL;
29
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
30
+ r->grouping.data = NULL;
31
}
32
33
void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value) {
34
- struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
34
+ struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.data;
35
36
g->count++;
37
@@ -62,7 +62,7 @@ static inline NETDATA_DOUBLE stddev(struct grouping_stddev *g) {
62
}
63
64
NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
65
- struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
65
+ struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.data;
66
67
NETDATA_DOUBLE value;
68
@@ -89,7 +89,7 @@ NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_optio
89
90
// https://en.wikipedia.org/wiki/Coefficient_of_variation
91
NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
92
- struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
92
+ struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.data;
93
94
NETDATA_DOUBLE value;
95
@@ -122,7 +122,7 @@ NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS
122
* Mean = average
123
*
124
NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
125
- struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
125
+ struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.grouping_data;
126
127
NETDATA_DOUBLE value;
128
@@ -149,7 +149,7 @@ NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options
149
* It is not advised to use this version of variance directly
150
*
151
NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
152
- struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
152
+ struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.grouping_data;
153
154
NETDATA_DOUBLE value;
155
web/api/queries/sum/sum.c
+6
-6
@@ -11,30 +11,30 @@ struct grouping_sum {
11
};
12
13
void grouping_create_sum(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_sum));
14
+ r->grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_sum));
15
}
16
17
// resets when switches dimensions
18
// so, clear everything to restart
19
void grouping_reset_sum(RRDR *r) {
20
- struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data;
20
+ struct grouping_sum *g = (struct grouping_sum *)r->grouping.data;
21
g->sum = 0;
22
g->count = 0;
23
}
24
25
void grouping_free_sum(RRDR *r) {
26
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
- r->internal.grouping_data = NULL;
26
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
27
+ r->grouping.data = NULL;
28
}
29
30
void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value) {
31
- struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data;
31
+ struct grouping_sum *g = (struct grouping_sum *)r->grouping.data;
32
g->sum += value;
33
g->count++;
34
}
35
36
NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
37
- struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data;
37
+ struct grouping_sum *g = (struct grouping_sum *)r->grouping.data;
38
39
NETDATA_DOUBLE value;
40
web/api/queries/trimmed_mean/trimmed_mean.c
+8
-8
@@ -14,7 +14,7 @@ struct grouping_trimmed_mean {
14
};
15
16
static void grouping_create_trimmed_mean_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17
- long entries = r->group;
17
+ long entries = r->view.group;
18
if(entries < 10) entries = 10;
19
20
struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_trimmed_mean));
@@ -30,7 +30,7 @@ static void grouping_create_trimmed_mean_internal(RRDR *r, const char *options,
30
}
31
32
g->percent = 1.0 - ((g->percent / 100.0) * 2.0);
33
- r->internal.grouping_data = g;
33
+ r->grouping.data = g;
34
}
35
36
void grouping_create_trimmed_mean1(RRDR *r, const char *options) {
@@ -61,20 +61,20 @@ void grouping_create_trimmed_mean25(RRDR *r, const char *options) {
61
// resets when switches dimensions
62
// so, clear everything to restart
63
void grouping_reset_trimmed_mean(RRDR *r) {
64
- struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
64
+ struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->grouping.data;
65
g->next_pos = 0;
66
}
67
68
void grouping_free_trimmed_mean(RRDR *r) {
69
- struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
69
+ struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->grouping.data;
70
if(g) onewayalloc_freez(r->internal.owa, g->series);
71
72
- onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
73
- r->internal.grouping_data = NULL;
72
+ onewayalloc_freez(r->internal.owa, r->grouping.data);
73
+ r->grouping.data = NULL;
74
}
75
76
void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value) {
77
- struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
77
+ struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->grouping.data;
78
79
if(unlikely(g->next_pos >= g->series_size)) {
80
g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
@@ -85,7 +85,7 @@ void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value) {
85
}
86
87
NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
88
- struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
88
+ struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->grouping.data;
89
90
NETDATA_DOUBLE value;
91
size_t available_slots = g->next_pos;
web/api/queries/weights.c
+4
-3
@@ -477,6 +477,7 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
477
NETDATA_DOUBLE *ret = NULL;
478
479
QUERY_TARGET_REQUEST qtr = {
480
+ .version = 1,
481
.host = host,
482
.rca = rca,
483
.ria = ria,
@@ -497,10 +498,10 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
498
goto cleanup;
499
500
stats->db_queries++;
500
- stats->result_points += r->internal.result_points_generated;
501
- stats->db_points += r->internal.db_points_read;
501
+ stats->result_points += r->stats.result_points_generated;
502
+ stats->db_points += r->stats.db_points_read;
503
for(size_t tr = 0; tr < storage_tiers ; tr++)
503
- stats->db_points_per_tier[tr] += r->internal.tier_points_read[tr];
504
+ stats->db_points_per_tier[tr] += r->stats.tier_points_read[tr];
505
506
if(r->d != 1) {
507
error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu", r->internal.qt->id, r->d);
web/api/web_api.c
new
+27
@@ -0,0 +1,27 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "web_api.h"
4
+
5
+int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url, struct web_api_command *api_commands) {
6
+ if(unlikely(!url || !*url)) {
7
+ buffer_flush(w->response.data);
8
+ buffer_sprintf(w->response.data, "Which API command?");
9
+ return HTTP_RESP_BAD_REQUEST;
10
+ }
11
+
12
+ uint32_t hash = simple_hash(url);
13
+
14
+ for(int i = 0; api_commands[i].command ; i++) {
15
+ if(unlikely(hash == api_commands[i].hash && !strcmp(url, api_commands[i].command))) {
16
+ if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))
17
+ return web_client_permission_denied(w);
18
+
19
+ return api_commands[i].callback(host, w, (w->decoded_query_string + 1));
20
+ }
21
+ }
22
+
23
+ buffer_flush(w->response.data);
24
+ buffer_strcat(w->response.data, "Unsupported API command: ");
25
+ buffer_strcat_htmlescape(w->response.data, url);
26
+ return HTTP_RESP_NOT_FOUND;
27
+}
web/api/web_api.h
new
+35
@@ -0,0 +1,35 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WEB_API_H
4
+#define NETDATA_WEB_API_H 1
5
+
6
+#include "daemon/common.h"
7
+#include "web/api/badges/web_buffer_svg.h"
8
+#include "web/api/formatters/rrd2json.h"
9
+#include "web/api/health/health_cmdapi.h"
10
+#include "web/api/queries/weights.h"
11
+
12
+struct web_api_command {
13
+ const char *command;
14
+ uint32_t hash;
15
+ WEB_CLIENT_ACL acl;
16
+ int (*callback)(RRDHOST *host, struct web_client *w, char *url);
17
+};
18
+
19
+struct web_client;
20
+
21
+int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url, struct web_api_command *api_commands);
22
+
23
+static inline void fix_google_param(char *s) {
24
+ if(unlikely(!s || !*s)) return;
25
+
26
+ for( ; *s ;s++) {
27
+ if(!isalnum(*s) && *s != '.' && *s != '_' && *s != '-')
28
+ *s = '_';
29
+ }
30
+}
31
+
32
+#include "web_api_v1.h"
33
+#include "web_api_v2.h"
34
+
35
+#endif //NETDATA_WEB_API_H
web/api/web_api_v1.c
+14
-52
@@ -42,6 +42,8 @@ static struct {
42
, {"virtual-points" , 0 , RRDR_OPTION_VIRTUAL_POINTS}
43
, {"all-dimensions" , 0 , RRDR_OPTION_ALL_DIMENSIONS}
44
, {"plan" , 0 , RRDR_OPTION_SHOW_PLAN}
45
+ , {"details" , 0 , RRDR_OPTION_SHOW_DETAILS}
46
+ , {"debug" , 0 , RRDR_OPTION_DEBUG}
47
, {NULL , 0 , 0}
48
};
49
@@ -565,18 +567,8 @@ inline int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w,
567
return web_client_api_request_single_chart(host, w, url, rrd_stats_api_v1_chart);
568
}
569
568
-void fix_google_param(char *s) {
569
- if(unlikely(!s)) return;
570
-
571
- for( ; *s ;s++) {
572
- if(!isalnum(*s) && *s != '.' && *s != '_' && *s != '-')
573
- *s = '_';
574
- }
575
-}
576
-
577
-
570
// returns the HTTP code
579
-inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url) {
571
+static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url) {
572
debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
573
574
int ret = HTTP_RESP_BAD_REQUEST;
@@ -722,6 +714,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
714
long group_time = (group_time_str && *group_time_str)?str2l(group_time_str):0;
715
716
QUERY_TARGET_REQUEST qtr = {
717
+ .version = 1,
718
.after = after,
719
.before = before,
720
.host = host,
@@ -739,7 +732,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
732
.resampling_time = group_time,
733
.tier = tier,
734
.chart_label_key = chart_label_key,
742
- .charts_labels_filter = chart_labels_filter,
735
+ .labels = chart_labels_filter,
736
.query_source = QUERY_SOURCE_API_DATA,
737
.priority = STORAGE_PRIORITY_NORMAL,
738
};
@@ -814,10 +807,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
807
buffer_strcat(w->response.data, ");");
808
809
cleanup:
817
- if(qt && qt->used) {
818
- internal_error(true, "QUERY_TARGET: left non-released on query '%s'", qt->id);
819
- query_target_release(qt);
820
- }
810
+ query_target_release(qt);
811
onewayalloc_destroy(owa);
812
buffer_free(dimensions);
813
return ret;
@@ -1193,8 +1183,10 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
1183
#endif
1184
1185
buffer_json_member_add_string(wb, "memory-mode", rrd_memory_mode_name(host->rrd_memory_mode));
1186
+#ifdef ENABLE_DBENGINE
1187
buffer_json_member_add_uint64(wb, "multidb-disk-quota", default_multidb_disk_quota_mb);
1188
buffer_json_member_add_uint64(wb, "page-cache-size", default_rrdeng_page_cache_mb);
1189
+#endif // ENABLE_DBENGINE
1190
buffer_json_member_add_boolean(wb, "web-enabled", web_server_mode != WEB_SERVER_MODE_NONE);
1191
buffer_json_member_add_boolean(wb, "stream-enabled", default_rrdpush_enabled);
1192
@@ -1539,12 +1531,7 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1531
#define ACL_DEV_OPEN_ACCESS 0
1532
#endif
1533
1542
-static struct api_command {
1543
- const char *command;
1544
- uint32_t hash;
1545
- WEB_CLIENT_ACL acl;
1546
- int (*callback)(RRDHOST *host, struct web_client *w, char *url);
1547
-} api_commands[] = {
1534
+static struct web_api_command api_commands_v1[] = {
1535
{ "info", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_info },
1536
{ "data", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_data },
1537
{ "chart", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_chart },
@@ -1566,8 +1553,8 @@ static struct api_command {
1553
{ "allmetrics", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_allmetrics },
1554
1555
#if defined(ENABLE_ML)
1569
- { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_ml_info },
1570
- { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models },
1556
+ { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_ml_info },
1557
+ { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models },
1558
#endif
1559
1560
{ "manage/health", 0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_mgmt_health },
@@ -1586,38 +1573,13 @@ static struct api_command {
1573
1574
inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url) {
1575
static int initialized = 0;
1589
- int i;
1576
1577
if(unlikely(initialized == 0)) {
1578
initialized = 1;
1579
1594
- for(i = 0; api_commands[i].command ; i++)
1595
- api_commands[i].hash = simple_hash(api_commands[i].command);
1580
+ for(int i = 0; api_commands_v1[i].command ; i++)
1581
+ api_commands_v1[i].hash = simple_hash(api_commands_v1[i].command);
1582
}
1583
1598
- // get the command
1599
- if(url) {
1600
- debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, url);
1601
- uint32_t hash = simple_hash(url);
1602
-
1603
- for(i = 0; api_commands[i].command ;i++) {
1604
- if(unlikely(hash == api_commands[i].hash && !strcmp(url, api_commands[i].command))) {
1605
- if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))
1606
- return web_client_permission_denied(w);
1607
-
1608
- //return api_commands[i].callback(host, w, url);
1609
- return api_commands[i].callback(host, w, (w->decoded_query_string + 1));
1610
- }
1611
- }
1612
-
1613
- buffer_flush(w->response.data);
1614
- buffer_strcat(w->response.data, "Unsupported v1 API command: ");
1615
- buffer_strcat_htmlescape(w->response.data, url);
1616
- return HTTP_RESP_NOT_FOUND;
1617
- }
1618
- else {
1619
- buffer_flush(w->response.data);
1620
- buffer_sprintf(w->response.data, "Which API v1 command?");
1621
- return HTTP_RESP_BAD_REQUEST;
1622
- }
1584
+ return web_client_api_request_vX(host, w, url, api_commands_v1);
1585
}
web/api/web_api_v1.h
+3
-6
@@ -3,11 +3,9 @@
3
#ifndef NETDATA_WEB_API_V1_H
4
#define NETDATA_WEB_API_V1_H 1
5
6
-#include "daemon/common.h"
7
-#include "web/api/badges/web_buffer_svg.h"
8
-#include "web/api/formatters/rrd2json.h"
9
-#include "web/api/health/health_cmdapi.h"
10
-#include "web/api/queries/weights.h"
6
+#include "web_api.h"
7
+
8
+struct web_client;
9
10
RRDR_OPTIONS web_client_api_request_v1_data_options(char *o);
11
void web_client_api_request_v1_data_options_to_buffer_json_array(BUFFER *wb, const char *key, RRDR_OPTIONS options);
@@ -24,7 +22,6 @@ int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_client *
22
int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_client *w, char *url);
23
int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url);
24
int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
27
-int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url);
25
int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);
26
int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url);
27
int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url);
web/api/web_api_v2.c
new
+270
@@ -0,0 +1,270 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "web_api_v2.h"
4
+
5
+
6
+static inline int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
7
+ debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
8
+
9
+ usec_t received_ut = now_monotonic_usec();
10
+
11
+ int ret = HTTP_RESP_BAD_REQUEST;
12
+
13
+ buffer_flush(w->response.data);
14
+
15
+ char *google_version = "0.6",
16
+ *google_reqId = "0",
17
+ *google_sig = "0",
18
+ *google_out = "json",
19
+ *responseHandler = NULL,
20
+ *outFileName = NULL;
21
+
22
+ time_t last_timestamp_in_data = 0, google_timestamp = 0;
23
+
24
+ char *scope_hosts = NULL;
25
+ char *scope_contexts = NULL;
26
+ char *hosts = NULL;
27
+ char *contexts = NULL;
28
+ char *instances = NULL;
29
+ char *dimensions = NULL;
30
+ char *before_str = NULL;
31
+ char *after_str = NULL;
32
+ char *resampling_time_str = NULL;
33
+ char *points_str = NULL;
34
+ char *timeout_str = NULL;
35
+ char *labels = NULL;
36
+ char *alerts = NULL;
37
+ char *time_group_options = NULL;
38
+ char *tier_str = NULL;
39
+ char *group_by_label = NULL;
40
+ size_t tier = 0;
41
+ RRDR_TIME_GROUPING time_group = RRDR_GROUPING_AVERAGE;
42
+ RRDR_GROUP_BY group_by = RRDR_GROUP_BY_DIMENSION;
43
+ RRDR_GROUP_BY_FUNCTION group_by_aggregate = RRDR_GROUP_BY_FUNCTION_AVERAGE;
44
+ DATASOURCE_FORMAT format = DATASOURCE_JSON;
45
+ RRDR_OPTIONS options = 0;
46
+
47
+ while(url) {
48
+ char *value = mystrsep(&url, "&");
49
+ if(!value || !*value) continue;
50
+
51
+ char *name = mystrsep(&value, "=");
52
+ if(!name || !*name) continue;
53
+ if(!value || !*value) continue;
54
+
55
+ // name and value are now the parameters
56
+ // they are not null and not empty
57
+
58
+ if(!strcmp(name, "scope_hosts")) scope_hosts = value;
59
+ else if(!strcmp(name, "scope_contexts")) scope_contexts = value;
60
+ else if(!strcmp(name, "hosts")) hosts = value;
61
+ else if(!strcmp(name, "contexts")) contexts = value;
62
+ else if(!strcmp(name, "instances")) instances = value;
63
+ else if(!strcmp(name, "dimensions")) dimensions = value;
64
+ else if(!strcmp(name, "labels")) labels = value;
65
+ else if(!strcmp(name, "alerts")) alerts = value;
66
+ else if(!strcmp(name, "after")) after_str = value;
67
+ else if(!strcmp(name, "before")) before_str = value;
68
+ else if(!strcmp(name, "points")) points_str = value;
69
+ else if(!strcmp(name, "timeout")) timeout_str = value;
70
+ else if(!strcmp(name, "group_by")) group_by = group_by_parse(value);
71
+ else if(!strcmp(name, "group_by_label")) group_by_label = value;
72
+ else if(!strcmp(name, "group_by_aggregate")) group_by_aggregate = group_by_aggregate_function_parse(value);
73
+ else if(!strcmp(name, "format")) format = web_client_api_request_v1_data_format(value);
74
+ else if(!strcmp(name, "options")) options |= web_client_api_request_v1_data_options(value);
75
+ else if(!strcmp(name, "time_group")) time_group = time_grouping_parse(value, RRDR_GROUPING_AVERAGE);
76
+ else if(!strcmp(name, "time_group_options")) time_group_options = value;
77
+ else if(!strcmp(name, "time_resampling")) resampling_time_str = value;
78
+ else if(!strcmp(name, "tier")) tier_str = value;
79
+ else if(!strcmp(name, "callback")) responseHandler = value;
80
+ else if(!strcmp(name, "filename")) outFileName = value;
81
+ else if(!strcmp(name, "tqx")) {
82
+ // parse Google Visualization API options
83
+ // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source
84
+ char *tqx_name, *tqx_value;
85
+
86
+ while(value) {
87
+ tqx_value = mystrsep(&value, ";");
88
+ if(!tqx_value || !*tqx_value) continue;
89
+
90
+ tqx_name = mystrsep(&tqx_value, ":");
91
+ if(!tqx_name || !*tqx_name) continue;
92
+ if(!tqx_value || !*tqx_value) continue;
93
+
94
+ if(!strcmp(tqx_name, "version"))
95
+ google_version = tqx_value;
96
+ else if(!strcmp(tqx_name, "reqId"))
97
+ google_reqId = tqx_value;
98
+ else if(!strcmp(tqx_name, "sig")) {
99
+ google_sig = tqx_value;
100
+ google_timestamp = strtoul(google_sig, NULL, 0);
101
+ }
102
+ else if(!strcmp(tqx_name, "out")) {
103
+ google_out = tqx_value;
104
+ format = web_client_api_request_v1_data_google_format(google_out);
105
+ }
106
+ else if(!strcmp(tqx_name, "responseHandler"))
107
+ responseHandler = tqx_value;
108
+ else if(!strcmp(tqx_name, "outFileName"))
109
+ outFileName = tqx_value;
110
+ }
111
+ }
112
+ }
113
+
114
+ // validate the google parameters given
115
+ fix_google_param(google_out);
116
+ fix_google_param(google_sig);
117
+ fix_google_param(google_reqId);
118
+ fix_google_param(google_version);
119
+ fix_google_param(responseHandler);
120
+ fix_google_param(outFileName);
121
+
122
+ if(group_by & ~(RRDR_GROUP_BY_DIMENSION))
123
+ options |= RRDR_OPTION_ABSOLUTE;
124
+
125
+ if(options & RRDR_OPTION_SHOW_PLAN)
126
+ options |= RRDR_OPTION_DEBUG;
127
+
128
+ if(group_by_label && *group_by_label)
129
+ group_by |= RRDR_GROUP_BY_LABEL;
130
+
131
+ if(tier_str && *tier_str) {
132
+ tier = str2ul(tier_str);
133
+ if(tier < storage_tiers)
134
+ options |= RRDR_OPTION_SELECTED_TIER;
135
+ else
136
+ tier = 0;
137
+ }
138
+
139
+ long long before = (before_str && *before_str)?str2l(before_str):0;
140
+ long long after = (after_str && *after_str) ?str2l(after_str):-600;
141
+ int points = (points_str && *points_str)?str2i(points_str):0;
142
+ int timeout = (timeout_str && *timeout_str)?str2i(timeout_str): 0;
143
+ long group_time = (resampling_time_str && *resampling_time_str) ? str2l(resampling_time_str) : 0;
144
+
145
+ QUERY_TARGET_REQUEST qtr = {
146
+ .version = 2,
147
+ .scope_hosts = scope_hosts,
148
+ .scope_contexts = scope_contexts,
149
+ .after = after,
150
+ .before = before,
151
+ .host = NULL,
152
+ .st = NULL,
153
+ .hosts = hosts,
154
+ .contexts = contexts,
155
+ .charts = instances,
156
+ .dimensions = dimensions,
157
+ .alerts = alerts,
158
+ .timeout = timeout,
159
+ .points = points,
160
+ .format = format,
161
+ .options = options,
162
+ .group_by = group_by,
163
+ .group_by_label = group_by_label,
164
+ .group_by_aggregate_function = group_by_aggregate,
165
+ .time_group_method = time_group,
166
+ .time_group_options = time_group_options,
167
+ .resampling_time = group_time,
168
+ .tier = tier,
169
+ .chart_label_key = NULL,
170
+ .labels = labels,
171
+ .query_source = QUERY_SOURCE_API_DATA,
172
+ .priority = STORAGE_PRIORITY_NORMAL,
173
+ .received_ut = received_ut,
174
+ };
175
+ QUERY_TARGET *qt = query_target_create(&qtr);
176
+ ONEWAYALLOC *owa = NULL;
177
+
178
+ if(!qt) {
179
+ buffer_sprintf(w->response.data, "Failed to prepare the query.");
180
+ ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
181
+ goto cleanup;
182
+ }
183
+
184
+ if (timeout) {
185
+ struct timeval now;
186
+ now_realtime_timeval(&now);
187
+ int inqueue = (int)dt_usec(&w->tv_in, &now) / 1000;
188
+ timeout -= inqueue;
189
+ if (timeout <= 0) {
190
+ buffer_flush(w->response.data);
191
+ buffer_strcat(w->response.data, "Query timeout exceeded");
192
+ ret = HTTP_RESP_BACKEND_FETCH_FAILED;
193
+ goto cleanup;
194
+ }
195
+ }
196
+
197
+ if(outFileName && *outFileName) {
198
+ buffer_sprintf(w->response.header, "Content-Disposition: attachment; filename=\"%s\"\r\n", outFileName);
199
+ debug(D_WEB_CLIENT, "%llu: generating outfilename header: '%s'", w->id, outFileName);
200
+ }
201
+
202
+ if(format == DATASOURCE_DATATABLE_JSONP) {
203
+ if(responseHandler == NULL)
204
+ responseHandler = "google.visualization.Query.setResponse";
205
+
206
+ debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSON/JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'",
207
+ w->id, google_version, google_reqId, google_sig, google_out, responseHandler, outFileName
208
+ );
209
+
210
+ buffer_sprintf(
211
+ w->response.data,
212
+ "%s({version:'%s',reqId:'%s',status:'ok',sig:'%"PRId64"',table:",
213
+ responseHandler,
214
+ google_version,
215
+ google_reqId,
216
+ (int64_t)now_realtime_sec());
217
+ }
218
+ else if(format == DATASOURCE_JSONP) {
219
+ if(responseHandler == NULL)
220
+ responseHandler = "callback";
221
+
222
+ buffer_strcat(w->response.data, responseHandler);
223
+ buffer_strcat(w->response.data, "(");
224
+ }
225
+
226
+ owa = onewayalloc_create(0);
227
+ ret = data_query_execute(owa, w->response.data, qt, &last_timestamp_in_data);
228
+
229
+ if(format == DATASOURCE_DATATABLE_JSONP) {
230
+ if(google_timestamp < last_timestamp_in_data)
231
+ buffer_strcat(w->response.data, "});");
232
+
233
+ else {
234
+ // the client already has the latest data
235
+ buffer_flush(w->response.data);
236
+ buffer_sprintf(w->response.data,
237
+ "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
238
+ responseHandler, google_version, google_reqId);
239
+ }
240
+ }
241
+ else if(format == DATASOURCE_JSONP)
242
+ buffer_strcat(w->response.data, ");");
243
+
244
+cleanup:
245
+ query_target_release(qt);
246
+ onewayalloc_destroy(owa);
247
+ return ret;
248
+}
249
+
250
+
251
+
252
+static struct web_api_command api_commands_v2[] = {
253
+ {"data", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_data},
254
+
255
+ // terminator
256
+ {NULL, 0, WEB_CLIENT_ACL_NONE, NULL},
257
+};
258
+
259
+inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url) {
260
+ static int initialized = 0;
261
+
262
+ if(unlikely(initialized == 0)) {
263
+ initialized = 1;
264
+
265
+ for(int i = 0; api_commands_v2[i].command ; i++)
266
+ api_commands_v2[i].hash = simple_hash(api_commands_v2[i].command);
267
+ }
268
+
269
+ return web_client_api_request_vX(host, w, url, api_commands_v2);
270
+}
web/api/web_api_v2.h
new
+12
@@ -0,0 +1,12 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WEB_API_V2_H
4
+#define NETDATA_WEB_API_V2_H 1
5
+
6
+#include "web_api.h"
7
+
8
+struct web_client;
9
+
10
+int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url);
11
+
12
+#endif //NETDATA_WEB_API_V2_H
web/server/web_client.c
+3
-1
@@ -545,7 +545,9 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url)
545
char *tok = mystrsep(&url, "/");
546
if(tok && *tok) {
547
debug(D_WEB_CLIENT, "%llu: Searching for API version '%s'.", w->id, tok);
548
- if(strcmp(tok, "v1") == 0)
548
+ if(strcmp(tok, "v2") == 0)
549
+ return web_client_api_request_v2(host, w, url);
550
+ else if(strcmp(tok, "v1") == 0)
551
return web_client_api_request_v1(host, w, url);
552
else {
553
buffer_flush(w->response.data);
web/server/web_client.h
+2
@@ -211,6 +211,8 @@ char *strip_control_characters(char *url);
211
212
int web_client_socket_is_now_used_for_streaming(struct web_client *w);
213
214
+#include "web/api/web_api_v1.h"
215
+#include "web/api/web_api_v2.h"
216
#include "daemon/common.h"
217
218
#endif