/api/v2/X improvements part 3 (#14665)
* max web request size to 64KB * fix the request too big message * increase max request reading tries to 100 * support for bigger web requests * add "avg" as a shortcut for "average" to both group by aggregation and time aggregation; discard the last partial points of a query in play mode, up to max update every; group by hidden dimensions too * better implementation for partial data trimming * added group_by=selected to return only one dimension for all selected metrics * fix acceptance of group_by=selected * passing option "raw" disables partial data trimming * remove obsolete option "plan"; use "debug" * fix view.min and view.max calculation - there were 2 bugs: a) min and max were reset for every row and b) min and max were corrupted by GBC and AR printing * per row annotations * added time column to point annotations * disable caching for /api/v2/contexts responses * added api format json2 that returns an array for each points, having all the point values and annotations in them * work on swagger about /api/v2 * prevent infinite loop * cleanup and swagger work * allow negative simple pattern expressions to work as expected * do not lookup in the dictionary empty names * garbage collect dictionaries * make query_target allocate less aggressively; queries fill the remaining points with nulls * reusable query ops to save memory on huge queries * move parts of query plans into query ops to save query target memory * remove storage engine from query metric tiers, to save memory, and recalculate it when it is needed
Costa Tsaousis committed
Mar 10, 2023 at 12:41 UTC
cf85c3b0e9fcda807a2a5e9c1834792d73725a50
31 files changed
+1376
-424
aclk/aclk_query.c
+2
-2
@@ -73,8 +73,8 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
73
74
struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
75
w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
76
- w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_aclk);
77
- w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_aclk);
76
+ w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
77
+ w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
78
strcpy(w->origin, "*"); // Simulate web_client_create_on_fd()
79
w->cookie1[0] = 0; // Simulate web_client_create_on_fd()
80
w->cookie2[0] = 0; // Simulate web_client_create_on_fd()
collectors/statsd.plugin/statsd.c
+1
-1
@@ -2129,7 +2129,7 @@ static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC
2129
strcpy(wildcarded, dim->name);
2130
char *ws = &wildcarded[dim_name_len];
2131
2132
- if(simple_pattern_matches_extract(dim->metric_pattern, m->name, ws, wildcarded_len - dim_name_len)) {
2132
+ if(simple_pattern_matches_extract(dim->metric_pattern, m->name, ws, wildcarded_len - dim_name_len) == SP_MATCHED_POSITIVE) {
2133
2134
char *final_name = NULL;
2135
daemon/main.c
+7
-3
@@ -1617,11 +1617,15 @@ int main(int argc, char **argv) {
1617
char wildcarded[len];
1618
1619
SIMPLE_PATTERN *p = simple_pattern_create(haystack, NULL, SIMPLE_PATTERN_EXACT, true);
1620
- int ret = simple_pattern_matches_extract(p, needle, wildcarded, len);
1620
+ SIMPLE_PATTERN_RESULT ret = simple_pattern_matches_extract(p, needle, wildcarded, len);
1621
simple_pattern_free(p);
1622
1623
- if(ret) {
1624
- fprintf(stdout, "RESULT: MATCHED - pattern '%s' matches '%s', wildcarded '%s'\n", haystack, needle, wildcarded);
1623
+ if(ret == SP_MATCHED_POSITIVE) {
1624
+ fprintf(stdout, "RESULT: POSITIVE MATCHED - pattern '%s' matches '%s', wildcarded '%s'\n", haystack, needle, wildcarded);
1625
+ return 0;
1626
+ }
1627
+ else if(ret == SP_MATCHED_NEGATIVE) {
1628
+ fprintf(stdout, "RESULT: NEGATIVE MATCHED - pattern '%s' matches '%s', wildcarded '%s'\n", haystack, needle, wildcarded);
1629
return 0;
1630
}
1631
else {
database/contexts/api_v1.c
+2
-2
@@ -366,7 +366,7 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
366
RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
367
368
if(after != 0 && before != 0)
369
- rrdr_relative_window_to_absolute(&after, &before);
369
+ rrdr_relative_window_to_absolute(&after, &before, NULL);
370
371
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
372
struct rrdcontext_to_json t_contexts = {
@@ -403,7 +403,7 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
403
uuid_unparse(*host->node_id, node_uuid);
404
405
if(after != 0 && before != 0)
406
- rrdr_relative_window_to_absolute(&after, &before);
406
+ rrdr_relative_window_to_absolute(&after, &before, NULL);
407
408
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
409
buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
database/contexts/query_scope.c
+26
-12
@@ -16,17 +16,27 @@ uint64_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN
16
if(host->node_id)
17
uuid_unparse_lower(*host->node_id, host_uuid_buffer);
18
19
- if(!scope_hosts_sp ||
20
- simple_pattern_matches_string(scope_hosts_sp, host->hostname) ||
21
- simple_pattern_matches(scope_hosts_sp, host->machine_guid) ||
22
- (*host_uuid_buffer && simple_pattern_matches(scope_hosts_sp, host_uuid_buffer))) {
23
-
24
- bool queryable_host = false;
25
- if(!hosts_sp ||
26
- simple_pattern_matches_string(hosts_sp, host->hostname) ||
27
- simple_pattern_matches(hosts_sp, host->machine_guid) ||
28
- (*host_uuid_buffer && simple_pattern_matches(hosts_sp, host_uuid_buffer)))
29
- queryable_host = true;
19
+ SIMPLE_PATTERN_RESULT ret = SP_MATCHED_POSITIVE;
20
+ if(scope_hosts_sp) {
21
+ ret = simple_pattern_matches_string_extract(scope_hosts_sp, host->hostname, NULL, 0);
22
+ if(ret == SP_NOT_MATCHED) {
23
+ ret = simple_pattern_matches_extract(scope_hosts_sp, host->machine_guid, NULL, 0);
24
+ if(ret == SP_NOT_MATCHED && *host_uuid_buffer)
25
+ ret = simple_pattern_matches_extract(scope_hosts_sp, host_uuid_buffer, NULL, 0);
26
+ }
27
+ }
28
+
29
+ if(ret == SP_MATCHED_POSITIVE) {
30
+ if(hosts_sp) {
31
+ ret = simple_pattern_matches_string_extract(hosts_sp, host->hostname, NULL, 0);
32
+ if(ret == SP_NOT_MATCHED) {
33
+ ret = simple_pattern_matches_extract(hosts_sp, host->machine_guid, NULL, 0);
34
+ if(ret == SP_NOT_MATCHED && *host_uuid_buffer)
35
+ ret = simple_pattern_matches_extract(hosts_sp, host_uuid_buffer, NULL, 0);
36
+ }
37
+ }
38
+
39
+ bool queryable_host = (ret == SP_MATCHED_POSITIVE);
40
41
count++;
42
v_hash += dictionary_version(host->rrdctx.contexts);
@@ -48,7 +58,11 @@ uint64_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN
58
size_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp, SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data) {
59
size_t added = 0;
60
51
- RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, scope_contexts);
61
+ RRDCONTEXT_ACQUIRED *rca = NULL;
62
+
63
+ if(scope_contexts)
64
+ rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, scope_contexts);
65
+
66
if(likely(rca)) {
67
// we found it!
68
database/contexts/query_target.c
+62
-44
@@ -2,7 +2,11 @@
2
3
#include "internal.h"
4
5
-static void query_metric_release(QUERY_METRIC *qm);
5
+#define QUERY_TARGET_MAX_REALLOC_INCREASE 500
6
+#define query_target_realloc_size(size, start) \
7
+ (size) ? ((size) < QUERY_TARGET_MAX_REALLOC_INCREASE ? (size) * 2 : (size) + QUERY_TARGET_MAX_REALLOC_INCREASE) : (start);
8
+
9
+static void query_metric_release(QUERY_TARGET *qt, QUERY_METRIC *qm);
10
static void query_dimension_release(QUERY_DIMENSION *qd);
11
static void query_instance_release(QUERY_INSTANCE *qi);
12
static void query_context_release(QUERY_CONTEXT *qc);
@@ -72,7 +76,7 @@ void query_target_release(QUERY_TARGET *qt) {
76
// release the query
77
for(size_t i = 0, used = qt->query.used; i < used ;i++) {
78
QUERY_METRIC *qm = query_metric(qt, i);
75
- query_metric_release(qm);
79
+ query_metric_release(qt, qm);
80
}
81
qt->query.used = 0;
82
@@ -151,26 +155,20 @@ void query_target_free(void) {
155
(((first_entry_s) - ((update_every_s) * 2) <= (qt)->window.before) && \
156
((last_entry_s) + ((update_every_s) * 2) >= (qt)->window.after))
157
158
+struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
159
+ QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
160
+ return qn->rrdhost->db[tier].eng;
161
+}
162
155
-static inline void query_metric_release(QUERY_METRIC *qm) {
156
- // reset the plans
157
- for(size_t p = 0; p < qm->plan.used; p++) {
158
- internal_fatal(qm->plan.array[p].initialized &&
159
- !qm->plan.array[p].finalized,
160
- "QUERY: left-over initialized plan");
161
-
162
- qm->plan.array[p].initialized = false;
163
- qm->plan.array[p].finalized = false;
164
- }
163
+static inline void query_metric_release(QUERY_TARGET *qt, QUERY_METRIC *qm) {
164
qm->plan.used = 0;
165
166
// reset the tiers
167
for(size_t tier = 0; tier < storage_tiers ;tier++) {
168
if(qm->tiers[tier].db_metric_handle) {
170
- STORAGE_ENGINE *eng = qm->tiers[tier].eng;
169
+ STORAGE_ENGINE *eng = query_metric_storage_engine(qt, qm, tier);
170
eng->api.metric_release(qm->tiers[tier].db_metric_handle);
171
qm->tiers[tier].db_metric_handle = NULL;
173
- qm->tiers[tier].eng = NULL;
172
}
173
}
174
}
@@ -253,7 +251,7 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
251
252
if (qt->query.used == qt->query.size) {
253
size_t old_mem = qt->query.size * sizeof(*qt->query.array);
256
- qt->query.size = (qt->query.size) ? qt->query.size * 2 : 1;
254
+ qt->query.size = query_target_realloc_size(qt->query.size, 4);
255
size_t new_mem = qt->query.size * sizeof(*qt->query.array);
256
qt->query.array = reallocz(qt->query.array, new_mem);
257
@@ -264,7 +262,7 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
262
263
qm->status = options;
264
267
- qm->link.query_host_id = qn->slot;
265
+ qm->link.query_node_id = qn->slot;
266
qm->link.query_context_id = qc->slot;
267
qm->link.query_instance_id = qi->slot;
268
qm->link.query_dimension_id = qd_slot;
@@ -276,7 +274,7 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
274
qt->db.last_time_s = common_last_time_s;
275
276
for (size_t tier = 0; tier < storage_tiers; tier++) {
279
- qm->tiers[tier].eng = tier_retention[tier].eng;
277
+ internal_fatal(tier_retention[tier].eng != query_metric_storage_engine(qt, qm, tier), "QUERY TARGET: storage engine mismatch");
278
qm->tiers[tier].db_metric_handle = tier_retention[tier].db_metric_handle;
279
qm->tiers[tier].db_first_time_s = tier_retention[tier].db_first_time_s;
280
qm->tiers[tier].db_last_time_s = tier_retention[tier].db_last_time_s;
@@ -285,15 +283,14 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
283
284
return true;
285
}
288
- else {
289
- // cleanup anything we allocated to the retention we will not use
290
- for(size_t tier = 0; tier < storage_tiers ;tier++) {
291
- if (tier_retention[tier].db_metric_handle)
292
- tier_retention[tier].eng->api.metric_release(tier_retention[tier].db_metric_handle);
293
- }
286
295
- return false;
287
+ // cleanup anything we allocated to the retention we will not use
288
+ for(size_t tier = 0; tier < storage_tiers ;tier++) {
289
+ if (tier_retention[tier].db_metric_handle)
290
+ tier_retention[tier].eng->api.metric_release(tier_retention[tier].db_metric_handle);
291
}
292
+
293
+ return false;
294
}
295
296
static inline bool rrdmetric_retention_matches_query(QUERY_TARGET *qt, RRDMETRIC *rm, time_t now_s) {
@@ -311,7 +308,7 @@ static inline void query_dimension_release(QUERY_DIMENSION *qd) {
308
static QUERY_DIMENSION *query_dimension_allocate(QUERY_TARGET *qt, RRDMETRIC_ACQUIRED *rma, QUERY_STATUS status) {
309
if(qt->dimensions.used == qt->dimensions.size) {
310
size_t old_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
314
- qt->dimensions.size = (qt->dimensions.size) ? qt->dimensions.size * 2 : 1;
311
+ qt->dimensions.size = query_target_realloc_size(qt->dimensions.size, 4);
312
size_t new_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
313
qt->dimensions.array = reallocz(qt->dimensions.array, new_mem);
314
@@ -355,10 +352,15 @@ static bool query_dimension_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_
352
if (qt->query.pattern) {
353
// the user asked for specific dimensions
354
358
- if ((qtl->match_ids && simple_pattern_matches_string(qt->query.pattern, rm->id)) ||
359
- (qtl->match_names && rm->name != rm->id &&
360
- simple_pattern_matches_string(qt->query.pattern, rm->name))) {
361
- // the user asked for this dimension
355
+ SIMPLE_PATTERN_RESULT ret = SP_NOT_MATCHED;
356
+
357
+ if(qtl->match_ids)
358
+ ret = simple_pattern_matches_string_extract(qt->query.pattern, rm->id, NULL, 0);
359
+
360
+ if(ret == SP_NOT_MATCHED && qtl->match_names && (rm->name != rm->id || !qtl->match_ids))
361
+ ret = simple_pattern_matches_string_extract(qt->query.pattern, rm->name, NULL, 0);
362
+
363
+ if(ret == SP_MATCHED_POSITIVE) {
364
needed = true;
365
options |= RRDR_DIMENSION_SELECTED | RRDR_DIMENSION_NONZERO;
366
}
@@ -568,12 +570,16 @@ static bool query_target_match_alert_pattern(QUERY_INSTANCE *qi, SIMPLE_PATTERN
570
netdata_rwlock_rdlock(&st->alerts.rwlock);
571
if (st->alerts.base) {
572
for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
571
- if(simple_pattern_matches_string(pattern, rc->name)) {
573
+ SIMPLE_PATTERN_RESULT ret = simple_pattern_matches_string_extract(pattern, rc->name, NULL, 0);
574
+
575
+ if(ret == SP_MATCHED_POSITIVE) {
576
matched = true;
577
break;
578
}
579
+ else if(ret == SP_MATCHED_NEGATIVE)
580
+ break;
581
576
- if(!wb)
582
+ if (!wb)
583
wb = buffer_create(0, NULL);
584
else
585
buffer_flush(wb);
@@ -582,10 +588,14 @@ static bool query_target_match_alert_pattern(QUERY_INSTANCE *qi, SIMPLE_PATTERN
588
buffer_fast_strcat(wb, ":", 1);
589
buffer_strcat(wb, rrdcalc_status2string(rc->status));
590
585
- if(simple_pattern_matches_buffer(pattern, wb)) {
591
+ ret = simple_pattern_matches_buffer_extract(pattern, wb, NULL, 0);
592
+
593
+ if(ret == SP_MATCHED_POSITIVE) {
594
matched = true;
595
break;
596
}
597
+ else if(ret == SP_MATCHED_NEGATIVE)
598
+ break;
599
}
600
}
601
netdata_rwlock_unlock(&st->alerts.rwlock);
@@ -608,7 +618,7 @@ static inline void query_instance_release(QUERY_INSTANCE *qi) {
618
static inline QUERY_INSTANCE *query_instance_allocate(QUERY_TARGET *qt, RRDINSTANCE_ACQUIRED *ria, size_t qn_slot) {
619
if(qt->instances.used == qt->instances.size) {
620
size_t old_mem = qt->instances.size * sizeof(*qt->instances.array);
611
- qt->instances.size = (qt->instances.size) ? qt->instances.size * 2 : 1;
621
+ qt->instances.size = query_target_realloc_size(qt->instances.size, 2);
622
size_t new_mem = qt->instances.size * sizeof(*qt->instances.array);
623
qt->instances.array = reallocz(qt->instances.array, new_mem);
624
@@ -638,14 +648,22 @@ static bool query_instance_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_C
648
qt->db.minimum_latest_update_every_s = ri->update_every_s;
649
650
if(queryable_instance && filter_instances) {
641
- queryable_instance = false;
642
- if(!qt->instances.pattern
643
- || (qtl->match_ids && simple_pattern_matches_string(qt->instances.pattern, ri->id))
644
- || (qtl->match_names && ri->name != ri->id && simple_pattern_matches_string(qt->instances.pattern, ri->name))
645
- || (qtl->match_ids && simple_pattern_matches_string(qt->instances.pattern, query_instance_id_fqdn(qt, qi)))
646
- || (qtl->match_names && simple_pattern_matches_string(qt->instances.pattern, query_instance_name_fqdn(qt, qi)))
647
- )
648
- queryable_instance = true;
651
+ SIMPLE_PATTERN_RESULT ret = SP_MATCHED_POSITIVE;
652
+
653
+ if(qt->instances.pattern) {
654
+ ret = SP_NOT_MATCHED;
655
+
656
+ if(qtl->match_ids)
657
+ ret = simple_pattern_matches_string_extract(qt->instances.pattern, ri->id, NULL, 0);
658
+ if (ret == SP_NOT_MATCHED && qtl->match_names && (ri->name != ri->id || !qtl->match_ids))
659
+ ret = simple_pattern_matches_string_extract(qt->instances.pattern, ri->name, NULL, 0);
660
+ if (ret == SP_NOT_MATCHED && qtl->match_ids)
661
+ ret = simple_pattern_matches_string_extract(qt->instances.pattern, query_instance_id_fqdn(qt, qi), NULL, 0);
662
+ if (ret == SP_NOT_MATCHED && qtl->match_names)
663
+ ret = simple_pattern_matches_string_extract(qt->instances.pattern, query_instance_name_fqdn(qt, qi), NULL, 0);
664
+ }
665
+
666
+ queryable_instance = (ret == SP_MATCHED_POSITIVE);
667
}
668
669
if(queryable_instance) {
@@ -707,7 +725,7 @@ static inline void query_context_release(QUERY_CONTEXT *qc) {
725
static inline QUERY_CONTEXT *query_context_allocate(QUERY_TARGET *qt, RRDCONTEXT_ACQUIRED *rca) {
726
if(qt->contexts.used == qt->contexts.size) {
727
size_t old_mem = qt->contexts.size * sizeof(*qt->contexts.array);
710
- qt->contexts.size = (qt->contexts.size) ? qt->contexts.size * 2 : 1;
728
+ qt->contexts.size = query_target_realloc_size(qt->contexts.size, 2);
729
size_t new_mem = qt->contexts.size * sizeof(*qt->contexts.array);
730
qt->contexts.array = reallocz(qt->contexts.array, new_mem);
731
@@ -766,7 +784,7 @@ static inline void query_node_release(QUERY_NODE *qn) {
784
static inline QUERY_NODE *query_node_allocate(QUERY_TARGET *qt, RRDHOST *host) {
785
if(qt->nodes.used == qt->nodes.size) {
786
size_t old_mem = qt->nodes.size * sizeof(*qt->nodes.array);
769
- qt->nodes.size = (qt->nodes.size) ? qt->nodes.size * 2 : 1;
787
+ qt->nodes.size = query_target_realloc_size(qt->nodes.size, 2);
788
size_t new_mem = qt->nodes.size * sizeof(*qt->nodes.array);
789
qt->nodes.array = reallocz(qt->nodes.array, new_mem);
790
@@ -952,7 +970,7 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
970
query_target_generate_name(qt);
971
qt->window.after = qt->request.after;
972
qt->window.before = qt->request.before;
955
- rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before);
973
+ rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now);
974
975
// prepare our local variables - we need these across all these functions
976
QUERY_TARGET_LOCALS qtl = {
database/contexts/rrdcontext.h
+5
-11
@@ -149,17 +149,9 @@ typedef struct query_plan_entry {
149
size_t tier;
150
time_t after;
151
time_t before;
152
- time_t expanded_after;
153
- time_t expanded_before;
154
- struct storage_engine_query_handle handle;
155
- STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
156
- int (*is_finished)(struct storage_engine_query_handle *handle);
157
- void (*finalize)(struct storage_engine_query_handle *handle);
158
- bool initialized;
159
- bool finalized;
152
} QUERY_PLAN_ENTRY;
153
162
-#define QUERY_PLANS_MAX (RRD_STORAGE_TIERS * 2)
154
+#define QUERY_PLANS_MAX (RRD_STORAGE_TIERS)
155
156
struct query_metrics_counts {
157
size_t selected;
@@ -234,7 +226,6 @@ typedef struct query_metric {
226
RRDR_DIMENSION_FLAGS status;
227
228
struct query_metric_tier {
237
- struct storage_engine *eng;
229
STORAGE_METRIC_HANDLE *db_metric_handle;
230
time_t db_first_time_s; // the oldest timestamp available for this tier
231
time_t db_last_time_s; // the latest timestamp available for this tier
@@ -248,7 +239,7 @@ typedef struct query_metric {
239
} plan;
240
241
struct {
251
- uint32_t query_host_id;
242
+ uint32_t query_node_id;
243
uint32_t query_context_id;
244
uint32_t query_instance_id;
245
uint32_t query_dimension_id;
@@ -334,6 +325,7 @@ typedef struct query_target {
325
size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries
326
327
struct {
328
+ time_t now; // the current timestamp, the absolute max for any query timestamp
329
bool relative; // true when the request made with relative timestamps, true if it was absolute
330
bool aligned;
331
time_t after; // the absolute timestamp this query is about
@@ -452,6 +444,8 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)
444
return rrdmetric_acquired_name(qd->rma);
445
}
446
447
+struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
448
+
449
STRING *query_instance_id_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);
450
STRING *query_instance_name_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);
451
database/contexts/worker.c
+5
-2
@@ -1069,15 +1069,18 @@ void *rrdcontext_main(void *ptr) {
1069
worker_is_busy(WORKER_JOB_HOSTS);
1070
1071
if(host->rrdctx.pp_queue) {
1072
- pp_queued_contexts_for_all_hosts +=
1073
- dictionary_entries(host->rrdctx.pp_queue);
1072
+ pp_queued_contexts_for_all_hosts += dictionary_entries(host->rrdctx.pp_queue);
1073
rrdcontext_post_process_queued_contexts(host);
1074
+ dictionary_garbage_collect(host->rrdctx.pp_queue);
1075
}
1076
1077
if(host->rrdctx.hub_queue) {
1078
hub_queued_contexts_for_all_hosts += dictionary_entries(host->rrdctx.hub_queue);
1079
rrdcontext_dispatch_queued_contexts_to_hub(host, now_ut);
1080
+ dictionary_garbage_collect(host->rrdctx.hub_queue);
1081
}
1082
+
1083
+ dictionary_garbage_collect(host->rrdctx.contexts);
1084
}
1085
dfe_done(host);
1086
database/rrdlabels.c
+1
-1
@@ -875,7 +875,7 @@ static int simple_pattern_match_name_and_value_callback(const DICTIONARY_ITEM *i
875
*dst = '\0';
876
877
t->searches++;
878
- if(simple_pattern_matches_length_extract(t->pattern, tmp, dst - tmp, NULL, 0))
878
+ if(simple_pattern_matches_length_extract(t->pattern, tmp, dst - tmp, NULL, 0) == SP_MATCHED_POSITIVE)
879
return -1;
880
881
return 0;
libnetdata/buffer/buffer.h
+9
@@ -732,6 +732,15 @@ static inline void buffer_json_add_array_item_time_t(BUFFER *wb, time_t value) {
732
wb->json.stack[wb->json.depth].count++;
733
}
734
735
+static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value) {
736
+ if(wb->json.stack[wb->json.depth].count)
737
+ buffer_fast_strcat(wb, ",", 1);
738
+
739
+ buffer_print_int64(wb, value);
740
+ buffer_fast_strcat(wb, "000", 3);
741
+ wb->json.stack[wb->json.depth].count++;
742
+}
743
+
744
static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value) {
745
if(wb->json.stack[wb->json.depth].count)
746
buffer_fast_strcat(wb, ",", 1);
libnetdata/dictionary/dictionary.c
+6
@@ -811,6 +811,10 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {
811
examined, deleted, pending);
812
}
813
814
+void dictionary_garbage_collect(DICTIONARY *dict) {
815
+ garbage_collect_pending_deletes(dict);
816
+}
817
+
818
// ----------------------------------------------------------------------------
819
// reference counters
820
@@ -2149,6 +2153,8 @@ DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(D
2153
if(unlikely(is_master_dictionary(dict)))
2154
fatal("DICTIONARY: this dictionary is a master, you cannot add items from other dictionaries.");
2155
2156
+ garbage_collect_pending_deletes(dict);
2157
+
2158
dictionary_acquired_item_dup(dict->master, master_item);
2159
DICTIONARY_ITEM *item = dict_item_add_or_reset_value_and_acquire(dict, name, name_len, NULL, 0, NULL, master_item);
2160
dictionary_acquired_item_release(dict->master, master_item);
libnetdata/dictionary/dictionary.h
+2
@@ -156,6 +156,8 @@ void dictionary_flush(DICTIONARY *dict);
156
157
void dictionary_version_increment(DICTIONARY *dict);
158
159
+void dictionary_garbage_collect(DICTIONARY *dict);
160
+
161
// ----------------------------------------------------------------------------
162
// Set an item in the dictionary
163
//
libnetdata/simple_pattern/simple_pattern.c
+12
-12
@@ -265,7 +265,7 @@ static inline bool match_pattern(struct simple_pattern *m, const char *str, size
265
return false;
266
}
267
268
-static inline int simple_pattern_matches_extract_with_length(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
268
+static inline SIMPLE_PATTERN_RESULT simple_pattern_matches_extract_with_length(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
269
struct simple_pattern *m, *root = (struct simple_pattern *)list;
270
271
for(m = root; m ; m = m->next) {
@@ -274,31 +274,31 @@ static inline int simple_pattern_matches_extract_with_length(SIMPLE_PATTERN *lis
274
if(unlikely(ws)) *ws = '\0';
275
276
if (match_pattern(m, str, len, ws, &wss)) {
277
- if (m->negative) return 0;
278
- return 1;
277
+ if (m->negative) return SP_MATCHED_NEGATIVE;
278
+ return SP_MATCHED_POSITIVE;
279
}
280
}
281
282
- return 0;
282
+ return SP_NOT_MATCHED;
283
}
284
285
-int simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size) {
286
- if(!list || !str || buffer_strlen(str)) return 0;
285
+SIMPLE_PATTERN_RESULT simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size) {
286
+ if(!list || !str || buffer_strlen(str)) return SP_NOT_MATCHED;
287
return simple_pattern_matches_extract_with_length(list, buffer_tostring(str), buffer_strlen(str), wildcarded, wildcarded_size);
288
}
289
290
-int simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, STRING *str, char *wildcarded, size_t wildcarded_size) {
291
- if(!list || !str) return 0;
290
+SIMPLE_PATTERN_RESULT simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, STRING *str, char *wildcarded, size_t wildcarded_size) {
291
+ if(!list || !str) return SP_NOT_MATCHED;
292
return simple_pattern_matches_extract_with_length(list, string2str(str), string_strlen(str), wildcarded, wildcarded_size);
293
}
294
295
-int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) {
296
- if(!list || !str || !*str) return 0;
295
+SIMPLE_PATTERN_RESULT simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) {
296
+ if(!list || !str || !*str) return SP_NOT_MATCHED;
297
return simple_pattern_matches_extract_with_length(list, str, strlen(str), wildcarded, wildcarded_size);
298
}
299
300
-int simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
301
- if(!list || !str || !*str || !len) return 0;
300
+SIMPLE_PATTERN_RESULT simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
301
+ if(!list || !str || !*str || !len) return SP_NOT_MATCHED;
302
return simple_pattern_matches_extract_with_length(list, str, len, wildcarded, wildcarded_size);
303
}
304
libnetdata/simple_pattern/simple_pattern.h
+15
-9
@@ -13,6 +13,12 @@ typedef enum __attribute__ ((__packed__)) {
13
SIMPLE_PATTERN_SUBSTRING
14
} SIMPLE_PREFIX_MODE;
15
16
+typedef enum __attribute__ ((__packed__)) {
17
+ SP_NOT_MATCHED,
18
+ SP_MATCHED_NEGATIVE,
19
+ SP_MATCHED_POSITIVE,
20
+} SIMPLE_PATTERN_RESULT;
21
+
22
typedef void SIMPLE_PATTERN;
23
24
// create a simple_pattern from the string given
@@ -20,18 +26,18 @@ typedef void SIMPLE_PATTERN;
26
// should be considered PREFIX matches.
27
SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode, bool case_sensitive);
28
23
-// test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*'
24
-int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size);
25
-
29
struct netdata_string;
27
-int simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, struct netdata_string *str, char *wildcarded, size_t wildcarded_size);
28
-int simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size);
29
-int simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size);
30
+
31
+// test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*'
32
+SIMPLE_PATTERN_RESULT simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size);
33
+SIMPLE_PATTERN_RESULT simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, struct netdata_string *str, char *wildcarded, size_t wildcarded_size);
34
+SIMPLE_PATTERN_RESULT simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size);
35
+SIMPLE_PATTERN_RESULT simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size);
36
37
// test if string str is matched from the pattern
32
-#define simple_pattern_matches(list, str) simple_pattern_matches_extract(list, str, NULL, 0)
33
-#define simple_pattern_matches_string(list, str) simple_pattern_matches_string_extract(list, str, NULL, 0)
34
-#define simple_pattern_matches_buffer(list, str) simple_pattern_matches_buffer_extract(list, str, NULL, 0)
38
+#define simple_pattern_matches(list, str) (simple_pattern_matches_extract(list, str, NULL, 0) == SP_MATCHED_POSITIVE)
39
+#define simple_pattern_matches_string(list, str) (simple_pattern_matches_string_extract(list, str, NULL, 0) == SP_MATCHED_POSITIVE)
40
+#define simple_pattern_matches_buffer(list, str) (simple_pattern_matches_buffer_extract(list, str, NULL, 0) == SP_MATCHED_POSITIVE)
41
42
// free a simple_pattern that was created with simple_pattern_create()
43
// list can be NULL, in which case, this does nothing.
web/api/formatters/csv/csv.c
+5
-7
@@ -84,7 +84,6 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
84
buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
85
}
86
87
- int set_min_max = 0;
87
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
88
total = 0;
89
for(c = 0; c < used ;c++) {
@@ -99,7 +98,6 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
98
}
99
// prevent a division by zero
100
if(total == 0) total = 1;
102
- set_min_max = 1;
101
}
102
103
// for each dimension
@@ -124,13 +122,13 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
122
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
123
n = n * 100 / total;
124
127
- if(unlikely(set_min_max)) {
125
+ if(unlikely(i == start && c == 0)) {
126
r->view.min = r->view.max = n;
129
- set_min_max = 0;
127
}
131
-
132
- if(n < r->view.min) r->view.min = n;
133
- if(n > r->view.max) r->view.max = n;
128
+ else {
129
+ if (n < r->view.min) r->view.min = n;
130
+ if (n > r->view.max) r->view.max = n;
131
+ }
132
}
133
134
buffer_print_netdata_double(wb, n);
web/api/formatters/json/json.c
+7
-9
@@ -211,8 +211,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
211
buffer_fast_strcat(wb, post_date, post_date_len);
212
}
213
214
- int set_min_max = 0;
215
- if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
214
+ if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_GBC|RRDR_OPTION_INTERNAL_AR)))) {
215
total = 0;
216
for(c = 0; c < used ;c++) {
217
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
@@ -232,7 +231,6 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
231
}
232
// prevent a division by zero
233
if(total == 0) total = 1;
235
- set_min_max = 1;
234
}
235
236
// for each dimension
@@ -263,16 +261,16 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
261
if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
262
n = -n;
263
266
- if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
264
+ if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_GBC|RRDR_OPTION_INTERNAL_AR)))) {
265
n = n * 100 / total;
266
269
- if(unlikely(set_min_max)) {
267
+ if(unlikely(i == start && c == 0)) {
268
r->view.min = r->view.max = n;
271
- set_min_max = 0;
269
}
273
-
274
- if(n < r->view.min) r->view.min = n;
275
- if(n > r->view.max) r->view.max = n;
270
+ else {
271
+ if (n < r->view.min) r->view.min = n;
272
+ if (n > r->view.max) r->view.max = n;
273
+ }
274
}
275
276
buffer_print_netdata_double(wb, n);
web/api/formatters/json_wrapper.c
+184
-52
@@ -213,7 +213,7 @@ static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, st
213
return;
214
215
buffer_json_member_add_object(wb, "sts");
216
- if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
216
+ if(query_target_aggregatable(qt)) {
217
buffer_json_member_add_uint64(wb, "cnt", d->group_points);
218
219
if(d->sum != 0.0)
@@ -801,7 +801,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR
801
buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points);
802
buffer_json_array_close(wb);
803
804
- if(options & RRDR_OPTION_SHOW_PLAN)
804
+ if(options & RRDR_OPTION_DEBUG)
805
jsonwrap_query_plan(r, wb);
806
}
807
@@ -1061,7 +1061,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
1061
1062
query_target_data_statistics(wb, qt, &qm->query_stats);
1063
1064
- if(options & RRDR_OPTION_SHOW_PLAN)
1064
+ if(options & RRDR_OPTION_DEBUG)
1065
jsonwrap_query_metric_plan(wb, qm);
1066
}
1067
}
@@ -1248,6 +1248,13 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1248
buffer_json_member_add_time_t(wb, "update_every", r->view.update_every);
1249
buffer_json_member_add_time_t(wb, "after", r->view.after);
1250
buffer_json_member_add_time_t(wb, "before", r->view.before);
1251
+
1252
+ buffer_json_member_add_object(wb, "partial_data_trimming");
1253
+ buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every);
1254
+ buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after);
1255
+ buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after);
1256
+ buffer_json_object_close(wb);
1257
+
1258
buffer_json_member_add_uint64(wb, "points", rows);
1259
query_target_combined_units_v2(wb, qt, contexts);
1260
query_target_combined_chart_type(wb, qt, contexts);
@@ -1266,77 +1273,202 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1273
buffer_json_object_close(wb);
1274
}
1275
1269
-static void annotations_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) {
1270
- const size_t dims = r->d, rows = r->rows;
1271
- size_t next_d_idx = 0;
1272
- for(size_t d = 0; d < dims ; d++) {
1276
+//static void annotations_range_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) {
1277
+// const size_t dims = r->d, rows = r->rows;
1278
+// size_t next_d_idx = 0;
1279
+// for(size_t d = 0; d < dims ; d++) {
1280
+// if(!rrdr_dimension_should_be_exposed(r->od[d], options))
1281
+// continue;
1282
+//
1283
+// size_t d_idx = next_d_idx++;
1284
+//
1285
+// size_t t = 0;
1286
+// while(t < rows) {
1287
+//
1288
+// // find the beginning
1289
+// time_t started = 0;
1290
+// for(; t < rows ;t++) {
1291
+// RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
1292
+// if(o & flags) {
1293
+// started = r->t[t];
1294
+// break;
1295
+// }
1296
+// }
1297
+//
1298
+// if(started) {
1299
+// time_t ended = 0;
1300
+// for(; t < rows ;t++) {
1301
+// RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
1302
+// if(!(o & flags)) {
1303
+// ended = r->t[t];
1304
+// break;
1305
+// }
1306
+// }
1307
+//
1308
+// if(!ended)
1309
+// ended = r->t[rows - 1];
1310
+//
1311
+// buffer_json_add_array_item_object(wb);
1312
+// buffer_json_member_add_string(wb, "t", type);
1313
+// // buffer_json_member_add_string(wb, "d", string2str(r->dn[d]));
1314
+// buffer_json_member_add_uint64(wb, "d", d_idx);
1315
+// if(started == ended) {
1316
+// if(options & RRDR_OPTION_MILLISECONDS)
1317
+// buffer_json_member_add_time_t2ms(wb, "x", started);
1318
+// else
1319
+// buffer_json_member_add_time_t(wb, "x", started);
1320
+// }
1321
+// else {
1322
+// buffer_json_member_add_array(wb, "x");
1323
+// if(options & RRDR_OPTION_MILLISECONDS) {
1324
+// buffer_json_add_array_item_time_t2ms(wb, started);
1325
+// buffer_json_add_array_item_time_t2ms(wb, ended);
1326
+// }
1327
+// else {
1328
+// buffer_json_add_array_item_time_t(wb, started);
1329
+// buffer_json_add_array_item_time_t(wb, ended);
1330
+// }
1331
+// buffer_json_array_close(wb);
1332
+// }
1333
+// buffer_json_object_close(wb);
1334
+// }
1335
+// }
1336
+// }
1337
+//}
1338
+//
1339
+//void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
1340
+// buffer_json_member_add_array(wb, "annotations");
1341
+//
1342
+// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap
1343
+// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow
1344
+// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial
1345
+//
1346
+// buffer_json_array_close(wb); // annotations
1347
+//}
1348
+
1349
+void rrdr2json_v2(RRDR *r __maybe_unused, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
1350
+ bool expose_gbc = query_target_aggregatable(r->internal.qt);
1351
+
1352
+ buffer_json_member_add_object(wb, "result");
1353
+
1354
+ buffer_json_member_add_array(wb, "labels");
1355
+ buffer_json_add_array_item_string(wb, "time");
1356
+ long d, i;
1357
+ const long used = (long)r->d;
1358
+ for(d = 0, i = 0; d < used ; d++) {
1359
if(!rrdr_dimension_should_be_exposed(r->od[d], options))
1360
continue;
1361
1276
- size_t d_idx = next_d_idx++;
1362
+ buffer_json_add_array_item_string(wb, string2str(r->dn[d]));
1363
+ i++;
1364
+ }
1365
+ buffer_json_array_close(wb); // labels
1366
+
1367
+ buffer_json_member_add_object(wb, "point");
1368
+ buffer_json_member_add_uint64(wb, "value", 0);
1369
+ buffer_json_member_add_uint64(wb, "ar", 1);
1370
+ buffer_json_member_add_uint64(wb, "pa", 2);
1371
+ if(expose_gbc)
1372
+ buffer_json_member_add_uint64(wb, "count", 3);
1373
+ buffer_json_object_close(wb);
1374
+
1375
+ buffer_json_member_add_array(wb, "data");
1376
+ if(i) {
1377
+ long start = 0, end = rrdr_rows(r), step = 1;
1378
+ if (!(options & RRDR_OPTION_REVERSED)) {
1379
+ start = rrdr_rows(r) - 1;
1380
+ end = -1;
1381
+ step = -1;
1382
+ }
1383
+
1384
+ // for each line in the array
1385
+ for (i = start; i != end; i += step) {
1386
+ NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
1387
+ RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
1388
+ NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
1389
+ uint32_t *gbc = &r->gbc [ i * r->d ];
1390
+ time_t now = r->t[i];
1391
+
1392
+ buffer_json_add_array_item_array(wb); // row
1393
+
1394
+ if (options & RRDR_OPTION_MILLISECONDS)
1395
+ buffer_json_add_array_item_time_ms(wb, now); // the time
1396
+ else
1397
+ buffer_json_add_array_item_time_t(wb, now); // the time
1398
+
1399
+ NETDATA_DOUBLE total = 1;
1400
+ if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_GBC|RRDR_OPTION_INTERNAL_AR)))) {
1401
+ total = 0;
1402
+ for(d = 0; d < used ; d++) {
1403
+ if(unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) continue;
1404
1278
- size_t t = 0;
1279
- while(t < rows) {
1405
+ NETDATA_DOUBLE n = cn[d];
1406
+ if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1407
+ n = -n;
1408
1281
- // find the beginning
1282
- time_t started = 0;
1283
- for(; t < rows ;t++) {
1284
- RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
1285
- if(o & flags) {
1286
- started = r->t[t];
1287
- break;
1409
+ total += n;
1410
}
1411
+
1412
+ // prevent a division by zero
1413
+ if(total == 0) total = 1;
1414
}
1415
1291
- if(started) {
1292
- time_t ended = 0;
1293
- for(; t < rows ;t++) {
1294
- RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
1295
- if(!(o & flags)) {
1296
- ended = r->t[t];
1297
- break;
1298
- }
1299
- }
1416
+ for (d = 0; d < used; d++) {
1417
+ if (!rrdr_dimension_should_be_exposed(r->od[d], options))
1418
+ continue;
1419
+
1420
+ RRDR_VALUE_FLAGS o = co[d];
1421
1301
- if(!ended)
1302
- ended = r->t[rows - 1];
1422
+ buffer_json_add_array_item_array(wb); // point
1423
1304
- buffer_json_add_array_item_object(wb);
1305
- buffer_json_member_add_string(wb, "t", type);
1306
- // buffer_json_member_add_string(wb, "d", string2str(r->dn[d]));
1307
- buffer_json_member_add_uint64(wb, "d", d_idx);
1308
- if(started == ended) {
1309
- if(options & RRDR_OPTION_MILLISECONDS)
1310
- buffer_json_member_add_time_t2ms(wb, "x", started);
1424
+ // add the value
1425
+ NETDATA_DOUBLE n = cn[d];
1426
+
1427
+ if(o & RRDR_VALUE_EMPTY) {
1428
+ if (unlikely(options & RRDR_OPTION_NULL2ZERO))
1429
+ buffer_json_add_array_item_double(wb, 0);
1430
else
1312
- buffer_json_member_add_time_t(wb, "x", started);
1431
+ buffer_json_add_array_item_double(wb, NAN);
1432
}
1433
else {
1315
- buffer_json_member_add_array(wb, "x");
1316
- if(options & RRDR_OPTION_MILLISECONDS) {
1317
- buffer_json_add_array_item_time_t2ms(wb, started);
1318
- buffer_json_add_array_item_time_t2ms(wb, ended);
1434
+ if (unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1435
+ n = -n;
1436
+
1437
+ if (unlikely((options & RRDR_OPTION_PERCENTAGE))) {
1438
+ n = n * 100 / total;
1439
+ }
1440
+
1441
+ if(unlikely(i == start && d == 0)) {
1442
+ r->view.min = r->view.max = n;
1443
}
1444
else {
1321
- buffer_json_add_array_item_time_t(wb, started);
1322
- buffer_json_add_array_item_time_t(wb, ended);
1445
+ if (n < r->view.min) r->view.min = n;
1446
+ if (n > r->view.max) r->view.max = n;
1447
}
1324
- buffer_json_array_close(wb);
1448
+
1449
+ buffer_json_add_array_item_double(wb, n);
1450
}
1326
- buffer_json_object_close(wb);
1451
+
1452
+ // add the anomaly
1453
+ buffer_json_add_array_item_double(wb, ar[d]);
1454
+
1455
+ // add the point annotations
1456
+ buffer_json_add_array_item_uint64(wb, o);
1457
+
1458
+ // add the count
1459
+ if(expose_gbc)
1460
+ buffer_json_add_array_item_uint64(wb, gbc[d]);
1461
+
1462
+ buffer_json_array_close(wb); // point
1463
}
1464
+
1465
+ buffer_json_array_close(wb); // row
1466
}
1467
}
1330
-}
1331
-
1332
-void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
1333
- buffer_json_member_add_array(wb, "annotations");
1468
1335
- annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap
1336
- annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow
1337
- annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial
1469
+ buffer_json_array_close(wb); // data
1470
1339
- buffer_json_array_close(wb); // annotations
1471
+ buffer_json_object_close(wb); // annotations
1472
}
1473
1474
void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options __maybe_unused) {
web/api/formatters/json_wrapper.h
+1
-1
@@ -10,7 +10,7 @@ typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, R
10
typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
11
12
void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method);
13
-void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
13
+void rrdr2json_v2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
14
void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
15
16
void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method);
web/api/formatters/rrd2json.c
+184
-115
@@ -11,6 +11,10 @@ inline bool query_target_has_percentage_units(struct query_target *qt) {
11
return false;
12
}
13
14
+bool query_target_aggregatable(struct query_target *qt) {
15
+ return (qt->request.options & RRDR_OPTION_RETURN_RAW);
16
+}
17
+
18
void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
19
rrdset2json(st, wb, NULL, NULL, 0);
20
}
@@ -20,6 +24,9 @@ const char *rrdr_format_to_string(DATASOURCE_FORMAT format) {
24
case DATASOURCE_JSON:
25
return DATASOURCE_FORMAT_JSON;
26
27
+ case DATASOURCE_JSON2:
28
+ return DATASOURCE_FORMAT_JSON2;
29
+
30
case DATASOURCE_DATATABLE_JSON:
31
return DATASOURCE_FORMAT_DATATABLE_JSON;
32
@@ -174,58 +181,74 @@ static RRDR *data_query_group_by(RRDR *r) {
181
if(!qt->group_by.used)
182
qt->request.group_by &= ~RRDR_GROUP_BY_LABEL;
183
177
- if(!(qt->request.group_by & (RRDR_GROUP_BY_NODE | RRDR_GROUP_BY_INSTANCE | RRDR_GROUP_BY_DIMENSION | RRDR_GROUP_BY_LABEL)))
184
+ if(!(qt->request.group_by & (RRDR_GROUP_BY_NODE | RRDR_GROUP_BY_INSTANCE | RRDR_GROUP_BY_DIMENSION | RRDR_GROUP_BY_LABEL | RRDR_GROUP_BY_SELECTED)))
185
qt->request.group_by = RRDR_GROUP_BY_DIMENSION;
186
187
int added = 0;
188
BUFFER *key = buffer_create(0, NULL);
189
QUERY_INSTANCE *last_qi = NULL;
190
size_t priority = 0;
184
- for(size_t c = 0; c < qt->query.used ;c++) {
185
- if(!rrdr_dimension_should_be_exposed(r->od[c], options))
191
+ time_t update_every_max = 0;
192
+ for(size_t d = 0; d < qt->query.used ; d++) {
193
+ if(unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
194
continue;
195
188
- QUERY_METRIC *qm = query_metric(qt, c);
196
+ QUERY_METRIC *qm = query_metric(qt, d);
197
QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
190
- QUERY_NODE *qn = query_node(qt, qm->link.query_host_id);
198
+ QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
199
200
if(qi != last_qi) {
201
priority = 0;
202
last_qi = qi;
203
+
204
+ time_t update_every = rrdinstance_acquired_update_every(qi->ria);
205
+ if(update_every > update_every_max)
206
+ update_every_max = update_every;
207
}
208
else
209
priority++;
210
211
+ // --------------------------------------------------------------------
212
// generate the group by key
213
214
buffer_flush(key);
202
- if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
203
- buffer_fast_strcat(key, "|", 1);
204
- buffer_strcat(key, query_metric_id(qt, qm));
215
+ if(unlikely(r->od[d] & RRDR_DIMENSION_HIDDEN)) {
216
+ buffer_strcat(key, "__hidden_dimensions__");
217
}
206
- if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
207
- buffer_fast_strcat(key, "|", 1);
208
- buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
218
+ else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
219
+ buffer_strcat(key, "selected");
220
}
210
- if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
211
- DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
212
- for(size_t l = 0; l < qt->group_by.used ;l++) {
221
+ else {
222
+ if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
223
buffer_fast_strcat(key, "|", 1);
214
- rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
224
+ buffer_strcat(key, query_metric_id(qt, qm));
225
}
216
- }
217
- if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
218
- buffer_fast_strcat(key, "|", 1);
219
- buffer_strcat(key, qn->rrdhost->machine_guid);
220
- }
226
222
- // append the units
223
- if(query_target_has_percentage_units(qt)) {
224
- buffer_fast_strcat(key, "|%", 2);
225
- }
226
- else {
227
- buffer_fast_strcat(key, "|", 1);
228
- buffer_strcat(key, rrdinstance_acquired_units(qi->ria));
227
+ if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
228
+ buffer_fast_strcat(key, "|", 1);
229
+ buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
230
+ }
231
+
232
+ if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
233
+ DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
234
+ for (size_t l = 0; l < qt->group_by.used; l++) {
235
+ buffer_fast_strcat(key, "|", 1);
236
+ rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
237
+ }
238
+ }
239
+
240
+ if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
241
+ buffer_fast_strcat(key, "|", 1);
242
+ buffer_strcat(key, qn->rrdhost->machine_guid);
243
+ }
244
+
245
+ // append the units
246
+ if (query_target_has_percentage_units(qt)) {
247
+ buffer_fast_strcat(key, "|%", 2);
248
+ } else {
249
+ buffer_fast_strcat(key, "|", 1);
250
+ buffer_strcat(key, rrdinstance_acquired_units(qi->ria));
251
+ }
252
}
253
254
// lookup the key in the dictionary
@@ -237,66 +260,92 @@ static RRDR *data_query_group_by(RRDR *r) {
260
261
*set = pos = added++;
262
263
+ // ----------------------------------------------------------------
264
// generate the dimension id
265
266
buffer_flush(key);
243
- if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
244
- buffer_strcat(key, query_metric_id(qt, qm));
267
+ if(unlikely(r->od[d] & RRDR_DIMENSION_HIDDEN)) {
268
+ buffer_strcat(key, "__hidden_dimensions__");
269
}
246
- if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
247
- if(buffer_strlen(key) != 0)
248
- buffer_fast_strcat(key, ",", 1);
249
-
250
- if(qt->request.group_by & RRDR_GROUP_BY_NODE)
251
- buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
252
- else
253
- buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
270
+ else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
271
+ buffer_strcat(key, "selected");
272
}
255
- if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
256
- DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
257
- for(size_t l = 0; l < qt->group_by.used ;l++) {
258
- if(buffer_strlen(key) != 0)
273
+ else {
274
+ if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
275
+ buffer_strcat(key, query_metric_id(qt, qm));
276
+ }
277
+
278
+ if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
279
+ if (buffer_strlen(key) != 0)
280
buffer_fast_strcat(key, ",", 1);
260
- rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
281
+
282
+ if (qt->request.group_by & RRDR_GROUP_BY_NODE)
283
+ buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
284
+ else
285
+ buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
286
}
262
- }
263
- if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
264
- if(buffer_strlen(key) != 0)
265
- buffer_fast_strcat(key, ",", 1);
287
267
- buffer_strcat(key, qn->rrdhost->machine_guid);
288
+ if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
289
+ DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
290
+ for (size_t l = 0; l < qt->group_by.used; l++) {
291
+ if (buffer_strlen(key) != 0)
292
+ buffer_fast_strcat(key, ",", 1);
293
+ rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
294
+ }
295
+ }
296
+
297
+ if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
298
+ if (buffer_strlen(key) != 0)
299
+ buffer_fast_strcat(key, ",", 1);
300
+
301
+ buffer_strcat(key, qn->rrdhost->machine_guid);
302
+ }
303
}
304
+
305
entries[pos].id = string_strdupz(buffer_tostring(key));
306
307
+ // ----------------------------------------------------------------
308
// generate the dimension name
309
310
buffer_flush(key);
274
- if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
275
- buffer_strcat(key, query_metric_name(qt, qm));
311
+ if(unlikely(r->od[d] & RRDR_DIMENSION_HIDDEN)) {
312
+ buffer_strcat(key, "__hidden_dimensions__");
313
}
277
- if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
278
- if(buffer_strlen(key) != 0)
279
- buffer_fast_strcat(key, ",", 1);
280
-
281
- if(qt->request.group_by & RRDR_GROUP_BY_NODE)
282
- buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
283
- else
284
- buffer_strcat(key, string2str(query_instance_name_fqdn(qt, qi)));
314
+ else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
315
+ buffer_strcat(key, "selected");
316
}
286
- if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
287
- DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
288
- for(size_t l = 0; l < qt->group_by.used ;l++) {
289
- if(buffer_strlen(key) != 0)
317
+ else {
318
+ if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
319
+ buffer_strcat(key, query_metric_name(qt, qm));
320
+ }
321
+
322
+ if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
323
+ if (buffer_strlen(key) != 0)
324
buffer_fast_strcat(key, ",", 1);
291
- rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
325
+
326
+ if (qt->request.group_by & RRDR_GROUP_BY_NODE)
327
+ buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
328
+ else
329
+ buffer_strcat(key, string2str(query_instance_name_fqdn(qt, qi)));
330
}
293
- }
294
- if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
295
- if(buffer_strlen(key) != 0)
296
- buffer_fast_strcat(key, ",", 1);
331
298
- buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
332
+ if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
333
+ DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
334
+ for (size_t l = 0; l < qt->group_by.used; l++) {
335
+ if (buffer_strlen(key) != 0)
336
+ buffer_fast_strcat(key, ",", 1);
337
+ rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
338
+ }
339
+ }
340
+
341
+ if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
342
+ if (buffer_strlen(key) != 0)
343
+ buffer_fast_strcat(key, ",", 1);
344
+
345
+ buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
346
+ }
347
}
348
+
349
entries[pos].name = string_strdupz(buffer_tostring(key));
350
351
// add the rest of the info
@@ -321,8 +370,8 @@ static RRDR *data_query_group_by(RRDR *r) {
370
// copy the dimension flags decided by the query target
371
// we need this, because if a dimension is explicitly selected
372
// the query target adds to it the non-zero flag
324
- qm->status |= RRDR_DIMENSION_GROUPED | r->od[c];
325
- entries[pos].od |= RRDR_DIMENSION_GROUPED | r->od[c];
373
+ qm->status |= RRDR_DIMENSION_GROUPED | r->od[d];
374
+ entries[pos].od |= RRDR_DIMENSION_GROUPED | r->od[d];
375
}
376
377
// check if we have multiple units
@@ -362,15 +411,22 @@ static RRDR *data_query_group_by(RRDR *r) {
411
r2->stats.result_points_generated = r2->d * r2->n;
412
413
// initialize r2 (dimension options, names, and ids)
365
- for(size_t c2 = 0; c2 < r2->d ; c2++) {
366
- r2->od[c2] = entries[c2].od;
367
- r2->di[c2] = entries[c2].id;
368
- r2->dn[c2] = entries[c2].name;
369
- r2->du[c2] = entries[c2].units;
370
- r2->dp[c2] = entries[c2].priority;
371
- r2->dgbc[c2] = entries[c2].count;
414
+ for(size_t d2 = 0; d2 < r2->d ; d2++) {
415
+ r2->od[d2] = entries[d2].od;
416
+ r2->di[d2] = entries[d2].id;
417
+ r2->dn[d2] = entries[d2].name;
418
+ r2->du[d2] = entries[d2].units;
419
+ r2->dp[d2] = entries[d2].priority;
420
+ r2->dgbc[d2] = entries[d2].count;
421
}
422
423
+ r2->partial_data_trimming.max_update_every = update_every_max;
424
+ r2->partial_data_trimming.expected_after =
425
+ (!(qt->request.options & RRDR_OPTION_RETURN_RAW) && qt->window.before >= qt->window.now - update_every_max) ?
426
+ qt->window.before - update_every_max :
427
+ qt->window.before;
428
+ r2->partial_data_trimming.trimmed_after = qt->window.before;
429
+
430
// initialize r2 (timestamps and value flags)
431
for(size_t i = 0; i != rows ;i++) {
432
// copy the timestamp
@@ -380,15 +436,17 @@ static RRDR *data_query_group_by(RRDR *r) {
436
NETDATA_DOUBLE *cn2 = &r2->v[ i * r2->d ];
437
RRDR_VALUE_FLAGS *co2 = &r2->o[ i * r2->d ];
438
NETDATA_DOUBLE *ar2 = &r2->ar[ i * r2->d ];
383
- for (size_t c2 = 0; c2 < r2->d; c2++) {
384
- cn2[c2] = 0.0;
385
- ar2[c2] = 0.0;
386
- co2[c2] = RRDR_VALUE_EMPTY;
439
+ for (size_t d2 = 0; d2 < r2->d; d2++) {
440
+ cn2[d2] = 0.0;
441
+ ar2[d2] = 0.0;
442
+ co2[d2] = RRDR_VALUE_EMPTY;
443
}
444
}
445
446
// do the group_by
447
+ size_t last_row_gbc = 0;
448
for(size_t i = 0; i != rows ;i++) {
449
+
450
size_t idx = i * r->d;
451
NETDATA_DOUBLE *cn_base = &r->v[ idx ];
452
RRDR_VALUE_FLAGS *co_base = &r->o[ idx ];
@@ -400,13 +458,14 @@ static RRDR *data_query_group_by(RRDR *r) {
458
NETDATA_DOUBLE *ar2_base = &r2->ar[ idx2 ];
459
uint32_t *gbc2_base = &r2->gbc[ idx2 ];
460
403
- for(size_t c = 0; c < r->d ;c++) {
404
- if (!rrdr_dimension_should_be_exposed(r->od[c], options))
461
+ size_t row_gbc = 0;
462
+ for(size_t d = 0; d < r->d ; d++) {
463
+ if(unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
464
continue;
465
407
- NETDATA_DOUBLE n = cn_base[c];
408
- RRDR_VALUE_FLAGS o = co_base[c];
409
- NETDATA_DOUBLE ar = ar_base[c];
466
+ NETDATA_DOUBLE n = cn_base[d];
467
+ RRDR_VALUE_FLAGS o = co_base[d];
468
+ NETDATA_DOUBLE ar = ar_base[d];
469
470
if(o & RRDR_VALUE_EMPTY) {
471
if(options & RRDR_OPTION_NULL2ZERO)
@@ -418,19 +477,18 @@ static RRDR *data_query_group_by(RRDR *r) {
477
if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
478
n = -n;
479
421
- QUERY_METRIC *qm = query_metric(qt, c);
422
- size_t c2 = qm->grouped_as.slot;
480
+ QUERY_METRIC *qm = query_metric(qt, d);
481
+ size_t d2 = qm->grouped_as.slot;
482
424
- NETDATA_DOUBLE *cn2 = &cn2_base[c2];
425
- RRDR_VALUE_FLAGS *co2 = &co2_base[c2];
426
- NETDATA_DOUBLE *ar2 = &ar2_base[c2];
427
- uint32_t *gbc2 = &gbc2_base[c2];
483
+ NETDATA_DOUBLE *cn2 = &cn2_base[d2];
484
+ RRDR_VALUE_FLAGS *co2 = &co2_base[d2];
485
+ NETDATA_DOUBLE *ar2 = &ar2_base[d2];
486
+ uint32_t *gbc2 = &gbc2_base[d2];
487
488
switch(qt->request.group_by_aggregate_function) {
489
default:
490
case RRDR_GROUP_BY_FUNCTION_AVERAGE:
491
case RRDR_GROUP_BY_FUNCTION_SUM:
433
- case RRDR_GROUP_BY_FUNCTION_SUM_COUNT:
492
*cn2 += n;
493
break;
494
@@ -448,17 +506,29 @@ static RRDR *data_query_group_by(RRDR *r) {
506
*co2 |= (o & (RRDR_VALUE_RESET|RRDR_VALUE_PARTIAL));
507
*ar2 += ar;
508
(*gbc2)++;
509
+
510
+ row_gbc++;
511
}
512
+
513
+ if(unlikely(r->t[i] > r2->partial_data_trimming.expected_after && row_gbc < last_row_gbc)) {
514
+ // discard the rest of the points
515
+ r2->partial_data_trimming.trimmed_after = r->t[i];
516
+ r2->rows = i;
517
+ rows = i;
518
+ break;
519
+ }
520
+ else
521
+ last_row_gbc = row_gbc;
522
}
523
524
// apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
525
size_t min_max_values = 0;
526
NETDATA_DOUBLE min = NAN, max = NAN;
457
- for (size_t c2 = 0; c2 < r2->d; c2++) {
527
+ for (size_t d2 = 0; d2 < r2->d; d2++) {
528
size_t non_zero = 0;
529
530
for(size_t i = 0; i != rows ;i++) {
461
- size_t idx2 = i * r2->d + c2;
531
+ size_t idx2 = i * r2->d + d2;
532
533
NETDATA_DOUBLE *cn2 = &r2->v[ idx2 ];
534
RRDR_VALUE_FLAGS *co2 = &r2->o[ idx2 ];
@@ -468,23 +538,18 @@ static RRDR *data_query_group_by(RRDR *r) {
538
if(likely(gbc2)) {
539
*co2 &= ~RRDR_VALUE_EMPTY;
540
471
- if(gbc2 != r2->dgbc[c2])
541
+ if(gbc2 != r2->dgbc[d2])
542
*co2 |= RRDR_VALUE_PARTIAL;
543
544
NETDATA_DOUBLE n;
545
476
- if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
477
- n = *cn2 / gbc2;
478
- }
479
- else if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_AVERAGE) {
480
- *ar2 /= gbc2;
481
- n = *cn2 / gbc2;
482
- *cn2 = n;
483
- }
484
- else {
485
- *ar2 /= gbc2;
546
+ if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_AVERAGE)
547
+ n = (*cn2 /= gbc2);
548
+ else
549
n = *cn2;
487
- }
550
+
551
+ if(!query_target_aggregatable(qt))
552
+ *ar2 /= gbc2;
553
554
if(islessgreater(n, 0.0))
555
non_zero++;
@@ -504,7 +569,7 @@ static RRDR *data_query_group_by(RRDR *r) {
569
}
570
571
if(non_zero)
507
- r2->od[c2] |= RRDR_DIMENSION_NONZERO;
572
+ r2->od[d2] |= RRDR_DIMENSION_NONZERO;
573
}
574
575
r2->view.min = min;
@@ -514,9 +579,9 @@ cleanup:
579
buffer_free(key);
580
581
if(!r2 && entries && added) {
517
- for(long c = 0; c < added ;c++) {
518
- string_freez(entries[c].id);
519
- string_freez(entries[c].name);
582
+ for(int d2 = 0; d2 < added ; d2++) {
583
+ string_freez(entries[d2].id);
584
+ string_freez(entries[d2].name);
585
}
586
}
587
onewayalloc_freez(r->internal.owa, entries);
@@ -759,20 +824,24 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
824
rrdr2json(r, wb, options, 0);
825
826
if(options & RRDR_OPTION_JSON_WRAP) {
762
- if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
827
+ if (query_target_aggregatable(qt)) {
828
buffer_json_member_add_key_only(wb, "group_by_count");
829
rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_GBC, false);
830
}
766
- if(options & RRDR_OPTION_RETURN_JWAR) {
831
+ if (options & RRDR_OPTION_RETURN_JWAR) {
832
buffer_json_member_add_key_only(wb, "anomaly_rates");
833
rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, false);
834
}
770
- if(options & RRDR_OPTION_JW_ANNOTATIONS) {
771
- rrdr_json_wrapper_annotations(r, wb, format, options);
772
- }
835
wrapper_end(r, wb, format, options);
836
}
837
break;
838
+
839
+ case DATASOURCE_JSON2:
840
+ wb->content_type = CT_APPLICATION_JSON;
841
+ wrapper_begin(r, wb, format, options, group_method);
842
+ rrdr2json_v2(r, wb, format, options);
843
+ wrapper_end(r, wb, format, options);
844
+ break;
845
}
846
847
if(r != r1)
web/api/formatters/rrd2json.h
+3
@@ -17,6 +17,7 @@ typedef enum {
17
DATASOURCE_SSV_COMMA = 9,
18
DATASOURCE_CSV_JSON_ARRAY = 10,
19
DATASOURCE_CSV_MARKDOWN = 11,
20
+ DATASOURCE_JSON2 = 12,
21
} DATASOURCE_FORMAT;
22
23
#include "web/api/web_api_v1.h"
@@ -40,6 +41,7 @@ typedef enum {
41
#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
42
43
#define DATASOURCE_FORMAT_JSON "json"
44
+#define DATASOURCE_FORMAT_JSON2 "json2"
45
#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
46
#define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource"
47
#define DATASOURCE_FORMAT_JSONP "jsonp"
@@ -59,6 +61,7 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, ti
61
62
struct query_target;
63
bool query_target_has_percentage_units(struct query_target *qt);
64
+bool query_target_aggregatable(struct query_target *qt);
65
66
int rrdset2value_api_v1(
67
RRDSET *st
web/api/formatters/value/value.c
+5
-7
@@ -16,7 +16,6 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
16
NETDATA_DOUBLE total = 1;
17
NETDATA_DOUBLE total_anomaly_rate = 0;
18
19
- int set_min_max = 0;
19
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
20
total = 0;
21
for (c = 0; c < r->d ; c++) {
@@ -30,7 +29,6 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
29
}
30
// prevent a division by zero
31
if(total == 0) total = 1;
33
- set_min_max = 1;
32
}
33
34
// for each dimension
@@ -46,13 +44,13 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
44
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
45
n = n * 100 / total;
46
49
- if(unlikely(set_min_max)) {
47
+ if(unlikely(c == 0)) {
48
r->view.min = r->view.max = n;
51
- set_min_max = 0;
49
}
53
-
54
- if(n < r->view.min) r->view.min = n;
55
- if(n > r->view.max) r->view.max = n;
50
+ else {
51
+ if (n < r->view.min) r->view.min = n;
52
+ if (n > r->view.max) r->view.max = n;
53
+ }
54
}
55
56
if(unlikely(init)) {
web/api/netdata-swagger.yaml
+599
-25
@@ -2,9 +2,9 @@ openapi: 3.0.0
2
info:
3
title: Netdata API
4
description: Real-time performance and health monitoring.
5
- version: 1.33.1
5
+ version: "1.38"
6
paths:
7
- /info:
7
+ /api/v1/info:
8
get:
9
summary: Get netdata basic information
10
description: |
@@ -28,7 +28,7 @@ paths:
28
$ref: "#/components/schemas/info"
29
"503":
30
description: netdata daemon not ready (used for health checks).
31
- /charts:
31
+ /api/v1/charts:
32
get:
33
summary: Get a list of all charts available at the server
34
description: The charts endpoint returns a summary about all charts stored in the
@@ -40,7 +40,7 @@ paths:
40
application/json:
41
schema:
42
$ref: "#/components/schemas/chart_summary"
43
- /chart:
43
+ /api/v1/chart:
44
get:
45
summary: Get info about a specific chart
46
description: The chart endpoint returns detailed information about a chart.
@@ -64,7 +64,7 @@ paths:
64
description: No chart id was supplied in the request.
65
"404":
66
description: No chart with the given id is found.
67
- /contexts:
67
+ /api/v1/contexts:
68
get:
69
summary: Get a list of all contexts available at the server
70
description: The contexts endpoint returns a summary about all contexts stored in the
@@ -135,7 +135,7 @@ paths:
135
application/json:
136
schema:
137
$ref: "#/components/schemas/context_summary"
138
- /context:
138
+ /api/v1/context:
139
get:
140
summary: Get info about a specific context
141
description: The context endpoint returns detailed information about a given context.
@@ -217,7 +217,7 @@ paths:
217
description: No context id was supplied in the request.
218
"404":
219
description: No context with the given id is found.
220
- /alarm_variables:
220
+ /api/v1/alarm_variables:
221
get:
222
summary: List variables available to configure alarms for a chart
223
description: Returns the basic information of a chart and all the variables that can
@@ -247,7 +247,327 @@ paths:
247
"500":
248
description: Internal server error. This usually means the server is out of
249
memory.
250
- /data:
250
+ /api/v2/data:
251
+ get:
252
+ summary: Query metrics data
253
+ description: |
254
+ Multi-node, multi-context, multi-instance, multi-dimension data queries, with time and metric aggregation.
255
+ parameters:
256
+ - name: scope_nodes
257
+ in: query
258
+ description: |
259
+ A simple pattern limiting the nodes scope of the query. The scope controls both data and metadata response. The simple pattern is checked against the nodes' machine guid, node id, hostname. The default nodes scope is all nodes for which this agent has data for. Usually the nodes scope is used to slice the entire dashboard (e.g. the Global Nodes Selector at the Netdata Cloud overview dashboard). Both positive and negative simple pattern expressions are supported.
260
+ required: false
261
+ schema:
262
+ type: string
263
+ format: simple pattern
264
+ default: "*"
265
+ - name: scope_contexts
266
+ in: query
267
+ description: |
268
+ A simple pattern limiting the contexts scope of the query. The scope controls both data and metadata response. The default contexts scope is all contexts for which this agent has data for. Usually the contexts scope is used to slice charts of the dashboard (e.g. each context based chart has its own contexts scope, limiting the chart to all the instances of the selected contexts). Both positive and negative simple pattern expressions are supported.
269
+ required: false
270
+ schema:
271
+ type: string
272
+ format: simple pattern
273
+ default: "*"
274
+ - name: nodes
275
+ in: query
276
+ description: |
277
+ A simple pattern matching the nodes to be queried. This only controls the data response, not the metadata. The simple pattern is checked against the nodes' machine guid, node id, hostname. The default nodes selector is all the nodes matched by the nodes scope. Both positive and negative simple pattern expressions are supported.
278
+ required: false
279
+ schema:
280
+ type: string
281
+ format: simple pattern
282
+ default: "*"
283
+ - name: contexts
284
+ in: query
285
+ description: |
286
+ A simple pattern matching the contexts to be queried. This only controls the data response, not the metadata. Both positive and negative simple pattern expressions are supported.
287
+ required: false
288
+ schema:
289
+ type: string
290
+ format: simple pattern
291
+ default: "*"
292
+ - name: instances
293
+ in: query
294
+ description: |
295
+ A simple pattern matching the instances to be queried. The simple pattern is checked against the instance `id`, the instance `name`, the fully qualified name of the instance `id` and `name`, like `instance@machine_guid`, where `instance` is either its `id` or `name`. Both positive and negative simple pattern expressions are supported.
296
+ required: false
297
+ schema:
298
+ type: string
299
+ format: simple pattern
300
+ default: "*"
301
+ - name: labels
302
+ in: query
303
+ description: |
304
+ A simple pattern matching the labels to be queried. The simple pattern is checked against `name:value` of all the labels of all the eligible instances (as filtered by all the above: scope nodes, scope contexts, nodes, contexts and instances). Negative simple patterns should not be used in this filter.
305
+ required: false
306
+ schema:
307
+ type: string
308
+ format: simple pattern
309
+ default: "*"
310
+ - name: alerts
311
+ in: query
312
+ description: |
313
+ A simple pattern matching the alerts to be queried. The simple pattern is checked against the `name` of alerts and the combination of `name:status`, when status is one of `CLEAR`, `WARNING`, `CRITICAL`, `REMOVED`, `UNDEFINED`, `UNINITIALIZED`, of all the alerts of all the eligible instances (as filtered by all the above). A negative simple pattern will exclude the instances having the labels matched.
314
+ required: false
315
+ schema:
316
+ type: string
317
+ format: simple pattern
318
+ default: "*"
319
+ - name: dimensions
320
+ in: query
321
+ description: |
322
+ A simple patterns matching the dimensions to be queried. The simple pattern is checked against and `id` and the `name` of the dimensions of the eligible instances (as filtered by all the above). Both positive and negative simple pattern expressions are supported.
323
+ required: false
324
+ schema:
325
+ type: string
326
+ format: simple pattern
327
+ default: "*"
328
+ - name: before
329
+ in: query
330
+ description: |
331
+ The end timestamp (unix epoch) of the data query, or a negative number specifying the number of seconds
332
+ in the past relative now.
333
+ required: false
334
+ schema:
335
+ type: number
336
+ format: integer
337
+ default: 0
338
+ - name: after
339
+ in: query
340
+ description: |
341
+ The start timestamp (unix epoch) of the data query, or a negative number specifying the number of seconds
342
+ in the past relative to parameter `before`.
343
+ required: false
344
+ schema:
345
+ type: number
346
+ format: integer
347
+ default: 0
348
+ - name: points
349
+ in: query
350
+ description: |
351
+ The number of points to be returned. If not given, or it is <= 0, or it is bigger than the points stored in the database for the given duration, all the available collected values for the given duration will be returned.
352
+ required: false
353
+ schema:
354
+ type: number
355
+ format: integer
356
+ default: 0
357
+ - name: group_by
358
+ in: query
359
+ description: |
360
+ A comma separated list of `dimension`, `label`, `instance`, `node`, `selected`. All possible values can be combined together, except `selected`. If `selected` is given in the list, all others are ignored. The order they are placed in the list is currently ignored. The result will always have in the order given here.
361
+ required: false
362
+ schema:
363
+ type: array
364
+ items:
365
+ type: string
366
+ enum:
367
+ - dimension
368
+ - label
369
+ - instance
370
+ - node
371
+ - selected
372
+ default:
373
+ - dimension
374
+ - name: group_by_label
375
+ in: query
376
+ description: |
377
+ A comma separated list of the label keys to group by their values. The order of the labels in the list is respected.
378
+ required: false
379
+ schema:
380
+ type: string
381
+ format: comma separated list of label keys to group by
382
+ default: ""
383
+ - name: aggregation
384
+ in: query
385
+ description: |
386
+ The aggregation function to apply when grouping metrics together.
387
+ required: false
388
+ schema:
389
+ type: string
390
+ enum:
391
+ - min
392
+ - max
393
+ - avg
394
+ - average
395
+ - sum
396
+ default: average
397
+ - name: time_group
398
+ in: query
399
+ description: |
400
+ Time aggregation function. If multiple collected values are to be grouped in order to return fewer points, this parameters defines the method of grouping. methods supported "min", "max", "average", "sum", "incremental-sum". "max" is actually calculated on the absolute value collected (so it works for both positive and negative dimensions to return the most extreme value in either direction).
401
+ required: true
402
+ schema:
403
+ type: string
404
+ enum:
405
+ - min
406
+ - max
407
+ - avg
408
+ - average
409
+ - median
410
+ - stddev
411
+ - sum
412
+ - incremental-sum
413
+ - ses
414
+ - des
415
+ - cv
416
+ - countif
417
+ - percentile
418
+ - percentile25
419
+ - percentile50
420
+ - percentile75
421
+ - percentile80
422
+ - percentile90
423
+ - percentile95
424
+ - percentile97
425
+ - percentile98
426
+ - percentile99
427
+ - trimmed-mean
428
+ - trimmed-mean1
429
+ - trimmed-mean2
430
+ - trimmed-mean3
431
+ - trimmed-mean5
432
+ - trimmed-mean10
433
+ - trimmed-mean15
434
+ - trimmed-mean20
435
+ - trimmed-mean25
436
+ - trimmed-median
437
+ - trimmed-median1
438
+ - trimmed-median2
439
+ - trimmed-median3
440
+ - trimmed-median5
441
+ - trimmed-median10
442
+ - trimmed-median15
443
+ - trimmed-median20
444
+ - trimmed-median25
445
+ default: average
446
+ - name: time_group_options
447
+ in: query
448
+ description: |
449
+ When the group function supports additional parameters, this field can be used to pass them to it. Currently `countif`, `trimmed-mean`, `trimmed-median` and `percentile` support this. For `countif` the string may start with `<`, `<=`, `<:`, `<>`, `!=`, `>`, `>=`, `>:`. For all others just a number is expected.
450
+ required: false
451
+ schema:
452
+ type: string
453
+ - name: time_resampling
454
+ in: query
455
+ description: |
456
+ For incremental values that are "per second", this value is used to resample them to "per minute` (60) or "per hour" (3600). It can only be used in conjunction with group=average.
457
+ required: false
458
+ schema:
459
+ type: number
460
+ format: integer
461
+ default: 0
462
+ - name: timeout
463
+ in: query
464
+ description: |
465
+ Specify a timeout value in milliseconds after which the agent will abort the query and return a 503 error. A value of 0 indicates no timeout.
466
+ required: false
467
+ schema:
468
+ type: number
469
+ format: integer
470
+ default: 0
471
+ - name: format
472
+ in: query
473
+ description: |
474
+ The format of the data to be returned.
475
+ required: true
476
+ allowEmptyValue: false
477
+ schema:
478
+ type: string
479
+ enum:
480
+ - json
481
+ - json2
482
+ - jsonp
483
+ - csv
484
+ - tsv
485
+ - tsv-excel
486
+ - ssv
487
+ - ssvcomma
488
+ - datatable
489
+ - datasource
490
+ - html
491
+ - markdown
492
+ - array
493
+ - csvjsonarray
494
+ default: json
495
+ - name: options
496
+ in: query
497
+ description: |
498
+ Options that affect data generation.
499
+ required: false
500
+ allowEmptyValue: false
501
+ schema:
502
+ type: array
503
+ items:
504
+ type: string
505
+ enum:
506
+ - nonzero
507
+ - flip
508
+ - jsonwrap
509
+ - min2max
510
+ - seconds
511
+ - milliseconds
512
+ - abs
513
+ - absolute
514
+ - absolute-sum
515
+ - null2zero
516
+ - objectrows
517
+ - google_json
518
+ - percentage
519
+ - unaligned
520
+ - match-ids
521
+ - match-names
522
+ - allow_past
523
+ - anomaly-bit
524
+ default:
525
+ - seconds
526
+ - jsonwrap
527
+ - name: tier
528
+ in: query
529
+ description: |
530
+ Use only the specified database tier.
531
+ required: false
532
+ schema:
533
+ type: number
534
+ format: integer
535
+ - name: callback
536
+ in: query
537
+ description: |
538
+ For JSONP responses, the callback function name.
539
+ required: false
540
+ schema:
541
+ type: string
542
+ - name: filename
543
+ in: query
544
+ description: |
545
+ Add `Content-Disposition: attachment; filename=` header to the response, that will instruct the browser to save the response with the given filename."
546
+ required: false
547
+ schema:
548
+ type: string
549
+ - name: tqx
550
+ in: query
551
+ description: |
552
+ [Google Visualization API](https://developers.google.com/chart/interactive/docs/dev/implementing_data_source?hl=en) formatted parameter.
553
+ required: false
554
+ schema:
555
+ type: string
556
+ responses:
557
+ "200":
558
+ description: |
559
+ The call was successful. The response includes the data in the format requested. Swagger2.0 does not process the discriminator field to show polymorphism. The response will be one of the sub-types of the data-schema according to the chosen format, e.g. json -> data_json.
560
+ content:
561
+ application/json:
562
+ schema:
563
+ $ref: "#/components/schemas/data_json2"
564
+ "400":
565
+ description: |
566
+ Bad request - the body will include a message stating what is wrong.
567
+ "500":
568
+ description: |
569
+ Internal server error. This usually means the server is out of memory.
570
+ /api/v1/data:
571
get:
572
summary: Get collected data for a specific chart
573
description: The data endpoint returns data stored in the round robin database of a
@@ -519,7 +839,7 @@ paths:
839
"500":
840
description: Internal server error. This usually means the server is out of
841
memory.
522
- /badge.svg:
842
+ /api/v1/badge.svg:
843
get:
844
summary: Generate a badge in form of SVG image for a chart (or dimension)
845
description: Successful responses are SVG images.
@@ -818,7 +1138,7 @@ paths:
1138
"500":
1139
description: Internal server error. This usually means the server is out of
1140
memory.
821
- /allmetrics:
1141
+ /api/v1/allmetrics:
1142
get:
1143
summary: Get a value of all the metrics maintained by netdata
1144
description: The allmetrics endpoint returns the latest value of all charts and
@@ -951,7 +1271,7 @@ paths:
1271
description: All the metrics returned in the format requested.
1272
"400":
1273
description: The format requested is not supported.
954
- /alarms:
1274
+ /api/v1/alarms:
1275
get:
1276
summary: Get a list of active or raised alarms on the server
1277
description: The alarms endpoint returns the list of all raised or enabled alarms on
@@ -980,7 +1300,7 @@ paths:
1300
application/json:
1301
schema:
1302
$ref: "#/components/schemas/alarms"
983
- /alarms_values:
1303
+ /api/v1/alarms_values:
1304
get:
1305
summary: Get a list of active or raised alarms on the server
1306
description: "The alarms_values endpoint returns the list of all raised or enabled alarms on
@@ -1011,7 +1331,7 @@ paths:
1331
application/json:
1332
schema:
1333
$ref: "#/components/schemas/alarms_values"
1014
- /alarm_log:
1334
+ /api/v1/alarm_log:
1335
get:
1336
summary: Retrieves the entries of the alarm log
1337
description: Returns an array of alarm_log entries, with historical information on
@@ -1036,7 +1356,7 @@ paths:
1356
type: array
1357
items:
1358
$ref: "#/components/schemas/alarm_log_entry"
1039
- /alarm_count:
1359
+ /api/v1/alarm_count:
1360
get:
1361
summary: Get an overall status of the chart
1362
description: Checks multiple charts with the same context and counts number of alarms
@@ -1082,7 +1402,7 @@ paths:
1402
"500":
1403
description: Internal server error. This usually means the server is out of
1404
memory.
1085
- /manage/health:
1405
+ /api/v1/manage/health:
1406
get:
1407
summary: "Accesses the health management API to control health checks and
1408
notifications at runtime."
@@ -1144,7 +1464,7 @@ paths:
1464
description: A plain text response based on the result of the command.
1465
"403":
1466
description: Bearer authentication error.
1147
- /aclk:
1467
+ /api/v1/aclk:
1468
get:
1469
summary: Get information about current ACLK state
1470
description: "ACLK endpoint returns detailed information
@@ -1156,7 +1476,7 @@ paths:
1476
application/json:
1477
schema:
1478
$ref: "#/components/schemas/aclk_state"
1159
- /metric_correlations:
1479
+ /api/v1/metric_correlations:
1480
get:
1481
summary: "Analyze all the metrics to find their correlations"
1482
description: "THIS ENDPOINT IS OBSOLETE. Use the /weights endpoint.
@@ -1350,7 +1670,7 @@ paths:
1670
that correlated the metrics did not produce any result.
1671
"504":
1672
description: Timeout - the query took too long and has been cancelled.
1353
- /function:
1673
+ /api/v1/function:
1674
get:
1675
summary: "Execute a collector function."
1676
parameters:
@@ -1384,14 +1704,14 @@ paths:
1704
description: Timeout while waiting for the collector to execute the function.
1705
"591":
1706
description: The collector sent a response, but it was invalid or corrupted.
1387
- /functions:
1707
+ /api/v1/functions:
1708
get:
1709
summary: Get a list of all registered collector functions.
1710
description: Collector functions are programs that can be executed on demand.
1711
responses:
1712
"200":
1713
description: A JSON object containing one object per supported function.
1394
- /weights:
1714
+ /api/v1/weights:
1715
get:
1716
summary: "Analyze all the metrics using an algorithm and score them accordingly"
1717
description: "This endpoint goes through all metrics and scores them according to an algorithm."
@@ -1595,8 +1915,9 @@ paths:
1915
"504":
1916
description: Timeout - the query took too long and has been cancelled.
1917
servers:
1598
- - url: https://registry.my-netdata.io/api/v1
1599
- - url: http://registry.my-netdata.io/api/v1
1918
+ - url: https://registry.my-netdata.io
1919
+ - url: http://registry.my-netdata.io
1920
+ - url: http://localhost:19999
1921
components:
1922
schemas:
1923
info:
@@ -2081,6 +2402,260 @@ components:
2402
type: object
2403
additionalProperties:
2404
$ref: "#/components/schemas/chart_variables"
2405
+ data_json2:
2406
+ description: |
2407
+ Data response with `format=json2`
2408
+ type: object
2409
+ properties:
2410
+ versions:
2411
+ description: |
2412
+ Hashes that allow the caller to detect important database changes of Netdata agents.
2413
+ type: object
2414
+ properties:
2415
+ contexts_hard_hash:
2416
+ description: |
2417
+ An auto-increment value that reflects the number of changes to the number of contexts maintained by the server. Everytime a context is added or removed, this number gets incremented.
2418
+ type: integer
2419
+ contexts_soft_hash:
2420
+ description: |
2421
+ An auto-increment value that reflects the number of changes to the queue that sends contexts updates to Netdata Cloud. Everytime the contents of a context are updated, this number gets incremented.
2422
+ type: integer
2423
+ summary:
2424
+ description: |
2425
+ Summarized information about nodes, contexts, instances, labels, alerts, and dimensions. The items returned are determined by the scope of the query only, however the statistical data in them are influenced by the filters of the query. Using this information the dashboard allows users to slice and dice the data by filtering and grouping.
2426
+ type: object
2427
+ properties:
2428
+ nodes:
2429
+ type: array
2430
+ items:
2431
+ type: object
2432
+ description: |
2433
+ An object describing a node. `is` stands for instances, `ds` for dimensions, `al` for alerts, `sts` for statistics.
2434
+ properties:
2435
+ ni:
2436
+ description: the node index id, a number that uniquely identifies this node for this query.
2437
+ type: integer
2438
+ mg:
2439
+ description: the machine guid of the node.
2440
+ type: string
2441
+ format: UUID
2442
+ nd:
2443
+ description: the node id of the node.
2444
+ type: string
2445
+ format: UUID
2446
+ nm:
2447
+ description: the name (hostname) of the node.
2448
+ type: string
2449
+ is:
2450
+ $ref: "#/components/schemas/data_json2_items_count"
2451
+ ds:
2452
+ $ref: "#/components/schemas/data_json2_items_count"
2453
+ al:
2454
+ $ref: "#/components/schemas/data_json2_alerts_count"
2455
+ sts:
2456
+ oneOf:
2457
+ - $ref: "#/components/schemas/data_json2_sts"
2458
+ - $ref: "#/components/schemas/data_json2_sts_raw"
2459
+ contexts:
2460
+ type: array
2461
+ items:
2462
+ type: object
2463
+ description: |
2464
+ An object describing a unique context. `is` stands for instances, `ds` for dimensions, `al` for alerts, `sts` for statistics.
2465
+ properties:
2466
+ id:
2467
+ description: the context id.
2468
+ type: string
2469
+ is:
2470
+ $ref: "#/components/schemas/data_json2_items_count"
2471
+ ds:
2472
+ $ref: "#/components/schemas/data_json2_items_count"
2473
+ al:
2474
+ $ref: "#/components/schemas/data_json2_alerts_count"
2475
+ sts:
2476
+ oneOf:
2477
+ - $ref: "#/components/schemas/data_json2_sts"
2478
+ - $ref: "#/components/schemas/data_json2_sts_raw"
2479
+ instances:
2480
+ type: array
2481
+ items:
2482
+ type: object
2483
+ description: |
2484
+ An object describing an instance. `ds` stands for dimensions, `al` for alerts, `sts` for statistics.
2485
+ properties:
2486
+ id:
2487
+ description: the id of the instance.
2488
+ type: string
2489
+ nm:
2490
+ description: the name of the instance (may be absent when it is the same with the id)
2491
+ type: string
2492
+ ni:
2493
+ description: the node index id this instance belongs to. The UI uses this to compone the fully qualified name of the instance, using the node hostname to present it to users and its machine guid to add it to filters.
2494
+ ds:
2495
+ $ref: "#/components/schemas/data_json2_items_count"
2496
+ al:
2497
+ $ref: "#/components/schemas/data_json2_alerts_count"
2498
+ sts:
2499
+ oneOf:
2500
+ - $ref: "#/components/schemas/data_json2_sts"
2501
+ - $ref: "#/components/schemas/data_json2_sts_raw"
2502
+ dimensions:
2503
+ type: array
2504
+ items:
2505
+ type: object
2506
+ description: |
2507
+ An object describing a unique dimension. `ds` stands for `dimensions`, `sts` for statistics.
2508
+ properties:
2509
+ id:
2510
+ description: the id of the dimension.
2511
+ type: string
2512
+ nm:
2513
+ description: the name of the dimension (may be absent when it is the same with the id)
2514
+ type: string
2515
+ ds:
2516
+ $ref: "#/components/schemas/data_json2_items_count"
2517
+ sts:
2518
+ oneOf:
2519
+ - $ref: "#/components/schemas/data_json2_sts"
2520
+ - $ref: "#/components/schemas/data_json2_sts_raw"
2521
+ labels:
2522
+ type: array
2523
+ items:
2524
+ type: object
2525
+ description: |
2526
+ An object describing a label key. `ds` stands for `dimensions`, `sts` for statistics.
2527
+ properties:
2528
+ id:
2529
+ description: the key of the label.
2530
+ type: string
2531
+ ds:
2532
+ $ref: "#/components/schemas/data_json2_items_count"
2533
+ sts:
2534
+ oneOf:
2535
+ - $ref: "#/components/schemas/data_json2_sts"
2536
+ - $ref: "#/components/schemas/data_json2_sts_raw"
2537
+ vl:
2538
+ description: |
2539
+ An array of values for this key.
2540
+ type: array
2541
+ items:
2542
+ type: object
2543
+ properties:
2544
+ id:
2545
+ description: The value string
2546
+ type: string
2547
+ ds:
2548
+ $ref: "#/components/schemas/data_json2_items_count"
2549
+ sts:
2550
+ oneOf:
2551
+ - $ref: "#/components/schemas/data_json2_sts"
2552
+ - $ref: "#/components/schemas/data_json2_sts_raw"
2553
+ alerts:
2554
+ description: |
2555
+ An array of all the unique alerts running, grouped by alert name (`nm` is available here)
2556
+ type: array
2557
+ items:
2558
+ $ref: "#/components/schemas/data_json2_alerts_count"
2559
+ totals:
2560
+ type: object
2561
+ properties:
2562
+ nodes:
2563
+ $ref: "#/components/schemas/data_json2_items_count"
2564
+ contexts:
2565
+ $ref: "#/components/schemas/data_json2_items_count"
2566
+ instances:
2567
+ $ref: "#/components/schemas/data_json2_items_count"
2568
+ dimensions:
2569
+ $ref: "#/components/schemas/data_json2_items_count"
2570
+ label_keys:
2571
+ $ref: "#/components/schemas/data_json2_items_count"
2572
+ label_key_values:
2573
+ $ref: "#/components/schemas/data_json2_items_count"
2574
+ functions:
2575
+ type: array
2576
+ items:
2577
+ type: string
2578
+ db:
2579
+ type: object
2580
+ view:
2581
+ type: object
2582
+ result:
2583
+ type: object
2584
+ min:
2585
+ type: number
2586
+ max:
2587
+ type: number
2588
+ timings:
2589
+ type: object
2590
+ data_json2_sts:
2591
+ description: |
2592
+ Statistical values
2593
+ type: object
2594
+ properties:
2595
+ avg:
2596
+ description: The average value of all metrics aggregated
2597
+ type: number
2598
+ arp:
2599
+ description: The average anomaly rate of all metrics aggregated
2600
+ type: number
2601
+ con:
2602
+ description: The contribution percentage of all the metrics aggregated
2603
+ type: number
2604
+ data_json2_sts_raw:
2605
+ description: |
2606
+ Statistical values when `raw` option is given.
2607
+ type: object
2608
+ properties:
2609
+ sum:
2610
+ description: The sum value of all metrics aggregated
2611
+ type: number
2612
+ ars:
2613
+ description: The sum anomaly rate of all metrics aggregated
2614
+ type: number
2615
+ vol:
2616
+ description: The volume of all the metrics aggregated
2617
+ type: number
2618
+ cnt:
2619
+ description: The count of all metrics aggregated
2620
+ type: integer
2621
+ data_json2_items_count:
2622
+ description: |
2623
+ Depending on the placement of this object, `items` may be `nodes`, `contexts`, `instances`, `dimensions`, `label keys`, `label key-value pairs`. Furthermore, if the whole object is missing it should be assumed that all its members are zero.
2624
+ type: object
2625
+ properties:
2626
+ sl:
2627
+ description: The number of items `selected` to query. If absent it is zero.
2628
+ type: integer
2629
+ ex:
2630
+ description: The number of items `excluded` from querying. If absent it is zero.
2631
+ type: integer
2632
+ qr:
2633
+ description: The number of items (out of `selected`) the query successfully `queried`. If absent it is zero.
2634
+ type: integer
2635
+ fl:
2636
+ description: The number of items (from `selected`) that `failed` to be queried. If absent it is zero.
2637
+ type: integer
2638
+ data_json2_alerts_count:
2639
+ description: |
2640
+ Counters about alert statuses. If this object is missing, it is assumed that all its members are zero.
2641
+ type: object
2642
+ properties:
2643
+ nm:
2644
+ description: The name of the alert. Can be absent when the counters refer to more than one alert instances.
2645
+ type: string
2646
+ cl:
2647
+ description: The number of CLEAR alerts. If absent, it is zero.
2648
+ type: integer
2649
+ wr:
2650
+ description: The number of WARNING alerts. If absent, it is zero.
2651
+ type: integer
2652
+ cr:
2653
+ description: The number of CRITICAL alerts. If absent, it is zero.
2654
+ type: integer
2655
+ ot:
2656
+ description: |
2657
+ The number of alerts that are not CLEAR, WARNING, CRITICAL (so, they are "other"). If absent, it is zero.
2658
+ type: integer
2659
data_json:
2660
description: Data response in json format.
2661
allOf:
@@ -2095,9 +2670,8 @@ components:
2670
items:
2671
type: string
2672
data:
2098
- description: The data requested, one element per sample with each element
2099
- containing the values of the dimensions described in the
2100
- labels value.
2673
+ description: |
2674
+ The data requested, one element per sample with each element containing the values of the dimensions described in the labels value.
2675
type: array
2676
items:
2677
type: number
web/api/queries/query.c
+162
-70
@@ -67,7 +67,18 @@ static struct {
67
.flush = grouping_flush_average,
68
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
69
},
70
- {.name = "mean", // alias on 'average'
70
+ {.name = "avg", // alias on 'average'
71
+ .hash = 0,
72
+ .value = RRDR_GROUPING_AVERAGE,
73
+ .init = NULL,
74
+ .create= grouping_create_average,
75
+ .reset = grouping_reset_average,
76
+ .free = grouping_free_average,
77
+ .add = grouping_add_average,
78
+ .flush = grouping_flush_average,
79
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
80
+ },
81
+ {.name = "mean", // alias on 'average'
82
.hash = 0,
83
.value = RRDR_GROUPING_AVERAGE,
84
.init = NULL,
@@ -677,12 +688,18 @@ RRDR_GROUP_BY group_by_parse(char *s) {
688
689
if (strcmp(key, "label") == 0)
690
group_by |= RRDR_GROUP_BY_LABEL;
691
+
692
+ if (strcmp(key, "selected") == 0)
693
+ group_by |= RRDR_GROUP_BY_SELECTED;
694
}
695
696
return group_by;
697
}
698
699
void buffer_json_group_by_to_array(BUFFER *wb, RRDR_GROUP_BY group_by) {
700
+ if(group_by & RRDR_GROUP_BY_SELECTED)
701
+ buffer_json_add_array_item_string(wb, "selected");
702
+
703
if(group_by & RRDR_GROUP_BY_DIMENSION)
704
buffer_json_add_array_item_string(wb, "dimension");
705
@@ -697,12 +714,12 @@ void buffer_json_group_by_to_array(BUFFER *wb, RRDR_GROUP_BY group_by) {
714
}
715
716
RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s) {
700
- if(strcmp(s, "sum-count") == 0)
701
- return RRDR_GROUP_BY_FUNCTION_SUM_COUNT;
702
-
717
if(strcmp(s, "average") == 0)
718
return RRDR_GROUP_BY_FUNCTION_AVERAGE;
719
720
+ if(strcmp(s, "avg") == 0)
721
+ return RRDR_GROUP_BY_FUNCTION_AVERAGE;
722
+
723
if(strcmp(s, "min") == 0)
724
return RRDR_GROUP_BY_FUNCTION_MIN;
725
@@ -721,9 +738,6 @@ const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_b
738
case RRDR_GROUP_BY_FUNCTION_AVERAGE:
739
return "average";
740
724
- case RRDR_GROUP_BY_FUNCTION_SUM_COUNT:
725
- return "sum-count";
726
-
741
case RRDR_GROUP_BY_FUNCTION_MIN:
742
return "min";
743
@@ -763,11 +777,6 @@ static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) {
777
return rrdr_line;
778
}
779
766
-static inline void rrdr_done(RRDR *r, long rrdr_line) {
767
- r->rows = rrdr_line + 1;
768
-}
769
-
770
-
780
// ----------------------------------------------------------------------------
781
// tier management
782
@@ -1028,6 +1037,19 @@ typedef struct query_engine_ops {
1037
// statistics
1038
size_t db_total_points_read;
1039
size_t db_points_read_per_tier[RRD_STORAGE_TIERS];
1040
+
1041
+ struct {
1042
+ time_t expanded_after;
1043
+ time_t expanded_before;
1044
+ struct storage_engine_query_handle handle;
1045
+ STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
1046
+ int (*is_finished)(struct storage_engine_query_handle *handle);
1047
+ void (*finalize)(struct storage_engine_query_handle *handle);
1048
+ bool initialized;
1049
+ bool finalized;
1050
+ } plans[QUERY_PLANS_MAX];
1051
+
1052
+ struct query_engine_ops *next;
1053
} QUERY_ENGINE_OPS;
1054
1055
@@ -1084,36 +1106,37 @@ static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) {
1106
time_t after = qm->plan.array[p].after - (time_t)(update_every * points_to_add_to_after);
1107
time_t before = qm->plan.array[p].before + (time_t)(update_every * points_to_add_to_before);
1108
1087
- qm->plan.array[p].expanded_after = after;
1088
- qm->plan.array[p].expanded_before = before;
1109
+ ops->plans[p].expanded_after = after;
1110
+ ops->plans[p].expanded_before = before;
1111
1112
ops->r->internal.qt->db.tiers[tier].queries++;
1113
1114
struct query_metric_tier *tier_ptr = &qm->tiers[tier];
1093
- tier_ptr->eng->api.query_ops.init(
1115
+ STORAGE_ENGINE *eng = query_metric_storage_engine(ops->r->internal.qt, qm, tier);
1116
+ eng->api.query_ops.init(
1117
tier_ptr->db_metric_handle,
1095
- &qm->plan.array[p].handle,
1118
+ &ops->plans[p].handle,
1119
after, before,
1120
ops->r->internal.qt->request.priority);
1121
1099
- qm->plan.array[p].next_metric = tier_ptr->eng->api.query_ops.next_metric;
1100
- qm->plan.array[p].is_finished = tier_ptr->eng->api.query_ops.is_finished;
1101
- qm->plan.array[p].finalize = tier_ptr->eng->api.query_ops.finalize;
1102
- qm->plan.array[p].initialized = true;
1103
- qm->plan.array[p].finalized = false;
1122
+ ops->plans[p].next_metric = eng->api.query_ops.next_metric;
1123
+ ops->plans[p].is_finished = eng->api.query_ops.is_finished;
1124
+ ops->plans[p].finalize = eng->api.query_ops.finalize;
1125
+ ops->plans[p].initialized = true;
1126
+ ops->plans[p].finalized = false;
1127
}
1128
}
1129
1130
static void query_planer_finalize_plan(QUERY_ENGINE_OPS *ops, size_t plan_id) {
1108
- QUERY_METRIC *qm = ops->qm;
1131
+ // QUERY_METRIC *qm = ops->qm;
1132
1110
- if(qm->plan.array[plan_id].initialized && !qm->plan.array[plan_id].finalized) {
1111
- qm->plan.array[plan_id].finalize(&qm->plan.array[plan_id].handle);
1112
- qm->plan.array[plan_id].initialized = false;
1113
- qm->plan.array[plan_id].finalized = true;
1114
- qm->plan.array[plan_id].next_metric = NULL;
1115
- qm->plan.array[plan_id].is_finished = NULL;
1116
- qm->plan.array[plan_id].finalize = NULL;
1133
+ if(ops->plans[plan_id].initialized && !ops->plans[plan_id].finalized) {
1134
+ ops->plans[plan_id].finalize(&ops->plans[plan_id].handle);
1135
+ ops->plans[plan_id].initialized = false;
1136
+ ops->plans[plan_id].finalized = true;
1137
+ ops->plans[plan_id].next_metric = NULL;
1138
+ ops->plans[plan_id].is_finished = NULL;
1139
+ ops->plans[plan_id].finalize = NULL;
1140
1141
if(ops->current_plan == plan_id) {
1142
ops->next_metric = NULL;
@@ -1134,17 +1157,17 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
1157
QUERY_METRIC *qm = ops->qm;
1158
1159
internal_fatal(plan_id >= qm->plan.used, "QUERY: invalid plan_id given");
1137
- internal_fatal(!qm->plan.array[plan_id].initialized, "QUERY: plan has not been initialized");
1138
- internal_fatal(qm->plan.array[plan_id].finalized, "QUERY: plan has been finalized");
1160
+ internal_fatal(!ops->plans[plan_id].initialized, "QUERY: plan has not been initialized");
1161
+ internal_fatal(ops->plans[plan_id].finalized, "QUERY: plan has been finalized");
1162
1163
internal_fatal(qm->plan.array[plan_id].after > qm->plan.array[plan_id].before, "QUERY: flipped after/before");
1164
1165
ops->tier = qm->plan.array[plan_id].tier;
1166
ops->tier_ptr = &qm->tiers[ops->tier];
1144
- ops->handle = &qm->plan.array[plan_id].handle;
1145
- ops->next_metric = qm->plan.array[plan_id].next_metric;
1146
- ops->is_finished = qm->plan.array[plan_id].is_finished;
1147
- ops->finalize = qm->plan.array[plan_id].finalize;
1167
+ ops->handle = &ops->plans[plan_id].handle;
1168
+ ops->next_metric = ops->plans[plan_id].next_metric;
1169
+ ops->is_finished = ops->plans[plan_id].is_finished;
1170
+ ops->finalize = ops->plans[plan_id].finalize;
1171
ops->current_plan = plan_id;
1172
1173
if(plan_id + 1 < qm->plan.used && qm->plan.array[plan_id + 1].after < qm->plan.array[plan_id].before)
@@ -1152,8 +1175,8 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
1175
else
1176
ops->current_plan_expire_time = qm->plan.array[plan_id].before;
1177
1155
- ops->plan_expanded_after = qm->plan.array[plan_id].expanded_after;
1156
- ops->plan_expanded_before = qm->plan.array[plan_id].expanded_before;
1178
+ ops->plan_expanded_after = ops->plans[plan_id].expanded_after;
1179
+ ops->plan_expanded_before = ops->plans[plan_id].expanded_before;
1180
}
1181
1182
static bool query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) {
@@ -1231,7 +1254,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1254
// check if our selected tier can start the query
1255
if (selected_tier_first_time_s > after_wanted) {
1256
// we need some help from other tiers
1234
- for (size_t tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
1257
+ for (size_t tr = (int)selected_tier + 1; tr < storage_tiers && qm->plan.used < QUERY_PLANS_MAX ; tr++) {
1258
if(!query_metric_is_valid_tier(qm, tr))
1259
continue;
1260
@@ -1245,9 +1268,9 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1268
.tier = tr,
1269
.after = (tier_first_time_s < after_wanted) ? after_wanted : tier_first_time_s,
1270
.before = selected_tier_first_time_s,
1248
- .initialized = false,
1249
- .finalized = false,
1271
};
1272
+ ops->plans[qm->plan.used].initialized = false;
1273
+ ops->plans[qm->plan.used].finalized = false;
1274
qm->plan.array[qm->plan.used++] = t;
1275
1276
internal_fatal(!t.after || !t.before, "QUERY: invalid plan selected");
@@ -1264,7 +1287,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1287
// check if our selected tier can finish the query
1288
if (selected_tier_last_time_s < before_wanted) {
1289
// we need some help from other tiers
1267
- for (int tr = (int)selected_tier - 1; tr >= 0; tr--) {
1290
+ for (int tr = (int)selected_tier - 1; tr >= 0 && qm->plan.used < QUERY_PLANS_MAX ; tr--) {
1291
if(!query_metric_is_valid_tier(qm, tr))
1292
continue;
1293
@@ -1280,9 +1303,9 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1303
.tier = tr,
1304
.after = selected_tier_last_time_s,
1305
.before = (tier_last_time_s > before_wanted) ? before_wanted : tier_last_time_s,
1283
- .initialized = false,
1284
- .finalized = false,
1306
};
1307
+ ops->plans[qm->plan.used].initialized = false;
1308
+ ops->plans[qm->plan.used].finalized = false;
1309
qm->plan.array[qm->plan.used++] = t;
1310
1311
// prepare for the tier
@@ -1356,10 +1379,42 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1379
(ops)->group_anomaly_all_points = (point).anomaly_all_points; \
1380
} while(0)
1381
1359
-static QUERY_ENGINE_OPS *rrd2rrdr_query_prep(RRDR *r, size_t dim_id_in_rrdr) {
1382
+static __thread QUERY_ENGINE_OPS *released_ops = NULL;
1383
+
1384
+static void rrd2rrdr_query_ops_freeall(RRDR *r __maybe_unused) {
1385
+ while(released_ops) {
1386
+ QUERY_ENGINE_OPS *ops = released_ops;
1387
+ released_ops = ops->next;
1388
+
1389
+ onewayalloc_freez(r->internal.owa, ops);
1390
+ }
1391
+}
1392
+
1393
+static void rrd2rrdr_query_ops_release(RRDR *r __maybe_unused, QUERY_ENGINE_OPS *ops) {
1394
+ if(!ops) return;
1395
+
1396
+ ops->next = released_ops;
1397
+ released_ops = ops;
1398
+}
1399
+
1400
+static QUERY_ENGINE_OPS *rrd2rrdr_query_ops_get(RRDR *r) {
1401
+ QUERY_ENGINE_OPS *ops;
1402
+ if(released_ops) {
1403
+ ops = released_ops;
1404
+ released_ops = ops->next;
1405
+ }
1406
+ else {
1407
+ ops = onewayalloc_mallocz(r->internal.owa, sizeof(QUERY_ENGINE_OPS));
1408
+ }
1409
+
1410
+ memset(ops, 0, sizeof(*ops));
1411
+ return ops;
1412
+}
1413
+
1414
+static QUERY_ENGINE_OPS *rrd2rrdr_query_ops_prep(RRDR *r, size_t dim_id_in_rrdr) {
1415
QUERY_TARGET *qt = r->internal.qt;
1416
1362
- QUERY_ENGINE_OPS *ops = onewayalloc_mallocz(r->internal.owa, sizeof(QUERY_ENGINE_OPS));
1417
+ QUERY_ENGINE_OPS *ops = rrd2rrdr_query_ops_get(r);
1418
*ops = (QUERY_ENGINE_OPS) {
1419
.r = r,
1420
.qm = query_metric(qt, dim_id_in_rrdr),
@@ -1371,8 +1426,10 @@ static QUERY_ENGINE_OPS *rrd2rrdr_query_prep(RRDR *r, size_t dim_id_in_rrdr) {
1426
.group_value_flags = RRDR_VALUE_NOTHING,
1427
};
1428
1374
- if(!query_plan(ops, qt->window.after, qt->window.before, qt->window.points))
1429
+ if(!query_plan(ops, qt->window.after, qt->window.before, qt->window.points)) {
1430
+ rrd2rrdr_query_ops_release(r, ops);
1431
return NULL;
1432
+ }
1433
1434
return ops;
1435
}
@@ -1412,9 +1469,10 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1469
time_t now_end_time = after_wanted + ops->view_update_every - ops->query_granularity;
1470
1471
size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch;
1472
+ size_t query_is_finished_counter = 0;
1473
1474
// The main loop, based on the query granularity we need
1417
- for( ; points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops->view_update_every) {
1475
+ for( ; points_added < points_wanted && query_is_finished_counter <= 10 ; now_start_time = now_end_time, now_end_time += ops->view_update_every) {
1476
1477
if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) {
1478
query_planer_next_plan(ops, now_end_time, new_point.end_time);
@@ -1431,6 +1489,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1489
}
1490
1491
if(unlikely(ops->is_finished(ops->handle))) {
1492
+ query_is_finished_counter++;
1493
+
1494
if(count_same_end_time != 0) {
1495
last2_point = last1_point;
1496
last1_point = new_point;
@@ -1443,6 +1503,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1503
//
1504
break;
1505
}
1506
+ else
1507
+ query_is_finished_counter = 0;
1508
1509
// fetch the new point
1510
{
@@ -1743,16 +1805,32 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1805
}
1806
query_planer_finalize_remaining_plans(ops);
1807
1746
- r->stats.result_points_generated += points_added;
1747
- r->stats.db_points_read += ops->db_total_points_read;
1748
- for(size_t tr = 0; tr < storage_tiers ; tr++)
1749
- qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
1808
+ // fill the rest of the points with empty values
1809
+ while (points_added < points_wanted) {
1810
+ if(!max_date)
1811
+ min_date = max_date = after_wanted + ops->view_update_every - ops->query_granularity;
1812
+ else
1813
+ max_date += ops->view_update_every;
1814
+
1815
+ rrdr_line = rrdr_line_init(r, max_date, rrdr_line);
1816
+ size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
1817
+ r->o[rrdr_o_v_index] = RRDR_VALUE_EMPTY;
1818
+ r->v[rrdr_o_v_index] = 0.0;
1819
+ r->ar[rrdr_o_v_index] = 0.0;
1820
+
1821
+ points_added++;
1822
+ }
1823
1824
r->view.min = min;
1825
r->view.max = max;
1826
r->view.before = max_date;
1827
r->view.after = min_date - ops->view_update_every + ops->query_granularity;
1755
- rrdr_done(r, rrdr_line);
1828
+ r->rows = rrdr_line + 1;
1829
+
1830
+ r->stats.result_points_generated += points_added;
1831
+ r->stats.db_points_read += ops->db_total_points_read;
1832
+ for(size_t tr = 0; tr < storage_tiers ; tr++)
1833
+ qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
1834
1835
internal_error(points_added != points_wanted,
1836
"QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).",
@@ -1891,9 +1969,12 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1969
#endif // NETDATA_INTERNAL_CHECKS
1970
1971
// Returns 1 if an absolute period was requested or 0 if it was a relative period
1894
-bool rrdr_relative_window_to_absolute(time_t *after, time_t *before) {
1972
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr) {
1973
time_t now = now_realtime_sec() - 1;
1974
1975
+ if(now_ptr)
1976
+ *now_ptr = now;
1977
+
1978
int absolute_period_requested = -1;
1979
long long after_requested, before_requested;
1980
@@ -2086,7 +2167,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
2167
}
2168
2169
// convert our before_wanted and after_wanted to absolute
2089
- rrdr_relative_window_to_absolute(&after_wanted, &before_wanted);
2170
+ rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL);
2171
query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
2172
2173
if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
@@ -2363,6 +2444,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2444
size_t last_db_points_read = 0;
2445
size_t last_result_points_generated = 0;
2446
2447
+ internal_fatal(released_ops, "QUERY: released_ops should be NULL when the query starts");
2448
+
2449
QUERY_ENGINE_OPS **ops = NULL;
2450
if(qt->query.used)
2451
ops = onewayalloc_callocz(r->internal.owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
@@ -2372,35 +2455,37 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2455
size_t queries_prepared = 0;
2456
while(queries_prepared < max_queries_to_prepare) {
2457
// preload another query
2375
- ops[queries_prepared] = rrd2rrdr_query_prep(r, queries_prepared);
2458
+ ops[queries_prepared] = rrd2rrdr_query_ops_prep(r, queries_prepared);
2459
queries_prepared++;
2460
}
2461
2379
- for(size_t c = 0, max = qt->query.used; c < max ; c++) {
2380
- QUERY_METRIC *qm = query_metric(qt, c);
2462
+ for(size_t d = 0, max = qt->query.used; d < max ; d++) {
2463
+ QUERY_METRIC *qm = query_metric(qt, d);
2464
QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
2465
QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
2466
QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id);
2384
- QUERY_NODE *qn = query_node(qt, qm->link.query_host_id);
2467
+ QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
2468
2469
if(queries_prepared < max) {
2470
// preload another query
2388
- ops[queries_prepared] = rrd2rrdr_query_prep(r, queries_prepared);
2471
+ ops[queries_prepared] = rrd2rrdr_query_ops_prep(r, queries_prepared);
2472
queries_prepared++;
2473
}
2474
2475
// set the query target dimension options to rrdr
2393
- r->od[c] = qm->status;
2476
+ r->od[d] = qm->status;
2477
2478
// reset the grouping for the new dimension
2479
r->grouping.reset(r);
2480
2398
- if(ops[c]) {
2399
- rrd2rrdr_query_execute(r, c, ops[c]);
2481
+ if(ops[d]) {
2482
+ rrd2rrdr_query_execute(r, d, ops[d]);
2483
+ rrd2rrdr_query_ops_release(r, ops[d]); // reuse this ops allocation
2484
+ ops[d] = NULL;
2485
2401
- r->od[c] |= RRDR_DIMENSION_QUERIED;
2402
- r->di[c] = rrdmetric_acquired_id_dup(qd->rma);
2403
- r->dn[c] = rrdmetric_acquired_name_dup(qd->rma);
2486
+ r->od[d] |= RRDR_DIMENSION_QUERIED;
2487
+ r->di[d] = rrdmetric_acquired_id_dup(qd->rma);
2488
+ r->dn[d] = rrdmetric_acquired_name_dup(qd->rma);
2489
2490
qi->metrics.queried++;
2491
qc->metrics.queried++;
@@ -2439,7 +2524,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2524
if (qt->request.timeout)
2525
now_realtime_timeval(&query_current_time);
2526
2442
- if(r->od[c] & RRDR_DIMENSION_NONZERO)
2527
+ if(r->od[d] & RRDR_DIMENSION_NONZERO)
2528
dimensions_nonzero++;
2529
2530
// verify all dimensions are aligned
@@ -2477,9 +2562,12 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2562
(NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout);
2563
r->view.flags |= RRDR_RESULT_FLAG_CANCEL;
2564
2480
- for(size_t i = c + 1; i < queries_prepared ; i++) {
2481
- if(ops[i])
2565
+ for(size_t i = d + 1; i < queries_prepared ; i++) {
2566
+ if(ops[i]) {
2567
query_planer_finalize_remaining_plans(ops[i]);
2568
+ rrd2rrdr_query_ops_release(r, ops[i]);
2569
+ ops[i] = NULL;
2570
+ }
2571
}
2572
2573
break;
@@ -2564,8 +2652,12 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2652
#endif
2653
2654
// free the query pipelining ops
2567
- for(size_t c = 0; c < qt->query.used ;c++)
2568
- onewayalloc_freez(owa, ops[c]);
2655
+ for(size_t d = 0; d < qt->query.used ; d++) {
2656
+ rrd2rrdr_query_ops_release(r, ops[d]);
2657
+ ops[d] = NULL;
2658
+ }
2659
+ rrd2rrdr_query_ops_freeall(r);
2660
+ internal_fatal(released_ops, "QUERY: released_ops should be NULL when the query ends");
2661
2662
onewayalloc_freez(owa, ops);
2663
web/api/queries/query.h
+1
-1
@@ -58,6 +58,7 @@ typedef enum rrdr_group_by {
58
RRDR_GROUP_BY_NODE = (1 << 1),
59
RRDR_GROUP_BY_INSTANCE = (1 << 2),
60
RRDR_GROUP_BY_LABEL = (1 << 3),
61
+ RRDR_GROUP_BY_SELECTED = (1 << 4),
62
} RRDR_GROUP_BY;
63
64
struct web_buffer;
@@ -70,7 +71,6 @@ typedef enum rrdr_group_by_function {
71
RRDR_GROUP_BY_FUNCTION_MIN,
72
RRDR_GROUP_BY_FUNCTION_MAX,
73
RRDR_GROUP_BY_FUNCTION_SUM,
73
- RRDR_GROUP_BY_FUNCTION_SUM_COUNT,
74
} RRDR_GROUP_BY_FUNCTION;
75
76
RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s);
web/api/queries/rrdr.h
+16
-8
@@ -41,20 +41,22 @@ typedef enum rrdr_options {
41
RRDR_OPTION_RETURN_JWAR = (1 << 20), // Return anomaly rates in jsonwrap
42
RRDR_OPTION_SELECTED_TIER = (1 << 21), // Use the selected tier for the query
43
RRDR_OPTION_ALL_DIMENSIONS = (1 << 22), // Return the full dimensions list
44
- RRDR_OPTION_SHOW_PLAN = (1 << 23), // Return the query plan in jsonwrap
45
- RRDR_OPTION_SHOW_DETAILS = (1 << 24), // v2 returns detailed object tree
46
- RRDR_OPTION_DEBUG = (1 << 25), // v2 returns request description
47
- RRDR_OPTION_MINIFY = (1 << 26), // remove JSON spaces and newlines from JSON output
48
- RRDR_OPTION_JW_ANNOTATIONS = (1 << 27), // add annotation array to the JSON output
44
+ RRDR_OPTION_SHOW_DETAILS = (1 << 23), // v2 returns detailed object tree
45
+ RRDR_OPTION_DEBUG = (1 << 24), // v2 returns request description
46
+ RRDR_OPTION_MINIFY = (1 << 25), // remove JSON spaces and newlines from JSON output
47
48
// internal ones - not to be exposed to the API
51
- RRDR_OPTION_HEALTH_RSRVD1 = (1 << 28), // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
52
- RRDR_OPTION_INTERNAL_ANNOTATIONS = (1 << 29), // internal use only, to let the formatters know we want to render the annotations
49
+ RRDR_OPTION_HEALTH_RSRVD1 = (1 << 29), // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
50
RRDR_OPTION_INTERNAL_AR = (1 << 30), // internal use only, to let the formatters know we want to render the anomaly rate
51
RRDR_OPTION_INTERNAL_GBC = (1 << 31), // internal use only, to let the formatters know we want to render the group by count
52
} RRDR_OPTIONS;
53
54
typedef enum __attribute__ ((__packed__)) rrdr_value_flag {
55
+
56
+ // IMPORTANT:
57
+ // THIS IS AN AGREED BIT MAP BETWEEN AGENT, CLOUD FRONT-END AND CLOUD BACK-END
58
+ // DO NOT CHANGE THE MAPPINGS !
59
+
60
RRDR_VALUE_NOTHING = 0, // no flag set (a good default)
61
RRDR_VALUE_EMPTY = (1 << 0), // the database value is empty
62
RRDR_VALUE_RESET = (1 << 1), // the database value is marked as reset (overflown)
@@ -132,6 +134,12 @@ typedef struct rrdresult {
134
NETDATA_DOUBLE resampling_divisor; // used by AVERAGE
135
} grouping;
136
137
+ struct {
138
+ time_t max_update_every;
139
+ time_t expected_after;
140
+ time_t trimmed_after;
141
+ } partial_data_trimming;
142
+
143
struct {
144
ONEWAYALLOC *owa; // the allocator used
145
struct query_target *qt; // the QUERY_TARGET
@@ -161,7 +169,7 @@ RRDR *rrd2rrdr_legacy(
169
RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
170
bool query_target_calculate_window(struct query_target *qt);
171
164
-bool rrdr_relative_window_to_absolute(time_t *after, time_t *before);
172
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr);
173
174
#ifdef __cplusplus
175
}
web/api/queries/weights.c
+2
-2
@@ -804,7 +804,7 @@ int web_api_v1_weights(
804
usec_t timeout_usec = timeout * USEC_PER_MS;
805
usec_t started_usec = now_realtime_usec();
806
807
- if(!rrdr_relative_window_to_absolute(&after, &before))
807
+ if(!rrdr_relative_window_to_absolute(&after, &before, NULL))
808
buffer_no_cacheable(wb);
809
810
if (before <= after) {
@@ -820,7 +820,7 @@ int web_api_v1_weights(
820
if(baseline_before <= API_RELATIVE_TIME_MAX)
821
baseline_before += after;
822
823
- rrdr_relative_window_to_absolute(&baseline_after, &baseline_before);
823
+ rrdr_relative_window_to_absolute(&baseline_after, &baseline_before, NULL);
824
825
if (baseline_before <= baseline_after) {
826
resp = HTTP_RESP_BAD_REQUEST;
web/api/web_api_v1.c
+4
-3
@@ -41,11 +41,9 @@ static struct {
41
, {"natural-points" , 0 , RRDR_OPTION_NATURAL_POINTS}
42
, {"virtual-points" , 0 , RRDR_OPTION_VIRTUAL_POINTS}
43
, {"all-dimensions" , 0 , RRDR_OPTION_ALL_DIMENSIONS}
44
- , {"plan" , 0 , RRDR_OPTION_SHOW_PLAN}
44
, {"details" , 0 , RRDR_OPTION_SHOW_DETAILS}
45
, {"debug" , 0 , RRDR_OPTION_DEBUG}
46
, {"minify" , 0 , RRDR_OPTION_MINIFY}
48
- , {"annotations" , 0 , RRDR_OPTION_JW_ANNOTATIONS}
47
, {NULL , 0 , 0}
48
};
49
@@ -57,6 +55,7 @@ static struct {
55
{ DATASOURCE_FORMAT_DATATABLE_JSON , 0 , DATASOURCE_DATATABLE_JSON}
56
, {DATASOURCE_FORMAT_DATATABLE_JSONP, 0 , DATASOURCE_DATATABLE_JSONP}
57
, {DATASOURCE_FORMAT_JSON , 0 , DATASOURCE_JSON}
58
+ , {DATASOURCE_FORMAT_JSON2 , 0 , DATASOURCE_JSON2}
59
, {DATASOURCE_FORMAT_JSONP , 0 , DATASOURCE_JSONP}
60
, {DATASOURCE_FORMAT_SSV , 0 , DATASOURCE_SSV}
61
, {DATASOURCE_FORMAT_CSV , 0 , DATASOURCE_CSV}
@@ -67,7 +66,9 @@ static struct {
66
, {DATASOURCE_FORMAT_SSV_COMMA , 0 , DATASOURCE_SSV_COMMA}
67
, {DATASOURCE_FORMAT_CSV_JSON_ARRAY , 0 , DATASOURCE_CSV_JSON_ARRAY}
68
, {DATASOURCE_FORMAT_CSV_MARKDOWN , 0 , DATASOURCE_CSV_MARKDOWN}
70
- , { NULL, 0, 0}
69
+
70
+ // terminator
71
+ , {NULL, 0, 0}
72
};
73
74
static struct {
web/api/web_api_v2.c
+7
-3
@@ -27,6 +27,7 @@ static int web_client_api_request_v2_contexts_internal(RRDHOST *host __maybe_unu
27
options |= CONTEXTS_V2_DEBUG;
28
29
buffer_flush(w->response.data);
30
+ buffer_no_cacheable(w->response.data);
31
return rrdcontext_to_json_v2(w->response.data, &req, options);
32
}
33
@@ -79,7 +80,7 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
80
RRDR_GROUP_BY group_by = RRDR_GROUP_BY_DIMENSION;
81
RRDR_GROUP_BY_FUNCTION group_by_aggregate = RRDR_GROUP_BY_FUNCTION_AVERAGE;
82
DATASOURCE_FORMAT format = DATASOURCE_JSON;
82
- RRDR_OPTIONS options = 0;
83
+ RRDR_OPTIONS options = RRDR_OPTION_VIRTUAL_POINTS | RRDR_OPTION_JSON_WRAP | RRDR_OPTION_RETURN_JWAR;
84
85
while(url) {
86
char *value = mystrsep(&url, "&");
@@ -162,11 +163,14 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
163
if(group_by == RRDR_GROUP_BY_NONE)
164
group_by = RRDR_GROUP_BY_DIMENSION;
165
166
+ if(group_by & RRDR_GROUP_BY_SELECTED)
167
+ group_by = RRDR_GROUP_BY_SELECTED; // remove all other groupings
168
+
169
if(group_by & ~(RRDR_GROUP_BY_DIMENSION))
170
options |= RRDR_OPTION_ABSOLUTE;
171
168
- if(options & RRDR_OPTION_SHOW_PLAN)
169
- options |= RRDR_OPTION_DEBUG;
172
+ if(options & RRDR_OPTION_DEBUG)
173
+ options &= ~RRDR_OPTION_MINIFY;
174
175
if(tier_str && *tier_str) {
176
tier = str2ul(tier_str);
web/server/web_client.c
+25
-10
@@ -919,12 +919,12 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
919
920
is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size);
921
if(!is_it_valid) {
922
- if(w->header_parse_tries > 10) {
922
+ if(w->header_parse_tries > HTTP_REQ_MAX_HEADER_FETCH_TRIES) {
923
info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
924
w->header_parse_tries = 0;
925
w->header_parse_last_size = 0;
926
web_client_disable_wait_receive(w);
927
- return HTTP_VALIDATION_NOT_SUPPORTED;
927
+ return HTTP_VALIDATION_TOO_MANY_READ_RETRIES;
928
}
929
930
return HTTP_VALIDATION_INCOMPLETE;
@@ -952,7 +952,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
952
w->header_parse_tries = 0;
953
w->header_parse_last_size = 0;
954
web_client_disable_wait_receive(w);
955
- return HTTP_VALIDATION_NOT_SUPPORTED;
955
+ return HTTP_VALIDATION_EXCESS_REQUEST_DATA;
956
}
957
}
958
web_client_enable_wait_receive(w);
@@ -1004,7 +1004,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
1004
web_client_split_path_query(w, encoded_url);
1005
1006
if (w->url_search_path && w->separator) {
1007
- *w->url_search_path = 0x00;
1007
+ *w->url_search_path = '\0';
1008
}
1009
1010
if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
@@ -1519,8 +1519,9 @@ void web_client_process_request(struct web_client *w) {
1519
1520
debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, w->response.data->len);
1521
1522
+ size_t len = w->response.data->len;
1523
buffer_flush(w->response.data);
1523
- buffer_sprintf(w->response.data, "Received request is too big (%zu bytes).\r\n", w->response.data->len);
1524
+ buffer_sprintf(w->response.data, "Received request is too big (received %zu bytes, max is %zu bytes).\r\n", len, (size_t)NETDATA_WEB_REQUEST_MAX_SIZE);
1525
w->response.code = HTTP_RESP_BAD_REQUEST;
1526
}
1527
else {
@@ -1550,17 +1551,31 @@ void web_client_process_request(struct web_client *w) {
1551
}
1552
#endif
1553
case HTTP_VALIDATION_MALFORMED_URL:
1553
- debug(D_WEB_CLIENT_ACCESS, "%llu: URL parsing failed (malformed URL). Cannot understand '%s'.", w->id, w->response.data->buffer);
1554
+ debug(D_WEB_CLIENT_ACCESS, "%llu: Malformed URL '%s'.", w->id, w->response.data->buffer);
1555
1556
buffer_flush(w->response.data);
1556
- buffer_strcat(w->response.data, "URL not valid. I don't understand you...\r\n");
1557
+ buffer_strcat(w->response.data, "Malformed URL...\r\n");
1558
+ w->response.code = HTTP_RESP_BAD_REQUEST;
1559
+ break;
1560
+ case HTTP_VALIDATION_EXCESS_REQUEST_DATA:
1561
+ debug(D_WEB_CLIENT_ACCESS, "%llu: Excess data in request '%s'.", w->id, w->response.data->buffer);
1562
+
1563
+ buffer_flush(w->response.data);
1564
+ buffer_strcat(w->response.data, "Excess data in request.\r\n");
1565
+ w->response.code = HTTP_RESP_BAD_REQUEST;
1566
+ break;
1567
+ case HTTP_VALIDATION_TOO_MANY_READ_RETRIES:
1568
+ debug(D_WEB_CLIENT_ACCESS, "%llu: Too many retries to read request '%s'.", w->id, w->response.data->buffer);
1569
+
1570
+ buffer_flush(w->response.data);
1571
+ buffer_strcat(w->response.data, "Too many retries to read request.\r\n");
1572
w->response.code = HTTP_RESP_BAD_REQUEST;
1573
break;
1574
case HTTP_VALIDATION_NOT_SUPPORTED:
1560
- debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, w->response.data->buffer);
1575
+ debug(D_WEB_CLIENT_ACCESS, "%llu: HTTP method requested is not supported '%s'.", w->id, w->response.data->buffer);
1576
1577
buffer_flush(w->response.data);
1563
- buffer_strcat(w->response.data, "I don't understand you...\r\n");
1578
+ buffer_strcat(w->response.data, "HTTP method requested is not supported...\r\n");
1579
w->response.code = HTTP_RESP_BAD_REQUEST;
1580
break;
1581
}
@@ -1919,7 +1934,7 @@ ssize_t web_client_receive(struct web_client *w)
1934
ssize_t left = (ssize_t)(w->response.data->size - w->response.data->len);
1935
1936
// do we have any space for more data?
1922
- buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_RECEIVE_SIZE);
1937
+ buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE);
1938
1939
#ifdef ENABLE_HTTPS
1940
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
web/server/web_client.h
+14
-10
@@ -32,6 +32,8 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy;
32
#define HTTP_RESP_GATEWAY_TIMEOUT 504
33
#define HTTP_RESP_BACKEND_RESPONSE_INVALID 591
34
35
+#define HTTP_REQ_MAX_HEADER_FETCH_TRIES 100
36
+
37
extern int respect_web_browser_do_not_track_policy;
38
extern char *web_x_frame_options;
39
@@ -45,12 +47,12 @@ typedef enum web_client_mode {
47
typedef enum {
48
HTTP_VALIDATION_OK,
49
HTTP_VALIDATION_NOT_SUPPORTED,
50
+ HTTP_VALIDATION_TOO_MANY_READ_RETRIES,
51
+ HTTP_VALIDATION_EXCESS_REQUEST_DATA,
52
HTTP_VALIDATION_MALFORMED_URL,
49
-#ifdef ENABLE_HTTPS
53
HTTP_VALIDATION_INCOMPLETE,
54
+#ifdef ENABLE_HTTPS
55
HTTP_VALIDATION_REDIRECT
52
-#else
53
- HTTP_VALIDATION_INCOMPLETE
56
#endif
57
} HTTP_VALIDATION;
58
@@ -118,14 +120,16 @@ typedef enum web_client_flags {
120
121
#define web_client_is_corkable(w) web_client_flag_check(w, WEB_CLIENT_FLAG_TCP_CLIENT)
122
121
-#define NETDATA_WEB_REQUEST_URL_SIZE 8192
123
+#define NETDATA_WEB_REQUEST_URL_SIZE 65536 // static allocation
124
+#define NETDATA_WEB_REQUEST_COOKIE_SIZE 1024 // static allocation
125
+#define NETDATA_WEB_REQUEST_ORIGIN_HEADER_SIZE 1024 // static allocation
126
+
127
#define NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE 16384
123
-#define NETDATA_WEB_RESPONSE_HEADER_SIZE 4096
124
-#define NETDATA_WEB_REQUEST_COOKIE_SIZE 1024
125
-#define NETDATA_WEB_REQUEST_ORIGIN_HEADER_SIZE 1024
126
-#define NETDATA_WEB_RESPONSE_INITIAL_SIZE 16384
127
-#define NETDATA_WEB_REQUEST_RECEIVE_SIZE 16384
128
-#define NETDATA_WEB_REQUEST_MAX_SIZE 16384
128
+
129
+#define NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE 4096
130
+#define NETDATA_WEB_RESPONSE_INITIAL_SIZE 8192
131
+#define NETDATA_WEB_REQUEST_INITIAL_SIZE 8192
132
+#define NETDATA_WEB_REQUEST_MAX_SIZE 65536
133
134
struct response {
135
BUFFER *header; // our response header
web/server/web_client_cache.c
+2
-2
@@ -72,8 +72,8 @@ static struct web_client *web_client_alloc(void) {
72
struct web_client *w = callocz(1, sizeof(struct web_client));
73
__atomic_add_fetch(&netdata_buffers_statistics.buffers_web, sizeof(struct web_client), __ATOMIC_RELAXED);
74
w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
75
- w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_web);
76
- w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_web);
75
+ w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
76
+ w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
77
return w;
78
}
79