Improved the data query when using the context parameter (#9978)
Stelios Fragkakis committed
Sep 24, 2020 at 13:05 UTC
80676423617d2587632eb69c9ea5a0ef0cb49d0c
9 files changed
+120
-81
database/engine/rrdengineapi.c
+2
-1
@@ -376,7 +376,7 @@ static inline uint32_t *pginfo_to_points(struct rrdeng_page_info *page_info)
376
* @return number of regions with different data collection intervals.
377
*/
378
unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
379
- struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, RRDDIM *temp_rd)
379
+ struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list)
380
{
381
struct pg_cache_page_index *page_index;
382
struct rrdengine_instance *ctx;
@@ -396,6 +396,7 @@ unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t e
396
*region_info_arrayp = NULL;
397
page_info_array = NULL;
398
399
+ RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
400
rrdset_rdlock(st);
401
for(rd_iter = temp_rd?temp_rd:st->dimensions, rd = NULL, min_time = (usec_t)-1 ; rd_iter ; rd_iter = rd_iter->next) {
402
/*
database/engine/rrdengineapi.h
+1
-1
@@ -43,7 +43,7 @@ extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_n
43
extern int rrdeng_store_metric_finalize(RRDDIM *rd);
44
extern unsigned
45
rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
46
- struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, RRDDIM *temp_rd);
46
+ struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
47
extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle,
48
time_t start_time, time_t end_time);
49
extern storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *current_time);
database/rrd.h
+1
@@ -13,6 +13,7 @@ typedef struct rrddimvar RRDDIMVAR;
13
typedef struct rrdcalc RRDCALC;
14
typedef struct rrdcalctemplate RRDCALCTEMPLATE;
15
typedef struct alarm_entry ALARM_ENTRY;
16
+typedef struct context_param CONTEXT_PARAM;
17
18
// forward declarations
19
struct rrddim_volatile;
web/api/formatters/rrd2json.c
+55
-42
@@ -22,6 +22,57 @@ static inline void free_temp_rrddim(RRDDIM *temp_rd)
22
}
23
}
24
25
+void free_context_param_list(struct context_param **param_list)
26
+{
27
+ if (unlikely(!param_list || !*param_list))
28
+ return;
29
+
30
+ free_temp_rrddim(((*param_list)->rd));
31
+ freez((*param_list));
32
+ *param_list = NULL;
33
+}
34
+
35
+void build_context_param_list(struct context_param **param_list, RRDSET *st)
36
+{
37
+ if (unlikely(!param_list || !st))
38
+ return;
39
+
40
+ if (unlikely(!(*param_list))) {
41
+ *param_list = mallocz(sizeof(struct context_param));
42
+ (*param_list)->first_entry_t = LONG_MAX;
43
+ (*param_list)->last_entry_t = 0;
44
+ (*param_list)->rd = NULL;
45
+ }
46
+
47
+ RRDDIM *rd1;
48
+ rrdset_rdlock(st);
49
+
50
+ st->last_accessed_time = now_realtime_sec();
51
+ (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t(st));
52
+ (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t(st));
53
+
54
+ rrddim_foreach_read(rd1, st) {
55
+ RRDDIM *rd = mallocz(rd1->memsize);
56
+ memcpy(rd, rd1, rd1->memsize);
57
+ rd->id = strdupz(rd1->id);
58
+ rd->name = strdupz(rd1->name);
59
+ rd->state = mallocz(sizeof(*rd->state));
60
+ memcpy(rd->state, rd1->state, sizeof(*rd->state));
61
+ memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops));
62
+ memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops));
63
+#ifdef ENABLE_DBENGINE
64
+ if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
65
+ rd->state->metric_uuid = mallocz(sizeof(uuid_t));
66
+ uuid_copy(*rd->state->metric_uuid, *rd1->state->metric_uuid);
67
+ }
68
+#endif
69
+ rd->next = (*param_list)->rd;
70
+ (*param_list)->rd = rd;
71
+ }
72
+
73
+ rrdset_unlock(st);
74
+}
75
+
76
void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
77
rrdset2json(st, wb, NULL, NULL, 0);
78
}
@@ -89,9 +140,8 @@ int rrdset2value_api_v1(
140
, time_t *db_before
141
, int *value_is_null
142
) {
92
- RRDDIM *temp_rd = NULL;
143
94
- RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, temp_rd);
144
+ RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL);
145
146
if(!r) {
147
if(value_is_null) *value_is_null = 1;
@@ -121,8 +171,6 @@ int rrdset2value_api_v1(
171
long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
172
*n = rrdr2value(r, i, options, value_is_null);
173
124
- free_temp_rrddim(temp_rd);
125
-
174
rrdr_free(r);
175
return HTTP_RESP_OK;
176
}
@@ -139,47 +187,14 @@ int rrdset2anything_api_v1(
187
, long group_time
188
, uint32_t options
189
, time_t *latest_timestamp
142
- , char *context
190
+ , struct context_param *context_param_list
191
) {
192
time_t last_accessed_time = now_realtime_sec();
193
st->last_accessed_time = last_accessed_time;
194
147
- RRDDIM *temp_rd = NULL;
148
-
149
- if (context) {
150
- rrdhost_rdlock(st->rrdhost);
151
- RRDSET *st1;
152
- rrdset_foreach_read(st1, st->rrdhost) {
153
- if (strcmp(st1->context, context) == 0) {
154
- // Loop the dimensions of the chart
155
- RRDDIM *rd1;
156
- rrdset_rdlock(st1);
157
- st1->last_accessed_time = last_accessed_time;
158
- rrddim_foreach_read(rd1, st1) {
159
- RRDDIM *rd = mallocz(rd1->memsize);
160
- memcpy(rd, rd1, rd1->memsize);
161
- rd->id = strdupz(rd1->id);
162
- rd->name = strdupz(rd1->name);
163
- rd->state = mallocz(sizeof(*rd->state));
164
- memcpy(rd->state, rd1->state, sizeof(*rd->state));
165
- memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops));
166
- memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops));
167
-#ifdef ENABLE_DBENGINE
168
- if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
169
- rd->state->metric_uuid = mallocz(sizeof(uuid_t));
170
- uuid_copy(*rd->state->metric_uuid, *rd1->state->metric_uuid);
171
- }
172
-#endif
173
- rd->next = temp_rd;
174
- temp_rd = rd;
175
- }
176
- rrdset_unlock(st1);
177
- }
178
- }
179
- rrdhost_unlock(st->rrdhost);
180
- }
195
+ RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
196
182
- RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, temp_rd);
197
+ RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list);
198
if(!r) {
199
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
200
return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -355,8 +370,6 @@ int rrdset2anything_api_v1(
370
break;
371
}
372
358
- free_temp_rrddim(temp_rd);
359
-
373
rrdr_free(r);
374
return HTTP_RESP_OK;
375
}
web/api/formatters/rrd2json.h
+10
-1
@@ -53,6 +53,12 @@
53
extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
54
extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
55
56
+typedef struct context_param {
57
+ RRDDIM *rd;
58
+ time_t first_entry_t;
59
+ time_t last_entry_t;
60
+} CONTEXT_PARAM;
61
+
62
extern int rrdset2anything_api_v1(
63
RRDSET *st
64
, BUFFER *wb
@@ -65,7 +71,7 @@ extern int rrdset2anything_api_v1(
71
, long group_time
72
, uint32_t options
73
, time_t *latest_timestamp
68
- , char *context
74
+ , struct context_param *context_param_list
75
);
76
77
extern int rrdset2value_api_v1(
@@ -84,4 +90,7 @@ extern int rrdset2value_api_v1(
90
, int *value_is_null
91
);
92
93
+extern void build_context_param_list(struct context_param **param_list, RRDSET *st);
94
+extern void free_context_param_list(struct context_param **param_list);
95
+
96
#endif /* NETDATA_RRD2JSON_H */
web/api/queries/query.c
+21
-15
@@ -815,8 +815,8 @@ static RRDR *rrd2rrdr_fixedstep(
815
, int update_every
816
, time_t first_entry_t
817
, time_t last_entry_t
818
- , int absolute_period_requested,
819
- RRDDIM *temp_rd
818
+ , int absolute_period_requested
819
+ , struct context_param *context_param_list
820
) {
821
int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
822
@@ -824,8 +824,10 @@ static RRDR *rrd2rrdr_fixedstep(
824
time_t duration = before_requested - after_requested;
825
long available_points = duration / update_every;
826
827
+ RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
828
+
829
if(duration <= 0 || available_points <= 0)
828
- return rrdr_create(st, 1, temp_rd);
830
+ return rrdr_create(st, 1, context_param_list);
831
832
// check the number of wanted points in the result
833
if(unlikely(points_requested < 0)) points_requested = -points_requested;
@@ -983,7 +985,7 @@ static RRDR *rrd2rrdr_fixedstep(
985
// initialize our result set
986
// this also locks the chart for us
987
986
- RRDR *r = rrdr_create(st, points_wanted, temp_rd);
988
+ RRDR *r = rrdr_create(st, points_wanted, context_param_list);
989
if(unlikely(!r)) {
990
#ifdef NETDATA_INTERNAL_CHECKS
991
error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
@@ -1185,7 +1187,7 @@ static RRDR *rrd2rrdr_variablestep(
1187
, time_t last_entry_t
1188
, int absolute_period_requested
1189
, struct rrdeng_region_info *region_info_array
1188
- , RRDDIM *temp_rd
1190
+ , struct context_param *context_param_list
1191
) {
1192
int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
1193
@@ -1193,9 +1195,11 @@ static RRDR *rrd2rrdr_variablestep(
1195
time_t duration = before_requested - after_requested;
1196
long available_points = duration / update_every;
1197
1198
+ RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
1199
+
1200
if(duration <= 0 || available_points <= 0) {
1201
freez(region_info_array);
1198
- return rrdr_create(st, 1, temp_rd);
1202
+ return rrdr_create(st, 1, context_param_list);
1203
}
1204
1205
// check the number of wanted points in the result
@@ -1354,7 +1358,7 @@ static RRDR *rrd2rrdr_variablestep(
1358
// initialize our result set
1359
// this also locks the chart for us
1360
1357
- RRDR *r = rrdr_create(st, points_wanted, temp_rd);
1361
+ RRDR *r = rrdr_create(st, points_wanted, context_param_list);
1362
if(unlikely(!r)) {
1363
#ifdef NETDATA_INTERNAL_CHECKS
1364
error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
@@ -1556,17 +1560,19 @@ RRDR *rrd2rrdr(
1560
, long resampling_time_requested
1561
, RRDR_OPTIONS options
1562
, const char *dimensions
1559
- , RRDDIM *temp_rd
1563
+ , struct context_param *context_param_list
1564
)
1565
{
1566
int rrd_update_every;
1567
int absolute_period_requested;
1568
1569
+// RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
1570
+
1571
time_t first_entry_t;
1572
time_t last_entry_t;
1567
- if (temp_rd) {
1568
- first_entry_t = rrddim_first_entry_t(temp_rd);
1569
- last_entry_t = rrddim_last_entry_t(temp_rd);
1573
+ if (context_param_list) {
1574
+ first_entry_t = context_param_list->first_entry_t;
1575
+ last_entry_t = context_param_list->last_entry_t;
1576
} else {
1577
first_entry_t = rrdset_first_entry_t(st);
1578
last_entry_t = rrdset_last_entry_t(st);
@@ -1583,7 +1589,7 @@ RRDR *rrd2rrdr(
1589
1590
/* This call takes the chart read-lock */
1591
regions = rrdeng_variable_step_boundaries(st, after_requested, before_requested,
1586
- ®ion_info_array, &max_interval, temp_rd);
1592
+ ®ion_info_array, &max_interval, context_param_list);
1593
if (1 == regions) {
1594
if (region_info_array) {
1595
if (rrd_update_every != region_info_array[0].update_every) {
@@ -1597,7 +1603,7 @@ RRDR *rrd2rrdr(
1603
}
1604
return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
1605
resampling_time_requested, options, dimensions, rrd_update_every,
1600
- first_entry_t, last_entry_t, absolute_period_requested, temp_rd);
1606
+ first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
1607
} else {
1608
if (rrd_update_every != (uint16_t)max_interval) {
1609
rrd_update_every = (uint16_t) max_interval;
@@ -1608,11 +1614,11 @@ RRDR *rrd2rrdr(
1614
}
1615
return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method,
1616
resampling_time_requested, options, dimensions, rrd_update_every,
1611
- first_entry_t, last_entry_t, absolute_period_requested, region_info_array, temp_rd);
1617
+ first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list);
1618
}
1619
}
1620
#endif
1621
return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
1622
resampling_time_requested, options, dimensions,
1617
- rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, temp_rd);
1623
+ rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
1624
}
\ No newline at end of file
web/api/queries/rrdr.c
+2
-1
@@ -98,7 +98,7 @@ inline void rrdr_free(RRDR *r)
98
freez(r);
99
}
100
101
-RRDR *rrdr_create(struct rrdset *st, long n, struct rrddim *temp_rd)
101
+RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list)
102
{
103
if(unlikely(!st)) {
104
error("NULL value given!");
@@ -110,6 +110,7 @@ RRDR *rrdr_create(struct rrdset *st, long n, struct rrddim *temp_rd)
110
111
rrdr_lock_rrdset(r);
112
113
+ RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
114
RRDDIM *rd;
115
if (temp_rd) {
116
RRDDIM *t = temp_rd;
web/api/queries/rrdr.h
+5
-2
@@ -99,12 +99,15 @@ typedef struct rrdresult {
99
100
#include "../../../database/rrd.h"
101
extern void rrdr_free(RRDR *r);
102
-extern RRDR *rrdr_create(struct rrdset *st, long n, struct rrddim *id);
102
+extern RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list);
103
104
#include "../web_api_v1.h"
105
#include "web/api/queries/query.h"
106
107
-extern RRDR *rrd2rrdr(RRDSET *st, long points_requested, long long after_requested, long long before_requested, RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions, RRDDIM *temp_rd);
107
+extern RRDR *rrd2rrdr(
108
+ RRDSET *st, long points_requested, long long after_requested, long long before_requested,
109
+ RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
110
+ struct context_param *context_param_list);
111
112
#include "query.h"
113
web/api/web_api_v1.c
+23
-18
@@ -491,36 +491,39 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
491
goto cleanup;
492
}
493
494
+ struct context_param *context_param_list = NULL;
495
if (context) {
495
- st = NULL;
496
- // TODO: Scan all charts of host
497
- rrdhost_rdlock(localhost);
496
RRDSET *st1;
497
+ uint32_t context_hash = simple_hash(context);
498
+ rrdhost_rdlock(localhost);
499
rrdset_foreach_read(st1, localhost) {
500
- if (strcmp(st1->context, context) == 0) {
501
- st = st1;
502
- break;
503
- }
500
+ if (st1->hash_context == context_hash && !strcmp(st1->context, context))
501
+ build_context_param_list(&context_param_list, st1);
502
}
503
rrdhost_unlock(localhost);
504
+ if (likely(context_param_list && context_param_list->rd)) // Just set the first one
505
+ st = context_param_list->rd->rrdset;
506
}
507
-
508
- if (!st) {
509
- if (!chart || !*chart) {
510
- buffer_sprintf(w->response.data, "No chart id is given at the request.");
511
- goto cleanup;
512
- }
507
+ else {
508
st = rrdset_find(host, chart);
509
if (!st)
510
st = rrdset_find_byname(host, chart);
516
- if (!st) {
511
+ if (likely(st))
512
+ st->last_accessed_time = now_realtime_sec();
513
+ }
514
+
515
+ if (!st && !context_param_list) {
516
+ if (context) {
517
+ buffer_strcat(w->response.data, "Context is not found: ");
518
+ buffer_strcat_htmlescape(w->response.data, context);
519
+ }
520
+ else {
521
buffer_strcat(w->response.data, "Chart is not found: ");
522
buffer_strcat_htmlescape(w->response.data, chart);
519
- ret = HTTP_RESP_NOT_FOUND;
520
- goto cleanup;
523
}
524
+ ret = HTTP_RESP_NOT_FOUND;
525
+ goto cleanup;
526
}
523
- st->last_accessed_time = now_realtime_sec();
527
528
long long before = (before_str && *before_str)?str2l(before_str):0;
529
long long after = (after_str && *after_str) ?str2l(after_str):-600;
@@ -565,7 +568,9 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
568
}
569
570
ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, points, after, before, group, group_time
568
- , options, &last_timestamp_in_data, context);
571
+ , options, &last_timestamp_in_data, context_param_list);
572
+
573
+ free_context_param_list(&context_param_list);
574
575
if(format == DATASOURCE_DATATABLE_JSONP) {
576
if(google_timestamp < last_timestamp_in_data)