Add a timeout parameter to data queries (#12649)
* Add timeout parameter in queries and in calling functions * Add CANCEL flag in RRDR and code to cancel a query * Update swagger * Format swagger file properly
Stelios Fragkakis committed
Apr 11, 2022 at 22:34 UTC
81b3d4b71e1e236f461cca9af0b588d385aefe05
10 files changed
+77
-15
daemon/unit_test.c
+2
-2
@@ -1732,7 +1732,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
1732
update_every = REGION_UPDATE_EVERY[current_region];
1733
long points = (time_end - time_start) / update_every;
1734
for (i = 0 ; i < CHARTS ; ++i) {
1735
- RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL);
1735
+ RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);
1736
if (!r) {
1737
fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
1738
return ++errors;
@@ -1851,7 +1851,7 @@ int test_dbengine(void)
1851
long points = (time_end[REGIONS - 1] - time_start[0]) / update_every; // cover all time regions with RRDR
1852
long point_offset = (time_start[current_region] - time_start[0]) / update_every;
1853
for (i = 0 ; i < CHARTS ; ++i) {
1854
- RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL);
1854
+ RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);
1855
if (!r) {
1856
fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
1857
++errors;
health/health.c
+1
-1
@@ -830,7 +830,7 @@ void *health_main(void *ptr) {
830
831
int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1, rc->after,
832
rc->before, rc->group, 0, rc->options, &rc->db_after,
833
- &rc->db_before, &value_is_null
833
+ &rc->db_before, &value_is_null, 0
834
);
835
836
if (unlikely(ret != 200)) {
web/api/badges/web_buffer_svg.c
+1
-1
@@ -1102,7 +1102,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
1102
// if the collected value is too old, don't calculate its value
1103
if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above)))
1104
ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL
1105
- , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null);
1105
+ , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null, 0);
1106
1107
// if the value cannot be calculated, show empty badge
1108
if (ret != HTTP_RESP_OK) {
web/api/formatters/rrd2json.c
+11
-4
@@ -167,9 +167,10 @@ int rrdset2value_api_v1(
167
, time_t *db_after
168
, time_t *db_before
169
, int *value_is_null
170
+ , int timeout
171
) {
172
172
- RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL);
173
+ RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout);
174
175
if(!r) {
176
if(value_is_null) *value_is_null = 1;
@@ -218,17 +219,23 @@ int rrdset2anything_api_v1(
219
, struct context_param *context_param_list
220
, char *chart_label_key
221
, int max_anomaly_rates
221
-) {
222
-
222
+ , int timeout
223
+)
224
+{
225
if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
226
st->last_accessed_time = now_realtime_sec();
227
226
- RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list);
228
+ RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
229
if(!r) {
230
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
231
return HTTP_RESP_INTERNAL_SERVER_ERROR;
232
}
233
234
+ if (r->result_options & RRDR_RESULT_OPTION_CANCEL) {
235
+ rrdr_free(r);
236
+ return HTTP_RESP_BACKEND_FETCH_FAILED;
237
+ }
238
+
239
if (st && st->state && st->state->is_ar_chart)
240
ml_process_rrdr(r, max_anomaly_rates);
241
web/api/formatters/rrd2json.h
+2
@@ -68,6 +68,7 @@ extern int rrdset2anything_api_v1(
68
, struct context_param *context_param_list
69
, char *chart_label_key
70
, int max_anomaly_rates
71
+ , int timeout
72
);
73
74
extern int rrdset2value_api_v1(
@@ -84,6 +85,7 @@ extern int rrdset2value_api_v1(
85
, time_t *db_after
86
, time_t *db_before
87
, int *value_is_null
88
+ , int timeout
89
);
90
91
extern void build_context_param_list(struct context_param **param_list, RRDSET *st);
web/api/netdata-swagger.json
+12
@@ -231,6 +231,18 @@
231
"default": 0
232
}
233
},
234
+ {
235
+ "name": "timeout",
236
+ "in": "query",
237
+ "description": "Specify a timeout value in milliseconds after which the agent will abort the query and return a 503 error. A value of 0 indicates no timeout.",
238
+ "required": false,
239
+ "allowEmptyValue": false,
240
+ "schema": {
241
+ "type": "number",
242
+ "format": "integer",
243
+ "default": 0
244
+ }
245
+ },
246
{
247
"name": "format",
248
"in": "query",
web/api/netdata-swagger.yaml
+10
@@ -202,6 +202,16 @@ paths:
202
type: number
203
format: integer
204
default: 0
205
+ - name: timeout
206
+ in: query
207
+ description: Specify a timeout value in milliseconds after which the agent will
208
+ abort the query and return a 503 error. A value of 0 indicates no timeout.
209
+ required: false
210
+ allowEmptyValue: false
211
+ schema:
212
+ type: number
213
+ format: integer
214
+ default: 0
215
- name: format
216
in: query
217
description: The format of the data to be returned.
web/api/queries/query.c
+32
-5
@@ -844,6 +844,7 @@ static RRDR *rrd2rrdr_fixedstep(
844
, time_t last_entry_t
845
, int absolute_period_requested
846
, struct context_param *context_param_list
847
+ , int timeout
848
) {
849
int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
850
@@ -1097,6 +1098,10 @@ static RRDR *rrd2rrdr_fixedstep(
1098
1099
RRDDIM *rd;
1100
long c, dimensions_used = 0, dimensions_nonzero = 0;
1101
+ struct timeval query_start_time;
1102
+ struct timeval query_current_time;
1103
+ if (timeout)
1104
+ now_realtime_timeval(&query_start_time);
1105
for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
1106
1107
// if we need a percentage, we need to calculate all dimensions
@@ -1118,6 +1123,8 @@ static RRDR *rrd2rrdr_fixedstep(
1123
, before_wanted
1124
, options
1125
);
1126
+ if (timeout)
1127
+ now_realtime_timeval(&query_current_time);
1128
1129
if(r->od[c] & RRDR_DIMENSION_NONZERO)
1130
dimensions_nonzero++;
@@ -1155,6 +1162,12 @@ static RRDR *rrd2rrdr_fixedstep(
1162
}
1163
1164
dimensions_used++;
1165
+ if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
1166
+ log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
1167
+ dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
1168
+ r->result_options |= RRDR_RESULT_OPTION_CANCEL;
1169
+ break;
1170
+ }
1171
}
1172
1173
#ifdef NETDATA_INTERNAL_CHECKS
@@ -1188,7 +1201,7 @@ static RRDR *rrd2rrdr_fixedstep(
1201
r->internal.grouping_free(r);
1202
1203
// when all the dimensions are zero, we should return all of them
1191
- if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) {
1204
+ if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
1205
// all the dimensions are zero
1206
// mark them as NONZERO to send them all
1207
for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
@@ -1217,6 +1230,7 @@ static RRDR *rrd2rrdr_variablestep(
1230
, int absolute_period_requested
1231
, struct rrdeng_region_info *region_info_array
1232
, struct context_param *context_param_list
1233
+ , int timeout
1234
) {
1235
int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
1236
@@ -1474,6 +1488,10 @@ static RRDR *rrd2rrdr_variablestep(
1488
1489
RRDDIM *rd;
1490
long c, dimensions_used = 0, dimensions_nonzero = 0;
1491
+ struct timeval query_start_time;
1492
+ struct timeval query_current_time;
1493
+ if (timeout)
1494
+ now_realtime_timeval(&query_start_time);
1495
for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
1496
1497
// if we need a percentage, we need to calculate all dimensions
@@ -1495,6 +1513,8 @@ static RRDR *rrd2rrdr_variablestep(
1513
, before_wanted
1514
, options
1515
);
1516
+ if (timeout)
1517
+ now_realtime_timeval(&query_current_time);
1518
1519
if(r->od[c] & RRDR_DIMENSION_NONZERO)
1520
dimensions_nonzero++;
@@ -1532,6 +1552,12 @@ static RRDR *rrd2rrdr_variablestep(
1552
}
1553
1554
dimensions_used++;
1555
+ if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
1556
+ log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
1557
+ dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
1558
+ r->result_options |= RRDR_RESULT_OPTION_CANCEL;
1559
+ break;
1560
+ }
1561
}
1562
1563
#ifdef NETDATA_INTERNAL_CHECKS
@@ -1566,7 +1592,7 @@ static RRDR *rrd2rrdr_variablestep(
1592
r->internal.grouping_free(r);
1593
1594
// when all the dimensions are zero, we should return all of them
1569
- if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) {
1595
+ if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
1596
// all the dimensions are zero
1597
// mark them as NONZERO to send them all
1598
for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
@@ -1591,6 +1617,7 @@ RRDR *rrd2rrdr(
1617
, RRDR_OPTIONS options
1618
, const char *dimensions
1619
, struct context_param *context_param_list
1620
+ , int timeout
1621
)
1622
{
1623
int rrd_update_every;
@@ -1644,7 +1671,7 @@ RRDR *rrd2rrdr(
1671
}
1672
return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
1673
resampling_time_requested, options, dimensions, rrd_update_every,
1647
- first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
1674
+ first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);
1675
} else {
1676
if (rrd_update_every != (uint16_t)max_interval) {
1677
rrd_update_every = (uint16_t) max_interval;
@@ -1655,11 +1682,11 @@ RRDR *rrd2rrdr(
1682
}
1683
return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method,
1684
resampling_time_requested, options, dimensions, rrd_update_every,
1658
- first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list);
1685
+ first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout);
1686
}
1687
}
1688
#endif
1689
return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
1690
resampling_time_requested, options, dimensions,
1664
- rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
1691
+ rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);
1692
}
web/api/queries/rrdr.h
+2
-1
@@ -47,6 +47,7 @@ typedef enum rrdr_result_flags {
47
RRDR_RESULT_OPTION_RELATIVE = 0x00000002, // the query uses relative time-frames
48
// (should not to be cached by browsers and proxies)
49
RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames
50
+ RRDR_RESULT_OPTION_CANCEL = 0x00000008, // the query needs to be cancelled
51
} RRDR_RESULT_FLAGS;
52
53
typedef struct rrdresult {
@@ -110,7 +111,7 @@ extern RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *contex
111
extern RRDR *rrd2rrdr(
112
RRDSET *st, long points_requested, long long after_requested, long long before_requested,
113
RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
113
- struct context_param *context_param_list);
114
+ struct context_param *context_param_list, int timeout);
115
116
#include "query.h"
117
web/api/web_api_v1.c
+4
-1
@@ -415,6 +415,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
415
char *after_str = NULL;
416
char *group_time_str = NULL;
417
char *points_str = NULL;
418
+ char *timeout_str = NULL;
419
char *max_anomaly_rates_str = NULL;
420
char *context = NULL;
421
char *chart_label_key = NULL;
@@ -447,6 +448,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
448
else if(!strcmp(name, "after")) after_str = value;
449
else if(!strcmp(name, "before")) before_str = value;
450
else if(!strcmp(name, "points")) points_str = value;
451
+ else if(!strcmp(name, "timeout")) timeout_str = value;
452
else if(!strcmp(name, "gtime")) group_time_str = value;
453
else if(!strcmp(name, "group")) {
454
group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE);
@@ -576,6 +578,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
578
long long before = (before_str && *before_str)?str2l(before_str):0;
579
long long after = (after_str && *after_str) ?str2l(after_str):-600;
580
int points = (points_str && *points_str)?str2i(points_str):0;
581
+ int timeout = (timeout_str && *timeout_str)?str2i(timeout_str): 0;
582
long group_time = (group_time_str && *group_time_str)?str2l(group_time_str):0;
583
int max_anomaly_rates = (max_anomaly_rates_str && *max_anomaly_rates_str) ? str2i(max_anomaly_rates_str) : 0;
584
@@ -623,7 +626,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
626
ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format,
627
points, after, before, group, group_time,
628
options, &last_timestamp_in_data, context_param_list,
626
- chart_label_key, max_anomaly_rates);
629
+ chart_label_key, max_anomaly_rates, timeout);
630
631
free_context_param_list(&context_param_list);
632