73x times faster metrics correlations at the agent (#13107)
* faster correlations * 4x times faster correlations * a little bit more help * 10x times faster metrics correlations * 6 digits precision; better comments * enabled metrics correlations by default * abstracted DIFFS_NUMBER to allow easily changing it * reworked the entire logic to have more accuracy and support a baseline that is power of two multiple of highlight * properly calculate shifts * even more improved version * added support for timeout; fixed another memory leak; skipped hidden dimensions * default timeout 1min * reduce memory even further * use dictionary for the list of charts and optimize locks * return 403 forbidden, when mc is not enabled * added query options * dont process zero dimensions * added volume method as an option to metric correlations ; now metric correlations can support multiple implementations * make sure we will never crash * spread results evenly for both kstwo and volume * fixed bug in query engine that was missing misaligned queries when a single point was requested from the db; improved comments; improved query flags * updated swagger and added sane defaults; query options are now supported, including anomaly-bit * added "raw" option to allow cross node correlations; added "group" option to allow different time aggregations; allowed calling metric correlations without any parameters; allowed calling metric correlations with relative timestamps; added timeout to volume method; properly handled timeout on ks2 method; json output now sends all parameters back - same for json_wrap; modified query engine to use present time for relative timestamps; modified "allow_past" to mean both past backwards and forwards * emulate the old behaviour about zero points * 100% accuracy against python ks_2samp(); now the default is volume and the default points are 500 * added config option to change default metric correlations method * removed work-arounds now that rrdlabels are merged
Costa Tsaousis committed
Jun 13, 2022 at 21:31 UTC
986a1abf68415cd0cb2bb0df8967ad0141356a34
19 files changed
+1440
-347
daemon/main.c
+4
@@ -625,6 +625,7 @@ static void get_netdata_configured_variables() {
625
// --------------------------------------------------------------------
626
// metric correlations
627
enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
628
+ default_metric_correlations_method = mc_string_to_method(config_get(CONFIG_SECTION_GLOBAL, "metric correlations method", mc_method_to_string(default_metric_correlations_method)));
629
630
// --------------------------------------------------------------------
631
// get various system parameters
@@ -895,6 +896,9 @@ int main(int argc, char **argv) {
896
}
897
#endif
898
#ifdef ENABLE_DBENGINE
899
+ else if(strcmp(optarg, "mctest") == 0) {
900
+ return mc_unittest();
901
+ }
902
else if(strcmp(optarg, "dicttest") == 0) {
903
return dictionary_unittest(10000);
904
}
database/metric_correlations.c
+829
-214
@@ -4,297 +4,912 @@
4
#include "KolmogorovSmirnovDist.h"
5
6
#define MAX_POINTS 10000
7
-int enable_metric_correlations = CONFIG_BOOLEAN_NO;
7
+int enable_metric_correlations = CONFIG_BOOLEAN_YES;
8
int metric_correlations_version = 1;
9
+METRIC_CORRELATIONS_METHOD default_metric_correlations_method = METRIC_CORRELATIONS_VOLUME;
10
+
11
+typedef struct mc_stats {
12
+ size_t db_points;
13
+ size_t result_points;
14
+ size_t db_queries;
15
+ size_t binary_searches;
16
+} MC_STATS;
17
+
18
+// ----------------------------------------------------------------------------
19
+// parse and render metric correlations methods
20
+
21
+static struct {
22
+ const char *name;
23
+ METRIC_CORRELATIONS_METHOD value;
24
+} metric_correlations_methods[] = {
25
+ { "ks2" , METRIC_CORRELATIONS_KS2 }
26
+ , { "volume" , METRIC_CORRELATIONS_VOLUME }
27
+ , { NULL , 0 }
28
+};
29
+
30
+METRIC_CORRELATIONS_METHOD mc_string_to_method(const char *method) {
31
+ for(int i = 0; metric_correlations_methods[i].name ;i++)
32
+ if(strcmp(method, metric_correlations_methods[i].name) == 0)
33
+ return metric_correlations_methods[i].value;
34
+
35
+ return default_metric_correlations_method;
36
+}
37
+
38
+const char *mc_method_to_string(METRIC_CORRELATIONS_METHOD method) {
39
+ for(int i = 0; metric_correlations_methods[i].name ;i++)
40
+ if(metric_correlations_methods[i].value == method)
41
+ return metric_correlations_methods[i].name;
42
10
-struct charts {
43
+ return "unknown";
44
+}
45
+
46
+// ----------------------------------------------------------------------------
47
+// The results per dimension are aggregated into a dictionary
48
+
49
+struct register_result {
50
RRDSET *st;
12
- struct charts *next;
51
+ const char *chart_id;
52
+ const char *context;
53
+ const char *dim_name;
54
+ calculated_number value;
55
};
56
15
-struct per_dim {
16
- char *dimension;
17
- calculated_number baseline[MAX_POINTS];
18
- calculated_number highlight[MAX_POINTS];
57
+static void register_result_insert_callback(const char *name, void *value, void *data) {
58
+ (void)name;
59
+ (void)data;
60
20
- double baseline_diffs[MAX_POINTS];
21
- double highlight_diffs[MAX_POINTS];
22
-};
61
+ struct register_result *t = (struct register_result *)value;
62
24
-int find_index(double arr[], long int n, double K, long int start)
25
-{
26
- for (long int i = start; i < n; i++) {
27
- if (K<arr[i]){
28
- return i;
29
- }
30
- }
31
- return n;
63
+ if(t->chart_id) t->chart_id = strdupz(t->chart_id);
64
+ if(t->context) t->context = strdupz(t->context);
65
+ if(t->dim_name) t->dim_name = strdupz(t->dim_name);
66
}
67
34
-int compare(const void *left, const void *right) {
35
- double lt = *(double *)left;
36
- double rt = *(double *)right;
68
+static void register_result_delete_callback(const char *name, void *value, void *data) {
69
+ (void)name;
70
+ (void)data;
71
+ struct register_result *t = (struct register_result *)value;
72
38
- if(unlikely(lt < rt)) return -1;
39
- if(unlikely(lt > rt)) return 1;
40
- return 0;
73
+ freez((void *)t->chart_id);
74
+ freez((void *)t->context);
75
+ freez((void *)t->dim_name);
76
}
77
43
-void kstwo(double data1[], long int n1, double data2[], long int n2, double *d, double *prob)
44
-{
45
- double en1, en2, en, data_all[MAX_POINTS*2], cdf1[MAX_POINTS], cdf2[MAX_POINTS], cddiffs[MAX_POINTS];
46
- double min = 0.0, max = 0.0;
47
- qsort(data1, n1, sizeof(double), compare);
48
- qsort(data2, n2, sizeof(double), compare);
78
+static DICTIONARY *register_result_init() {
79
+ DICTIONARY *results = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
80
+ dictionary_register_insert_callback(results, register_result_insert_callback, results);
81
+ dictionary_register_delete_callback(results, register_result_delete_callback, results);
82
+ return results;
83
+}
84
50
- for (int i = 0; i < n1; i++)
51
- data_all[i] = data1[i];
52
- for (int i = 0; i < n2; i++)
53
- data_all[n1 + i] = data2[i];
85
+static void register_result_destroy(DICTIONARY *results) {
86
+ dictionary_destroy(results);
87
+}
88
55
- en1 = (double)n1;
56
- en2 = (double)n2;
57
- *d = 0.0;
58
- cddiffs[0]=0; //for uninitialized warning
89
+static void register_result(DICTIONARY *results, RRDSET *st, RRDDIM *d, calculated_number value) {
90
+ struct register_result t = {
91
+ .st = st,
92
+ .chart_id = st->id,
93
+ .context = st->context,
94
+ .dim_name = d->name,
95
+ .value = value
96
+ };
97
+
98
+ char buf[5000 + 1];
99
+ snprintfz(buf, 5000, "%s:%s", st->id, d->name);
100
+ dictionary_set(results, buf, &t, sizeof(struct register_result));
101
+}
102
60
- for (int i=0; i<n1+n2;i++)
61
- cdf1[i] = find_index(data1, n1, data_all[i], 0) / en1; //TODO, use the start to reduce loops
103
+// ----------------------------------------------------------------------------
104
+// Generation of JSON output for the results
105
+
106
+static size_t registered_results_to_json(DICTIONARY *results, BUFFER *wb,
107
+ long long after, long long before,
108
+ long long baseline_after, long long baseline_before,
109
+ long points, METRIC_CORRELATIONS_METHOD method,
110
+ RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
111
+ size_t correlated_dimensions, usec_t duration, MC_STATS *stats) {
112
+
113
+ buffer_sprintf(wb, "{\n"
114
+ "\t\"after\": %lld,\n"
115
+ "\t\"before\": %lld,\n"
116
+ "\t\"duration\": %lld,\n"
117
+ "\t\"points\": %ld,\n"
118
+ "\t\"baseline_after\": %lld,\n"
119
+ "\t\"baseline_before\": %lld,\n"
120
+ "\t\"baseline_duration\": %lld,\n"
121
+ "\t\"baseline_points\": %ld,\n"
122
+ "\t\"statistics\": {\n"
123
+ "\t\t\"query_time_ms\": %f,\n"
124
+ "\t\t\"db_queries\": %zu,\n"
125
+ "\t\t\"db_points_read\": %zu,\n"
126
+ "\t\t\"query_result_points\": %zu,\n"
127
+ "\t\t\"binary_searches\": %zu\n"
128
+ "\t},\n"
129
+ "\t\"group\": \"%s\",\n"
130
+ "\t\"method\": \"%s\",\n"
131
+ "\t\"options\": \"",
132
+ after,
133
+ before,
134
+ before - after,
135
+ points,
136
+ baseline_after,
137
+ baseline_before,
138
+ baseline_before - baseline_after,
139
+ points << shifts,
140
+ (double)duration / (double)USEC_PER_MS,
141
+ stats->db_queries,
142
+ stats->db_points,
143
+ stats->result_points,
144
+ stats->binary_searches,
145
+ web_client_api_request_v1_data_group_to_string(group),
146
+ mc_method_to_string(method));
147
+
148
+ web_client_api_request_v1_data_options_to_string(wb, options);
149
+ buffer_strcat(wb, "\",\n\t\"correlated_charts\": {\n");
150
+
151
+ size_t charts = 0, chart_dims = 0, total_dimensions = 0;
152
+ struct register_result *t;
153
+ RRDSET *last_st = NULL; // never access this - we use it only for comparison
154
+ dfe_start_read(results, t) {
155
+ if(!last_st || t->st != last_st) {
156
+ last_st = t->st;
157
+
158
+ if(charts) buffer_strcat(wb, "\n\t\t\t}\n\t\t},\n");
159
+ buffer_strcat(wb, "\t\t\"");
160
+ buffer_strcat(wb, t->chart_id);
161
+ buffer_strcat(wb, "\": {\n");
162
+ buffer_strcat(wb, "\t\t\t\"context\": \"");
163
+ buffer_strcat(wb, t->context);
164
+ buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n");
165
+ charts++;
166
+ chart_dims = 0;
167
+ }
168
+ if (chart_dims) buffer_sprintf(wb, ",\n");
169
+ buffer_sprintf(wb, "\t\t\t\t\"%s\": " CALCULATED_NUMBER_FORMAT, t->dim_name, t->value);
170
+ chart_dims++;
171
+ total_dimensions++;
172
+ }
173
+ dfe_done(t);
174
+
175
+ // close dimensions and chart
176
+ if (total_dimensions)
177
+ buffer_strcat(wb, "\n\t\t\t}\n\t\t}\n");
178
+
179
+ // close correlated_charts
180
+ buffer_sprintf(wb, "\t},\n"
181
+ "\t\"correlated_dimensions\": %zu,\n"
182
+ "\t\"total_dimensions_count\": %zu\n"
183
+ "}\n",
184
+ total_dimensions,
185
+ correlated_dimensions // yes, we flip them
186
+ );
187
+
188
+ return total_dimensions;
189
+}
190
63
- for (int i=0; i<n1+n2;i++)
64
- cdf2[i] = find_index(data2, n2, data_all[i], 0) / en2;
191
+// ----------------------------------------------------------------------------
192
+// KS2 algorithm functions
193
66
- for ( int i=0;i<n2+n1;i++)
67
- cddiffs[i] = cdf1[i] - cdf2[i];
194
+typedef long int DIFFS_NUMBERS;
195
+#define DOUBLE_TO_INT_MULTIPLIER 100000
196
69
- min = cddiffs[0];
70
- for ( int i=0;i<n2+n1;i++) {
71
- if (cddiffs[i] < min)
72
- min = cddiffs[i];
73
- }
197
+static inline int binary_search_bigger_than(const DIFFS_NUMBERS arr[], int left, int size, DIFFS_NUMBERS K) {
198
+ // binary search to find the index the smallest index
199
+ // of the first value in the array that is greater than K
200
75
- //clip min
76
- if (fabs(min) < 0) min = 0;
77
- else if (fabs(min) > 1) min = 1;
201
+ int right = size;
202
+ while(left < right) {
203
+ int middle = (int)(((unsigned int)(left + right)) >> 1);
204
79
- max = fabs(cddiffs[0]);
80
- for ( int i=0;i<n2+n1;i++)
81
- if (cddiffs[i] >= max) max = cddiffs[i];
205
+ if(arr[middle] > K)
206
+ right = middle;
207
83
- if (fabs(min) < max)
84
- *d = max;
85
- else
86
- *d = fabs(min);
208
+ else
209
+ left = middle + 1;
210
+ }
211
88
-
89
-
90
- en = (en1*en2 / (en1 + en2));
91
- *prob = KSfbar(round(en), *d);
212
+ return left;
213
}
214
94
-void fill_nan (struct per_dim *d, long int hp, long int bp)
95
-{
96
- int k;
215
+int compare_diffs(const void *left, const void *right) {
216
+ DIFFS_NUMBERS lt = *(DIFFS_NUMBERS *)left;
217
+ DIFFS_NUMBERS rt = *(DIFFS_NUMBERS *)right;
218
+
219
+ // https://stackoverflow.com/a/3886497/1114110
220
+ return (lt > rt) - (lt < rt);
221
+}
222
+
223
+static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, calculated_number *arr, size_t size) {
224
+ calculated_number *last = &arr[size - 1];
225
+ size_t added = 0;
226
+
227
+ while(last > arr) {
228
+ calculated_number second = *last--;
229
+ calculated_number first = *last;
230
+ *diffs++ = (DIFFS_NUMBERS)((first - second) * (calculated_number)DOUBLE_TO_INT_MULTIPLIER);
231
+ added++;
232
+ }
233
+
234
+ return added;
235
+}
236
98
- for (k = 0; k < bp; k++) {
99
- if (isnan(d->baseline[k])) {
100
- d->baseline[k] = 0.0;
237
+static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMBERS highlight_diffs[], int high_size, uint32_t base_shifts) {
238
+
239
+ qsort(baseline_diffs, base_size, sizeof(DIFFS_NUMBERS), compare_diffs);
240
+ qsort(highlight_diffs, high_size, sizeof(DIFFS_NUMBERS), compare_diffs);
241
+
242
+ // Now we should be calculating this:
243
+ //
244
+ // For each number in the diffs arrays, we should find the index of the
245
+ // number bigger than them in both arrays and calculate the % of this index
246
+ // vs the total array size. Once we have the 2 percentages, we should find
247
+ // the min and max across the delta of all of them.
248
+ //
249
+ // It should look like this:
250
+ //
251
+ // base_pcent = binary_search_bigger_than(...) / base_size;
252
+ // high_pcent = binary_search_bigger_than(...) / high_size;
253
+ // delta = base_pcent - high_pcent;
254
+ // if(delta < min) min = delta;
255
+ // if(delta > max) max = delta;
256
+ //
257
+ // This would require a lot of multiplications and divisions.
258
+ //
259
+ // To speed it up, we do the binary search to find the index of each number
260
+ // but then we divide the base index by the power of two number (shifts) it
261
+ // is bigger than high index. So the 2 indexes are now comparable.
262
+ // We also keep track of the original indexes with min and max, to properly
263
+ // calculate their percentages once the loops finish.
264
+
265
+
266
+ // initialize min and max using the first number of baseline_diffs
267
+ DIFFS_NUMBERS K = baseline_diffs[0];
268
+ int base_idx = binary_search_bigger_than(baseline_diffs, 1, base_size, K);
269
+ int high_idx = binary_search_bigger_than(highlight_diffs, 0, high_size, K);
270
+ int delta = base_idx - (high_idx << base_shifts);
271
+ int min = delta, max = delta;
272
+ int base_min_idx = base_idx;
273
+ int base_max_idx = base_idx;
274
+ int high_min_idx = high_idx;
275
+ int high_max_idx = high_idx;
276
+
277
+ // do the baseline_diffs starting from 1 (we did position 0 above)
278
+ for(int i = 1; i < base_size; i++) {
279
+ K = baseline_diffs[i];
280
+ base_idx = binary_search_bigger_than(baseline_diffs, i + 1, base_size, K); // starting from i, since data1 is sorted
281
+ high_idx = binary_search_bigger_than(highlight_diffs, 0, high_size, K);
282
+
283
+ delta = base_idx - (high_idx << base_shifts);
284
+ if(delta < min) {
285
+ min = delta;
286
+ base_min_idx = base_idx;
287
+ high_min_idx = high_idx;
288
+ }
289
+ else if(delta > max) {
290
+ max = delta;
291
+ base_max_idx = base_idx;
292
+ high_max_idx = high_idx;
293
}
294
}
295
104
- for (k = 0; k < hp; k++) {
105
- if (isnan(d->highlight[k])) {
106
- d->highlight[k] = 0.0;
296
+ // do the highlight_diffs starting from 0
297
+ for(int i = 0; i < high_size; i++) {
298
+ K = highlight_diffs[i];
299
+ base_idx = binary_search_bigger_than(baseline_diffs, 0, base_size, K);
300
+ high_idx = binary_search_bigger_than(highlight_diffs, i + 1, high_size, K); // starting from i, since data2 is sorted
301
+
302
+ delta = base_idx - (high_idx << base_shifts);
303
+ if(delta < min) {
304
+ min = delta;
305
+ base_min_idx = base_idx;
306
+ high_min_idx = high_idx;
307
+ }
308
+ else if(delta > max) {
309
+ max = delta;
310
+ base_max_idx = base_idx;
311
+ high_max_idx = high_idx;
312
}
313
}
314
+
315
+ // now we have the min, max and their indexes
316
+ // properly calculate min and max as dmin and dmax
317
+ double dbase_size = (double)base_size;
318
+ double dhigh_size = (double)high_size;
319
+ double dmin = ((double)base_min_idx / dbase_size) - ((double)high_min_idx / dhigh_size);
320
+ double dmax = ((double)base_max_idx / dbase_size) - ((double)high_max_idx / dhigh_size);
321
+
322
+ dmin = -dmin;
323
+ if(islessequal(dmin, 0.0)) dmin = 0.0;
324
+ else if(isgreaterequal(dmin, 1.0)) dmin = 1.0;
325
+
326
+ double d;
327
+ if(isgreaterequal(dmin, dmax)) d = dmin;
328
+ else d = dmax;
329
+
330
+ double en = round(dbase_size * dhigh_size / (dbase_size + dhigh_size));
331
+
332
+ // under these conditions, KSfbar() crashes
333
+ if(unlikely(isnan(en) || isinf(en) || en == 0.0 || isnan(d) || isinf(d)))
334
+ return NAN;
335
+
336
+ return KSfbar((int)en, d);
337
}
338
111
-//TODO check counters
112
-void run_diffs_and_rev (struct per_dim *d, long int hp, long int bp)
113
-{
114
- int k, j;
339
+static double kstwo(calculated_number baseline[], int baseline_points, calculated_number highlight[], int highlight_points, uint32_t base_shifts) {
340
+ // -1 in size, since the calculate_pairs_diffs() returns one less point
341
+ DIFFS_NUMBERS baseline_diffs[baseline_points - 1];
342
+ DIFFS_NUMBERS highlight_diffs[highlight_points - 1];
343
116
- for (k = 0, j = bp; k < bp - 1; k++, j--)
117
- d->baseline_diffs[k] = (double)d->baseline[j - 2] - (double)d->baseline[j - 1];
118
- for (k = 0, j = hp; k < hp - 1; k++, j--) {
119
- d->highlight_diffs[k] = (double)d->highlight[j - 2] - (double)d->highlight[j - 1];
344
+ int base_size = (int)calculate_pairs_diff(baseline_diffs, baseline, baseline_points);
345
+ int high_size = (int)calculate_pairs_diff(highlight_diffs, highlight, highlight_points);
346
+
347
+ if(unlikely(!base_size || !high_size))
348
+ return NAN;
349
+
350
+ if(unlikely(base_size != baseline_points - 1 || high_size != highlight_points - 1)) {
351
+ error("Metric correlations: internal error - calculate_pairs_diff() returns the wrong number of entries");
352
+ return NAN;
353
}
354
+
355
+ return ks_2samp(baseline_diffs, base_size, highlight_diffs, high_size, base_shifts);
356
}
357
123
-int run_metric_correlations (BUFFER *wb, RRDSET *st, long long baseline_after, long long baseline_before, long long highlight_after, long long highlight_before, long long max_points)
124
-{
125
- uint32_t options = 0x00000000;
126
- int group_method = RRDR_GROUPING_AVERAGE;
358
+
359
+static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
360
+ long long baseline_after, long long baseline_before,
361
+ long long after, long long before,
362
+ long long points, RRDR_OPTIONS options, RRDR_GROUPING group,
363
+ uint32_t shifts, int timeout, MC_STATS *stats) {
364
long group_time = 0;
365
struct context_param *context_param_list = NULL;
129
- long c;
130
- int i=0, j=0;
131
- int b_dims = 0;
132
- long int baseline_points = 0, highlight_points = 0;
366
134
- struct per_dim *pd = NULL;
367
+ int correlated_dimensions = 0;
368
+
369
+ RRDR *high_rrdr = NULL;
370
+ RRDR *base_rrdr = NULL;
371
136
- //TODO get everything in one go, when baseline is right before highlight
137
- //get baseline
372
+ // get first the highlight to find the number of points available
373
+ stats->db_queries++;
374
+ usec_t started_usec = now_realtime_usec();
375
ONEWAYALLOC *owa = onewayalloc_create(0);
139
- RRDR *rb = rrd2rrdr(owa, st, max_points, baseline_after, baseline_before, group_method, group_time, options, NULL, context_param_list, 0);
140
- if(!rb) {
141
- info("Cannot generate metric correlations output with these parameters on this chart.");
142
- onewayalloc_destroy(owa);
143
- return 0;
144
- } else {
145
- baseline_points = rrdr_rows(rb);
146
- pd = mallocz(sizeof(struct per_dim) * rb->d);
147
- b_dims = rb->d;
148
- for (c = 0; c != rrdr_rows(rb) ; ++c) {
149
- RRDDIM *d;
150
- for (j = 0, d = rb->st->dimensions ; d && j < rb->d ; ++j, d = d->next) {
151
- calculated_number *cn = &rb->v[ c * rb->d ];
152
- if (!c) {
153
- //TODO use points from query
154
- pd[j].dimension = strdupz (d->name);
155
- pd[j].baseline[c] = cn[j];
156
- } else {
157
- pd[j].baseline[c] = cn[j];
158
- }
376
+ high_rrdr = rrd2rrdr(owa, st, points,
377
+ after, before, group,
378
+ group_time, options, NULL, context_param_list, timeout);
379
+ if(!high_rrdr) {
380
+ info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name);
381
+ goto cleanup;
382
+ }
383
+ stats->db_points += high_rrdr->internal.db_points_read;
384
+ stats->result_points += high_rrdr->internal.result_points_generated;
385
+ if(!high_rrdr->d) {
386
+ info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
387
+ goto cleanup;
388
+ }
389
+ if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
390
+ info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", st->name);
391
+ goto cleanup;
392
+ }
393
+ int high_points = rrdr_rows(high_rrdr);
394
+
395
+ usec_t now_usec = now_realtime_usec();
396
+ if(now_usec - started_usec > timeout * USEC_PER_MS)
397
+ goto cleanup;
398
+
399
+ // get the baseline, requesting the same number of points as the highlight
400
+ stats->db_queries++;
401
+ base_rrdr = rrd2rrdr(owa, st,high_points << shifts,
402
+ baseline_after, baseline_before, group,
403
+ group_time, options, NULL, context_param_list,
404
+ (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)));
405
+ if(!base_rrdr) {
406
+ info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name);
407
+ goto cleanup;
408
+ }
409
+ stats->db_points += base_rrdr->internal.db_points_read;
410
+ stats->result_points += base_rrdr->internal.result_points_generated;
411
+ if(!base_rrdr->d) {
412
+ info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
413
+ goto cleanup;
414
+ }
415
+ if (base_rrdr->d != high_rrdr->d) {
416
+ info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", st->name);
417
+ goto cleanup;
418
+ }
419
+ if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
420
+ info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", st->name);
421
+ goto cleanup;
422
+ }
423
+ int base_points = rrdr_rows(base_rrdr);
424
+
425
+ now_usec = now_realtime_usec();
426
+ if(now_usec - started_usec > timeout * USEC_PER_MS)
427
+ goto cleanup;
428
+
429
+ // we need at least 2 points to do the job
430
+ if(base_points < 2 || high_points < 2)
431
+ goto cleanup;
432
+
433
+ // for each dimension
434
+ RRDDIM *d;
435
+ int i;
436
+ for(i = 0, d = base_rrdr->st->dimensions ; d && i < base_rrdr->d; i++, d = d->next) {
437
+
438
+ // skip the not evaluated ones
439
+ if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN))
440
+ continue;
441
+
442
+ correlated_dimensions++;
443
+
444
+ // skip the dimensions that are just zero for both the baseline and the highlight
445
+ if(unlikely(!(base_rrdr->od[i] & RRDR_DIMENSION_NONZERO) && !(high_rrdr->od[i] & RRDR_DIMENSION_NONZERO)))
446
+ continue;
447
+
448
+ // copy the baseline points of the dimension to a contiguous array
449
+ // there is no need to check for empty values, since empty are already zero
450
+ calculated_number baseline[base_points];
451
+ for(int c = 0; c < base_points; c++)
452
+ baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ];
453
+
454
+ // copy the highlight points of the dimension to a contiguous array
455
+ // there is no need to check for empty values, since empty values are already zero
456
+ // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
457
+ calculated_number highlight[high_points];
458
+ for(int c = 0; c < high_points; c++)
459
+ highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ];
460
+
461
+ stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1);
462
+
463
+ double prob = kstwo(baseline, base_points, highlight, high_points, shifts);
464
+ if(!isnan(prob) && !isinf(prob)) {
465
+
466
+ // these conditions should never happen, but still let's check
467
+ if(unlikely(prob < 0.0)) {
468
+ error("Metric correlations: kstwo() returned a negative number: %f", prob);
469
+ prob = -prob;
470
+ }
471
+ if(unlikely(prob > 1.0)) {
472
+ error("Metric correlations: kstwo() returned a number above 1.0: %f", prob);
473
+ prob = 1.0;
474
}
475
+
476
+ // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated
477
+ // so we flip the result of kstwo()
478
+ register_result(results, base_rrdr->st, d, 1.0 - prob);
479
}
480
}
162
- rrdr_free(owa, rb);
481
+
482
+cleanup:
483
+ rrdr_free(owa, high_rrdr);
484
+ rrdr_free(owa, base_rrdr);
485
onewayalloc_destroy(owa);
164
- if (!pd)
165
- return 0;
166
-
167
- //get highlight
168
- owa = onewayalloc_create(0);
169
- RRDR *rh = rrd2rrdr(owa, st, max_points, highlight_after, highlight_before, group_method, group_time, options, NULL, context_param_list, 0);
170
- if(!rh) {
171
- info("Cannot generate metric correlations output with these parameters on this chart.");
172
- freez(pd);
173
- onewayalloc_destroy(owa);
174
- return 0;
175
- } else {
176
- if (rh->d != b_dims) {
177
- //TODO handle different dims
178
- rrdr_free(owa, rh);
179
- onewayalloc_destroy(owa);
180
- freez(pd);
181
- return 0;
486
+ return correlated_dimensions;
487
+}
488
+
489
+// ----------------------------------------------------------------------------
490
+// VOLUME algorithm functions
491
+
492
+static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
493
+ long long baseline_after, long long baseline_before,
494
+ long long after, long long before,
495
+ RRDR_OPTIONS options, RRDR_GROUPING group, int timeout, MC_STATS *stats) {
496
+ options |= RRDR_OPTION_MATCH_IDS;
497
+ long group_time = 0;
498
+
499
+ int correlated_dimensions = 0;
500
+ int ret, value_is_null;
501
+ usec_t started_usec = now_realtime_usec();
502
+
503
+ RRDDIM *d;
504
+ for(d = st->dimensions; d ; d = d->next) {
505
+ usec_t now_usec = now_realtime_usec();
506
+ if(now_usec - started_usec > timeout * USEC_PER_MS)
507
+ return correlated_dimensions;
508
+
509
+ // we count how many metrics we evaluated
510
+ correlated_dimensions++;
511
+
512
+ // there is no point to pass a timeout to these queries
513
+ // since the query engine checks for a timeout between
514
+ // dimensions, and we query a single dimension at a time.
515
+
516
+ stats->db_queries++;
517
+ calculated_number highlight_average = NAN;
518
+ value_is_null = 1;
519
+ ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1,
520
+ after, before,
521
+ group, group_time, options,
522
+ NULL, NULL,
523
+ &stats->db_points, &stats->result_points,
524
+ &value_is_null, 0);
525
+
526
+ if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(highlight_average)) {
527
+ // error("Metric correlations: cannot query highlight duration of dimension '%s' of chart '%s', %d %s %s %s", st->name, d->name, ret, (ret != HTTP_RESP_OK)?"response failed":"", (value_is_null)?"value is null":"", (!calculated_number_isnumber(highlight_average))?"result is NAN":"");
528
+ // this means no data for the highlighted duration - so skip it
529
+ continue;
530
}
183
- highlight_points = rrdr_rows(rh);
184
- for (c = 0; c != rrdr_rows(rh) ; ++c) {
185
- RRDDIM *d;
186
- for (j = 0, d = rh->st->dimensions ; d && j < rh->d ; ++j, d = d->next) {
187
- calculated_number *cn = &rh->v[ c * rh->d ];
188
- pd[j].highlight[c] = cn[j];
189
- }
531
+
532
+ stats->db_queries++;
533
+ calculated_number baseline_average = NAN;
534
+ value_is_null = 1;
535
+ ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,
536
+ baseline_after, baseline_before,
537
+ group, group_time, options,
538
+ NULL, NULL,
539
+ &stats->db_points, &stats->result_points,
540
+ &value_is_null, 0);
541
+
542
+ if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(baseline_average)) {
543
+ // error("Metric correlations: cannot query baseline duration of dimension '%s' of chart '%s', %d %s %s %s", st->name, d->name, ret, (ret != HTTP_RESP_OK)?"response failed":"", (value_is_null)?"value is null":"", (!calculated_number_isnumber(baseline_average))?"result is NAN":"");
544
+ // continue;
545
+ // this means no data for the baseline window, but we have data for the highlighted one - assume zero
546
+ baseline_average = 0.0;
547
}
548
+
549
+ calculated_number pcent = NAN;
550
+ if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0))
551
+ pcent = (highlight_average - baseline_average) / baseline_average;
552
+
553
+ else if(isgreater(highlight_average, 0.0) || isless(highlight_average, 0.0))
554
+ pcent = highlight_average;
555
+
556
+ if(!isnan(pcent))
557
+ register_result(results, st, d, pcent);
558
}
192
- rrdr_free(owa, rh);
193
- onewayalloc_destroy(owa);
559
195
- for (i = 0; i < b_dims; i++) {
196
- fill_nan(&pd[i], highlight_points, baseline_points);
560
+ return correlated_dimensions;
561
+}
562
+
563
+int compare_calculated_numbers(const void *left, const void *right) {
564
+ calculated_number lt = *(calculated_number *)left;
565
+ calculated_number rt = *(calculated_number *)right;
566
+
567
+ // https://stackoverflow.com/a/3886497/1114110
568
+ return (lt > rt) - (lt < rt);
569
+}
570
+
571
+static inline int binary_search_bigger_than_calculated_number(const calculated_number arr[], int left, int size, calculated_number K) {
572
+ // binary search to find the index the smallest index
573
+ // of the first value in the array that is greater than K
574
+
575
+ int right = size;
576
+ while(left < right) {
577
+ int middle = (int)(((unsigned int)(left + right)) >> 1);
578
+
579
+ if(arr[middle] > K)
580
+ right = middle;
581
+
582
+ else
583
+ left = middle + 1;
584
}
585
199
- for (i = 0; i < b_dims; i++) {
200
- run_diffs_and_rev(&pd[i], highlight_points, baseline_points);
586
+ return left;
587
+}
588
+
589
+// ----------------------------------------------------------------------------
590
+// spread the results evenly according to their value
591
+
592
+static size_t spread_results_evenly(DICTIONARY *results) {
593
+ struct register_result *t;
594
+
595
+ // count the dimensions
596
+ size_t dimensions = dictionary_stats_entries(results);
597
+ if(!dimensions) return 0;
598
+
599
+ // create an array of the right size and copy all the values in it
600
+ calculated_number slots[dimensions];
601
+ dimensions = 0;
602
+ dfe_start_read(results, t) {
603
+ t->value = calculated_number_fabs(t->value);
604
+ slots[dimensions++] = t->value;
605
}
606
+ dfe_done(t);
607
203
- double d=0, prob=0;
204
- for (i=0;i < j ;i++) {
205
- if (baseline_points && highlight_points) {
206
- kstwo(pd[i].baseline_diffs, baseline_points-1, pd[i].highlight_diffs, highlight_points-1, &d, &prob);
207
- buffer_sprintf(wb, "\t\t\t\t\"%s\": %f", pd[i].dimension, prob);
208
- if (i != j-1)
209
- buffer_sprintf(wb, ",\n");
210
- else
211
- buffer_sprintf(wb, "\n");
212
- }
608
+ // sort the array with the values of all dimensions
609
+ qsort(slots, dimensions, sizeof(calculated_number), compare_calculated_numbers);
610
+
611
+ // skip the duplicates in the sorted array
612
+ calculated_number last_value = NAN;
613
+ size_t unique_values = 0;
614
+ for(size_t i = 0; i < dimensions ;i++) {
615
+ if(likely(slots[i] != last_value))
616
+ slots[unique_values++] = last_value = slots[i];
617
}
618
215
- freez(pd);
216
- return j;
619
+ // calculate the weight of each slot, using the number of unique values
620
+ calculated_number slot_weight = 1.0 / (calculated_number)unique_values;
621
+
622
+ dfe_start_read(results, t) {
623
+ int slot = binary_search_bigger_than_calculated_number(slots, 0, (int)unique_values, t->value);
624
+ calculated_number v = slot * slot_weight;
625
+ if(unlikely(v > 1.0)) v = 1.0;
626
+ v = 1.0 - v;
627
+ t->value = v;
628
+ }
629
+ dfe_done(t);
630
+
631
+ return dimensions;
632
}
633
219
-void metric_correlations (RRDHOST *host, BUFFER *wb, long long baseline_after, long long baseline_before, long long highlight_after, long long highlight_before, long long max_points)
220
-{
221
- info ("Running metric correlations, highlight_after: %lld, highlight_before: %lld, baseline_after: %lld, baseline_before: %lld, max_points: %lld", highlight_after, highlight_before, baseline_after, baseline_before, max_points);
634
+// ----------------------------------------------------------------------------
635
+// The main function
636
+
637
+int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD method, RRDR_GROUPING group,
638
+ long long baseline_after, long long baseline_before,
639
+ long long after, long long before,
640
+ long long points, RRDR_OPTIONS options, int timeout) {
641
223
- if (!enable_metric_correlations) {
224
- error("Metric correlations functionality is not enabled.");
642
+ // method = METRIC_CORRELATIONS_VOLUME;
643
+ // options |= RRDR_OPTION_ANOMALY_BIT;
644
+
645
+ MC_STATS stats = {};
646
+
647
+ if (enable_metric_correlations == CONFIG_BOOLEAN_NO) {
648
buffer_strcat(wb, "{\"error\": \"Metric correlations functionality is not enabled.\" }");
226
- return;
649
+ return HTTP_RESP_FORBIDDEN;
650
}
651
229
- if (highlight_before <= highlight_after || baseline_before <= baseline_after) {
230
- error("Invalid baseline or highlight ranges.");
652
+ // if the user didn't give a timeout
653
+ // assume 60 seconds
654
+ if(!timeout)
655
+ timeout = 60 * MSEC_PER_SEC;
656
+
657
+ // if the timeout is less than 1 second
658
+ // make it at least 1 second
659
+ if(timeout < (long)(1 * MSEC_PER_SEC))
660
+ timeout = 1 * MSEC_PER_SEC;
661
+
662
+ usec_t timeout_usec = timeout * USEC_PER_MS;
663
+ usec_t started_usec = now_realtime_usec();
664
+
665
+ if(!points) points = 500;
666
+
667
+ rrdr_relative_window_to_absolute(&after, &before, default_rrd_update_every, points);
668
+
669
+ if(baseline_before <= API_RELATIVE_TIME_MAX)
670
+ baseline_before += after;
671
+
672
+ rrdr_relative_window_to_absolute(&baseline_after, &baseline_before, default_rrd_update_every, points * 4);
673
+
674
+ if (before <= after || baseline_before <= baseline_after) {
675
buffer_strcat(wb, "{\"error\": \"Invalid baseline or highlight ranges.\" }");
232
- return;
676
+ return HTTP_RESP_BAD_REQUEST;
677
}
678
235
- long long dims = 0, total_dims = 0;
236
- RRDSET *st;
237
- size_t c = 0;
238
- BUFFER *wdims = buffer_create(1000);
679
+ DICTIONARY *results = register_result_init();
680
+ DICTIONARY *charts = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);;
681
+
682
+ char *error = NULL;
683
+ int resp = HTTP_RESP_OK;
684
+
685
+ // baseline should be a power of two multiple of highlight
686
+ uint32_t shifts = 0;
687
+ {
688
+ long long base_delta = baseline_before - baseline_after;
689
+ long long high_delta = before - after;
690
+ uint32_t multiplier = (uint32_t)round((double)base_delta / (double)high_delta);
691
+
692
+ // check if the multiplier is a power of two
693
+ // https://stackoverflow.com/a/600306/1114110
694
+ if((multiplier & (multiplier - 1)) != 0) {
695
+ // it is not power of two
696
+ // let's find the closest power of two
697
+ // https://stackoverflow.com/a/466242/1114110
698
+ multiplier--;
699
+ multiplier |= multiplier >> 1;
700
+ multiplier |= multiplier >> 2;
701
+ multiplier |= multiplier >> 4;
702
+ multiplier |= multiplier >> 8;
703
+ multiplier |= multiplier >> 16;
704
+ multiplier++;
705
+ }
706
+
707
+ // convert the multiplier to the number of shifts
708
+ // we need to do, to divide baseline numbers to match
709
+ // the highlight ones
710
+ while(multiplier > 1) {
711
+ shifts++;
712
+ multiplier = multiplier >> 1;
713
+ }
714
+
715
+ // if the baseline size will not comply to MAX_POINTS
716
+ // lower the window of the baseline
717
+ while(shifts && (points << shifts) > MAX_POINTS)
718
+ shifts--;
719
240
- if (!max_points || max_points > MAX_POINTS)
241
- max_points = MAX_POINTS;
720
+ // if the baseline size still does not comply to MAX_POINTS
721
+ // lower the resolution of the highlight and the baseline
722
+ while((points << shifts) > MAX_POINTS)
723
+ points = points >> 1;
724
+
725
+ if(points < 100) {
726
+ // error = "cannot comply to at least 100 points";
727
+ resp = HTTP_RESP_BAD_REQUEST;
728
+ goto cleanup;
729
+ }
730
+
731
+ // adjust the baseline to be multiplier times bigger than the highlight
732
+ baseline_after = baseline_before - (high_delta << shifts);
733
+ }
734
243
- //dont lock here and wait for results
244
- //get the charts and run mc after
245
- //should not be a problem for the query
246
- struct charts *charts = NULL;
735
+ // dont lock here and wait for results
736
+ // get the charts and run mc after
737
+ RRDSET *st;
738
rrdhost_rdlock(host);
739
rrdset_foreach_read(st, host) {
249
- if (rrdset_is_available_for_viewers(st)) {
250
- rrdset_rdlock(st);
251
- struct charts *chart = callocz(1, sizeof(struct charts));
252
- chart->st = st;
253
- chart->next = NULL;
254
- if (charts) {
255
- chart->next = charts;
256
- }
257
- charts = chart;
258
- }
740
+ if (rrdset_is_available_for_viewers(st))
741
+ dictionary_set(charts, st->name, "", 1);
742
}
743
rrdhost_unlock(host);
744
262
- buffer_strcat(wb, "{\n\t\"correlated_charts\": {");
745
+ size_t correlated_dimensions = 0;
746
+ void *ptr;
747
264
- for (struct charts *ch = charts; ch; ch = ch->next) {
265
- buffer_flush(wdims);
266
- dims = run_metric_correlations(wdims, ch->st, baseline_after, baseline_before, highlight_after, highlight_before, max_points);
267
- if (dims) {
268
- if (c)
269
- buffer_strcat(wb, "\t\t},");
270
- buffer_strcat(wb, "\n\t\t\"");
271
- buffer_strcat(wb, ch->st->id);
272
- buffer_strcat(wb, "\": {\n");
273
- buffer_strcat(wb, "\t\t\t\"context\": \"");
274
- buffer_strcat(wb, ch->st->context);
275
- buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n");
276
- buffer_sprintf(wb, "%s", buffer_tostring(wdims));
277
- buffer_strcat(wb, "\t\t\t}\n");
278
- total_dims += dims;
279
- c++;
748
+ // for every chart in the dictionary
749
+ dfe_start_read(charts, ptr) {
750
+ usec_t now_usec = now_realtime_usec();
751
+ if(now_usec - started_usec > timeout_usec) {
752
+ error = "timed out";
753
+ resp = HTTP_RESP_GATEWAY_TIMEOUT;
754
+ goto cleanup;
755
+ }
756
+
757
+ st = rrdset_find_byname(host, ptr_name);
758
+ if(!st) continue;
759
+
760
+ rrdset_rdlock(st);
761
+
762
+ switch(method) {
763
+ case METRIC_CORRELATIONS_VOLUME:
764
+ correlated_dimensions += rrdset_metric_correlations_volume(st, results,
765
+ baseline_after, baseline_before,
766
+ after, before,
767
+ options, group,
768
+ (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
769
+ &stats);
770
+ break;
771
+
772
+ default:
773
+ case METRIC_CORRELATIONS_KS2:
774
+ correlated_dimensions += rrdset_metric_correlations_ks2(st, results,
775
+ baseline_after, baseline_before,
776
+ after, before,
777
+ points, options, group, shifts,
778
+ (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
779
+ &stats);
780
+ break;
781
}
782
+
783
+ rrdset_unlock(st);
784
}
282
- buffer_strcat(wb, "\t\t}\n");
283
- buffer_sprintf(wb, "\t},\n\t\"total_dimensions_count\": %lld\n}", total_dims);
785
+ dfe_done(ptr);
786
285
- if (!total_dims) {
286
- buffer_flush(wb);
287
- buffer_strcat(wb, "{\"error\": \"No results from metric correlations.\" }");
787
+ if(!(options & RRDR_OPTION_RETURN_RAW))
788
+ spread_results_evenly(results);
789
+
790
+ usec_t ended_usec = now_realtime_usec();
791
+
792
+ // generate the json output we need
793
+ buffer_flush(wb);
794
+ size_t added_dimensions = registered_results_to_json(results, wb,
795
+ after, before,
796
+ baseline_after, baseline_before,
797
+ points, method, group, options, shifts, correlated_dimensions,
798
+ ended_usec - started_usec, &stats);
799
+
800
+ if(!added_dimensions) {
801
+ error = "no results produced from correlations";
802
+ resp = HTTP_RESP_NOT_FOUND;
803
}
804
290
- struct charts* ch;
291
- while(charts){
292
- ch = charts;
293
- charts = charts->next;
294
- rrdset_unlock(ch->st);
295
- free(ch);
805
+cleanup:
806
+ if(charts) dictionary_destroy(charts);
807
+ if(results) register_result_destroy(results);
808
+
809
+ if(error) {
810
+ buffer_flush(wb);
811
+ buffer_sprintf(wb, "{\"error\": \"%s\" }", error);
812
}
813
298
- buffer_free(wdims);
299
- info ("Done running metric correlations");
814
+ return resp;
815
+}
816
+
817
+
818
+
819
+// ----------------------------------------------------------------------------
820
+// unittest
821
+
822
+/*
823
+
824
+Unit tests against the output of this:
825
+
826
+https://github.com/scipy/scipy/blob/4cf21e753cf937d1c6c2d2a0e372fbc1dbbeea81/scipy/stats/_stats_py.py#L7275-L7449
827
+
828
+import matplotlib.pyplot as plt
829
+import pandas as pd
830
+import numpy as np
831
+import scipy as sp
832
+from scipy import stats
833
+
834
+data1 = np.array([ 1111, -2222, 33, 100, 100, 15555, -1, 19999, 888, 755, -1, -730 ])
835
+data2 = np.array([365, -123, 0])
836
+data1 = np.sort(data1)
837
+data2 = np.sort(data2)
838
+n1 = data1.shape[0]
839
+n2 = data2.shape[0]
840
+data_all = np.concatenate([data1, data2])
841
+cdf1 = np.searchsorted(data1, data_all, side='right') / n1
842
+cdf2 = np.searchsorted(data2, data_all, side='right') / n2
843
+print(data_all)
844
+print("\ndata1", data1, cdf1)
845
+print("\ndata2", data2, cdf2)
846
+cddiffs = cdf1 - cdf2
847
+print("\ncddiffs", cddiffs)
848
+minS = np.clip(-np.min(cddiffs), 0, 1)
849
+maxS = np.max(cddiffs)
850
+print("\nmin", minS)
851
+print("max", maxS)
852
+m, n = sorted([float(n1), float(n2)], reverse=True)
853
+en = m * n / (m + n)
854
+d = max(minS, maxS)
855
+prob = stats.distributions.kstwo.sf(d, np.round(en))
856
+print("\nprob", prob)
857
+
858
+*/
859
+
860
+static int double_expect(double v, const char *str, const char *descr) {
861
+ char buf[100 + 1];
862
+ snprintfz(buf, 100, "%0.6f", v);
863
+ int ret = strcmp(buf, str) ? 1 : 0;
864
+
865
+ fprintf(stderr, "%s %s, expected %s, got %s\n", ret?"FAILED":"OK", descr, str, buf);
866
+ return ret;
867
+}
868
+
869
+static int mc_unittest1(void) {
870
+ int bs = 3, hs = 3;
871
+ DIFFS_NUMBERS base[3] = { 1, 2, 3 };
872
+ DIFFS_NUMBERS high[3] = { 3, 4, 6 };
873
+
874
+ double prob = ks_2samp(base, bs, high, hs, 0);
875
+ return double_expect(prob, "0.222222", "3x3");
876
+}
877
+
878
+static int mc_unittest2(void) {
879
+ int bs = 6, hs = 3;
880
+ DIFFS_NUMBERS base[6] = { 1, 2, 3, 10, 10, 15 };
881
+ DIFFS_NUMBERS high[3] = { 3, 4, 6 };
882
+
883
+ double prob = ks_2samp(base, bs, high, hs, 1);
884
+ return double_expect(prob, "0.500000", "6x3");
885
+}
886
+
887
+static int mc_unittest3(void) {
888
+ int bs = 12, hs = 3;
889
+ DIFFS_NUMBERS base[12] = { 1, 2, 3, 10, 10, 15, 111, 19999, 8, 55, -1, -73 };
890
+ DIFFS_NUMBERS high[3] = { 3, 4, 6 };
891
+
892
+ double prob = ks_2samp(base, bs, high, hs, 2);
893
+ return double_expect(prob, "0.347222", "12x3");
894
}
895
+
896
+static int mc_unittest4(void) {
897
+ int bs = 12, hs = 3;
898
+ DIFFS_NUMBERS base[12] = { 1111, -2222, 33, 100, 100, 15555, -1, 19999, 888, 755, -1, -730 };
899
+ DIFFS_NUMBERS high[3] = { 365, -123, 0 };
900
+
901
+ double prob = ks_2samp(base, bs, high, hs, 2);
902
+ return double_expect(prob, "0.777778", "12x3");
903
+}
904
+
905
+int mc_unittest(void) {
906
+ int errors = 0;
907
+
908
+ errors += mc_unittest1();
909
+ errors += mc_unittest2();
910
+ errors += mc_unittest3();
911
+ errors += mc_unittest4();
912
+
913
+ return errors;
914
+}
915
+
database/metric_correlations.h
+16
-1
@@ -3,9 +3,24 @@
3
#ifndef NETDATA_METRIC_CORRELATIONS_H
4
#define NETDATA_METRIC_CORRELATIONS_H 1
5
6
+#include "web/api/queries/query.h"
7
+
8
+typedef enum {
9
+ METRIC_CORRELATIONS_KS2 = 1,
10
+ METRIC_CORRELATIONS_VOLUME = 2,
11
+} METRIC_CORRELATIONS_METHOD;
12
+
13
extern int enable_metric_correlations;
14
extern int metric_correlations_version;
15
+extern METRIC_CORRELATIONS_METHOD default_metric_correlations_method;
16
+
17
+extern int metric_correlations (RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD method, RRDR_GROUPING group,
18
+ long long baseline_after, long long baseline_before,
19
+ long long after, long long before,
20
+ long long points, RRDR_OPTIONS options, int timeout);
21
9
-void metric_correlations (RRDHOST *host, BUFFER *wb, long long selected_after, long long selected_before, long long reference_after, long long reference_before, long long max_points);
22
+extern METRIC_CORRELATIONS_METHOD mc_string_to_method(const char *method);
23
+extern const char *mc_method_to_string(METRIC_CORRELATIONS_METHOD method);
24
+extern int mc_unittest(void);
25
26
#endif //NETDATA_METRIC_CORRELATIONS_H
health/health.c
+6
-4
@@ -855,10 +855,12 @@ void *health_main(void *ptr) {
855
/* time_t old_db_timestamp = rc->db_before; */
856
int value_is_null = 0;
857
858
- int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1, rc->after,
859
- rc->before, rc->group, 0, rc->options, &rc->db_after,
860
- &rc->db_before, &value_is_null, 0
861
- );
858
+ int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1,
859
+ rc->after, rc->before, rc->group,
860
+ 0, rc->options,
861
+ &rc->db_after,&rc->db_before,
862
+ NULL, NULL,
863
+ &value_is_null, 0);
864
865
if (unlikely(ret != 200)) {
866
// database lookup failed
libnetdata/storage_number/storage_number.h
+4
@@ -23,6 +23,8 @@ typedef double calculated_number;
23
#define LONG_DOUBLE_MODIFIER "f"
24
typedef double LONG_DOUBLE;
25
26
+#define CALCULATED_NUMBER_MAX DBL_MAX
27
+
28
#else // NETDATA_WITHOUT_LONG_DOUBLE
29
30
typedef long double calculated_number;
@@ -33,6 +35,8 @@ typedef long double calculated_number;
35
#define LONG_DOUBLE_MODIFIER "Lf"
36
typedef long double LONG_DOUBLE;
37
38
+#define CALCULATED_NUMBER_MAX LDBL_MAX
39
+
40
#endif // NETDATA_WITHOUT_LONG_DOUBLE
41
42
//typedef long long calculated_number;
web/api/badges/web_buffer_svg.c
+6
-2
@@ -1101,8 +1101,12 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
1101
1102
// if the collected value is too old, don't calculate its value
1103
if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above)))
1104
- ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL
1105
- , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null, 0);
1104
+ ret = rrdset2value_api_v1(st, w->response.data, &n,
1105
+ (dimensions) ? buffer_tostring(dimensions) : NULL,
1106
+ points, after, before, group, 0, options,
1107
+ NULL, &latest_timestamp,
1108
+ NULL, NULL,
1109
+ &value_is_null, 0);
1110
1111
// if the value cannot be calculated, show empty badge
1112
if (ret != HTTP_RESP_OK) {
web/api/formatters/json_wrapper.c
+10
-3
@@ -35,7 +35,7 @@ static int fill_formatted_callback(const char *name, const char *value, RRDLABEL
35
}
36
37
void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
38
- QUERY_PARAMS *rrdset_query_data)
38
+ RRDR_GROUPING group_method, QUERY_PARAMS *rrdset_query_data)
39
{
40
struct context_param *context_param_list = rrdset_query_data->context_param_list;
41
char *chart_label_key = rrdset_query_data->chart_label_key;
@@ -76,7 +76,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
76
" %slast_entry%s: %u,\n"
77
" %sbefore%s: %u,\n"
78
" %safter%s: %u,\n"
79
- " %sdimension_names%s: ["
79
+ " %sgroup%s: %s%s%s,\n"
80
+ " %soptions%s: %s"
81
, kq, kq
82
, kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq
83
, kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq
@@ -86,7 +87,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
87
, kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t_nolock(r->st))
88
, kq, kq, (uint32_t)r->before
89
, kq, kq, (uint32_t)r->after
89
- , kq, kq);
90
+ , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq
91
+ , kq, kq, sq);
92
+
93
+ web_client_api_request_v1_data_options_to_string(wb, options);
94
+
95
+ buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq);
96
+
97
if (should_lock)
98
rrdset_unlock(r->st);
99
web/api/formatters/json_wrapper.h
+3
-1
@@ -4,9 +4,11 @@
4
#define NETDATA_API_FORMATTER_JSON_WRAPPER_H
5
6
#include "rrd2json.h"
7
+#include "web/api/queries/query.h"
8
+
9
10
extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
9
- QUERY_PARAMS *query_params);
11
+ RRDR_GROUPING group_method, QUERY_PARAMS *query_params);
12
extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
13
14
#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
web/api/formatters/rrd2json.c
+20
-14
@@ -162,6 +162,8 @@ int rrdset2value_api_v1(
162
, uint32_t options
163
, time_t *db_after
164
, time_t *db_before
165
+ , size_t *db_points_read
166
+ , size_t *result_points_generated
167
, int *value_is_null
168
, int timeout
169
) {
@@ -177,9 +179,13 @@ int rrdset2value_api_v1(
179
goto cleanup;
180
}
181
180
- if(rrdr_rows(r) == 0) {
181
- rrdr_free(owa, r);
182
+ if(db_points_read)
183
+ *db_points_read += r->internal.db_points_read;
184
185
+ if(result_points_generated)
186
+ *result_points_generated += r->internal.result_points_generated;
187
+
188
+ if(rrdr_rows(r) == 0) {
189
if(db_after) *db_after = 0;
190
if(db_before) *db_before = 0;
191
if(value_is_null) *value_is_null = 1;
@@ -266,7 +272,7 @@ int rrdset2anything_api_v1(
272
case DATASOURCE_SSV:
273
if(options & RRDR_OPTION_JSON_WRAP) {
274
wb->contenttype = CT_APPLICATION_JSON;
269
- rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
275
+ rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
276
rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
277
rrdr_json_wrapper_end(r, wb, format, options, 1);
278
}
@@ -279,7 +285,7 @@ int rrdset2anything_api_v1(
285
case DATASOURCE_SSV_COMMA:
286
if(options & RRDR_OPTION_JSON_WRAP) {
287
wb->contenttype = CT_APPLICATION_JSON;
282
- rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
288
+ rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
289
rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
290
rrdr_json_wrapper_end(r, wb, format, options, 1);
291
}
@@ -292,7 +298,7 @@ int rrdset2anything_api_v1(
298
case DATASOURCE_JS_ARRAY:
299
if(options & RRDR_OPTION_JSON_WRAP) {
300
wb->contenttype = CT_APPLICATION_JSON;
295
- rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
301
+ rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
302
rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
303
rrdr_json_wrapper_end(r, wb, format, options, 0);
304
}
@@ -305,7 +311,7 @@ int rrdset2anything_api_v1(
311
case DATASOURCE_CSV:
312
if(options & RRDR_OPTION_JSON_WRAP) {
313
wb->contenttype = CT_APPLICATION_JSON;
308
- rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
314
+ rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
315
rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
316
rrdr_json_wrapper_end(r, wb, format, options, 1);
317
}
@@ -318,7 +324,7 @@ int rrdset2anything_api_v1(
324
case DATASOURCE_CSV_MARKDOWN:
325
if(options & RRDR_OPTION_JSON_WRAP) {
326
wb->contenttype = CT_APPLICATION_JSON;
321
- rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
327
+ rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
328
rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
329
rrdr_json_wrapper_end(r, wb, format, options, 1);
330
}
@@ -331,7 +337,7 @@ int rrdset2anything_api_v1(
337
case DATASOURCE_CSV_JSON_ARRAY:
338
wb->contenttype = CT_APPLICATION_JSON;
339
if(options & RRDR_OPTION_JSON_WRAP) {
334
- rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
340
+ rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
341
buffer_strcat(wb, "[\n");
342
rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
343
buffer_strcat(wb, "\n]");
@@ -348,7 +354,7 @@ int rrdset2anything_api_v1(
354
case DATASOURCE_TSV:
355
if(options & RRDR_OPTION_JSON_WRAP) {
356
wb->contenttype = CT_APPLICATION_JSON;
351
- rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
357
+ rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
358
rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
359
rrdr_json_wrapper_end(r, wb, format, options, 1);
360
}
@@ -361,7 +367,7 @@ int rrdset2anything_api_v1(
367
case DATASOURCE_HTML:
368
if(options & RRDR_OPTION_JSON_WRAP) {
369
wb->contenttype = CT_APPLICATION_JSON;
364
- rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
370
+ rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
371
buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
372
rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
373
buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
@@ -379,7 +385,7 @@ int rrdset2anything_api_v1(
385
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
386
387
if(options & RRDR_OPTION_JSON_WRAP)
382
- rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
388
+ rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
389
390
rrdr2json(r, wb, options, 1, query_params->context_param_list);
391
@@ -391,7 +397,7 @@ int rrdset2anything_api_v1(
397
wb->contenttype = CT_APPLICATION_JSON;
398
399
if(options & RRDR_OPTION_JSON_WRAP)
394
- rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
400
+ rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
401
402
rrdr2json(r, wb, options, 1, query_params->context_param_list);
403
@@ -402,7 +408,7 @@ int rrdset2anything_api_v1(
408
case DATASOURCE_JSONP:
409
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
410
if(options & RRDR_OPTION_JSON_WRAP)
405
- rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
411
+ rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
412
413
rrdr2json(r, wb, options, 0, query_params->context_param_list);
414
@@ -415,7 +421,7 @@ int rrdset2anything_api_v1(
421
wb->contenttype = CT_APPLICATION_JSON;
422
423
if(options & RRDR_OPTION_JSON_WRAP)
418
- rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
424
+ rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
425
426
rrdr2json(r, wb, options, 0, query_params->context_param_list);
427
web/api/formatters/rrd2json.h
+2
@@ -92,6 +92,8 @@ extern int rrdset2value_api_v1(
92
, uint32_t options
93
, time_t *db_after
94
, time_t *db_before
95
+ , size_t *db_points_read
96
+ , size_t *result_points_generated
97
, int *value_is_null
98
, int timeout
99
);
web/api/netdata-swagger.json
+210
-7
@@ -319,7 +319,8 @@
319
"match-ids",
320
"match-names",
321
"showcustomvars",
322
- "allow_past"
322
+ "allow_past",
323
+ "anomaly-bit"
324
]
325
},
326
"default": [
@@ -484,7 +485,8 @@
485
"absolute-sum",
486
"null2zero",
487
"percentage",
487
- "unaligned"
488
+ "unaligned",
489
+ "anomaly-bit"
490
]
491
},
492
"default": [
@@ -908,7 +910,7 @@
910
"/alarms_values": {
911
"get": {
912
"summary": "Get a list of active or raised alarms on the server",
911
- "description": "The alarms_values endpoint returns the list of all raised or enabled alarms on the netdata server. Called without any parameters, the raised alarms in state WARNING or CRITICAL are returned. By passing \"?all\", all the enabled alarms are returned. This option output differs from `/alarms` in the number of variables delivered. This endpoint gives to user `id`, `value`, `last_updated` time and alarm `status`.",
913
+ "description": "The alarms_values endpoint returns the list of all raised or enabled alarms on the netdata server. Called without any parameters, the raised alarms in state WARNING or CRITICAL are returned. By passing \"?all\", all the enabled alarms are returned. This option output differs from `/alarms` in the number of variables delivered. This endpoint gives to user `id`, `value`, `last_updated` time, and alarm `status`.",
914
"parameters": [
915
{
916
"name": "all",
@@ -1129,6 +1131,154 @@
1131
}
1132
}
1133
}
1134
+ },
1135
+ "/metric_correlations": {
1136
+ "get": {
1137
+ "summary": "Analyze all the metrics to find their correlations",
1138
+ "description": "Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
1139
+ "parameters": [
1140
+ {
1141
+ "name": "baseline_after",
1142
+ "in": "query",
1143
+ "description": "This parameter can either be an absolute timestamp specifying the starting point of baseline window, or a relative number of seconds (negative, relative to parameter baseline_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",
1144
+ "required": false,
1145
+ "allowEmptyValue": false,
1146
+ "schema": {
1147
+ "type": "number",
1148
+ "format": "integer",
1149
+ "default": -300
1150
+ }
1151
+ },
1152
+ {
1153
+ "name": "baseline_before",
1154
+ "in": "query",
1155
+ "description": "This parameter can either be an absolute timestamp specifying the ending point of the baseline window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",
1156
+ "required": false,
1157
+ "schema": {
1158
+ "type": "number",
1159
+ "format": "integer",
1160
+ "default": -60
1161
+ }
1162
+ },
1163
+ {
1164
+ "name": "highlight_after",
1165
+ "in": "query",
1166
+ "description": "This parameter can either be an absolute timestamp specifying the starting point of highlighted window, or a relative number of seconds (negative, relative to parameter highlight_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",
1167
+ "required": false,
1168
+ "allowEmptyValue": false,
1169
+ "schema": {
1170
+ "type": "number",
1171
+ "format": "integer",
1172
+ "default": -60
1173
+ }
1174
+ },
1175
+ {
1176
+ "name": "highlight_before",
1177
+ "in": "query",
1178
+ "description": "This parameter can either be an absolute timestamp specifying the ending point of the highlighted window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",
1179
+ "required": false,
1180
+ "schema": {
1181
+ "type": "number",
1182
+ "format": "integer",
1183
+ "default": 0
1184
+ }
1185
+ },
1186
+ {
1187
+ "name": "points",
1188
+ "in": "query",
1189
+ "description": "The number of points to be evaluated for the highlighted window. The baseline window will be adjusted automatically to receive a proportional amount of points.",
1190
+ "required": false,
1191
+ "allowEmptyValue": false,
1192
+ "schema": {
1193
+ "type": "number",
1194
+ "format": "integer",
1195
+ "default": 500
1196
+ }
1197
+ },
1198
+ {
1199
+ "name": "method",
1200
+ "in": "query",
1201
+ "description": "the algorithm to run",
1202
+ "required": false,
1203
+ "schema": {
1204
+ "type": "string",
1205
+ "enum": [
1206
+ "ks2",
1207
+ "volume"
1208
+ ],
1209
+ "default": "volume"
1210
+ }
1211
+ },
1212
+ {
1213
+ "name": "timeout",
1214
+ "in": "query",
1215
+ "description": "Cancel the query if to takes more that this amount of milliseconds.",
1216
+ "required": false,
1217
+ "allowEmptyValue": false,
1218
+ "schema": {
1219
+ "type": "number",
1220
+ "format": "integer",
1221
+ "default": 60000
1222
+ }
1223
+ },
1224
+ {
1225
+ "name": "options",
1226
+ "in": "query",
1227
+ "description": "Options that affect data generation.",
1228
+ "required": false,
1229
+ "allowEmptyValue": false,
1230
+ "schema": {
1231
+ "type": "array",
1232
+ "items": {
1233
+ "type": "string",
1234
+ "enum": [
1235
+ "min2max",
1236
+ "abs",
1237
+ "absolute",
1238
+ "absolute-sum",
1239
+ "null2zero",
1240
+ "percentage",
1241
+ "unaligned",
1242
+ "allow_past",
1243
+ "nonzero",
1244
+ "anomaly-bit",
1245
+ "raw"
1246
+ ]
1247
+ },
1248
+ "default": [
1249
+ "null2zero",
1250
+ "allow_past",
1251
+ "nonzero",
1252
+ "unaligned"
1253
+ ]
1254
+ }
1255
+ }
1256
+ ],
1257
+ "responses": {
1258
+ "200": {
1259
+ "description": "JSON object with weights for each chart and dimension.",
1260
+ "content": {
1261
+ "application/json": {
1262
+ "schema": {
1263
+ "$ref": "#/components/schemas/metric_correlations"
1264
+ }
1265
+ }
1266
+ }
1267
+ },
1268
+ "400": {
1269
+ "description": "The given parameters are invalid."
1270
+ },
1271
+ "403": {
1272
+ "description": "metrics correlations are not enabled on this Netdata Agent."
1273
+ },
1274
+ "404": {
1275
+ "description": "No charts could be found, or the method that correlated the metrics did not produce any result."
1276
+ },
1277
+ "504": {
1278
+ "description": "Timeout - the query took too long and has been cancelled."
1279
+ }
1280
+ }
1281
+ }
1282
}
1283
},
1284
"servers": [
@@ -1267,7 +1417,7 @@
1417
"stream_compression": {
1418
"type": "boolean",
1419
"description": "Stream transmission compression method.",
1270
- "example": "true"
1420
+ "example": true
1421
},
1422
"labels": {
1423
"type": "object",
@@ -2135,7 +2285,7 @@
2285
"type": "object",
2286
"properties": {
2287
"aclk-available": {
2138
- "type": "boolean",
2288
+ "type": "string",
2289
"description": "Describes whether this agent is capable of connection to the Cloud. False means agent has been built without ACLK component either on purpose (user choice) or due to missing dependency."
2290
},
2291
"aclk-version": {
@@ -2153,7 +2303,7 @@
2303
"type": "boolean",
2304
"description": "Informs whether this agent has been added to a space in the cloud (User has to perform claiming). If false (user didn't perform claiming) agent will never attempt any cloud connection."
2305
},
2156
- "claimed-id": {
2306
+ "claimed_id": {
2307
"type": "string",
2308
"format": "uuid",
2309
"description": "Unique ID this agent uses to identify when connecting to cloud"
@@ -2171,7 +2321,60 @@
2321
]
2322
}
2323
}
2324
+ },
2325
+ "metric_correlations": {
2326
+ "type": "object",
2327
+ "properties": {
2328
+ "correlated_charts": {
2329
+ "type": "object",
2330
+ "description": "An object containing chart objects with their metrics correlations.",
2331
+ "properties": {
2332
+ "chart-id1": {
2333
+ "type": "object",
2334
+ "properties": {
2335
+ "context": {
2336
+ "type": "string"
2337
+ },
2338
+ "dimensions": {
2339
+ "type": "object",
2340
+ "properties": {
2341
+ "dimension1-name": {
2342
+ "type": "number"
2343
+ },
2344
+ "dimension2-name": {
2345
+ "type": "number"
2346
+ }
2347
+ }
2348
+ }
2349
+ }
2350
+ },
2351
+ "chart-id2": {
2352
+ "type": "object",
2353
+ "properties": {
2354
+ "context": {
2355
+ "type": "string"
2356
+ },
2357
+ "dimensions": {
2358
+ "type": "object",
2359
+ "properties": {
2360
+ "dimension1-name": {
2361
+ "type": "number"
2362
+ },
2363
+ "dimension2-name": {
2364
+ "type": "number"
2365
+ }
2366
+ }
2367
+ }
2368
+ }
2369
+ }
2370
+ }
2371
+ },
2372
+ "total_dimensions_count": {
2373
+ "description": "The number of dimensions correlated",
2374
+ "type": "integer"
2375
+ }
2376
+ }
2377
}
2378
}
2379
}
2177
-}
2380
+}
\ No newline at end of file
web/api/netdata-swagger.yaml
+168
@@ -280,6 +280,7 @@ paths:
280
- match-names
281
- showcustomvars
282
- allow_past
283
+ - anomaly-bit
284
default:
285
- seconds
286
- jsonwrap
@@ -427,6 +428,7 @@ paths:
428
- null2zero
429
- percentage
430
- unaligned
431
+ - anomaly-bit
432
default:
433
- absolute
434
- name: label
@@ -913,6 +915,138 @@ paths:
915
application/json:
916
schema:
917
$ref: "#/components/schemas/aclk_state"
918
+ /metric_correlations:
919
+ get:
920
+ summary: Analyze all the metrics to find their correlations
921
+ description: Given two time-windows (baseline, highlight), it goes
922
+ through all the available metrics, querying both windows and tries to find
923
+ how these two windows relate to each other. It supports
924
+ multiple algorithms to do so. The result is a list of all
925
+ metrics evaluated, weighted for 0.0 (the two windows are
926
+ more different) to 1.0 (the two windows are similar).
927
+ The algorithm adjusts automatically the baseline window to be
928
+ a power of two multiple of the highlighted (1, 2, 4, 8, etc).
929
+ parameters:
930
+ - name: baseline_after
931
+ in: query
932
+ description: This parameter can either be an absolute timestamp specifying the
933
+ starting point of baseline window, or a relative number of
934
+ seconds (negative, relative to parameter baseline_before). Netdata will
935
+ assume it is a relative number if it is less that 3 years (in seconds).
936
+ required: false
937
+ allowEmptyValue: false
938
+ schema:
939
+ type: number
940
+ format: integer
941
+ default: -300
942
+ - name: baseline_before
943
+ in: query
944
+ description: This parameter can either be an absolute timestamp specifying the
945
+ ending point of the baseline window, or a relative number of
946
+ seconds (negative), relative to the last collected timestamp.
947
+ Netdata will assume it is a relative number if it is less than 3
948
+ years (in seconds).
949
+ required: false
950
+ schema:
951
+ type: number
952
+ format: integer
953
+ default: -60
954
+ - name: highlight_after
955
+ in: query
956
+ description: This parameter can either be an absolute timestamp specifying the
957
+ starting point of highlighted window, or a relative number of
958
+ seconds (negative, relative to parameter highlight_before). Netdata will
959
+ assume it is a relative number if it is less that 3 years (in seconds).
960
+ required: false
961
+ allowEmptyValue: false
962
+ schema:
963
+ type: number
964
+ format: integer
965
+ default: -60
966
+ - name: highlight_before
967
+ in: query
968
+ description: This parameter can either be an absolute timestamp specifying the
969
+ ending point of the highlighted window, or a relative number of
970
+ seconds (negative), relative to the last collected timestamp.
971
+ Netdata will assume it is a relative number if it is less than 3
972
+ years (in seconds).
973
+ required: false
974
+ schema:
975
+ type: number
976
+ format: integer
977
+ default: 0
978
+ - name: points
979
+ in: query
980
+ description: The number of points to be evaluated for the highlighted window.
981
+ The baseline window will be adjusted automatically to receive a proportional
982
+ amount of points.
983
+ required: false
984
+ allowEmptyValue: false
985
+ schema:
986
+ type: number
987
+ format: integer
988
+ default: 500
989
+ - name: method
990
+ in: query
991
+ description: the algorithm to run
992
+ required: false
993
+ schema:
994
+ type: string
995
+ enum:
996
+ - ks2
997
+ - volume
998
+ default: volume
999
+ - name: timeout
1000
+ in: query
1001
+ description: Cancel the query if to takes more that this amount of milliseconds.
1002
+ required: false
1003
+ allowEmptyValue: false
1004
+ schema:
1005
+ type: number
1006
+ format: integer
1007
+ default: 60000
1008
+ - name: options
1009
+ in: query
1010
+ description: Options that affect data generation.
1011
+ required: false
1012
+ allowEmptyValue: false
1013
+ schema:
1014
+ type: array
1015
+ items:
1016
+ type: string
1017
+ enum:
1018
+ - min2max
1019
+ - abs
1020
+ - absolute
1021
+ - absolute-sum
1022
+ - null2zero
1023
+ - percentage
1024
+ - unaligned
1025
+ - allow_past
1026
+ - nonzero
1027
+ - anomaly-bit
1028
+ - raw
1029
+ default:
1030
+ - null2zero
1031
+ - allow_past
1032
+ - nonzero
1033
+ - unaligned
1034
+ responses:
1035
+ "200":
1036
+ description: JSON object with weights for each chart and dimension.
1037
+ content:
1038
+ application/json:
1039
+ schema:
1040
+ $ref: "#/components/schemas/metric_correlations"
1041
+ "400":
1042
+ description: The given parameters are invalid.
1043
+ "403":
1044
+ description: metrics correlations are not enabled on this Netdata Agent.
1045
+ "404":
1046
+ description: No charts could be found, or the method
1047
+ that correlated the metrics did not produce any result.
1048
+ "504":
1049
+ description: Timeout - the query took too long and has been cancelled.
1050
servers:
1051
- url: https://registry.my-netdata.io/api/v1
1052
- url: http://registry.my-netdata.io/api/v1
@@ -1696,3 +1830,37 @@ components:
1830
enum:
1831
- Old
1832
- New
1833
+ metric_correlations:
1834
+ type: object
1835
+ properties:
1836
+ correlated_charts:
1837
+ type: object
1838
+ description: An object containing chart objects with their metrics correlations.
1839
+ properties:
1840
+ chart-id1:
1841
+ type: object
1842
+ properties:
1843
+ context:
1844
+ type: string
1845
+ dimensions:
1846
+ type: object
1847
+ properties:
1848
+ dimension1-name:
1849
+ type: number
1850
+ dimension2-name:
1851
+ type: number
1852
+ chart-id2:
1853
+ type: object
1854
+ properties:
1855
+ context:
1856
+ type: string
1857
+ dimensions:
1858
+ type: object
1859
+ properties:
1860
+ dimension1-name:
1861
+ type: number
1862
+ dimension2-name:
1863
+ type: number
1864
+ total_dimensions_count:
1865
+ description: The number of dimensions correlated
1866
+ type: integer
web/api/queries/query.c
+105
-74
@@ -280,6 +280,16 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI
280
return def;
281
}
282
283
+const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group) {
284
+ int i;
285
+
286
+ for(i = 0; api_v1_data_groups[i].name ; i++)
287
+ if(unlikely(group == api_v1_data_groups[i].value))
288
+ return api_v1_data_groups[i].name;
289
+
290
+ return "unknown";
291
+}
292
+
293
// ----------------------------------------------------------------------------
294
295
static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,
@@ -791,49 +801,36 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
801
#endif // NETDATA_INTERNAL_CHECKS
802
803
// Returns 1 if an absolute period was requested or 0 if it was a relative period
794
-static int rrdr_convert_before_after_to_absolute(
795
- long long *after_requestedp
796
- , long long *before_requestedp
797
- , int update_every
798
- , time_t first_entry_t
799
- , time_t last_entry_t
800
- , RRDR_OPTIONS options
801
-) {
804
+int rrdr_relative_window_to_absolute(long long *after, long long *before, int update_every, long points) {
805
+ time_t now = now_realtime_sec() - 1;
806
+
807
int absolute_period_requested = -1;
808
long long after_requested, before_requested;
809
805
- before_requested = *before_requestedp;
806
- after_requested = *after_requestedp;
807
-
808
- if(before_requested == 0 && after_requested == 0) {
809
- // dump the all the data
810
- before_requested = last_entry_t;
811
- after_requested = first_entry_t;
812
- absolute_period_requested = 0;
813
- }
810
+ before_requested = *before;
811
+ after_requested = *after;
812
813
// allow relative for before (smaller than API_RELATIVE_TIME_MAX)
814
if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
817
- if(ABS(before_requested) % update_every) {
818
- // make sure it is multiple of st->update_every
819
- if(before_requested < 0) before_requested = before_requested - update_every -
820
- before_requested % update_every;
821
- else before_requested = before_requested + update_every - before_requested % update_every;
822
- }
823
- if(before_requested > 0) before_requested = first_entry_t + before_requested;
824
- else before_requested = last_entry_t + before_requested; //last_entry_t is not really now_t
825
- //TODO: fix before_requested to be relative to now_t
815
+ // if the user asked for a positive relative time,
816
+ // flip it to a negative
817
+ if(before_requested > 0)
818
+ before_requested = -before_requested;
819
+
820
+ before_requested = now + before_requested;
821
absolute_period_requested = 0;
822
}
823
824
// allow relative for after (smaller than API_RELATIVE_TIME_MAX)
825
if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
831
- if(after_requested == 0) after_requested = -update_every;
832
- if(ABS(after_requested) % update_every) {
833
- // make sure it is multiple of st->update_every
834
- if(after_requested < 0) after_requested = after_requested - update_every - after_requested % update_every;
835
- else after_requested = after_requested + update_every - after_requested % update_every;
836
- }
826
+ if(after_requested > 0)
827
+ after_requested = -after_requested;
828
+
829
+ // if the user didn't give an after, use the number of points
830
+ // to give a sane default
831
+ if(after_requested == 0)
832
+ after_requested = -(points * update_every);
833
+
834
after_requested = before_requested + after_requested;
835
absolute_period_requested = 0;
836
}
@@ -841,24 +838,37 @@ static int rrdr_convert_before_after_to_absolute(
838
if(absolute_period_requested == -1)
839
absolute_period_requested = 1;
840
844
- // make sure they are within our timeframe
845
- if(before_requested > last_entry_t) before_requested = last_entry_t;
846
- if(before_requested < first_entry_t && !(options & RRDR_OPTION_ALLOW_PAST))
847
- before_requested = first_entry_t;
848
-
849
- if(after_requested > last_entry_t) after_requested = last_entry_t;
850
- if(after_requested < first_entry_t && !(options & RRDR_OPTION_ALLOW_PAST))
851
- after_requested = first_entry_t;
852
-
853
- // check if they are reversed
854
- if(after_requested > before_requested) {
855
- time_t tmp = before_requested;
841
+ // check if the parameters are flipped
842
+ if(after_requested >= before_requested) {
843
+ long long t = before_requested;
844
before_requested = after_requested;
857
- after_requested = tmp;
845
+ after_requested = t;
846
+ }
847
+
848
+ // we need to make sure that the query is aligned
849
+ // with the database update every, otherwise when the user
850
+ // requests just 1 point for the entire duration, it may not
851
+ // be created (the last 1 point may be misaligned with the
852
+ // query).
853
+ if(before_requested % update_every)
854
+ before_requested += update_every - (before_requested % update_every);
855
+
856
+ if(after_requested % update_every)
857
+ after_requested -= after_requested % update_every;
858
+
859
+ // if the query requests future data
860
+ // shift the query back to be in the present time
861
+ // (this may also happen because of the rules above)
862
+ if(before_requested > now) {
863
+ long long delta = before_requested - now;
864
+ if(delta % update_every)
865
+ delta += update_every - (delta % update_every);
866
+ before_requested -= delta;
867
+ after_requested -= delta;
868
}
869
860
- *before_requestedp = before_requested;
861
- *after_requestedp = after_requested;
870
+ *before = before_requested;
871
+ *after = after_requested;
872
873
return absolute_period_requested;
874
}
@@ -881,25 +891,25 @@ static RRDR *rrd2rrdr_fixedstep(
891
, int timeout
892
) {
893
int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
894
+ RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
895
896
// the duration of the chart
897
time_t duration = before_requested - after_requested;
898
long available_points = duration / update_every;
899
889
- RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
890
-
900
if(duration <= 0 || available_points <= 0)
892
- return rrdr_create(owa, st, 1, context_param_list);
901
+ return NULL;
902
894
- // check the number of wanted points in the result
895
- if(unlikely(points_requested < 0)) points_requested = -points_requested;
896
- if(unlikely(points_requested > available_points)) points_requested = available_points;
897
- if(unlikely(points_requested == 0)) points_requested = available_points;
903
+ if(unlikely(points_requested > available_points))
904
+ points_requested = available_points;
905
906
// calculate the desired grouping of source data points
907
long group = available_points / points_requested;
908
if(unlikely(group <= 0)) group = 1;
902
- if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer
909
+
910
+ // round "group" to the closest integer
911
+ if(unlikely(available_points % points_requested > points_requested / 2))
912
+ group++;
913
914
// resampling_time_requested enforces a certain grouping multiple
915
calculated_number resampling_divisor = 1.0;
@@ -955,7 +965,7 @@ static RRDR *rrd2rrdr_fixedstep(
965
if(aligned) {
966
// alignment has been requested, so align the values
967
before_requested -= before_requested % (group * update_every);
958
- after_requested -= after_requested % (group * update_every);
968
+ after_requested -= after_requested % (group * update_every);
969
}
970
971
// we align the request on requested_before
@@ -967,7 +977,6 @@ static RRDR *rrd2rrdr_fixedstep(
977
978
before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * update_every ));
979
}
970
- //size_t before_slot = rrdset_time2slot(st, before_wanted);
980
981
// we need to estimate the number of points, for having
982
// an integer number of values per point
@@ -989,7 +998,6 @@ static RRDR *rrd2rrdr_fixedstep(
998
after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every );
999
}
1000
}
992
- //size_t after_slot = rrdset_time2slot(st, after_wanted);
1001
1002
// check if they are reversed
1003
if(unlikely(after_wanted > before_wanted)) {
@@ -1656,29 +1664,53 @@ RRDR *rrd2rrdr(
1664
, int timeout
1665
)
1666
{
1659
- int rrd_update_every;
1667
+ int rrd_update_every = st->update_every;
1668
int absolute_period_requested;
1669
1670
+ if(unlikely(points_requested < 0))
1671
+ points_requested = -points_requested;
1672
+
1673
+ long points_original = points_requested;
1674
+ if(unlikely(!points_requested))
1675
+ points_requested = (before_requested - after_requested) / rrd_update_every;
1676
+
1677
+ if(unlikely(!points_requested))
1678
+ points_requested = 1;
1679
+
1680
time_t first_entry_t;
1681
time_t last_entry_t;
1682
if (context_param_list) {
1683
first_entry_t = context_param_list->first_entry_t;
1666
- last_entry_t = context_param_list->last_entry_t;
1667
- } else {
1684
+ last_entry_t = context_param_list->last_entry_t;
1685
+ }
1686
+ else {
1687
rrdset_rdlock(st);
1688
first_entry_t = rrdset_first_entry_t_nolock(st);
1670
- last_entry_t = rrdset_last_entry_t_nolock(st);
1689
+ last_entry_t = rrdset_last_entry_t_nolock(st);
1690
rrdset_unlock(st);
1691
}
1692
1674
- rrd_update_every = st->update_every;
1675
- absolute_period_requested = rrdr_convert_before_after_to_absolute(&after_requested, &before_requested,
1676
- rrd_update_every, first_entry_t,
1677
- last_entry_t, options);
1678
- if (options & RRDR_OPTION_ALLOW_PAST)
1693
+ absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested,
1694
+ rrd_update_every, points_requested);
1695
+
1696
+ if(options & RRDR_OPTION_ALLOW_PAST) {
1697
if (first_entry_t > after_requested)
1698
first_entry_t = after_requested;
1699
1700
+ if (last_entry_t < before_requested)
1701
+ last_entry_t = before_requested;
1702
+ }
1703
+ else {
1704
+ if(after_requested < first_entry_t)
1705
+ after_requested = first_entry_t;
1706
+
1707
+ if(before_requested > last_entry_t)
1708
+ before_requested = last_entry_t;
1709
+ }
1710
+
1711
+ if(!points_original)
1712
+ points_requested = (before_requested - after_requested) / rrd_update_every;
1713
+
1714
if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
1715
rebuild_context_param_list(owa, context_param_list, after_requested);
1716
st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
@@ -1699,22 +1731,21 @@ RRDR *rrd2rrdr(
1731
if (rrd_update_every != region_info_array[0].update_every) {
1732
rrd_update_every = region_info_array[0].update_every;
1733
/* recalculate query alignment */
1702
- absolute_period_requested =
1703
- rrdr_convert_before_after_to_absolute(&after_requested, &before_requested, rrd_update_every,
1704
- first_entry_t, last_entry_t, options);
1734
+ absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested,
1735
+ rrd_update_every, points_requested);
1736
}
1737
freez(region_info_array);
1738
}
1739
return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method,
1740
resampling_time_requested, options, dimensions, rrd_update_every,
1741
first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);
1711
- } else {
1742
+ }
1743
+ else {
1744
if (rrd_update_every != (uint16_t)max_interval) {
1745
rrd_update_every = (uint16_t) max_interval;
1746
/* recalculate query alignment */
1715
- absolute_period_requested = rrdr_convert_before_after_to_absolute(&after_requested, &before_requested,
1716
- rrd_update_every, first_entry_t,
1717
- last_entry_t, options);
1747
+ absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested,
1748
+ rrd_update_every, points_requested);
1749
}
1750
return rrd2rrdr_variablestep(owa, st, points_requested, after_requested, before_requested, group_method,
1751
resampling_time_requested, options, dimensions, rrd_update_every,
web/api/queries/query.h
+1
@@ -20,5 +20,6 @@ typedef enum rrdr_grouping {
20
extern const char *group_method2string(RRDR_GROUPING group);
21
extern void web_client_api_v1_init_grouping(void);
22
extern RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def);
23
+extern const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group);
24
25
#endif //NETDATA_API_DATA_QUERY_H
web/api/queries/rrdr.c
+2
-6
@@ -83,12 +83,8 @@ inline static void rrdr_unlock_rrdset(RRDR *r) {
83
}
84
}
85
86
-inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r)
87
-{
88
- if(unlikely(!r)) {
89
- error("NULL value given!");
90
- return;
91
- }
86
+inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
87
+ if(unlikely(!r)) return;
88
89
rrdr_unlock_rrdset(r);
90
onewayalloc_freez(owa, r->t);
web/api/queries/rrdr.h
+3
@@ -25,6 +25,7 @@ typedef enum rrdr_options {
25
RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response
26
RRDR_OPTION_ALLOW_PAST = 0x00020000, // The after parameter can extend in the past before the first entry
27
RRDR_OPTION_ANOMALY_BIT = 0x00040000, // Return the anomaly bit stored in each collected_number
28
+ RRDR_OPTION_RETURN_RAW = 0x00080000, // Return raw data for aggregating across multiple nodes
29
} RRDR_OPTIONS;
30
31
typedef enum rrdr_value_flag {
@@ -114,6 +115,8 @@ extern RRDR *rrd2rrdr(
115
RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
116
struct context_param *context_param_list, int timeout);
117
118
+extern int rrdr_relative_window_to_absolute(long long *after, long long *before, int update_every, long points);
119
+
120
#include "query.h"
121
122
#endif //NETDATA_QUERIES_RRDR_H
web/api/web_api_v1.c
+47
-20
@@ -36,9 +36,9 @@ static struct {
36
, {"match_names" , 0 , RRDR_OPTION_MATCH_NAMES}
37
, {"match-names" , 0 , RRDR_OPTION_MATCH_NAMES}
38
, {"showcustomvars" , 0 , RRDR_OPTION_CUSTOM_VARS}
39
- , {"allow_past" , 0 , RRDR_OPTION_ALLOW_PAST}
39
, {"anomaly-bit" , 0 , RRDR_OPTION_ANOMALY_BIT}
41
- , { NULL, 0, 0}
40
+ , {"raw" , 0 , RRDR_OPTION_RETURN_RAW}
41
+ , {NULL , 0 , 0}
42
};
43
44
static struct {
@@ -162,8 +162,8 @@ void web_client_api_v1_management_init(void) {
162
api_secret = get_mgmt_api_key();
163
}
164
165
-inline uint32_t web_client_api_request_v1_data_options(char *o) {
166
- uint32_t ret = 0x00000000;
165
+inline RRDR_OPTIONS web_client_api_request_v1_data_options(char *o) {
166
+ RRDR_OPTIONS ret = 0x00000000;
167
char *tok;
168
169
while(o && *o && (tok = mystrsep(&o, ", |"))) {
@@ -182,6 +182,19 @@ inline uint32_t web_client_api_request_v1_data_options(char *o) {
182
return ret;
183
}
184
185
+void web_client_api_request_v1_data_options_to_string(BUFFER *wb, RRDR_OPTIONS options) {
186
+ RRDR_OPTIONS used = 0; // to prevent adding duplicates
187
+ int added = 0;
188
+ for(int i = 0; api_v1_data_options[i].name ; i++) {
189
+ if (unlikely((api_v1_data_options[i].value & options) && !(api_v1_data_options[i].value & used))) {
190
+ if(added) buffer_strcat(wb, ",");
191
+ buffer_strcat(wb, api_v1_data_options[i].name);
192
+ used |= api_v1_data_options[i].value;
193
+ added++;
194
+ }
195
+ }
196
+}
197
+
198
inline uint32_t web_client_api_request_v1_data_format(char *name) {
199
uint32_t hash = simple_hash(name);
200
int i;
@@ -1319,8 +1332,12 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie
1332
if (!netdata_ready)
1333
return HTTP_RESP_BACKEND_FETCH_FAILED;
1334
1322
- long long baseline_after = 0, baseline_before = 0, highlight_after = 0, highlight_before = 0, max_points = 0;
1323
-
1335
+ long long baseline_after = 0, baseline_before = 0, after = 0, before = 0, points = 0;
1336
+ RRDR_OPTIONS options = RRDR_OPTION_NOT_ALIGNED | RRDR_OPTION_NONZERO | RRDR_OPTION_NULL2ZERO | RRDR_OPTION_ALLOW_PAST;
1337
+ METRIC_CORRELATIONS_METHOD method = default_metric_correlations_method;
1338
+ RRDR_GROUPING group = RRDR_GROUPING_AVERAGE;
1339
+ int timeout = 0;
1340
+
1341
while (url) {
1342
char *value = mystrsep(&url, "&");
1343
if (!value || !*value)
@@ -1334,15 +1351,31 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie
1351
1352
if (!strcmp(name, "baseline_after"))
1353
baseline_after = (long long) strtoul(value, NULL, 0);
1354
+
1355
else if (!strcmp(name, "baseline_before"))
1356
baseline_before = (long long) strtoul(value, NULL, 0);
1339
- else if (!strcmp(name, "highlight_after"))
1340
- highlight_after = (long long) strtoul(value, NULL, 0);
1341
- else if (!strcmp(name, "highlight_before"))
1342
- highlight_before = (long long) strtoul(value, NULL, 0);
1343
- else if (!strcmp(name, "max_points"))
1344
- max_points = (long long) strtoul(value, NULL, 0);
1345
-
1357
+
1358
+ else if (!strcmp(name, "after") || !strcmp(name, "highlight_after"))
1359
+ after = (long long) strtoul(value, NULL, 0);
1360
+
1361
+ else if (!strcmp(name, "before") || !strcmp(name, "highlight_before"))
1362
+ before = (long long) strtoul(value, NULL, 0);
1363
+
1364
+ else if (!strcmp(name, "points") || !strcmp(name, "max_points"))
1365
+ points = (long long) strtoul(value, NULL, 0);
1366
+
1367
+ else if (!strcmp(name, "timeout"))
1368
+ timeout = (int) strtoul(value, NULL, 0);
1369
+
1370
+ else if(!strcmp(name, "group"))
1371
+ group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE);
1372
+
1373
+ else if(!strcmp(name, "options"))
1374
+ options |= web_client_api_request_v1_data_options(value);
1375
+
1376
+ else if(!strcmp(name, "method"))
1377
+ method = mc_string_to_method(value);
1378
+
1379
}
1380
1381
BUFFER *wb = w->response.data;
@@ -1350,13 +1383,7 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie
1383
wb->contenttype = CT_APPLICATION_JSON;
1384
buffer_no_cacheable(wb);
1385
1353
- if (!highlight_after || !highlight_before)
1354
- buffer_strcat(wb, "{\"error\": \"Missing or invalid required highlight after and before parameters.\" }");
1355
- else {
1356
- metric_correlations(host, wb, baseline_after, baseline_before, highlight_after, highlight_before, max_points);
1357
- }
1358
-
1359
- return HTTP_RESP_OK;
1386
+ return metric_correlations(host, wb, method, group, baseline_after, baseline_before, after, before, points, options, timeout);
1387
}
1388
1389
static struct api_command {
web/api/web_api_v1.h
+3
-1
@@ -9,7 +9,9 @@
9
#include "web/api/health/health_cmdapi.h"
10
11
#define MAX_CHART_LABELS_FILTER (32)
12
-extern uint32_t web_client_api_request_v1_data_options(char *o);
12
+extern RRDR_OPTIONS web_client_api_request_v1_data_options(char *o);
13
+extern void web_client_api_request_v1_data_options_to_string(BUFFER *wb, RRDR_OPTIONS options);
14
+
15
extern uint32_t web_client_api_request_v1_data_format(char *name);
16
extern uint32_t web_client_api_request_v1_data_google_format(char *name);
17
web/server/web_client.h
+1
@@ -26,6 +26,7 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy;
26
// HTTP_CODES 5XX Server Errors
27
#define HTTP_RESP_INTERNAL_SERVER_ERROR 500
28
#define HTTP_RESP_BACKEND_FETCH_FAILED 503
29
+#define HTTP_RESP_GATEWAY_TIMEOUT 504
30
31
extern int respect_web_browser_do_not_track_policy;
32
extern char *web_x_frame_options;