@cryptotaxi247 / netdata-1 / commits / 4078661e3

Metric correlations (#12582)

* initial attempt at metric correlations * fix loop * simplify struct * change json * get points from query * comment * dont lock the host as much * add a configuration option to enable/disable metric correlations * remove KSfbar from header file * lock charts * add timeout * cast multiplication * add licencing info * better licencing * use onewayalloc * destroy owa

Emmanuel Vasilakis committed May 4, 2022 at 13:59 UTC 4078661e3342d7b311b2ec314f5323b6229c689c
10 files changed +1255 -2
CMakeLists.txt
+4
@@ -676,6 +676,10 @@ set(RRD_PLUGIN_FILES
676 database/engine/metadata_log/metalogpluginsd.h
677 database/engine/metadata_log/compaction.c
678 database/engine/metadata_log/compaction.h
679 + database/KolmogorovSmirnovDist.c
680 + database/KolmogorovSmirnovDist.h
681 + database/metric_correlations.c
682 + database/metric_correlations.h
683 )
684
685 set(WEB_PLUGIN_FILES
Makefile.am
+4
@@ -460,6 +460,10 @@ RRD_PLUGIN_FILES = \
460 database/sqlite/sqlite_aclk_alert.h \
461 database/sqlite/sqlite3.c \
462 database/sqlite/sqlite3.h \
463 + database/KolmogorovSmirnovDist.c \
464 + database/KolmogorovSmirnovDist.h \
465 + database/metric_correlations.c \
466 + database/metric_correlations.h \
467 $(NULL)
468
469 if ENABLE_DBENGINE
REDISTRIBUTED.md
+5
@@ -180,4 +180,9 @@ connectivity is not available.
180
181 Copyright 2015, Benedikt Schmitt [Unlicense License](https://unlicense.org/)
182
183 +- [Kolmogorov-Smirnov distribution](http://simul.iro.umontreal.ca/ksdir/)
184 +
185 + Copyright March 2010 by Université de Montréal, Richard Simard and Pierre L'Ecuyer
186 + [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.en.html)
187 +
188
daemon/common.h
+3
@@ -84,6 +84,9 @@
84 #include "commands.h"
85 #include "analytics.h"
86
87 +// metric correlations
88 +#include "database/metric_correlations.h"
89 +
90 // global netdata daemon variables
91 extern char *netdata_configured_hostname;
92 extern char *netdata_configured_user_config_dir;
daemon/main.c
+4
@@ -553,6 +553,10 @@ static void get_netdata_configured_variables() {
553 enable_ksm = config_get_boolean(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", enable_ksm);
554 #endif
555
556 + // --------------------------------------------------------------------
557 + // metric correlations
558 + enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
559 +
560 // --------------------------------------------------------------------
561 // get various system parameters
562
database/KolmogorovSmirnovDist.c new
+788
@@ -0,0 +1,788 @@
1 +// SPDX-License-Identifier: GPL-3.0
2 +
3 +/********************************************************************
4 + *
5 + * File: KolmogorovSmirnovDist.c
6 + * Environment: ISO C99 or ANSI C89
7 + * Author: Richard Simard
8 + * Organization: DIRO, Université de Montréal
9 + * Date: 1 February 2012
10 + * Version 1.1
11 +
12 + * Copyright 1 march 2010 by Université de Montréal,
13 + Richard Simard and Pierre L'Ecuyer
14 + =====================================================================
15 +
16 + This program is free software: you can redistribute it and/or modify
17 + it under the terms of the GNU General Public License as published by
18 + the Free Software Foundation, version 3 of the License.
19 +
20 + This program is distributed in the hope that it will be useful,
21 + but WITHOUT ANY WARRANTY; without even the implied warranty of
22 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23 + GNU General Public License for more details.
24 +
25 + You should have received a copy of the GNU General Public License
26 + along with this program. If not, see <http://www.gnu.org/licenses/>.
27 +
28 + =====================================================================*/
29 +
30 +#include "KolmogorovSmirnovDist.h"
31 +#include <math.h>
32 +#include <stdlib.h>
33 +
34 +#define num_Pi 3.14159265358979323846 /* PI */
35 +#define num_Ln2 0.69314718055994530941 /* log(2) */
36 +
37 +/* For x close to 0 or 1, we use the exact formulae of Ruben-Gambino in all
38 + cases. For n <= NEXACT, we use exact algorithms: the Durbin matrix and
39 + the Pomeranz algorithms. For n > NEXACT, we use asymptotic methods
40 + except for x close to 0 where we still use the method of Durbin
41 + for n <= NKOLMO. For n > NKOLMO, we use asymptotic methods only and
42 + so the precision is less for x close to 0.
43 + We could increase the limit NKOLMO to 10^6 to get better precision
44 + for x close to 0, but at the price of a slower speed. */
45 +#define NEXACT 500
46 +#define NKOLMO 100000
47 +
48 +/* The Durbin matrix algorithm for the Kolmogorov-Smirnov distribution */
49 +static double DurbinMatrix (int n, double d);
50 +
51 +
52 +/*========================================================================*/
53 +#if 0
54 +
55 +/* For ANSI C89 only, not for ISO C99 */
56 +#define MAXI 50
57 +#define EPSILON 1.0e-15
58 +
59 +double log1p (double x)
60 +{
61 + /* returns a value equivalent to log(1 + x) accurate also for small x. */
62 + if (fabs (x) > 0.1) {
63 + return log (1.0 + x);
64 + } else {
65 + double term = x;
66 + double sum = x;
67 + int s = 2;
68 + while ((fabs (term) > EPSILON * fabs (sum)) && (s < MAXI)) {
69 + term *= -x;
70 + sum += term / s;
71 + s++;
72 + }
73 + return sum;
74 + }
75 +}
76 +
77 +#undef MAXI
78 +#undef EPSILON
79 +
80 +#endif
81 +
82 +/*========================================================================*/
83 +#define MFACT 30
84 +
85 +/* The natural logarithm of factorial n! for 0 <= n <= MFACT */
86 +static double LnFactorial[MFACT + 1] = {
87 + 0.,
88 + 0.,
89 + 0.6931471805599453,
90 + 1.791759469228055,
91 + 3.178053830347946,
92 + 4.787491742782046,
93 + 6.579251212010101,
94 + 8.525161361065415,
95 + 10.60460290274525,
96 + 12.80182748008147,
97 + 15.10441257307552,
98 + 17.50230784587389,
99 + 19.98721449566188,
100 + 22.55216385312342,
101 + 25.19122118273868,
102 + 27.89927138384088,
103 + 30.67186010608066,
104 + 33.50507345013688,
105 + 36.39544520803305,
106 + 39.33988418719949,
107 + 42.33561646075348,
108 + 45.3801388984769,
109 + 48.47118135183522,
110 + 51.60667556776437,
111 + 54.7847293981123,
112 + 58.00360522298051,
113 + 61.26170176100199,
114 + 64.55753862700632,
115 + 67.88974313718154,
116 + 71.257038967168,
117 + 74.65823634883016
118 +};
119 +
120 +/*------------------------------------------------------------------------*/
121 +
122 +static double getLogFactorial (int n)
123 +{
124 + /* Returns the natural logarithm of factorial n! */
125 + if (n <= MFACT) {
126 + return LnFactorial[n];
127 +
128 + } else {
129 + double x = (double) (n + 1);
130 + double y = 1.0 / (x * x);
131 + double z = ((-(5.95238095238E-4 * y) + 7.936500793651E-4) * y -
132 + 2.7777777777778E-3) * y + 8.3333333333333E-2;
133 + z = ((x - 0.5) * log (x) - x) + 9.1893853320467E-1 + z / x;
134 + return z;
135 + }
136 +}
137 +
138 +/*------------------------------------------------------------------------*/
139 +
140 +static double rapfac (int n)
141 +{
142 + /* Computes n! / n^n */
143 + int i;
144 + double res = 1.0 / n;
145 + for (i = 2; i <= n; i++) {
146 + res *= (double) i / n;
147 + }
148 + return res;
149 +}
150 +
151 +
152 +/*========================================================================*/
153 +
154 +static double **CreateMatrixD (int N, int M)
155 +{
156 + int i;
157 + double **T2;
158 +
159 + T2 = (double **) malloc (N * sizeof (double *));
160 + T2[0] = (double *) malloc ((size_t) N * M * sizeof (double));
161 + for (i = 1; i < N; i++)
162 + T2[i] = T2[0] + i * M;
163 + return T2;
164 +}
165 +
166 +
167 +static void DeleteMatrixD (double **T)
168 +{
169 + free (T[0]);
170 + free (T);
171 +}
172 +
173 +
174 +/*========================================================================*/
175 +
176 +static double KSPlusbarAsymp (int n, double x)
177 +{
178 + /* Compute the probability of the KS+ distribution using an asymptotic
179 + formula */
180 + double t = (6.0 * n * x + 1);
181 + double z = t * t / (18.0 * n);
182 + double v = 1.0 - (2.0 * z * z - 4.0 * z - 1.0) / (18.0 * n);
183 + if (v <= 0.0)
184 + return 0.0;
185 + v = v * exp (-z);
186 + if (v >= 1.0)
187 + return 1.0;
188 + return v;
189 +}
190 +
191 +
192 +/*-------------------------------------------------------------------------*/
193 +
194 +static double KSPlusbarUpper (int n, double x)
195 +{
196 + /* Compute the probability of the KS+ distribution in the upper tail using
197 + Smirnov's stable formula */
198 + const double EPSILON = 1.0E-12;
199 + double q;
200 + double Sum = 0.0;
201 + double term;
202 + double t;
203 + double LogCom;
204 + double LOGJMAX;
205 + int j;
206 + int jdiv;
207 + int jmax = (int) (n * (1.0 - x));
208 +
209 + if (n > 200000)
210 + return KSPlusbarAsymp (n, x);
211 +
212 + /* Avoid log(0) for j = jmax and q ~ 1.0 */
213 + if ((1.0 - x - (double) jmax / n) <= 0.0)
214 + jmax--;
215 +
216 + if (n > 3000)
217 + jdiv = 2;
218 + else
219 + jdiv = 3;
220 +
221 + j = jmax / jdiv + 1;
222 + LogCom = getLogFactorial (n) - getLogFactorial (j) -
223 + getLogFactorial (n - j);
224 + LOGJMAX = LogCom;
225 +
226 + while (j <= jmax) {
227 + q = (double) j / n + x;
228 + term = LogCom + (j - 1) * log (q) + (n - j) * log1p (-q);
229 + t = exp (term);
230 + Sum += t;
231 + LogCom += log ((double) (n - j) / (j + 1));
232 + if (t <= Sum * EPSILON)
233 + break;
234 + j++;
235 + }
236 +
237 + j = jmax / jdiv;
238 + LogCom = LOGJMAX + log ((double) (j + 1) / (n - j));
239 +
240 + while (j > 0) {
241 + q = (double) j / n + x;
242 + term = LogCom + (j - 1) * log (q) + (n - j) * log1p (-q);
243 + t = exp (term);
244 + Sum += t;
245 + LogCom += log ((double) j / (n - j + 1));
246 + if (t <= Sum * EPSILON)
247 + break;
248 + j--;
249 + }
250 +
251 + Sum *= x;
252 + /* add the term j = 0 */
253 + Sum += exp (n * log1p (-x));
254 + return Sum;
255 +}
256 +
257 +
258 +/*========================================================================*/
259 +
260 +static double Pelz (int n, double x)
261 +{
262 + /* Approximating the Lower Tail-Areas of the Kolmogorov-Smirnov One-Sample
263 + Statistic,
264 + Wolfgang Pelz and I. J. Good,
265 + Journal of the Royal Statistical Society, Series B.
266 + Vol. 38, No. 2 (1976), pp. 152-156
267 + */
268 +
269 + const int JMAX = 20;
270 + const double EPS = 1.0e-10;
271 + const double C = 2.506628274631001; /* sqrt(2*Pi) */
272 + const double C2 = 1.2533141373155001; /* sqrt(Pi/2) */
273 + const double PI2 = num_Pi * num_Pi;
274 + const double PI4 = PI2 * PI2;
275 + const double RACN = sqrt ((double) n);
276 + const double z = RACN * x;
277 + const double z2 = z * z;
278 + const double z4 = z2 * z2;
279 + const double z6 = z4 * z2;
280 + const double w = PI2 / (2.0 * z * z);
281 + double ti, term, tom;
282 + double sum;
283 + int j;
284 +
285 + term = 1;
286 + j = 0;
287 + sum = 0;
288 + while (j <= JMAX && term > EPS * sum) {
289 + ti = j + 0.5;
290 + term = exp (-ti * ti * w);
291 + sum += term;
292 + j++;
293 + }
294 + sum *= C / z;
295 +
296 + term = 1;
297 + tom = 0;
298 + j = 0;
299 + while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
300 + ti = j + 0.5;
301 + term = (PI2 * ti * ti - z2) * exp (-ti * ti * w);
302 + tom += term;
303 + j++;
304 + }
305 + sum += tom * C2 / (RACN * 3.0 * z4);
306 +
307 + term = 1;
308 + tom = 0;
309 + j = 0;
310 + while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
311 + ti = j + 0.5;
312 + term = 6 * z6 + 2 * z4 + PI2 * (2 * z4 - 5 * z2) * ti * ti +
313 + PI4 * (1 - 2 * z2) * ti * ti * ti * ti;
314 + term *= exp (-ti * ti * w);
315 + tom += term;
316 + j++;
317 + }
318 + sum += tom * C2 / (n * 36.0 * z * z6);
319 +
320 + term = 1;
321 + tom = 0;
322 + j = 1;
323 + while (j <= JMAX && term > EPS * tom) {
324 + ti = j;
325 + term = PI2 * ti * ti * exp (-ti * ti * w);
326 + tom += term;
327 + j++;
328 + }
329 + sum -= tom * C2 / (n * 18.0 * z * z2);
330 +
331 + term = 1;
332 + tom = 0;
333 + j = 0;
334 + while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
335 + ti = j + 0.5;
336 + ti = ti * ti;
337 + term = -30 * z6 - 90 * z6 * z2 + PI2 * (135 * z4 - 96 * z6) * ti +
338 + PI4 * (212 * z4 - 60 * z2) * ti * ti + PI2 * PI4 * ti * ti * ti * (5 -
339 + 30 * z2);
340 + term *= exp (-ti * w);
341 + tom += term;
342 + j++;
343 + }
344 + sum += tom * C2 / (RACN * n * 3240.0 * z4 * z6);
345 +
346 + term = 1;
347 + tom = 0;
348 + j = 1;
349 + while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
350 + ti = j * j;
351 + term = (3 * PI2 * ti * z2 - PI4 * ti * ti) * exp (-ti * w);
352 + tom += term;
353 + j++;
354 + }
355 + sum += tom * C2 / (RACN * n * 108.0 * z6);
356 +
357 + return sum;
358 +}
359 +
360 +
361 +/*=========================================================================*/
362 +
363 +static void CalcFloorCeil (
364 + int n, /* sample size */
365 + double t, /* = nx */
366 + double *A, /* A_i */
367 + double *Atflo, /* floor (A_i - t) */
368 + double *Atcei /* ceiling (A_i + t) */
369 + )
370 +{
371 + /* Precompute A_i, floors, and ceilings for limits of sums in the Pomeranz
372 + algorithm */
373 + int i;
374 + int ell = (int) t; /* floor (t) */
375 + double z = t - ell; /* t - floor (t) */
376 + double w = ceil (t) - t;
377 +
378 + if (z > 0.5) {
379 + for (i = 2; i <= 2 * n + 2; i += 2)
380 + Atflo[i] = i / 2 - 2 - ell;
381 + for (i = 1; i <= 2 * n + 2; i += 2)
382 + Atflo[i] = i / 2 - 1 - ell;
383 +
384 + for (i = 2; i <= 2 * n + 2; i += 2)
385 + Atcei[i] = i / 2 + ell;
386 + for (i = 1; i <= 2 * n + 2; i += 2)
387 + Atcei[i] = i / 2 + 1 + ell;
388 +
389 + } else if (z > 0.0) {
390 + for (i = 1; i <= 2 * n + 2; i++)
391 + Atflo[i] = i / 2 - 1 - ell;
392 +
393 + for (i = 2; i <= 2 * n + 2; i++)
394 + Atcei[i] = i / 2 + ell;
395 + Atcei[1] = 1 + ell;
396 +
397 + } else { /* z == 0 */
398 + for (i = 2; i <= 2 * n + 2; i += 2)
399 + Atflo[i] = i / 2 - 1 - ell;
400 + for (i = 1; i <= 2 * n + 2; i += 2)
401 + Atflo[i] = i / 2 - ell;
402 +
403 + for (i = 2; i <= 2 * n + 2; i += 2)
404 + Atcei[i] = i / 2 - 1 + ell;
405 + for (i = 1; i <= 2 * n + 2; i += 2)
406 + Atcei[i] = i / 2 + ell;
407 + }
408 +
409 + if (w < z)
410 + z = w;
411 + A[0] = A[1] = 0;
412 + A[2] = z;
413 + A[3] = 1 - A[2];
414 + for (i = 4; i <= 2 * n + 1; i++)
415 + A[i] = A[i - 2] + 1;
416 + A[2 * n + 2] = n;
417 +}
418 +
419 +
420 +/*========================================================================*/
421 +
422 +static double Pomeranz (int n, double x)
423 +{
424 + /* The Pomeranz algorithm to compute the KS distribution */
425 + const double EPS = 1.0e-15;
426 + const int ENO = 350;
427 + const double RENO = ldexp (1.0, ENO); /* for renormalization of V */
428 + int coreno; /* counter: how many renormalizations */
429 + const double t = n * x;
430 + double w, sum, minsum;
431 + int i, j, k, s;
432 + int r1, r2; /* Indices i and i-1 for V[i][] */
433 + int jlow, jup, klow, kup, kup0;
434 + double *A;
435 + double *Atflo;
436 + double *Atcei;
437 + double **V;
438 + double **H; /* = pow(w, j) / Factorial(j) */
439 +
440 + A = (double *) calloc ((size_t) (2 * n + 3), sizeof (double));
441 + Atflo = (double *) calloc ((size_t) (2 * n + 3), sizeof (double));
442 + Atcei = (double *) calloc ((size_t) (2 * n + 3), sizeof (double));
443 + V = (double **) CreateMatrixD (2, n + 2);
444 + H = (double **) CreateMatrixD (4, n + 2);
445 +
446 + CalcFloorCeil (n, t, A, Atflo, Atcei);
447 +
448 + for (j = 1; j <= n + 1; j++)
449 + V[0][j] = 0;
450 + for (j = 2; j <= n + 1; j++)
451 + V[1][j] = 0;
452 + V[1][1] = RENO;
453 + coreno = 1;
454 +
455 + /* Precompute H[][] = (A[j] - A[j-1]^k / k! for speed */
456 + H[0][0] = 1;
457 + w = 2.0 * A[2] / n;
458 + for (j = 1; j <= n + 1; j++)
459 + H[0][j] = w * H[0][j - 1] / j;
460 +
461 + H[1][0] = 1;
462 + w = (1.0 - 2.0 * A[2]) / n;
463 + for (j = 1; j <= n + 1; j++)
464 + H[1][j] = w * H[1][j - 1] / j;
465 +
466 + H[2][0] = 1;
467 + w = A[2] / n;
468 + for (j = 1; j <= n + 1; j++)
469 + H[2][j] = w * H[2][j - 1] / j;
470 +
471 + H[3][0] = 1;
472 + for (j = 1; j <= n + 1; j++)
473 + H[3][j] = 0;
474 +
475 + r1 = 0;
476 + r2 = 1;
477 + for (i = 2; i <= 2 * n + 2; i++) {
478 + jlow = 2 + (int) Atflo[i];
479 + if (jlow < 1)
480 + jlow = 1;
481 + jup = (int) Atcei[i];
482 + if (jup > n + 1)
483 + jup = n + 1;
484 +
485 + klow = 2 + (int) Atflo[i - 1];
486 + if (klow < 1)
487 + klow = 1;
488 + kup0 = (int) Atcei[i - 1];
489 +
490 + /* Find to which case it corresponds */
491 + w = (A[i] - A[i - 1]) / n;
492 + s = -1;
493 + for (j = 0; j < 4; j++) {
494 + if (fabs (w - H[j][1]) <= EPS) {
495 + s = j;
496 + break;
497 + }
498 + }
499 + /* assert (s >= 0, "Pomeranz: s < 0"); */
500 +
501 + minsum = RENO;
502 + r1 = (r1 + 1) & 1; /* i - 1 */
503 + r2 = (r2 + 1) & 1; /* i */
504 +
505 + for (j = jlow; j <= jup; j++) {
506 + kup = kup0;
507 + if (kup > j)
508 + kup = j;
509 + sum = 0;
510 + for (k = kup; k >= klow; k--)
511 + sum += V[r1][k] * H[s][j - k];
512 + V[r2][j] = sum;
513 + if (sum < minsum)
514 + minsum = sum;
515 + }
516 +
517 + if (minsum < 1.0e-280) {
518 + /* V is too small: renormalize to avoid underflow of probabilities */
519 + for (j = jlow; j <= jup; j++)
520 + V[r2][j] *= RENO;
521 + coreno++; /* keep track of log of RENO */
522 + }
523 + }
524 +
525 + sum = V[r2][n + 1];
526 + free (A);
527 + free (Atflo);
528 + free (Atcei);
529 + DeleteMatrixD (H);
530 + DeleteMatrixD (V);
531 + w = getLogFactorial (n) - coreno * ENO * num_Ln2 + log (sum);
532 + if (w >= 0.)
533 + return 1.;
534 + return exp (w);
535 +}
536 +
537 +
538 +/*========================================================================*/
539 +
540 +static double cdfSpecial (int n, double x)
541 +{
542 + /* The KS distribution is known exactly for these cases */
543 +
544 + /* For nx^2 > 18, KSfbar(n, x) is smaller than 5e-16 */
545 + if ((n * x * x >= 18.0) || (x >= 1.0))
546 + return 1.0;
547 +
548 + if (x <= 0.5 / n)
549 + return 0.0;
550 +
551 + if (n == 1)
552 + return 2.0 * x - 1.0;
553 +
554 + if (x <= 1.0 / n) {
555 + double t = 2.0 * x * n - 1.0;
556 + double w;
557 + if (n <= NEXACT) {
558 + w = rapfac (n);
559 + return w * pow (t, (double) n);
560 + }
561 + w = getLogFactorial (n) + n * log (t / n);
562 + return exp (w);
563 + }
564 +
565 + if (x >= 1.0 - 1.0 / n) {
566 + return 1.0 - 2.0 * pow (1.0 - x, (double) n);
567 + }
568 +
569 + return -1.0;
570 +}
571 +
572 +
573 +/*========================================================================*/
574 +
575 +double KScdf (int n, double x)
576 +{
577 + const double w = n * x * x;
578 + double u = cdfSpecial (n, x);
579 + if (u >= 0.0)
580 + return u;
581 +
582 + if (n <= NEXACT) {
583 + if (w < 0.754693)
584 + return DurbinMatrix (n, x);
585 + if (w < 4.0)
586 + return Pomeranz (n, x);
587 + return 1.0 - KSfbar (n, x);
588 + }
589 +
590 + if ((w * x * n <= 7.0) && (n <= NKOLMO))
591 + return DurbinMatrix (n, x);
592 +
593 + return Pelz (n, x);
594 +}
595 +
596 +
597 +/*=========================================================================*/
598 +
599 +static double fbarSpecial (int n, double x)
600 +{
601 + const double w = n * x * x;
602 +
603 + if ((w >= 370.0) || (x >= 1.0))
604 + return 0.0;
605 + if ((w <= 0.0274) || (x <= 0.5 / n))
606 + return 1.0;
607 + if (n == 1)
608 + return 2.0 - 2.0 * x;
609 +
610 + if (x <= 1.0 / n) {
611 + double z;
612 + double t = 2.0 * x * n - 1.0;
613 + if (n <= NEXACT) {
614 + z = rapfac (n);
615 + return 1.0 - z * pow (t, (double) n);
616 + }
617 + z = getLogFactorial (n) + n * log (t / n);
618 + return 1.0 - exp (z);
619 + }
620 +
621 + if (x >= 1.0 - 1.0 / n) {
622 + return 2.0 * pow (1.0 - x, (double) n);
623 + }
624 + return -1.0;
625 +}
626 +
627 +
628 +/*========================================================================*/
629 +
630 +double KSfbar (int n, double x)
631 +{
632 + const double w = n * x * x;
633 + double v = fbarSpecial (n, x);
634 + if (v >= 0.0)
635 + return v;
636 +
637 + if (n <= NEXACT) {
638 + if (w < 4.0)
639 + return 1.0 - KScdf (n, x);
640 + else
641 + return 2.0 * KSPlusbarUpper (n, x);
642 + }
643 +
644 + if (w >= 2.65)
645 + return 2.0 * KSPlusbarUpper (n, x);
646 +
647 + return 1.0 - KScdf (n, x);
648 +}
649 +
650 +
651 +/*=========================================================================
652 +
653 +The following implements the Durbin matrix algorithm and was programmed by
654 +G. Marsaglia, Wai Wan Tsang and Jingbo Wong.
655 +
656 +I have made small modifications in their program. (Richard Simard)
657 +
658 +
659 +
660 +=========================================================================*/
661 +
662 +/*
663 + The C program to compute Kolmogorov's distribution
664 +
665 + K(n,d) = Prob(D_n < d), where
666 +
667 + D_n = max(x_1-0/n,x_2-1/n...,x_n-(n-1)/n,1/n-x_1,2/n-x_2,...,n/n-x_n)
668 +
669 + with x_1<x_2,...<x_n a purported set of n independent uniform [0,1)
670 + random variables sorted into increasing order.
671 + See G. Marsaglia, Wai Wan Tsang and Jingbo Wong,
672 + J.Stat.Software, 8, 18, pp 1--4, (2003).
673 +*/
674 +
675 +#define NORM 1.0e140
676 +#define INORM 1.0e-140
677 +#define LOGNORM 140
678 +
679 +
680 +/* Matrix product */
681 +static void mMultiply (double *A, double *B, double *C, int m);
682 +
683 +/* Matrix power */
684 +static void mPower (double *A, int eA, double *V, int *eV, int m, int n);
685 +
686 +
687 +static double DurbinMatrix (int n, double d)
688 +{
689 + int k, m, i, j, g, eH, eQ;
690 + double h, s, *H, *Q;
691 + /* OMIT NEXT TWO LINES IF YOU REQUIRE >7 DIGIT ACCURACY IN THE RIGHT TAIL */
692 +#if 0
693 + s = d * d * n;
694 + if (s > 7.24 || (s > 3.76 && n > 99))
695 + return 1 - 2 * exp (-(2.000071 + .331 / sqrt (n) + 1.409 / n) * s);
696 +#endif
697 + k = (int) (n * d) + 1;
698 + m = 2 * k - 1;
699 + h = k - n * d;
700 + H = (double *) malloc ((m * m) * sizeof (double));
701 + Q = (double *) malloc ((m * m) * sizeof (double));
702 + for (i = 0; i < m; i++)
703 + for (j = 0; j < m; j++)
704 + if (i - j + 1 < 0)
705 + H[i * m + j] = 0;
706 + else
707 + H[i * m + j] = 1;
708 + for (i = 0; i < m; i++) {
709 + H[i * m] -= pow (h, (double) (i + 1));
710 + H[(m - 1) * m + i] -= pow (h, (double) (m - i));
711 + }
712 + H[(m - 1) * m] += (2 * h - 1 > 0 ? pow (2 * h - 1, (double) m) : 0);
713 + for (i = 0; i < m; i++)
714 + for (j = 0; j < m; j++)
715 + if (i - j + 1 > 0)
716 + for (g = 1; g <= i - j + 1; g++)
717 + H[i * m + j] /= g;
718 + eH = 0;
719 + mPower (H, eH, Q, &eQ, m, n);
720 + s = Q[(k - 1) * m + k - 1];
721 +
722 + for (i = 1; i <= n; i++) {
723 + s = s * (double) i / n;
724 + if (s < INORM) {
725 + s *= NORM;
726 + eQ -= LOGNORM;
727 + }
728 + }
729 + s *= pow (10., (double) eQ);
730 + free (H);
731 + free (Q);
732 + return s;
733 +}
734 +
735 +
736 +static void mMultiply (double *A, double *B, double *C, int m)
737 +{
738 + int i, j, k;
739 + double s;
740 + for (i = 0; i < m; i++)
741 + for (j = 0; j < m; j++) {
742 + s = 0.;
743 + for (k = 0; k < m; k++)
744 + s += A[i * m + k] * B[k * m + j];
745 + C[i * m + j] = s;
746 + }
747 +}
748 +
749 +
750 +static void renormalize (double *V, int m, int *p)
751 +{
752 + int i;
753 + for (i = 0; i < m * m; i++)
754 + V[i] *= INORM;
755 + *p += LOGNORM;
756 +}
757 +
758 +
759 +static void mPower (double *A, int eA, double *V, int *eV, int m, int n)
760 +{
761 + double *B;
762 + int eB, i;
763 + if (n == 1) {
764 + for (i = 0; i < m * m; i++)
765 + V[i] = A[i];
766 + *eV = eA;
767 + return;
768 + }
769 + mPower (A, eA, V, eV, m, n / 2);
770 + B = (double *) malloc ((m * m) * sizeof (double));
771 + mMultiply (V, V, B, m);
772 + eB = 2 * (*eV);
773 + if (B[(m / 2) * m + (m / 2)] > NORM)
774 + renormalize (B, m, &eB);
775 +
776 + if (n % 2 == 0) {
777 + for (i = 0; i < m * m; i++)
778 + V[i] = B[i];
779 + *eV = eB;
780 + } else {
781 + mMultiply (A, B, V, m);
782 + *eV = eA + eB;
783 + }
784 +
785 + if (V[(m / 2) * m + (m / 2)] > NORM)
786 + renormalize (V, m, eV);
787 + free (B);
788 +}
database/KolmogorovSmirnovDist.h new
+91
@@ -0,0 +1,91 @@
1 +// SPDX-License-Identifier: GPL-3.0
2 +
3 +#ifndef KOLMOGOROVSMIRNOVDIST_H
4 +#define KOLMOGOROVSMIRNOVDIST_H
5 +
6 +#ifdef __cplusplus
7 +extern "C" {
8 +#endif
9 +
10 +
11 +/********************************************************************
12 + *
13 + * File: KolmogorovSmirnovDist.h
14 + * Environment: ISO C99 or ANSI C89
15 + * Author: Richard Simard
16 + * Organization: DIRO, Université de Montréal
17 + * Date: 1 February 2012
18 + * Version 1.1
19 + *
20 + * Copyright March 2010 by Université de Montréal,
21 + Richard Simard and Pierre L'Ecuyer
22 + =====================================================================
23 +
24 + This program is free software: you can redistribute it and/or modify
25 + it under the terms of the GNU General Public License as published by
26 + the Free Software Foundation, version 3 of the License.
27 +
28 + This program is distributed in the hope that it will be useful,
29 + but WITHOUT ANY WARRANTY; without even the implied warranty of
30 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 + GNU General Public License for more details.
32 +
33 + You should have received a copy of the GNU General Public License
34 + along with this program. If not, see <http://www.gnu.org/licenses/>.
35 +
36 + =====================================================================*/
37 +/*
38 + *
39 + * The Kolmogorov-Smirnov test statistic D_n is defined by
40 + *
41 + * D_n = sup_x |F(x) - S_n(x)|
42 + *
43 + * where n is the sample size, F(x) is a completely specified theoretical
44 + * distribution, and S_n(x) is an empirical distribution function.
45 + *
46 + *
47 + * The function
48 + *
49 + * double KScdf (int n, double x);
50 + *
51 + * computes the cumulative probability P[D_n <= x] of the 2-sided 1-sample
52 + * Kolmogorov-Smirnov distribution with sample size n at x.
53 + * It returns at least 13 decimal digits of precision for n <= 500,
54 + * at least 7 decimal digits of precision for 500 < n <= 100000,
55 + * and a few correct decimal digits for n > 100000.
56 + *
57 + */
58 +
59 +double KScdf (int n, double x);
60 +
61 +
62 +/*
63 + * The function
64 + *
65 + * double KSfbar (int n, double x);
66 + *
67 + * computes the complementary cumulative probability P[D_n >= x] of the
68 + * 2-sided 1-sample Kolmogorov-Smirnov distribution with sample size n at x.
69 + * It returns at least 10 decimal digits of precision for n <= 500,
70 + * at least 6 decimal digits of precision for 500 < n <= 200000,
71 + * and a few correct decimal digits for n > 200000.
72 + *
73 + */
74 +
75 +double KSfbar (int n, double x);
76 +
77 +
78 +/*
79 + * NOTE:
80 + * The ISO C99 function log1p of the standard math library does not exist in
81 + * ANSI C89. Here, it is programmed explicitly in KolmogorovSmirnovDist.c.
82 +
83 + * For ANSI C89 compilers, change the preprocessor condition to make it
84 + * available.
85 + */
86 +
87 +#ifdef __cplusplus
88 +}
89 +#endif
90 +
91 +#endif
database/metric_correlations.c new
+299
@@ -0,0 +1,299 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "daemon/common.h"
4 +#include "KolmogorovSmirnovDist.h"
5 +
6 +#define MAX_POINTS 10000
7 +int enable_metric_correlations = CONFIG_BOOLEAN_YES;
8 +
9 +struct charts {
10 + RRDSET *st;
11 + struct charts *next;
12 +};
13 +
14 +struct per_dim {
15 + char *dimension;
16 + calculated_number baseline[MAX_POINTS];
17 + calculated_number highlight[MAX_POINTS];
18 +
19 + double baseline_diffs[MAX_POINTS];
20 + double highlight_diffs[MAX_POINTS];
21 +};
22 +
23 +int find_index(double arr[], long int n, double K, long int start)
24 +{
25 + for (long int i = start; i < n; i++) {
26 + if (K<arr[i]){
27 + return i;
28 + }
29 + }
30 + return n;
31 +}
32 +
33 +int compare(const void *left, const void *right) {
34 + double lt = *(double *)left;
35 + double rt = *(double *)right;
36 +
37 + if(unlikely(lt < rt)) return -1;
38 + if(unlikely(lt > rt)) return 1;
39 + return 0;
40 +}
41 +
42 +void kstwo(double data1[], long int n1, double data2[], long int n2, double *d, double *prob)
43 +{
44 + double en1, en2, en, data_all[MAX_POINTS*2], cdf1[MAX_POINTS], cdf2[MAX_POINTS], cddiffs[MAX_POINTS];
45 + double min = 0.0, max = 0.0;
46 + qsort(data1, n1, sizeof(double), compare);
47 + qsort(data2, n2, sizeof(double), compare);
48 +
49 + for (int i = 0; i < n1; i++)
50 + data_all[i] = data1[i];
51 + for (int i = 0; i < n2; i++)
52 + data_all[n1 + i] = data2[i];
53 +
54 + en1 = (double)n1;
55 + en2 = (double)n2;
56 + *d = 0.0;
57 + cddiffs[0]=0; //for uninitialized warning
58 +
59 + for (int i=0; i<n1+n2;i++)
60 + cdf1[i] = find_index(data1, n1, data_all[i], 0) / en1; //TODO, use the start to reduce loops
61 +
62 + for (int i=0; i<n1+n2;i++)
63 + cdf2[i] = find_index(data2, n2, data_all[i], 0) / en2;
64 +
65 + for ( int i=0;i<n2+n1;i++)
66 + cddiffs[i] = cdf1[i] - cdf2[i];
67 +
68 + min = cddiffs[0];
69 + for ( int i=0;i<n2+n1;i++) {
70 + if (cddiffs[i] < min)
71 + min = cddiffs[i];
72 + }
73 +
74 + //clip min
75 + if (fabs(min) < 0) min = 0;
76 + else if (fabs(min) > 1) min = 1;
77 +
78 + max = fabs(cddiffs[0]);
79 + for ( int i=0;i<n2+n1;i++)
80 + if (cddiffs[i] >= max) max = cddiffs[i];
81 +
82 + if (fabs(min) < max)
83 + *d = max;
84 + else
85 + *d = fabs(min);
86 +
87 +
88 +
89 + en = (en1*en2 / (en1 + en2));
90 + *prob = KSfbar(round(en), *d);
91 +}
92 +
93 +void fill_nan (struct per_dim *d, long int hp, long int bp)
94 +{
95 + int k;
96 +
97 + for (k = 0; k < bp; k++) {
98 + if (isnan(d->baseline[k])) {
99 + d->baseline[k] = 0.0;
100 + }
101 + }
102 +
103 + for (k = 0; k < hp; k++) {
104 + if (isnan(d->highlight[k])) {
105 + d->highlight[k] = 0.0;
106 + }
107 + }
108 +}
109 +
110 +//TODO check counters
111 +void run_diffs_and_rev (struct per_dim *d, long int hp, long int bp)
112 +{
113 + int k, j;
114 +
115 + for (k = 0, j = bp; k < bp - 1; k++, j--)
116 + d->baseline_diffs[k] = (double)d->baseline[j - 2] - (double)d->baseline[j - 1];
117 + for (k = 0, j = hp; k < hp - 1; k++, j--) {
118 + d->highlight_diffs[k] = (double)d->highlight[j - 2] - (double)d->highlight[j - 1];
119 + }
120 +}
121 +
122 +int run_metric_correlations (BUFFER *wb, RRDSET *st, long long baseline_after, long long baseline_before, long long highlight_after, long long highlight_before, long long max_points)
123 +{
124 + uint32_t options = 0x00000000;
125 + int group_method = RRDR_GROUPING_AVERAGE;
126 + long group_time = 0;
127 + struct context_param *context_param_list = NULL;
128 + long c;
129 + int i=0, j=0;
130 + int b_dims = 0;
131 + long int baseline_points = 0, highlight_points = 0;
132 +
133 + struct per_dim *pd = NULL;
134 +
135 + //TODO get everything in one go, when baseline is right before highlight
136 + //get baseline
137 + ONEWAYALLOC *owa = onewayalloc_create(0);
138 + RRDR *rb = rrd2rrdr(owa, st, max_points, baseline_after, baseline_before, group_method, group_time, options, NULL, context_param_list, 0);
139 + if(!rb) {
140 + info("Cannot generate metric correlations output with these parameters on this chart.");
141 + onewayalloc_destroy(owa);
142 + return 0;
143 + } else {
144 + baseline_points = rrdr_rows(rb);
145 + pd = mallocz(sizeof(struct per_dim) * rb->d);
146 + b_dims = rb->d;
147 + for (c = 0; c != rrdr_rows(rb) ; ++c) {
148 + RRDDIM *d;
149 + for (j = 0, d = rb->st->dimensions ; d && j < rb->d ; ++j, d = d->next) {
150 + calculated_number *cn = &rb->v[ c * rb->d ];
151 + if (!c) {
152 + //TODO use points from query
153 + pd[j].dimension = strdupz (d->name);
154 + pd[j].baseline[c] = cn[j];
155 + } else {
156 + pd[j].baseline[c] = cn[j];
157 + }
158 + }
159 + }
160 + }
161 + rrdr_free(owa, rb);
162 + onewayalloc_destroy(owa);
163 + if (!pd)
164 + return 0;
165 +
166 + //get highlight
167 + owa = onewayalloc_create(0);
168 + RRDR *rh = rrd2rrdr(owa, st, max_points, highlight_after, highlight_before, group_method, group_time, options, NULL, context_param_list, 0);
169 + if(!rh) {
170 + info("Cannot generate metric correlations output with these parameters on this chart.");
171 + freez(pd);
172 + onewayalloc_destroy(owa);
173 + return 0;
174 + } else {
175 + if (rh->d != b_dims) {
176 + //TODO handle different dims
177 + rrdr_free(owa, rh);
178 + onewayalloc_destroy(owa);
179 + freez(pd);
180 + return 0;
181 + }
182 + highlight_points = rrdr_rows(rh);
183 + for (c = 0; c != rrdr_rows(rh) ; ++c) {
184 + RRDDIM *d;
185 + for (j = 0, d = rh->st->dimensions ; d && j < rh->d ; ++j, d = d->next) {
186 + calculated_number *cn = &rh->v[ c * rh->d ];
187 + pd[j].highlight[c] = cn[j];
188 + }
189 + }
190 + }
191 + rrdr_free(owa, rh);
192 + onewayalloc_destroy(owa);
193 +
194 + for (i = 0; i < b_dims; i++) {
195 + fill_nan(&pd[i], highlight_points, baseline_points);
196 + }
197 +
198 + for (i = 0; i < b_dims; i++) {
199 + run_diffs_and_rev(&pd[i], highlight_points, baseline_points);
200 + }
201 +
202 + double d=0, prob=0;
203 + for (i=0;i < j ;i++) {
204 + if (baseline_points && highlight_points) {
205 + kstwo(pd[i].baseline_diffs, baseline_points-1, pd[i].highlight_diffs, highlight_points-1, &d, &prob);
206 + buffer_sprintf(wb, "\t\t\t\t\"%s\": %f", pd[i].dimension, prob);
207 + if (i != j-1)
208 + buffer_sprintf(wb, ",\n");
209 + else
210 + buffer_sprintf(wb, "\n");
211 + }
212 + }
213 +
214 + freez(pd);
215 + return j;
216 +}
217 +
218 +void metric_correlations (RRDHOST *host, BUFFER *wb, long long baseline_after, long long baseline_before, long long highlight_after, long long highlight_before, long long max_points)
219 +{
220 + info ("Running metric correlations, highlight_after: %lld, highlight_before: %lld, baseline_after: %lld, baseline_before: %lld, max_points: %lld", highlight_after, highlight_before, baseline_after, baseline_before, max_points);
221 +
222 + if (!enable_metric_correlations) {
223 + error("Metric correlations functionality is not enabled.");
224 + buffer_strcat(wb, "{\"error\": \"Metric correlations functionality is not enabled.\" }");
225 + return;
226 + }
227 +
228 + if (highlight_before <= highlight_after || baseline_before <= baseline_after) {
229 + error("Invalid baseline or highlight ranges.");
230 + buffer_strcat(wb, "{\"error\": \"Invalid baseline or highlight ranges.\" }");
231 + return;
232 + }
233 +
234 + long long dims = 0, total_dims = 0;
235 + RRDSET *st;
236 + size_t c = 0;
237 + BUFFER *wdims = buffer_create(1000);
238 +
239 + if (!max_points || max_points > MAX_POINTS)
240 + max_points = MAX_POINTS;
241 +
242 + //dont lock here and wait for results
243 + //get the charts and run mc after
244 + //should not be a problem for the query
245 + struct charts *charts = NULL;
246 + rrdhost_rdlock(host);
247 + rrdset_foreach_read(st, host) {
248 + if (rrdset_is_available_for_viewers(st)) {
249 + rrdset_rdlock(st);
250 + struct charts *chart = callocz(1, sizeof(struct charts));
251 + chart->st = st;
252 + chart->next = NULL;
253 + if (charts) {
254 + chart->next = charts;
255 + }
256 + charts = chart;
257 + }
258 + }
259 + rrdhost_unlock(host);
260 +
261 + buffer_strcat(wb, "{\n\t\"correlated_charts\": {");
262 +
263 + for (struct charts *ch = charts; ch; ch = ch->next) {
264 + buffer_flush(wdims);
265 + dims = run_metric_correlations(wdims, ch->st, baseline_after, baseline_before, highlight_after, highlight_before, max_points);
266 + if (dims) {
267 + if (c)
268 + buffer_strcat(wb, "\t\t},");
269 + buffer_strcat(wb, "\n\t\t\"");
270 + buffer_strcat(wb, ch->st->id);
271 + buffer_strcat(wb, "\": {\n");
272 + buffer_strcat(wb, "\t\t\t\"context\": \"");
273 + buffer_strcat(wb, ch->st->context);
274 + buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n");
275 + buffer_sprintf(wb, "%s", buffer_tostring(wdims));
276 + buffer_strcat(wb, "\t\t\t}\n");
277 + total_dims += dims;
278 + c++;
279 + }
280 + }
281 + buffer_strcat(wb, "\t\t}\n");
282 + buffer_sprintf(wb, "\t},\n\t\"total_dimensions_count\": %lld\n}", total_dims);
283 +
284 + if (!total_dims) {
285 + buffer_flush(wb);
286 + buffer_strcat(wb, "{\"error\": \"No results from metric correlations.\" }");
287 + }
288 +
289 + struct charts* ch;
290 + while(charts){
291 + ch = charts;
292 + charts = charts->next;
293 + rrdset_unlock(ch->st);
294 + free(ch);
295 + }
296 +
297 + buffer_free(wdims);
298 + info ("Done running metric correlations");
299 +}
database/metric_correlations.h new
+10
@@ -0,0 +1,10 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_METRIC_CORRELATIONS_H
4 +#define NETDATA_METRIC_CORRELATIONS_H 1
5 +
6 +extern int enable_metric_correlations;
7 +
8 +void metric_correlations (RRDHOST *host, BUFFER *wb, long long selected_after, long long selected_before, long long reference_after, long long reference_before, long long max_points);
9 +
10 +#endif //NETDATA_METRIC_CORRELATIONS_H
web/api/web_api_v1.c
+47 -2
@@ -1299,6 +1299,50 @@ static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client
1299 return HTTP_RESP_OK;
1300 }
1301
1302 +int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_client *w, char *url) {
1303 + if (!netdata_ready)
1304 + return HTTP_RESP_BACKEND_FETCH_FAILED;
1305 +
1306 + long long baseline_after = 0, baseline_before = 0, highlight_after = 0, highlight_before = 0, max_points = 0;
1307 +
1308 + while (url) {
1309 + char *value = mystrsep(&url, "&");
1310 + if (!value || !*value)
1311 + continue;
1312 +
1313 + char *name = mystrsep(&value, "=");
1314 + if (!name || !*name)
1315 + continue;
1316 + if (!value || !*value)
1317 + continue;
1318 +
1319 + if (!strcmp(name, "baseline_after"))
1320 + baseline_after = (long long) strtoul(value, NULL, 0);
1321 + else if (!strcmp(name, "baseline_before"))
1322 + baseline_before = (long long) strtoul(value, NULL, 0);
1323 + else if (!strcmp(name, "highlight_after"))
1324 + highlight_after = (long long) strtoul(value, NULL, 0);
1325 + else if (!strcmp(name, "highlight_before"))
1326 + highlight_before = (long long) strtoul(value, NULL, 0);
1327 + else if (!strcmp(name, "max_points"))
1328 + max_points = (long long) strtoul(value, NULL, 0);
1329 +
1330 + }
1331 +
1332 + BUFFER *wb = w->response.data;
1333 + buffer_flush(wb);
1334 + wb->contenttype = CT_APPLICATION_JSON;
1335 + buffer_no_cacheable(wb);
1336 +
1337 + if (!highlight_after || !highlight_before)
1338 + buffer_strcat(wb, "{\"error\": \"Missing or invalid required highlight after and before parameters.\" }");
1339 + else {
1340 + metric_correlations(host, wb, baseline_after, baseline_before, highlight_after, highlight_before, max_points);
1341 + }
1342 +
1343 + return HTTP_RESP_OK;
1344 +}
1345 +
1346 static struct api_command {
1347 const char *command;
1348 uint32_t hash;
@@ -1330,8 +1374,9 @@ static struct api_command {
1374 { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info },
1375 #endif
1376
1333 - { "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health },
1334 - { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state },
1377 + { "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health },
1378 + { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state },
1379 + { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_metric_correlations },
1380 // terminator
1381 { NULL, 0, WEB_CLIENT_ACL_NONE, NULL },
1382 };