query code cleanup (#4480)
* queries code cleanup; renaming of variables; user configurable; tuning for defaults * reformatted main queries array * added documentation about all queries * empty doc * changed resampling variable names * added documentation to query module functions * fixed typos * renames * identation * more renames * fixed a faulty function definition at backends
Costa Tsaousis committed
Oct 24, 2018 at 23:06 UTC
c9234ccc8cea6023c190370ea7e7e95fb5433dab
32 files changed
+733
-180
backends/backends.c
+2
-1
@@ -1,5 +1,6 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
+#define BACKENDS_INTERNALS
4
#include "backends.h"
5
6
// ----------------------------------------------------------------------------
@@ -48,7 +49,7 @@ size_t backend_name_copy(char *d, const char *s, size_t usable) {
49
// calculate the SUM or AVERAGE of a dimension, for any timeframe
50
// may return NAN if the database does not have any value in the give timeframe
51
51
-inline calculated_number backend_calculate_value_from_stored_data(
52
+calculated_number backend_calculate_value_from_stored_data(
53
RRDSET *st // the chart
54
, RRDDIM *rd // the dimension
55
, time_t after // the start timestamp
backends/backends.h
+5
-3
@@ -24,22 +24,24 @@ extern const char *global_backend_prefix;
24
25
extern void *backends_main(void *ptr);
26
27
-extern int backends_can_send_rrdset(BACKEND_OPTIONS backend_options, RRDSET *st);
27
extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options);
28
29
+#ifdef BACKENDS_INTERNALS
30
+
31
+extern int backends_can_send_rrdset(BACKEND_OPTIONS backend_options, RRDSET *st);
32
extern calculated_number backend_calculate_value_from_stored_data(
33
RRDSET *st // the chart
34
, RRDDIM *rd // the dimension
35
, time_t after // the start timestamp
36
, time_t before // the end timestamp
35
- , uint32_t backend_options // BACKEND_SOURCE_* bitmap
37
+ , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
38
, time_t *first_timestamp // the timestamp of the first point used in this response
39
, time_t *last_timestamp // the timestamp that should be reported to backend
40
);
41
40
-#ifdef BACKENDS_INTERNALS
42
extern size_t backend_name_copy(char *d, const char *s, size_t usable);
43
extern int discard_response(BUFFER *b, const char *backend);
44
+
45
#endif // BACKENDS_INTERNALS
46
47
#include "backends/prometheus/backend_prometheus.h"
backends/prometheus/backend_prometheus.c
+1
@@ -1,5 +1,6 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
+#define BACKENDS_INTERNALS
4
#include "backend_prometheus.h"
5
6
// ----------------------------------------------------------------------------
web/api/queries/README.md
+4
@@ -0,0 +1,4 @@
1
+# Database Queries
2
+
3
+TBD
4
+
web/api/queries/average/README.md
+39
@@ -0,0 +1,39 @@
1
+# Average or Mean
2
+
3
+> This query is available as `average` and `mean`.
4
+
5
+An average is a single number taken as representative of a list of numbers.
6
+
7
+It is calculated as:
8
+
9
+```
10
+average = sum(numbers) / count(numbers)
11
+```
12
+
13
+## how to use
14
+
15
+Use it in alarms like this:
16
+
17
+```
18
+ alarm: my_alarm
19
+ on: my_chart
20
+lookup: average -1m unaligned of my_dimension
21
+ warn: $this > 1000
22
+```
23
+
24
+`average` does not change the units. For example, if the chart units is `requests/sec`, the result
25
+will be again expressed in the same units.
26
+
27
+It can also be used in APIs and badges as `&group=average` in the URL.
28
+
29
+## Examples
30
+
31
+Examining last 1 minute `successful` web server responses:
32
+
33
+- 
34
+- 
35
+- 
36
+
37
+## References
38
+
39
+- [https://en.wikipedia.org/wiki/Average](https://en.wikipedia.org/wiki/Average).
web/api/queries/average/average.c
+8
-7
@@ -10,7 +10,7 @@ struct grouping_average {
10
size_t count;
11
};
12
13
-void *grouping_init_average(RRDR *r) {
13
+void *grouping_create_average(RRDR *r) {
14
(void)r;
15
return callocz(1, sizeof(struct grouping_average));
16
}
@@ -18,25 +18,26 @@ void *grouping_init_average(RRDR *r) {
18
// resets when switches dimensions
19
// so, clear everything to restart
20
void grouping_reset_average(RRDR *r) {
21
- struct grouping_average *g = (struct grouping_average *)r->grouping_data;
21
+ struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
22
g->sum = 0;
23
g->count = 0;
24
}
25
26
void grouping_free_average(RRDR *r) {
27
- freez(r->grouping_data);
27
+ freez(r->internal.grouping_data);
28
+ r->internal.grouping_data = NULL;
29
}
30
31
void grouping_add_average(RRDR *r, calculated_number value) {
32
if(!isnan(value)) {
32
- struct grouping_average *g = (struct grouping_average *)r->grouping_data;
33
+ struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
34
g->sum += value;
35
g->count++;
36
}
37
}
38
39
calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
39
- struct grouping_average *g = (struct grouping_average *)r->grouping_data;
40
+ struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
41
42
calculated_number value;
43
@@ -45,8 +46,8 @@ calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_
46
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
47
}
48
else {
48
- if(unlikely(r->group_points != 1))
49
- value = g->sum / r->group_sum_divisor;
49
+ if(unlikely(r->internal.resampling_group != 1))
50
+ value = g->sum / r->internal.resampling_divisor;
51
else
52
value = g->sum / g->count;
53
}
web/api/queries/average/average.h
+1
-1
@@ -6,7 +6,7 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_average(RRDR *r);
9
+extern void *grouping_create_average(RRDR *r);
10
extern void grouping_reset_average(RRDR *r);
11
extern void grouping_free_average(RRDR *r);
12
extern void grouping_add_average(RRDR *r, calculated_number value);
web/api/queries/des/README.md
+65
@@ -1 +1,66 @@
1
# double exponential smoothing
2
+
3
+Exponential smoothing is one of many window functions commonly applied to smooth data in signal
4
+processing, acting as low-pass filters to remove high frequency noise.
5
+
6
+Simple exponential smoothing does not do well when there is a trend in the data.
7
+In such situations, several methods were devised under the name "double exponential smoothing"
8
+or "second-order exponential smoothing.", which is the recursive application of an exponential
9
+filter twice, thus being termed "double exponential smoothing".
10
+
11
+In simple terms, this is like an average value, but more recent values are given more weight
12
+and the trend of the values influences significantly the result.
13
+
14
+> **IMPORTANT**
15
+>
16
+> It is common for `des` to provide "average" values that far beyond the minimum or the maximum
17
+> values found in the time-series.
18
+> `des` estimates these values because of it takes into account the trend.
19
+
20
+This module implements the "Holt-Winters double exponential smoothing".
21
+
22
+Netdata automatically adjusts the weight (`alpha`) and the trend (`beta`) based on the number
23
+of values processed, using the formula:
24
+
25
+```
26
+window = max(number of values, 15)
27
+alpha = 2 / (window + 1)
28
+beta = 2 / (window + 1)
29
+```
30
+
31
+You can change the fixed value `15` by setting in `netdata.conf`:
32
+
33
+```
34
+[web]
35
+ des max window = 15
36
+```
37
+
38
+## how to use
39
+
40
+Use it in alarms like this:
41
+
42
+```
43
+ alarm: my_alarm
44
+ on: my_chart
45
+lookup: des -1m unaligned of my_dimension
46
+ warn: $this > 1000
47
+```
48
+
49
+`des` does not change the units. For example, if the chart units is `requests/sec`, the result
50
+will be again expressed in the same units.
51
+
52
+It can also be used in APIs and badges as `&group=des` in the URL.
53
+
54
+## Examples
55
+
56
+Examining last 1 minute `successful` web server responses:
57
+
58
+- 
59
+- 
60
+- 
61
+- 
62
+- 
63
+
64
+## References
65
+
66
+- [https://en.wikipedia.org/wiki/Exponential_smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing).
web/api/queries/des/des.c
+38
-11
@@ -1,5 +1,6 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
+#include <web/api/queries/rrdr.h>
4
#include "des.h"
5
6
@@ -18,14 +19,41 @@ struct grouping_des {
19
size_t count;
20
};
21
21
-#define MAX_WINDOW_SIZE 10
22
+static size_t max_window_size = 15;
23
+
24
+void grouping_init_des(void) {
25
+ long long ret = config_get_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size);
26
+ if(ret <= 1) {
27
+ config_set_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size);
28
+ }
29
+ else {
30
+ max_window_size = (size_t) ret;
31
+ }
32
+}
33
+
34
+static inline calculated_number window(RRDR *r, struct grouping_des *g) {
35
+ (void)g;
36
+
37
+ calculated_number points;
38
+ if(r->group == 1) {
39
+ // provide a running DES
40
+ points = r->internal.points_wanted;
41
+ }
42
+ else {
43
+ // provide a SES with flush points
44
+ points = r->group;
45
+ }
46
+
47
+ // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
48
+ // A commonly used value for alpha is 2 / (N + 1)
49
+ return (points > max_window_size) ? max_window_size : points;
50
+}
51
52
static inline void set_alpha(RRDR *r, struct grouping_des *g) {
53
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
54
// A commonly used value for alpha is 2 / (N + 1)
26
- calculated_number window = (r->group > MAX_WINDOW_SIZE) ? MAX_WINDOW_SIZE : r->group;
55
28
- g->alpha = 2.0 / ((calculated_number)window + 1.0);
56
+ g->alpha = 2.0 / (window(r, g) + 1.0);
57
g->alpha_other = 1.0 - g->alpha;
58
59
//info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha);
@@ -34,15 +62,14 @@ static inline void set_alpha(RRDR *r, struct grouping_des *g) {
62
static inline void set_beta(RRDR *r, struct grouping_des *g) {
63
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
64
// A commonly used value for alpha is 2 / (N + 1)
37
- calculated_number window = (r->group > MAX_WINDOW_SIZE) ? MAX_WINDOW_SIZE : r->group;
65
39
- g->beta = 2.0 / ((calculated_number)window + 1.0);
66
+ g->beta = 2.0 / (window(r, g) + 1.0);
67
g->beta_other = 1.0 - g->beta;
68
69
//info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta);
70
}
71
45
-void *grouping_init_des(RRDR *r) {
72
+void *grouping_create_des(RRDR *r) {
73
struct grouping_des *g = (struct grouping_des *)malloc(sizeof(struct grouping_des));
74
set_alpha(r, g);
75
set_beta(r, g);
@@ -55,7 +82,7 @@ void *grouping_init_des(RRDR *r) {
82
// resets when switches dimensions
83
// so, clear everything to restart
84
void grouping_reset_des(RRDR *r) {
58
- struct grouping_des *g = (struct grouping_des *)r->grouping_data;
85
+ struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
86
g->level = 0.0;
87
g->trend = 0.0;
88
g->count = 0;
@@ -65,12 +92,12 @@ void grouping_reset_des(RRDR *r) {
92
}
93
94
void grouping_free_des(RRDR *r) {
68
- freez(r->grouping_data);
69
- r->grouping_data = NULL;
95
+ freez(r->internal.grouping_data);
96
+ r->internal.grouping_data = NULL;
97
}
98
99
void grouping_add_des(RRDR *r, calculated_number value) {
73
- struct grouping_des *g = (struct grouping_des *)r->grouping_data;
100
+ struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
101
102
if(isnormal(value)) {
103
if(likely(g->count > 0)) {
@@ -99,7 +126,7 @@ void grouping_add_des(RRDR *r, calculated_number value) {
126
}
127
128
calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
102
- struct grouping_des *g = (struct grouping_des *)r->grouping_data;
129
+ struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
130
131
if(unlikely(!g->count || !isnormal(g->level))) {
132
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
web/api/queries/des/des.h
+3
-1
@@ -6,7 +6,9 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_des(RRDR *r);
9
+extern void grouping_init_des(void);
10
+
11
+extern void *grouping_create_des(RRDR *r);
12
extern void grouping_reset_des(RRDR *r);
13
extern void grouping_free_des(RRDR *r);
14
extern void grouping_add_des(RRDR *r, calculated_number value);
web/api/queries/incremental_sum/README.md
+34
@@ -0,0 +1,34 @@
1
+# Incremental Sum (`incremental_sum`)
2
+
3
+This modules finds the incremental sum of a period, which `last value - first value`.
4
+
5
+The result may be positive (rising) or negative (falling) depending on the first and last values.
6
+
7
+## how to use
8
+
9
+Use it in alarms like this:
10
+
11
+```
12
+ alarm: my_alarm
13
+ on: my_chart
14
+lookup: incremental_sum -1m unaligned of my_dimension
15
+ warn: $this > 1000
16
+```
17
+
18
+`incremental_sum` does not change the units. For example, if the chart units is `requests/sec`, the result
19
+will be again expressed in the same units.
20
+
21
+It can also be used in APIs and badges as `&group=incremental_sum` in the URL.
22
+
23
+## Examples
24
+
25
+Examining last 1 minute `successful` web server responses:
26
+
27
+- 
28
+- 
29
+- 
30
+- 
31
+
32
+## References
33
+
34
+- none
web/api/queries/incremental_sum/incremental_sum.c
+6
-5
@@ -11,7 +11,7 @@ struct grouping_incremental_sum {
11
size_t count;
12
};
13
14
-void *grouping_init_incremental_sum(RRDR *r) {
14
+void *grouping_create_incremental_sum(RRDR *r) {
15
(void)r;
16
return callocz(1, sizeof(struct grouping_incremental_sum));
17
}
@@ -19,19 +19,20 @@ void *grouping_init_incremental_sum(RRDR *r) {
19
// resets when switches dimensions
20
// so, clear everything to restart
21
void grouping_reset_incremental_sum(RRDR *r) {
22
- struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->grouping_data;
22
+ struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
23
g->first = 0;
24
g->last = 0;
25
g->count = 0;
26
}
27
28
void grouping_free_incremental_sum(RRDR *r) {
29
- freez(r->grouping_data);
29
+ freez(r->internal.grouping_data);
30
+ r->internal.grouping_data = NULL;
31
}
32
33
void grouping_add_incremental_sum(RRDR *r, calculated_number value) {
34
if(!isnan(value)) {
34
- struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->grouping_data;
35
+ struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
36
37
if(unlikely(!g->count)) {
38
g->first = value;
@@ -45,7 +46,7 @@ void grouping_add_incremental_sum(RRDR *r, calculated_number value) {
46
}
47
48
calculated_number grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
48
- struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->grouping_data;
49
+ struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
50
51
calculated_number value;
52
web/api/queries/incremental_sum/incremental_sum.h
+1
-1
@@ -6,7 +6,7 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_incremental_sum(RRDR *r);
9
+extern void *grouping_create_incremental_sum(RRDR *r);
10
extern void grouping_reset_incremental_sum(RRDR *r);
11
extern void grouping_free_incremental_sum(RRDR *r);
12
extern void grouping_add_incremental_sum(RRDR *r, calculated_number value);
web/api/queries/max/README.md
+31
@@ -0,0 +1,31 @@
1
+# Max
2
+
3
+This module finds the max value in the time-frame given.
4
+
5
+## how to use
6
+
7
+Use it in alarms like this:
8
+
9
+```
10
+ alarm: my_alarm
11
+ on: my_chart
12
+lookup: max -1m unaligned of my_dimension
13
+ warn: $this > 1000
14
+```
15
+
16
+`max` does not change the units. For example, if the chart units is `requests/sec`, the result
17
+will be again expressed in the same units.
18
+
19
+It can also be used in APIs and badges as `&group=max` in the URL.
20
+
21
+## Examples
22
+
23
+Examining last 1 minute `successful` web server responses:
24
+
25
+- 
26
+- 
27
+- 
28
+
29
+## References
30
+
31
+- [https://en.wikipedia.org/wiki/Sample_maximum_and_minimum](https://en.wikipedia.org/wiki/Sample_maximum_and_minimum).
web/api/queries/max/max.c
+6
-5
@@ -10,7 +10,7 @@ struct grouping_max {
10
size_t count;
11
};
12
13
-void *grouping_init_max(RRDR *r) {
13
+void *grouping_create_max(RRDR *r) {
14
(void)r;
15
return callocz(1, sizeof(struct grouping_max));
16
}
@@ -18,18 +18,19 @@ void *grouping_init_max(RRDR *r) {
18
// resets when switches dimensions
19
// so, clear everything to restart
20
void grouping_reset_max(RRDR *r) {
21
- struct grouping_max *g = (struct grouping_max *)r->grouping_data;
21
+ struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
22
g->max = 0;
23
g->count = 0;
24
}
25
26
void grouping_free_max(RRDR *r) {
27
- freez(r->grouping_data);
27
+ freez(r->internal.grouping_data);
28
+ r->internal.grouping_data = NULL;
29
}
30
31
void grouping_add_max(RRDR *r, calculated_number value) {
32
if(!isnan(value)) {
32
- struct grouping_max *g = (struct grouping_max *)r->grouping_data;
33
+ struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
34
35
if(!g->count || calculated_number_fabs(value) > calculated_number_fabs(g->max)) {
36
g->max = value;
@@ -39,7 +40,7 @@ void grouping_add_max(RRDR *r, calculated_number value) {
40
}
41
42
calculated_number grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
42
- struct grouping_max *g = (struct grouping_max *)r->grouping_data;
43
+ struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
44
45
calculated_number value;
46
web/api/queries/max/max.h
+1
-1
@@ -6,7 +6,7 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_max(RRDR *r);
9
+extern void *grouping_create_max(RRDR *r);
10
extern void grouping_reset_max(RRDR *r);
11
extern void grouping_free_max(RRDR *r);
12
extern void grouping_add_max(RRDR *r, calculated_number value);
web/api/queries/median/README.md
+37
@@ -0,0 +1,37 @@
1
+# Median
2
+
3
+The median is the value separating the higher half from the lower half of a data sample
4
+(a population or a probability distribution). For a data set, it may be thought of as the
5
+"middle" value.
6
+
7
+`median` is not an accurate average. However, it eliminates all spikes, by sorting
8
+all the values in a period, and selecting the value in the middle of the sorted array.
9
+
10
+## how to use
11
+
12
+Use it in alarms like this:
13
+
14
+```
15
+ alarm: my_alarm
16
+ on: my_chart
17
+lookup: median -1m unaligned of my_dimension
18
+ warn: $this > 1000
19
+```
20
+
21
+`median` does not change the units. For example, if the chart units is `requests/sec`, the result
22
+will be again expressed in the same units.
23
+
24
+It can also be used in APIs and badges as `&group=median` in the URL.
25
+
26
+## Examples
27
+
28
+Examining last 1 minute `successful` web server responses:
29
+
30
+- 
31
+- 
32
+- 
33
+- 
34
+
35
+## References
36
+
37
+- [https://en.wikipedia.org/wiki/Median](https://en.wikipedia.org/wiki/Median).
web/api/queries/median/median.c
+8
-7
@@ -13,8 +13,8 @@ struct grouping_median {
13
LONG_DOUBLE series[];
14
};
15
16
-void *grouping_init_median(RRDR *r) {
17
- long entries = (r->group > r->group_points) ? r->group : r->group_points;
16
+void *grouping_create_median(RRDR *r) {
17
+ long entries = r->group;
18
if(entries < 0) entries = 0;
19
20
struct grouping_median *g = (struct grouping_median *)callocz(1, sizeof(struct grouping_median) + entries * sizeof(LONG_DOUBLE));
@@ -26,19 +26,20 @@ void *grouping_init_median(RRDR *r) {
26
// resets when switches dimensions
27
// so, clear everything to restart
28
void grouping_reset_median(RRDR *r) {
29
- struct grouping_median *g = (struct grouping_median *)r->grouping_data;
29
+ struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
30
g->next_pos = 0;
31
}
32
33
void grouping_free_median(RRDR *r) {
34
- freez(r->grouping_data);
34
+ freez(r->internal.grouping_data);
35
+ r->internal.grouping_data = NULL;
36
}
37
38
void grouping_add_median(RRDR *r, calculated_number value) {
38
- struct grouping_median *g = (struct grouping_median *)r->grouping_data;
39
+ struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
40
41
if(unlikely(g->next_pos >= g->series_size)) {
41
- error("INTERNAL ERROR: median buffer overflow on chart '%s' - next_pos = %zu, series_size = %zu, r->group = %ld, r->group_points = %ld.", r->st->name, g->next_pos, g->series_size, r->group, r->group_points);
42
+ error("INTERNAL ERROR: median buffer overflow on chart '%s' - next_pos = %zu, series_size = %zu, r->group = %ld.", r->st->name, g->next_pos, g->series_size, r->group);
43
}
44
else {
45
if(isnormal(value))
@@ -47,7 +48,7 @@ void grouping_add_median(RRDR *r, calculated_number value) {
48
}
49
50
calculated_number grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
50
- struct grouping_median *g = (struct grouping_median *)r->grouping_data;
51
+ struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
52
53
calculated_number value;
54
web/api/queries/median/median.h
+1
-1
@@ -6,7 +6,7 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_median(RRDR *r);
9
+extern void *grouping_create_median(RRDR *r);
10
extern void grouping_reset_median(RRDR *r);
11
extern void grouping_free_median(RRDR *r);
12
extern void grouping_add_median(RRDR *r, calculated_number value);
web/api/queries/min/README.md
+31
@@ -0,0 +1,31 @@
1
+# Min
2
+
3
+This module finds the min value in the time-frame given.
4
+
5
+## how to use
6
+
7
+Use it in alarms like this:
8
+
9
+```
10
+ alarm: my_alarm
11
+ on: my_chart
12
+lookup: min -1m unaligned of my_dimension
13
+ warn: $this > 1000
14
+```
15
+
16
+`min` does not change the units. For example, if the chart units is `requests/sec`, the result
17
+will be again expressed in the same units.
18
+
19
+It can also be used in APIs and badges as `&group=min` in the URL.
20
+
21
+## Examples
22
+
23
+Examining last 1 minute `successful` web server responses:
24
+
25
+- 
26
+- 
27
+- 
28
+
29
+## References
30
+
31
+- [https://en.wikipedia.org/wiki/Sample_maximum_and_minimum](https://en.wikipedia.org/wiki/Sample_maximum_and_minimum).
web/api/queries/min/min.c
+6
-5
@@ -10,7 +10,7 @@ struct grouping_min {
10
size_t count;
11
};
12
13
-void *grouping_init_min(RRDR *r) {
13
+void *grouping_create_min(RRDR *r) {
14
(void)r;
15
return callocz(1, sizeof(struct grouping_min));
16
}
@@ -18,18 +18,19 @@ void *grouping_init_min(RRDR *r) {
18
// resets when switches dimensions
19
// so, clear everything to restart
20
void grouping_reset_min(RRDR *r) {
21
- struct grouping_min *g = (struct grouping_min *)r->grouping_data;
21
+ struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data;
22
g->min = 0;
23
g->count = 0;
24
}
25
26
void grouping_free_min(RRDR *r) {
27
- freez(r->grouping_data);
27
+ freez(r->internal.grouping_data);
28
+ r->internal.grouping_data = NULL;
29
}
30
31
void grouping_add_min(RRDR *r, calculated_number value) {
32
if(!isnan(value)) {
32
- struct grouping_min *g = (struct grouping_min *)r->grouping_data;
33
+ struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data;
34
35
if(!g->count || calculated_number_fabs(value) < calculated_number_fabs(g->min)) {
36
g->min = value;
@@ -39,7 +40,7 @@ void grouping_add_min(RRDR *r, calculated_number value) {
40
}
41
42
calculated_number grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
42
- struct grouping_min *g = (struct grouping_min *)r->grouping_data;
43
+ struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data;
44
45
calculated_number value;
46
web/api/queries/min/min.h
+1
-1
@@ -6,7 +6,7 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_min(RRDR *r);
9
+extern void *grouping_create_min(RRDR *r);
10
extern void grouping_reset_min(RRDR *r);
11
extern void grouping_free_min(RRDR *r);
12
extern void grouping_add_min(RRDR *r, calculated_number value);
web/api/queries/query.c
+262
-74
@@ -20,43 +20,230 @@ static struct {
20
const char *name;
21
uint32_t hash;
22
RRDR_GROUPING value;
23
- void *(*init)(struct rrdresult *r);
23
+
24
+ // One time initialization for the module.
25
+ // This is called once, when netdata starts.
26
+ void (*init)(void);
27
+
28
+ // Allocate all required structures for a query.
29
+ // This is called once for each netdata query.
30
+ void *(*create)(struct rrdresult *r);
31
+
32
+ // Cleanup collected values, but don't destroy the structures.
33
+ // This is called when the query engine switches dimensions,
34
+ // as part of the same query (so same chart, switching metric).
35
void (*reset)(struct rrdresult *r);
36
+
37
+ // Free all resources allocated for the query.
38
void (*free)(struct rrdresult *r);
39
+
40
+ // Add a single value into the calculation.
41
+ // The module may decide to cache it, or use it in the fly.
42
void (*add)(struct rrdresult *r, calculated_number value);
43
+
44
+ // Generate a single result for the values added so far.
45
+ // More values and points may be requested later.
46
+ // It is up to the module to reset its internal structures
47
+ // when flushing it (so for a few modules it may be better to
48
+ // continue after a flush as if nothing changed, for others a
49
+ // cleanup of the internal structures may be required).
50
calculated_number (*flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
51
} api_v1_data_groups[] = {
29
- { "average" , 0, RRDR_GROUPING_AVERAGE , grouping_init_average , grouping_reset_average , grouping_free_average , grouping_add_average , grouping_flush_average }
30
- , { "incremental_sum" , 0, RRDR_GROUPING_INCREMENTAL_SUM, grouping_init_incremental_sum, grouping_reset_incremental_sum, grouping_free_incremental_sum, grouping_add_incremental_sum, grouping_flush_incremental_sum }
31
- , { "incremental-sum" , 0, RRDR_GROUPING_INCREMENTAL_SUM, grouping_init_incremental_sum, grouping_reset_incremental_sum, grouping_free_incremental_sum, grouping_add_incremental_sum, grouping_flush_incremental_sum }
32
- , { "median" , 0, RRDR_GROUPING_MEDIAN , grouping_init_median , grouping_reset_median , grouping_free_median , grouping_add_median , grouping_flush_median }
33
- , { "min" , 0, RRDR_GROUPING_MIN , grouping_init_min , grouping_reset_min , grouping_free_min , grouping_add_min , grouping_flush_min }
34
- , { "max" , 0, RRDR_GROUPING_MAX , grouping_init_max , grouping_reset_max , grouping_free_max , grouping_add_max , grouping_flush_max }
35
- , { "sum" , 0, RRDR_GROUPING_SUM , grouping_init_sum , grouping_reset_sum , grouping_free_sum , grouping_add_sum , grouping_flush_sum }
36
-
37
- // stddev module provides mean, variance and coefficient of variation
38
- , { "stddev" , 0, RRDR_GROUPING_STDDEV , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_stddev }
39
- , { "cv" , 0, RRDR_GROUPING_CV , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_coefficient_of_variation }
40
- //, { "mean" , 0, RRDR_GROUPING_MEAN , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_mean }
41
- //, { "variance" , 0, RRDR_GROUPING_VARIANCE , grouping_init_stddev , grouping_reset_stddev , grouping_free_stddev , grouping_add_stddev , grouping_flush_variance }
42
-
43
- // single exponential smoothing or exponential weighted moving average
44
- , { "ses" , 0, RRDR_GROUPING_SES , grouping_init_ses , grouping_reset_ses , grouping_free_ses , grouping_add_ses , grouping_flush_ses }
45
- , { "ema" , 0, RRDR_GROUPING_SES , grouping_init_ses , grouping_reset_ses , grouping_free_ses , grouping_add_ses , grouping_flush_ses }
46
- , { "ewma" , 0, RRDR_GROUPING_SES , grouping_init_ses , grouping_reset_ses , grouping_free_ses , grouping_add_ses , grouping_flush_ses }
52
+ {.name = "average",
53
+ .hash = 0,
54
+ .value = RRDR_GROUPING_AVERAGE,
55
+ .init = NULL,
56
+ .create= grouping_create_average,
57
+ .reset = grouping_reset_average,
58
+ .free = grouping_free_average,
59
+ .add = grouping_add_average,
60
+ .flush = grouping_flush_average
61
+ },
62
+ {.name = "mean", // alias on 'average'
63
+ .hash = 0,
64
+ .value = RRDR_GROUPING_AVERAGE,
65
+ .init = NULL,
66
+ .create= grouping_create_average,
67
+ .reset = grouping_reset_average,
68
+ .free = grouping_free_average,
69
+ .add = grouping_add_average,
70
+ .flush = grouping_flush_average
71
+ },
72
+ {.name = "incremental_sum",
73
+ .hash = 0,
74
+ .value = RRDR_GROUPING_INCREMENTAL_SUM,
75
+ .init = NULL,
76
+ .create= grouping_create_incremental_sum,
77
+ .reset = grouping_reset_incremental_sum,
78
+ .free = grouping_free_incremental_sum,
79
+ .add = grouping_add_incremental_sum,
80
+ .flush = grouping_flush_incremental_sum
81
+ },
82
+ {.name = "incremental-sum",
83
+ .hash = 0,
84
+ .value = RRDR_GROUPING_INCREMENTAL_SUM,
85
+ .init = NULL,
86
+ .create= grouping_create_incremental_sum,
87
+ .reset = grouping_reset_incremental_sum,
88
+ .free = grouping_free_incremental_sum,
89
+ .add = grouping_add_incremental_sum,
90
+ .flush = grouping_flush_incremental_sum
91
+ },
92
+ {.name = "median",
93
+ .hash = 0,
94
+ .value = RRDR_GROUPING_MEDIAN,
95
+ .init = NULL,
96
+ .create= grouping_create_median,
97
+ .reset = grouping_reset_median,
98
+ .free = grouping_free_median,
99
+ .add = grouping_add_median,
100
+ .flush = grouping_flush_median
101
+ },
102
+ {.name = "min",
103
+ .hash = 0,
104
+ .value = RRDR_GROUPING_MIN,
105
+ .init = NULL,
106
+ .create= grouping_create_min,
107
+ .reset = grouping_reset_min,
108
+ .free = grouping_free_min,
109
+ .add = grouping_add_min,
110
+ .flush = grouping_flush_min
111
+ },
112
+ {.name = "max",
113
+ .hash = 0,
114
+ .value = RRDR_GROUPING_MAX,
115
+ .init = NULL,
116
+ .create= grouping_create_max,
117
+ .reset = grouping_reset_max,
118
+ .free = grouping_free_max,
119
+ .add = grouping_add_max,
120
+ .flush = grouping_flush_max
121
+ },
122
+ {.name = "sum",
123
+ .hash = 0,
124
+ .value = RRDR_GROUPING_SUM,
125
+ .init = NULL,
126
+ .create= grouping_create_sum,
127
+ .reset = grouping_reset_sum,
128
+ .free = grouping_free_sum,
129
+ .add = grouping_add_sum,
130
+ .flush = grouping_flush_sum
131
+ },
132
+
133
+ // standard deviation
134
+ {.name = "stddev",
135
+ .hash = 0,
136
+ .value = RRDR_GROUPING_STDDEV,
137
+ .init = NULL,
138
+ .create= grouping_create_stddev,
139
+ .reset = grouping_reset_stddev,
140
+ .free = grouping_free_stddev,
141
+ .add = grouping_add_stddev,
142
+ .flush = grouping_flush_stddev
143
+ },
144
+ {.name = "cv", // coefficient variation is calculated by stddev
145
+ .hash = 0,
146
+ .value = RRDR_GROUPING_CV,
147
+ .init = NULL,
148
+ .create= grouping_create_stddev, // not an error, stddev calculates this too
149
+ .reset = grouping_reset_stddev, // not an error, stddev calculates this too
150
+ .free = grouping_free_stddev, // not an error, stddev calculates this too
151
+ .add = grouping_add_stddev, // not an error, stddev calculates this too
152
+ .flush = grouping_flush_coefficient_of_variation
153
+ },
154
+
155
+ /*
156
+ {.name = "mean", // same as average, no need to define it again
157
+ .hash = 0,
158
+ .value = RRDR_GROUPING_MEAN,
159
+ .setup = NULL,
160
+ .create= grouping_create_stddev,
161
+ .reset = grouping_reset_stddev,
162
+ .free = grouping_free_stddev,
163
+ .add = grouping_add_stddev,
164
+ .flush = grouping_flush_mean
165
+ },
166
+ */
167
+
168
+ /*
169
+ {.name = "variance", // meaningless to offer
170
+ .hash = 0,
171
+ .value = RRDR_GROUPING_VARIANCE,
172
+ .setup = NULL,
173
+ .create= grouping_create_stddev,
174
+ .reset = grouping_reset_stddev,
175
+ .free = grouping_free_stddev,
176
+ .add = grouping_add_stddev,
177
+ .flush = grouping_flush_variance
178
+ },
179
+ */
180
+
181
+ // single exponential smoothing
182
+ {.name = "ses",
183
+ .hash = 0,
184
+ .value = RRDR_GROUPING_SES,
185
+ .init = grouping_init_ses,
186
+ .create= grouping_create_ses,
187
+ .reset = grouping_reset_ses,
188
+ .free = grouping_free_ses,
189
+ .add = grouping_add_ses,
190
+ .flush = grouping_flush_ses
191
+ },
192
+ {.name = "ema", // alias for 'ses'
193
+ .hash = 0,
194
+ .value = RRDR_GROUPING_SES,
195
+ .init = NULL,
196
+ .create= grouping_create_ses,
197
+ .reset = grouping_reset_ses,
198
+ .free = grouping_free_ses,
199
+ .add = grouping_add_ses,
200
+ .flush = grouping_flush_ses
201
+ },
202
+ {.name = "ewma", // alias for ses
203
+ .hash = 0,
204
+ .value = RRDR_GROUPING_SES,
205
+ .init = NULL,
206
+ .create= grouping_create_ses,
207
+ .reset = grouping_reset_ses,
208
+ .free = grouping_free_ses,
209
+ .add = grouping_add_ses,
210
+ .flush = grouping_flush_ses
211
+ },
212
213
// double exponential smoothing
49
- , { "des" , 0, RRDR_GROUPING_DES , grouping_init_des , grouping_reset_des , grouping_free_des , grouping_add_des , grouping_flush_des }
214
+ {.name = "des",
215
+ .hash = 0,
216
+ .value = RRDR_GROUPING_DES,
217
+ .init = grouping_init_des,
218
+ .create= grouping_create_des,
219
+ .reset = grouping_reset_des,
220
+ .free = grouping_free_des,
221
+ .add = grouping_add_des,
222
+ .flush = grouping_flush_des
223
+ },
224
225
// terminator
52
- , { NULL , 0, RRDR_GROUPING_UNDEFINED , grouping_init_average , grouping_reset_average , grouping_free_average , grouping_add_average , grouping_flush_average }
226
+ {.name = NULL,
227
+ .hash = 0,
228
+ .value = RRDR_GROUPING_UNDEFINED,
229
+ .init = NULL,
230
+ .create= grouping_create_average,
231
+ .reset = grouping_reset_average,
232
+ .free = grouping_free_average,
233
+ .add = grouping_add_average,
234
+ .flush = grouping_flush_average
235
+ }
236
};
237
238
void web_client_api_v1_init_grouping(void) {
239
int i;
240
58
- for(i = 0; api_v1_data_groups[i].name ; i++)
241
+ for(i = 0; api_v1_data_groups[i].name ; i++) {
242
api_v1_data_groups[i].hash = simple_hash(api_v1_data_groups[i].name);
243
+
244
+ if(api_v1_data_groups[i].init)
245
+ api_v1_data_groups[i].init();
246
+ }
247
}
248
249
const char *group_method2string(RRDR_GROUPING group) {
@@ -217,13 +404,13 @@ static inline void do_dimension(
404
// make sure we return data in the proper time range
405
if(unlikely(now > before_wanted)) {
406
#ifdef NETDATA_INTERNAL_CHECKS
220
- r->log = "stopped, because attempted to access the db after 'wanted before'";
407
+ r->internal.log = "stopped, because attempted to access the db after 'wanted before'";
408
#endif
409
break;
410
}
411
if(unlikely(now < after_wanted)) {
412
#ifdef NETDATA_INTERNAL_CHECKS
226
- r->log = "skipped, because attempted to access the db before 'wanted after'";
413
+ r->internal.log = "skipped, because attempted to access the db before 'wanted after'";
414
#endif
415
continue;
416
}
@@ -243,7 +430,7 @@ static inline void do_dimension(
430
}
431
432
// add this value for grouping
246
- r->grouping_add(r, value);
433
+ r->internal.grouping_add(r, value);
434
values_in_group++;
435
436
if(unlikely(values_in_group == group_size)) {
@@ -263,7 +450,7 @@ static inline void do_dimension(
450
*rrdr_value_options_ptr = group_value_flags;
451
452
// store the value
266
- r->v[rrdr_line * r->d + dim_id_in_rrdr] = r->grouping_flush(r, rrdr_value_options_ptr);
453
+ r->v[rrdr_line * r->d + dim_id_in_rrdr] = r->internal.grouping_flush(r, rrdr_value_options_ptr);
454
455
points_added++;
456
values_in_group = 0;
@@ -290,8 +477,8 @@ static void rrd2rrdr_log_request_response_metdata(RRDR *r
477
, RRDR_GROUPING group_method
478
, int aligned
479
, long group
293
- , long group_time
294
- , long group_points
480
+ , long resampling_time
481
+ , long resampling_group
482
, time_t after_wanted
483
, time_t after_requested
484
, time_t before_wanted
@@ -302,7 +489,7 @@ static void rrd2rrdr_log_request_response_metdata(RRDR *r
489
, size_t before_slot
490
, const char *msg
491
) {
305
- info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, gtime: %ld, gpoints: %ld), "
492
+ info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), "
493
"after (got: %zu, want: %zu, req: %zu, db: %zu), "
494
"before (got: %zu, want: %zu, req: %zu, db: %zu), "
495
"duration (got: %zu, want: %zu, req: %zu, db: %zu), "
@@ -316,8 +503,8 @@ static void rrd2rrdr_log_request_response_metdata(RRDR *r
503
, (aligned) ? "aligned" : "unaligned"
504
, group_method2string(group_method)
505
, group
319
- , group_time
320
- , group_points
506
+ , resampling_time
507
+ , resampling_group
508
509
// after
510
, (size_t)r->after - (group - 1) * r->st->update_every
@@ -359,7 +546,7 @@ RRDR *rrd2rrdr(
546
, long long after_requested
547
, long long before_requested
548
, RRDR_GROUPING group_method
362
- , long group_time_requested
549
+ , long resampling_time_requested
550
, RRDR_OPTIONS options
551
, const char *dimensions
552
) {
@@ -435,36 +622,36 @@ RRDR *rrd2rrdr(
622
if(unlikely(group <= 0)) group = 1;
623
if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer
624
438
- // group_time enforces a certain grouping multiple
439
- calculated_number group_sum_divisor = 1.0;
440
- long group_points = 1;
441
- if(unlikely(group_time_requested > st->update_every)) {
442
- if (unlikely(group_time_requested > duration)) {
625
+ // resampling_time_requested enforces a certain grouping multiple
626
+ calculated_number resampling_divisor = 1.0;
627
+ long resampling_group = 1;
628
+ if(unlikely(resampling_time_requested > st->update_every)) {
629
+ if (unlikely(resampling_time_requested > duration)) {
630
// group_time is above the available duration
631
632
#ifdef NETDATA_INTERNAL_CHECKS
446
- info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, group_time_requested, duration);
633
+ info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, resampling_time_requested, duration);
634
#endif
635
636
group = available_points; // use all the points
637
}
638
else {
639
// the points we should group to satisfy gtime
453
- group_points = group_time_requested / st->update_every;
454
- if(unlikely(group_time_requested % st->update_every)) {
640
+ resampling_group = resampling_time_requested / st->update_every;
641
+ if(unlikely(resampling_time_requested % st->update_every)) {
642
#ifdef NETDATA_INTERNAL_CHECKS
456
- info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, group_time_requested, st->update_every);
643
+ info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, st->update_every);
644
#endif
645
459
- group_points++;
646
+ resampling_group++;
647
}
648
462
- // adapt group according to group_points
463
- if(unlikely(group < group_points)) group = group_points; // do not allow grouping below the desired one
464
- if(unlikely(group % group_points)) group += group_points - (group % group_points); // make sure group is multiple of group_points
649
+ // adapt group according to resampling_group
650
+ if(unlikely(group < resampling_group)) group = resampling_group; // do not allow grouping below the desired one
651
+ if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
652
466
- //group_sum_divisor = group / group_points;
467
- group_sum_divisor = (calculated_number)(group * st->update_every) / (calculated_number)group_time_requested;
653
+ //resampling_divisor = group / resampling_group;
654
+ resampling_divisor = (calculated_number)(group * st->update_every) / (calculated_number)resampling_time_requested;
655
}
656
}
657
@@ -553,11 +740,11 @@ RRDR *rrd2rrdr(
740
if(points_wanted > (before_wanted - after_wanted) / group / st->update_every + 1)
741
error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / st->update_every + 1);
742
556
- if(group < group_points)
557
- error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, group_points);
743
+ if(group < resampling_group)
744
+ error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group);
745
559
- if(group > group_points && group % group_points)
560
- error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, group_points);
746
+ if(group > resampling_group && group % resampling_group)
747
+ error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, resampling_group);
748
#endif
749
750
// -------------------------------------------------------------------------
@@ -594,8 +781,9 @@ RRDR *rrd2rrdr(
781
r->update_every = (int)group * st->update_every;
782
r->before = before_wanted;
783
r->after = after_wanted;
597
- r->group_points = group_points;
598
- r->group_sum_divisor = group_sum_divisor;
784
+ r->internal.points_wanted = points_wanted;
785
+ r->internal.resampling_group = resampling_group;
786
+ r->internal.resampling_divisor = resampling_divisor;
787
788
789
// -------------------------------------------------------------------------
@@ -605,11 +793,11 @@ RRDR *rrd2rrdr(
793
int i, found = 0;
794
for(i = 0; !found && api_v1_data_groups[i].name ;i++) {
795
if(api_v1_data_groups[i].value == group_method) {
608
- r->grouping_init = api_v1_data_groups[i].init;
609
- r->grouping_reset = api_v1_data_groups[i].reset;
610
- r->grouping_free = api_v1_data_groups[i].free;
611
- r->grouping_add = api_v1_data_groups[i].add;
612
- r->grouping_flush = api_v1_data_groups[i].flush;
796
+ r->internal.grouping_create= api_v1_data_groups[i].create;
797
+ r->internal.grouping_reset = api_v1_data_groups[i].reset;
798
+ r->internal.grouping_free = api_v1_data_groups[i].free;
799
+ r->internal.grouping_add = api_v1_data_groups[i].add;
800
+ r->internal.grouping_flush = api_v1_data_groups[i].flush;
801
found = 1;
802
}
803
}
@@ -618,16 +806,16 @@ RRDR *rrd2rrdr(
806
#ifdef NETDATA_INTERNAL_CHECKS
807
error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name);
808
#endif
621
- r->grouping_init = grouping_init_average;
622
- r->grouping_reset = grouping_reset_average;
623
- r->grouping_free = grouping_free_average;
624
- r->grouping_add = grouping_add_average;
625
- r->grouping_flush = grouping_flush_average;
809
+ r->internal.grouping_create= grouping_create_average;
810
+ r->internal.grouping_reset = grouping_reset_average;
811
+ r->internal.grouping_free = grouping_free_average;
812
+ r->internal.grouping_add = grouping_add_average;
813
+ r->internal.grouping_flush = grouping_flush_average;
814
}
815
}
816
817
// allocate any memory required by the grouping method
630
- r->grouping_data = r->grouping_init(r);
818
+ r->internal.grouping_data = r->internal.grouping_create(r);
819
820
821
// -------------------------------------------------------------------------
@@ -654,7 +842,7 @@ RRDR *rrd2rrdr(
842
continue;
843
844
// reset the grouping for the new dimension
657
- r->grouping_reset(r);
845
+ r->internal.grouping_reset(r);
846
847
do_dimension(
848
r
@@ -707,33 +895,33 @@ RRDR *rrd2rrdr(
895
896
#ifdef NETDATA_INTERNAL_CHECKS
897
710
- if(r->log)
711
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, r->log);
898
+ if(r->internal.log)
899
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, r->internal.log);
900
901
if(r->rows != points_wanted)
714
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'points' is not wanted 'points'");
902
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'points' is not wanted 'points'");
903
904
if(aligned && (r->before % group) != 0)
717
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'before' is not aligned but alignment is required");
905
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'before' is not aligned but alignment is required");
906
907
// 'after' should not be aligned, since we start inside the first group
908
//if(aligned && (r->after % group) != 0)
721
- // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
909
+ // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
910
911
if(r->before != before_requested)
724
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "chart is not aligned to requested 'before'");
912
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "chart is not aligned to requested 'before'");
913
914
if(r->before != before_wanted)
727
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'before' is not wanted 'before'");
915
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'before' is not wanted 'before'");
916
917
// reported 'after' varies, depending on group
918
if((r->after - (group - 1) * r->st->update_every) != after_wanted)
731
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'after' is not wanted 'after'");
919
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'after' is not wanted 'after'");
920
921
#endif
922
923
// free all resources used by the grouping method
736
- r->grouping_free(r);
924
+ r->internal.grouping_free(r);
925
926
// when all the dimensions are zero, we should return all of them
927
if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) {
web/api/queries/rrdr.h
+17
-14
@@ -70,19 +70,22 @@ typedef struct rrdresult {
70
int has_st_lock; // if st is read locked by us
71
72
// internal rrd2rrdr() members below this point
73
- long group_points;
74
- calculated_number group_sum_divisor;
75
-
76
- void *(*grouping_init)(struct rrdresult *r);
77
- void (*grouping_reset)(struct rrdresult *r);
78
- void (*grouping_free)(struct rrdresult *r);
79
- void (*grouping_add)(struct rrdresult *r, calculated_number value);
80
- calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
81
- void *grouping_data;
82
-
83
-#ifdef NETDATA_INTERNAL_CHECKS
84
- const char *log;
85
-#endif
73
+ struct {
74
+ long points_wanted;
75
+ long resampling_group;
76
+ calculated_number resampling_divisor;
77
+
78
+ void *(*grouping_create)(struct rrdresult *r);
79
+ void (*grouping_reset)(struct rrdresult *r);
80
+ void (*grouping_free)(struct rrdresult *r);
81
+ void (*grouping_add)(struct rrdresult *r, calculated_number value);
82
+ calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
83
+ void *grouping_data;
84
+
85
+ #ifdef NETDATA_INTERNAL_CHECKS
86
+ const char *log;
87
+ #endif
88
+ } internal;
89
} RRDR;
90
91
#define rrdr_rows(r) ((r)->rows)
@@ -98,7 +101,7 @@ extern RRDR *rrdr_create(RRDSET *st, long n);
101
102
#include "web/api/queries/query.h"
103
101
-extern RRDR *rrd2rrdr(RRDSET *st, long points_requested, long long after_requested, long long before_requested, RRDR_GROUPING group_method, long group_time_requested, RRDR_OPTIONS options, const char *dimensions);
104
+extern RRDR *rrd2rrdr(RRDSET *st, long points_requested, long long after_requested, long long before_requested, RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions);
105
106
#include "query.h"
107
web/api/queries/ses/README.md
+15
-6
@@ -8,10 +8,19 @@ exponentially. The weighting for each older datum decreases exponentially, never
8
9
In simple terms, this is like an average value, but more recent values are given more weight.
10
11
-Netdata automatically adjusts the weight based on the number of values processed, using the formula:
11
+Netdata automatically adjusts the weight (`alpha`) based on the number of values processed,
12
+using the formula:
13
14
```
14
-alpha = 2 / (number_of_values + 1)
15
+window = max(number of values, 15)
16
+alpha = 2 / (window + 1)
17
+```
18
+
19
+You can change the fixed value `15` by setting in `netdata.conf`:
20
+
21
+```
22
+[web]
23
+ ses max window = 15
24
```
25
26
## how to use
@@ -34,10 +43,10 @@ It can also be used in APIs and badges as `&group=ses` in the URL.
43
44
Examining last 1 minute `successful` web server responses:
45
37
-- 
38
-- 
39
-- 
40
-- 
46
+- 
47
+- 
48
+- 
49
+- 
50
51
## References
52
web/api/queries/ses/ses.c
+36
-8
@@ -13,14 +13,42 @@ struct grouping_ses {
13
size_t count;
14
};
15
16
+static size_t max_window_size = 15;
17
+
18
+void grouping_init_ses(void) {
19
+ long long ret = config_get_number(CONFIG_SECTION_WEB, "ses max window", (long long)max_window_size);
20
+ if(ret <= 1) {
21
+ config_set_number(CONFIG_SECTION_WEB, "ses max window", (long long)max_window_size);
22
+ }
23
+ else {
24
+ max_window_size = (size_t) ret;
25
+ }
26
+}
27
+
28
+static inline calculated_number window(RRDR *r, struct grouping_ses *g) {
29
+ (void)g;
30
+
31
+ calculated_number points;
32
+ if(r->group == 1) {
33
+ // provide a running DES
34
+ points = r->internal.points_wanted;
35
+ }
36
+ else {
37
+ // provide a SES with flush points
38
+ points = r->group;
39
+ }
40
+
41
+ return (points > max_window_size) ? max_window_size : points;
42
+}
43
+
44
static inline void set_alpha(RRDR *r, struct grouping_ses *g) {
45
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
46
// A commonly used value for alpha is 2 / (N + 1)
19
- g->alpha = 2.0 / ((calculated_number)r->group + 1.0);
20
- g->alpha_other = 1 - g->alpha;
47
+ g->alpha = 2.0 / (window(r, g) + 1.0);
48
+ g->alpha_other = 1.0 - g->alpha;
49
}
50
23
-void *grouping_init_ses(RRDR *r) {
51
+void *grouping_create_ses(RRDR *r) {
52
struct grouping_ses *g = (struct grouping_ses *)callocz(1, sizeof(struct grouping_ses));
53
set_alpha(r, g);
54
g->level = 0.0;
@@ -30,18 +58,18 @@ void *grouping_init_ses(RRDR *r) {
58
// resets when switches dimensions
59
// so, clear everything to restart
60
void grouping_reset_ses(RRDR *r) {
33
- struct grouping_ses *g = (struct grouping_ses *)r->grouping_data;
61
+ struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data;
62
g->level = 0.0;
63
g->count = 0;
64
}
65
66
void grouping_free_ses(RRDR *r) {
39
- freez(r->grouping_data);
40
- r->grouping_data = NULL;
67
+ freez(r->internal.grouping_data);
68
+ r->internal.grouping_data = NULL;
69
}
70
71
void grouping_add_ses(RRDR *r, calculated_number value) {
44
- struct grouping_ses *g = (struct grouping_ses *)r->grouping_data;
72
+ struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data;
73
74
if(isnormal(value)) {
75
if(unlikely(!g->count))
@@ -53,7 +81,7 @@ void grouping_add_ses(RRDR *r, calculated_number value) {
81
}
82
83
calculated_number grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
56
- struct grouping_ses *g = (struct grouping_ses *)r->grouping_data;
84
+ struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data;
85
86
if(unlikely(!g->count || !isnormal(g->level))) {
87
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
web/api/queries/ses/ses.h
+3
-1
@@ -6,7 +6,9 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_ses(RRDR *r);
9
+extern void grouping_init_ses(void);
10
+
11
+extern void *grouping_create_ses(RRDR *r);
12
extern void grouping_reset_ses(RRDR *r);
13
extern void grouping_free_ses(RRDR *r);
14
extern void grouping_add_ses(RRDR *r, calculated_number value);
web/api/queries/stddev/stddev.c
+29
-20
@@ -14,8 +14,8 @@ struct grouping_stddev {
14
calculated_number m_oldM, m_newM, m_oldS, m_newS;
15
};
16
17
-void *grouping_init_stddev(RRDR *r) {
18
- long entries = (r->group > r->group_points) ? r->group : r->group_points;
17
+void *grouping_create_stddev(RRDR *r) {
18
+ long entries = r->group;
19
if(entries < 0) entries = 0;
20
21
return callocz(1, sizeof(struct grouping_stddev) + entries * sizeof(LONG_DOUBLE));
@@ -24,16 +24,17 @@ void *grouping_init_stddev(RRDR *r) {
24
// resets when switches dimensions
25
// so, clear everything to restart
26
void grouping_reset_stddev(RRDR *r) {
27
- struct grouping_stddev *g = (struct grouping_stddev *)r->grouping_data;
27
+ struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
28
g->count = 0;
29
}
30
31
void grouping_free_stddev(RRDR *r) {
32
- freez(r->grouping_data);
32
+ freez(r->internal.grouping_data);
33
+ r->internal.grouping_data = NULL;
34
}
35
36
void grouping_add_stddev(RRDR *r, calculated_number value) {
36
- struct grouping_stddev *g = (struct grouping_stddev *)r->grouping_data;
37
+ struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
38
39
if(isnormal(value)) {
40
g->count++;
@@ -66,15 +67,11 @@ static inline calculated_number stddev(struct grouping_stddev *g) {
67
}
68
69
calculated_number grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
69
- struct grouping_stddev *g = (struct grouping_stddev *)r->grouping_data;
70
+ struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
71
72
calculated_number value;
73
73
- if(unlikely(!g->count)) {
74
- value = 0.0;
75
- *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
76
- }
77
- else {
74
+ if(likely(g->count > 1)) {
75
value = stddev(g);
76
77
if(!isnormal(value)) {
@@ -82,6 +79,13 @@ calculated_number grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_op
79
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
80
}
81
}
82
+ else if(g->count == 1) {
83
+ value = 0.0;
84
+ }
85
+ else {
86
+ value = 0.0;
87
+ *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
88
+ }
89
90
grouping_reset_stddev(r);
91
@@ -90,23 +94,28 @@ calculated_number grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_op
94
95
// https://en.wikipedia.org/wiki/Coefficient_of_variation
96
calculated_number grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
93
- struct grouping_stddev *g = (struct grouping_stddev *)r->grouping_data;
97
+ struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
98
99
calculated_number value;
100
97
- if(unlikely(!g->count)) {
98
- value = 0.0;
99
- *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
100
- }
101
- else {
101
+ if(likely(g->count > 1)) {
102
calculated_number m = mean(g);
103
value = 100.0 * stddev(g) / ((m < 0)? -m : m);
104
105
- if(!isnormal(value)) {
105
+ if(unlikely(!isnormal(value))) {
106
value = 0.0;
107
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
108
}
109
}
110
+ else if(g->count == 1) {
111
+ // one value collected
112
+ value = 0.0;
113
+ }
114
+ else {
115
+ // no values collected
116
+ value = 0.0;
117
+ *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
118
+ }
119
120
grouping_reset_stddev(r);
121
@@ -118,7 +127,7 @@ calculated_number grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FL
127
* Mean = average
128
*
129
calculated_number grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
121
- struct grouping_stddev *g = (struct grouping_stddev *)r->grouping_data;
130
+ struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
131
132
calculated_number value;
133
@@ -145,7 +154,7 @@ calculated_number grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_opti
154
* It is not advised to use this version of variance directly
155
*
156
calculated_number grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
148
- struct grouping_stddev *g = (struct grouping_stddev *)r->grouping_data;
157
+ struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data;
158
159
calculated_number value;
160
web/api/queries/stddev/stddev.h
+1
-1
@@ -6,7 +6,7 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_stddev(RRDR *r);
9
+extern void *grouping_create_stddev(RRDR *r);
10
extern void grouping_reset_stddev(RRDR *r);
11
extern void grouping_free_stddev(RRDR *r);
12
extern void grouping_add_stddev(RRDR *r, calculated_number value);
web/api/queries/sum/README.md
+34
@@ -0,0 +1,34 @@
1
+# Sum
2
+
3
+This module sums all the values in the time-frame requested.
4
+
5
+You can use `sum` to find the volume of something over a period.
6
+
7
+## how to use
8
+
9
+Use it in alarms like this:
10
+
11
+```
12
+ alarm: my_alarm
13
+ on: my_chart
14
+lookup: sum -1m unaligned of my_dimension
15
+ warn: $this > 1000
16
+```
17
+
18
+`sum` does not change the units. For example, if the chart units is `requests/sec`, the result
19
+will be again expressed in the same units.
20
+
21
+It can also be used in APIs and badges as `&group=sum` in the URL.
22
+
23
+## Examples
24
+
25
+Examining last 1 minute `successful` web server responses:
26
+
27
+- 
28
+- 
29
+- 
30
+- 
31
+
32
+## References
33
+
34
+- [https://en.wikipedia.org/wiki/Summation](https://en.wikipedia.org/wiki/Summation).
web/api/queries/sum/sum.c
+6
-5
@@ -10,7 +10,7 @@ struct grouping_sum {
10
size_t count;
11
};
12
13
-void *grouping_init_sum(RRDR *r) {
13
+void *grouping_create_sum(RRDR *r) {
14
(void)r;
15
return callocz(1, sizeof(struct grouping_sum));
16
}
@@ -18,18 +18,19 @@ void *grouping_init_sum(RRDR *r) {
18
// resets when switches dimensions
19
// so, clear everything to restart
20
void grouping_reset_sum(RRDR *r) {
21
- struct grouping_sum *g = (struct grouping_sum *)r->grouping_data;
21
+ struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data;
22
g->sum = 0;
23
g->count = 0;
24
}
25
26
void grouping_free_sum(RRDR *r) {
27
- freez(r->grouping_data);
27
+ freez(r->internal.grouping_data);
28
+ r->internal.grouping_data = NULL;
29
}
30
31
void grouping_add_sum(RRDR *r, calculated_number value) {
32
if(!isnan(value)) {
32
- struct grouping_sum *g = (struct grouping_sum *)r->grouping_data;
33
+ struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data;
34
35
if(!g->count || calculated_number_fabs(value) > calculated_number_fabs(g->sum)) {
36
g->sum += value;
@@ -39,7 +40,7 @@ void grouping_add_sum(RRDR *r, calculated_number value) {
40
}
41
42
calculated_number grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
42
- struct grouping_sum *g = (struct grouping_sum *)r->grouping_data;
43
+ struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data;
44
45
calculated_number value;
46
web/api/queries/sum/sum.h
+1
-1
@@ -6,7 +6,7 @@
6
#include "../query.h"
7
#include "../rrdr.h"
8
9
-extern void *grouping_init_sum(RRDR *r);
9
+extern void *grouping_create_sum(RRDR *r);
10
extern void grouping_reset_sum(RRDR *r);
11
extern void grouping_free_sum(RRDR *r);
12
extern void grouping_add_sum(RRDR *r, calculated_number value);