query engine documentation and stats (#4483)
Costa Tsaousis committed
Oct 25, 2018 at 03:34 UTC
93467af78c91bf0b0898e3b28b3a7647e7442f7c
5 files changed
+262
-35
daemon/global_statistics.c
+131
-14
@@ -2,7 +2,26 @@
2
3
#include "common.h"
4
5
-volatile struct global_statistics global_statistics = {
5
+#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01
6
+
7
+
8
+static struct global_statistics {
9
+ volatile uint16_t connected_clients;
10
+
11
+ volatile uint64_t web_requests;
12
+ volatile uint64_t web_usec;
13
+ volatile uint64_t web_usec_max;
14
+ volatile uint64_t bytes_received;
15
+ volatile uint64_t bytes_sent;
16
+ volatile uint64_t content_size;
17
+ volatile uint64_t compressed_content_size;
18
+
19
+ volatile uint64_t web_client_count;
20
+
21
+ volatile uint64_t rrdr_queries_made;
22
+ volatile uint64_t rrdr_db_points_read;
23
+ volatile uint64_t rrdr_result_points_generated;
24
+} global_statistics = {
25
.connected_clients = 0,
26
.web_requests = 0,
27
.web_usec = 0,
@@ -10,18 +29,45 @@ volatile struct global_statistics global_statistics = {
29
.bytes_sent = 0,
30
.content_size = 0,
31
.compressed_content_size = 0,
13
- .web_client_count = 1
32
+ .web_client_count = 1,
33
+
34
+ .rrdr_queries_made = 0,
35
+ .rrdr_db_points_read = 0,
36
+ .rrdr_result_points_generated = 0,
37
};
38
39
+#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
40
+#else
41
netdata_mutex_t global_statistics_mutex = NETDATA_MUTEX_INITIALIZER;
42
18
-inline void global_statistics_lock(void) {
43
+static inline void global_statistics_lock(void) {
44
netdata_mutex_lock(&global_statistics_mutex);
45
}
46
22
-inline void global_statistics_unlock(void) {
47
+static inline void global_statistics_unlock(void) {
48
netdata_mutex_unlock(&global_statistics_mutex);
49
}
50
+#endif
51
+
52
+
53
+void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated) {
54
+#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
55
+ __atomic_fetch_add(&global_statistics.rrdr_queries_made, 1, __ATOMIC_SEQ_CST);
56
+ __atomic_fetch_add(&global_statistics.rrdr_db_points_read, db_points_read, __ATOMIC_SEQ_CST);
57
+ __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, result_points_generated, __ATOMIC_SEQ_CST);
58
+#else
59
+ #warning NOT using atomic operations - using locks for global statistics
60
+ if (web_server_is_multithreaded)
61
+ global_statistics_lock();
62
+
63
+ global_statistics.rrdr_queries_made++;
64
+ global_statistics.rrdr_db_points_read += db_points_read;
65
+ global_statistics.rrdr_result_points_generated += result_points_generated;
66
+
67
+ if (web_server_is_multithreaded)
68
+ global_statistics_unlock();
69
+#endif
70
+}
71
72
void finished_web_request_statistics(uint64_t dt,
73
uint64_t bytes_received,
@@ -92,17 +138,21 @@ void web_client_disconnected(void) {
138
}
139
140
95
-inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) {
141
+static inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) {
142
#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
97
- gs->connected_clients = __atomic_fetch_add(&global_statistics.connected_clients, 0, __ATOMIC_SEQ_CST);
98
- gs->web_requests = __atomic_fetch_add(&global_statistics.web_requests, 0, __ATOMIC_SEQ_CST);
99
- gs->web_usec = __atomic_fetch_add(&global_statistics.web_usec, 0, __ATOMIC_SEQ_CST);
100
- gs->web_usec_max = __atomic_fetch_add(&global_statistics.web_usec_max, 0, __ATOMIC_SEQ_CST);
101
- gs->bytes_received = __atomic_fetch_add(&global_statistics.bytes_received, 0, __ATOMIC_SEQ_CST);
102
- gs->bytes_sent = __atomic_fetch_add(&global_statistics.bytes_sent, 0, __ATOMIC_SEQ_CST);
103
- gs->content_size = __atomic_fetch_add(&global_statistics.content_size, 0, __ATOMIC_SEQ_CST);
104
- gs->compressed_content_size = __atomic_fetch_add(&global_statistics.compressed_content_size, 0, __ATOMIC_SEQ_CST);
105
- gs->web_client_count = __atomic_fetch_add(&global_statistics.web_client_count, 0, __ATOMIC_SEQ_CST);
143
+ gs->connected_clients = __atomic_fetch_add(&global_statistics.connected_clients, 0, __ATOMIC_SEQ_CST);
144
+ gs->web_requests = __atomic_fetch_add(&global_statistics.web_requests, 0, __ATOMIC_SEQ_CST);
145
+ gs->web_usec = __atomic_fetch_add(&global_statistics.web_usec, 0, __ATOMIC_SEQ_CST);
146
+ gs->web_usec_max = __atomic_fetch_add(&global_statistics.web_usec_max, 0, __ATOMIC_SEQ_CST);
147
+ gs->bytes_received = __atomic_fetch_add(&global_statistics.bytes_received, 0, __ATOMIC_SEQ_CST);
148
+ gs->bytes_sent = __atomic_fetch_add(&global_statistics.bytes_sent, 0, __ATOMIC_SEQ_CST);
149
+ gs->content_size = __atomic_fetch_add(&global_statistics.content_size, 0, __ATOMIC_SEQ_CST);
150
+ gs->compressed_content_size = __atomic_fetch_add(&global_statistics.compressed_content_size, 0, __ATOMIC_SEQ_CST);
151
+ gs->web_client_count = __atomic_fetch_add(&global_statistics.web_client_count, 0, __ATOMIC_SEQ_CST);
152
+
153
+ gs->rrdr_queries_made = __atomic_fetch_add(&global_statistics.rrdr_queries_made, 0, __ATOMIC_SEQ_CST);
154
+ gs->rrdr_db_points_read = __atomic_fetch_add(&global_statistics.rrdr_db_points_read, 0, __ATOMIC_SEQ_CST);
155
+ gs->rrdr_result_points_generated = __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, 0, __ATOMIC_SEQ_CST);
156
157
if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) {
158
uint64_t n = 0;
@@ -413,4 +463,71 @@ void global_statistics_charts(void) {
463
464
rrdset_done(st_compression);
465
}
466
+
467
+ // ----------------------------------------------------------------
468
+
469
+ if(gs.rrdr_queries_made) {
470
+ static RRDSET *st_rrdr_queries = NULL;
471
+ static RRDDIM *rd_queries = NULL;
472
+
473
+ if (unlikely(!st_rrdr_queries)) {
474
+ st_rrdr_queries = rrdset_create_localhost(
475
+ "netdata"
476
+ , "queries"
477
+ , NULL
478
+ , "queries"
479
+ , NULL
480
+ , "NetData API Queries"
481
+ , "queries/s"
482
+ , "netdata"
483
+ , "stats"
484
+ , 130500
485
+ , localhost->rrd_update_every
486
+ , RRDSET_TYPE_LINE
487
+ );
488
+
489
+ rd_queries = rrddim_add(st_rrdr_queries, "queries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
490
+ }
491
+ else
492
+ rrdset_next(st_rrdr_queries);
493
+
494
+ rrddim_set_by_pointer(st_rrdr_queries, rd_queries, (collected_number)gs.rrdr_queries_made);
495
+
496
+ rrdset_done(st_rrdr_queries);
497
+ }
498
+
499
+ // ----------------------------------------------------------------
500
+
501
+ if(gs.rrdr_db_points_read || gs.rrdr_result_points_generated) {
502
+ static RRDSET *st_rrdr_points = NULL;
503
+ static RRDDIM *rd_points_read = NULL;
504
+ static RRDDIM *rd_points_generated = NULL;
505
+
506
+ if (unlikely(!st_rrdr_points)) {
507
+ st_rrdr_points = rrdset_create_localhost(
508
+ "netdata"
509
+ , "db_points"
510
+ , NULL
511
+ , "queries"
512
+ , NULL
513
+ , "NetData API Points"
514
+ , "points/s"
515
+ , "netdata"
516
+ , "stats"
517
+ , 130501
518
+ , localhost->rrd_update_every
519
+ , RRDSET_TYPE_AREA
520
+ );
521
+
522
+ rd_points_read = rrddim_add(st_rrdr_points, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
523
+ rd_points_generated = rrddim_add(st_rrdr_points, "generated", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
524
+ }
525
+ else
526
+ rrdset_next(st_rrdr_points);
527
+
528
+ rrddim_set_by_pointer(st_rrdr_points, rd_points_read, (collected_number)gs.rrdr_db_points_read);
529
+ rrddim_set_by_pointer(st_rrdr_points, rd_points_generated, (collected_number)gs.rrdr_result_points_generated);
530
+
531
+ rrdset_done(st_rrdr_points);
532
+ }
533
}
daemon/global_statistics.h
+1
-20
@@ -8,24 +8,8 @@
8
// ----------------------------------------------------------------------------
9
// global statistics
10
11
-struct global_statistics {
12
- volatile uint16_t connected_clients;
11
+extern void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated);
12
14
- volatile uint64_t web_requests;
15
- volatile uint64_t web_usec;
16
- volatile uint64_t web_usec_max;
17
- volatile uint64_t bytes_received;
18
- volatile uint64_t bytes_sent;
19
- volatile uint64_t content_size;
20
- volatile uint64_t compressed_content_size;
21
-
22
- volatile uint64_t web_client_count;
23
-};
24
-
25
-extern volatile struct global_statistics global_statistics;
26
-
27
-extern void global_statistics_lock(void);
28
-extern void global_statistics_unlock(void);
13
extern void finished_web_request_statistics(uint64_t dt,
14
uint64_t bytes_received,
15
uint64_t bytes_sent,
@@ -34,9 +18,6 @@ extern void finished_web_request_statistics(uint64_t dt,
18
19
extern uint64_t web_client_connected(void);
20
extern void web_client_disconnected(void);
37
-
38
-#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01
39
-extern void global_statistics_copy(struct global_statistics *gs, uint8_t options);
21
extern void global_statistics_charts(void);
22
23
#endif /* NETDATA_GLOBAL_STATISTICS_H */
web/api/queries/README.md
+121
-1
@@ -1,4 +1,124 @@
1
# Database Queries
2
3
-TBD
3
+Netdata database can be queried with `/api/v1/data` and `/api/v1/badge.svg` API methods.
4
+
5
+Every data query accepts the following parameters:
6
+
7
+name|description
8
+:----:|:----:
9
+`chart`|The chart to be queried.
10
+`points`|The number of points to be returned. Netdata can reduce number of points by applying query grouping methods.
11
+`before`|The absolute timestamp or the relative (to now) time the query should finish evaluating data.
12
+`after`|The absolute timestamp or the relative (to `before`) time the query should start evaluating data.
13
+`group`|The grouping method to use when reducing the points the database has.
14
+`gtime`|A resampling period to change the units of the metrics (i.e. setting this to `60` will convert `per second` metrics to `per minute`.
15
+`options`|A bitmap of options that can affect the operation of the query. Only 2 options are used by the query engine: `unaligned` and `percentage`. All the other options are used by the output formatters.
16
+`dimensions`|A simple pattern to filter the dimensions to be queried.
17
+
18
+## Operation
19
+
20
+The query engine works as follows (in this order):
21
+
22
+1. **Identify the exact time-frame required, in absolute timestamps.**
23
+
24
+ `after` and `before` define a time-frame:
25
+
26
+ - in **absolute timestamps** (unix timestamps, i.e. seconds since epoch).
27
+
28
+ - in **relative timestamps**:
29
+
30
+ `before` is relative to now and `after` is relative to `before`.
31
+
32
+ So, `before=-60&after=-60` evaluates to the time-frame from -120 up to -60 seconds in
33
+ the past, relative to now.
34
+
35
+ At the end of this operation, `after` and `before` are absolute timestamps.
36
+ The engine verifies that the time-frame is available at the database. If it is not,
37
+ it will adjust `after` and `before` accordingly so that usable data can be returned,
38
+ or no data at all if the time-frame is entirely outside the current range of the
39
+ database.
40
+
41
+2. **Identify the grouping of database points required.**
42
+
43
+ Grouping database points is used when the caller requests a longer time-frame to be
44
+ expressed with fewer points, compared to what is available at the database.
45
+
46
+ There are 2 uses of this (that can be combined):
47
+
48
+ - The caller requests a specific number of `points` to be returned.
49
+
50
+ For example, for a time-frame of 10 minutes, the database has 600 points (1/sec),
51
+ while the caller requested these 10 minutes to be expressed in 200 points.
52
+
53
+ This feature is used by netdata dashboards when you zoom-out the charts.
54
+ The dashboard is requesting the number of points the user's screen has, and netdata
55
+ returns that many points to perfectly match the screen. This saves bandwidth
56
+ and makes drawing the charts a lot faster.
57
+
58
+ - The caller requests a **re-sampling** of the database, by setting `gtime` to any value
59
+ above `1`. For example, the database maintains the metrics in the form of `X/sec`
60
+ but the caller set `gtime=60` to get `X/min`.
61
+
62
+ Using the above information the query engine tries to find a best fit for database-points
63
+ to result-points ratio (we call this `group points`). It always tries to keep `group points`
64
+ an integer. Keep in mind the query engine may alter a bit `after` if required. So, the engine
65
+ may decide to shift the starting point of the time-frame to keep the query optimal.
66
+
67
+3. **Align the time-frame.**
68
+
69
+ Alignment is a very important aspect of netdata queries. Without it, the animated
70
+ charts on the dashboards would constantly change shape during incremental updates.
71
+ To provide consistent grouping of all points, the query engine (by default) aligns
72
+ `after` and `before` to be a multiple of `group points`.
73
+
74
+ For example, if `group points` is 60 and alignment is enabled, the engine will return
75
+ each point with durations XX:XX:00 - XX:XX:59 matching minutes. Of course, depending
76
+ on the database granularity for the specific chart and the requested points to be
77
+ returned, the engine may use any integer number for `group points`.
78
+
79
+ To disable alignment, pass `&options=unaligned` to the query.
80
+
81
+4. **Execute the query**
82
+
83
+ To execute the query, the engine evaluates all dimensions of the chart, one after another.
84
+ The engine will not evaluate dimensions that do not match the simple pattern given at
85
+ the `dimensions` parameter, except when `options=percentage` is given (this option requires
86
+ all the dimensions to be evaluated to find the percentage of each dimension vs to chart
87
+ total).
88
+
89
+ For each dimension, it starts evaluating values from `after` towards `before`.
90
+ For each value it calls the **grouping method** specified (the default is `average`).
91
+
92
+## Grouping methods
93
+
94
+The following grouping methods are supported. These are given all the values in the time-frame
95
+and they group the values every `group points`.
96
+
97
+name|identifier(s)|description
98
+:---:|:---:|:---:
99
+Min|`min`|finds the minimum value
100
+Max|`max`|finds the maximum value
101
+Average|`average` `mean`|finds the average value
102
+Sum|`sum`|adds all the values and returns the sum
103
+Median|`median`|sorts the values and returns the value in the middle of the list
104
+Standard Deviation|`stddev`|finds the standard deviation of the values
105
+Coefficient of Variation|`cv` `rds`|finds the relative standard deviation of the values
106
+Single Exponential Smoothing|`ses` `ema` `ewma`|finds the exponential weighted moving average of the values
107
+Double Exponential Smoothing|`des`|applies Holt-Winters double exponential smoothing
108
+Incremental Sum|`incremental_sum` `incremental-sum`|find the difference of the last vs the first values
109
+
110
+## Further processing
111
+
112
+The result of the query engine is always a structure that has dimensions and values
113
+for each dimension.
114
+
115
+Formatting modules are then used to convert this result in many different formats and return it
116
+to the caller.
117
+
118
+## Performance
119
+
120
+The query engine is highly optimized for speed. Most of its modules implement "online"
121
+versions of the algorithms, requiring just one pass on the database values to produce
122
+the result.
123
+
124
web/api/queries/query.c
+6
@@ -408,6 +408,7 @@ static inline void do_dimension(
408
RRDR_VALUE_FLAGS
409
group_value_flags = RRDR_VALUE_NOTHING;
410
411
+ size_t db_points_read = 0;
412
for( ; points_added < points_wanted ; now += dt, slot++ ) {
413
if(unlikely(slot >= entries)) slot = 0;
414
@@ -442,6 +443,7 @@ static inline void do_dimension(
443
// add this value for grouping
444
r->internal.grouping_add(r, value);
445
values_in_group++;
446
+ db_points_read++;
447
448
if(unlikely(values_in_group == group_size)) {
449
rrdr_line = rrdr_line_init(r, now, rrdr_line);
@@ -469,6 +471,9 @@ static inline void do_dimension(
471
}
472
}
473
474
+ r->internal.db_points_read += db_points_read;
475
+ r->internal.result_points_generated += points_added;
476
+
477
r->before = max_date;
478
r->after = min_date;
479
rrdr_done(r, rrdr_line);
@@ -943,5 +948,6 @@ RRDR *rrd2rrdr(
948
}
949
}
950
951
+ rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);
952
return r;
953
}
web/api/queries/rrdr.h
+3
@@ -85,6 +85,9 @@ typedef struct rrdresult {
85
#ifdef NETDATA_INTERNAL_CHECKS
86
const char *log;
87
#endif
88
+
89
+ size_t db_points_read;
90
+ size_t result_points_generated;
91
} internal;
92
} RRDR;
93