metrics cardinality - more statistics and groupings (#19368)
* added percentages * added grouping by context and by node
Costa Tsaousis committed
Jan 10, 2025 at 13:17 UTC
da5ac37d35d25d2386761bf3c758621631277714
1 file changed
+216
-53
src/web/api/functions/function-metrics-cardinality.c
+216
-53
@@ -3,7 +3,7 @@
3
#include "function-metrics-cardinality.h"
4
#include "database/contexts/internal.h"
5
6
-struct context_counts {
6
+struct counts {
7
size_t nodes;
8
size_t instances;
9
size_t metrics;
@@ -25,13 +25,73 @@ int function_metrics_cardinality(BUFFER *wb, const char *function __maybe_unused
25
buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
26
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
27
buffer_json_member_add_string(wb, "type", "table");
28
- buffer_json_member_add_time_t(wb, "update_every", 1);
28
+ buffer_json_member_add_time_t(wb, "update_every", 10);
29
buffer_json_member_add_boolean(wb, "has_history", false);
30
buffer_json_member_add_string(wb, "help", RRDFUNCTIONS_METRICS_CARDINALITY_HELP);
31
- buffer_json_member_add_array(wb, "data");
31
+
32
+ // Add parameter definition similar to network-viewer
33
+ buffer_json_member_add_array(wb, "accepted_params");
34
+ {
35
+ buffer_json_add_array_item_string(wb, "group");
36
+ }
37
+ buffer_json_array_close(wb);
38
+
39
+ buffer_json_member_add_array(wb, "required_params");
40
+ {
41
+ buffer_json_add_array_item_object(wb);
42
+ {
43
+ buffer_json_member_add_string(wb, "id", "group");
44
+ buffer_json_member_add_string(wb, "name", "Grouping");
45
+ buffer_json_member_add_string(wb, "help", "Select how to group the metrics");
46
+ buffer_json_member_add_boolean(wb, "unique_view", true);
47
+ buffer_json_member_add_string(wb, "type", "select");
48
+ buffer_json_member_add_array(wb, "options");
49
+ {
50
+ buffer_json_add_array_item_object(wb);
51
+ {
52
+ buffer_json_member_add_string(wb, "id", "by-context");
53
+ buffer_json_member_add_string(wb, "name", "Group by Context");
54
+ }
55
+ buffer_json_object_close(wb);
56
+ buffer_json_add_array_item_object(wb);
57
+ {
58
+ buffer_json_member_add_string(wb, "id", "by-node");
59
+ buffer_json_member_add_string(wb, "name", "Group by Node");
60
+ }
61
+ buffer_json_object_close(wb);
62
+ }
63
+ buffer_json_array_close(wb);
64
+ }
65
+ buffer_json_object_close(wb);
66
+ }
67
+ buffer_json_array_close(wb);
68
+
69
+ // Parse function parameters
70
+ bool by_node = false;
71
+ {
72
+ char function_copy[strlen(function) + 1];
73
+ memcpy(function_copy, function, sizeof(function_copy));
74
+ char *words[1024];
75
+ size_t num_words = quoted_strings_splitter_whitespace(function_copy, words, 1024);
76
+ for (size_t i = 1; i < num_words; i++) {
77
+ char *param = get_word(words, num_words, i);
78
+ if (strcmp(param, "group:by-node") == 0) {
79
+ by_node = true;
80
+ } else if (strcmp(param, "group:by-context") == 0) {
81
+ by_node = false;
82
+ } else if (strcmp(param, "info") == 0) {
83
+ buffer_json_finalize(wb);
84
+ return HTTP_RESP_OK;
85
+ }
86
+ }
87
+ }
88
89
DICTIONARY *contexts_dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
90
91
+ buffer_json_member_add_array(wb, "data");
92
+
93
+ struct counts all = { 0 };
94
+
95
// Collect stats for each context across all nodes
96
RRDHOST *host;
97
dfe_start_read(rrdhost_root_index, host) {
@@ -41,61 +101,92 @@ int function_metrics_cardinality(BUFFER *wb, const char *function __maybe_unused
101
102
RRDCONTEXT *rc;
103
dfe_start_read(host->rrdctx.contexts, rc) {
44
- const char *context_id = string2str(rc->id);
104
+ const char *aggregation_key = by_node ? string2str(host->hostname) :string2str(rc->id);
105
46
- struct context_counts tmp = { 0 };
47
- struct context_counts *cnt = dictionary_set(contexts_dict, context_id, &tmp, sizeof(tmp));
106
+ struct counts cnt = { 0 };
107
49
- cnt->nodes++;
108
+ cnt.nodes++;
109
51
- if(host_online)
52
- cnt->online_nodes++;
110
+ if (host_online)
111
+ cnt.online_nodes++;
112
else
54
- cnt->offline_nodes++;
113
+ cnt.offline_nodes++;
114
115
RRDINSTANCE *ri;
116
dfe_start_read(rc->rrdinstances, ri) {
58
- cnt->instances++;
117
+ cnt.instances++;
118
60
- if(rrd_flag_check(ri, RRD_FLAG_COLLECTED)) {
61
- cnt->online_instances++;
119
+ if (rrd_flag_check(ri, RRD_FLAG_COLLECTED)) {
120
+ cnt.online_instances++;
121
122
RRDMETRIC *rm;
123
dfe_start_read(ri->rrdmetrics, rm) {
65
- cnt->metrics++;
124
+ cnt.metrics++;
125
67
- if(rrd_flag_check(rm, RRD_FLAG_COLLECTED))
68
- cnt->online_metrics++;
126
+ if (rrd_flag_check(rm, RRD_FLAG_COLLECTED))
127
+ cnt.online_metrics++;
128
else
70
- cnt->offline_metrics++;
129
+ cnt.offline_metrics++;
130
}
131
dfe_done(rm);
132
}
133
else {
75
- cnt->offline_instances++;
134
+ cnt.offline_instances++;
135
size_t metrics = dictionary_entries(ri->rrdmetrics);
77
- cnt->metrics += metrics;
78
- cnt->offline_metrics += metrics;
136
+ cnt.metrics += metrics;
137
+ cnt.offline_metrics += metrics;
138
}
139
}
140
dfe_done(ri);
141
+
142
+ struct counts *cc = dictionary_get(contexts_dict, aggregation_key);
143
+ if (cc) {
144
+ cc->nodes += cnt.nodes;
145
+ cc->online_nodes += cnt.online_nodes;
146
+ cc->offline_nodes += cnt.offline_nodes;
147
+
148
+ cc->instances += cnt.instances;
149
+ cc->online_instances += cnt.online_instances;
150
+ cc->offline_instances += cnt.offline_instances;
151
+
152
+ cc->metrics += cnt.metrics;
153
+ cc->online_metrics += cnt.online_metrics;
154
+ cc->offline_metrics += cnt.offline_metrics;
155
+ }
156
+ else
157
+ dictionary_set(contexts_dict, aggregation_key, &cnt, sizeof(cnt));
158
+
159
+ // keep track of the total
160
+ all.nodes += cnt.nodes;
161
+ all.online_nodes += cnt.online_nodes;
162
+ all.offline_nodes += cnt.offline_nodes;
163
+
164
+ all.instances += cnt.instances;
165
+ all.online_instances += cnt.online_instances;
166
+ all.offline_instances += cnt.offline_instances;
167
+
168
+ all.metrics += cnt.metrics;
169
+ all.online_metrics += cnt.online_metrics;
170
+ all.offline_metrics += cnt.offline_metrics;
171
}
172
dfe_done(rc);
173
}
174
dfe_done(host);
175
87
- struct context_counts max = { 0 };
176
+ struct counts max = { 0 };
177
178
// Output collected stats
90
- struct context_counts *cnt;
179
+ struct counts *cnt;
180
dfe_start_read(contexts_dict, cnt) {
181
buffer_json_add_array_item_array(wb);
182
{
183
buffer_json_add_array_item_string(wb, cnt_dfe.name);
184
96
- buffer_json_add_array_item_uint64(wb, cnt->nodes);
97
- buffer_json_add_array_item_uint64(wb, cnt->online_nodes);
98
- buffer_json_add_array_item_uint64(wb, cnt->offline_nodes);
185
+ if(!by_node) {
186
+ buffer_json_add_array_item_uint64(wb, cnt->nodes);
187
+ buffer_json_add_array_item_uint64(wb, cnt->online_nodes);
188
+ buffer_json_add_array_item_uint64(wb, cnt->offline_nodes);
189
+ }
190
191
buffer_json_add_array_item_uint64(wb, cnt->instances);
192
buffer_json_add_array_item_uint64(wb, cnt->online_instances);
@@ -110,6 +201,24 @@ int function_metrics_cardinality(BUFFER *wb, const char *function __maybe_unused
201
202
double metrics_ephemerality = (cnt->metrics > 0) ? ((double)cnt->offline_metrics * 100.0) / (double)cnt->metrics : 0.0;
203
buffer_json_add_array_item_double(wb, metrics_ephemerality);
204
+
205
+ double percentage_of_all_instances = (all.instances) ? ((double)cnt->instances * 100.0) / (double)all.instances : 0.0;
206
+ buffer_json_add_array_item_double(wb, percentage_of_all_instances);
207
+
208
+ double percentage_of_online_instances = (all.online_instances) ? ((double)cnt->online_instances * 100.0) / (double)all.online_instances : 0.0;
209
+ buffer_json_add_array_item_double(wb, percentage_of_online_instances);
210
+
211
+ double percentage_of_offline_instances = (all.offline_instances) ? ((double)cnt->offline_instances * 100.0) / (double)all.offline_instances : 0.0;
212
+ buffer_json_add_array_item_double(wb, percentage_of_offline_instances);
213
+
214
+ double percentage_of_all_metrics = (all.metrics) ? ((double)cnt->metrics * 100.0) / (double)all.metrics : 0.0;
215
+ buffer_json_add_array_item_double(wb, percentage_of_all_metrics);
216
+
217
+ double percentage_of_online_metrics = (all.online_metrics) ? ((double)cnt->online_metrics * 100.0) / (double)all.online_metrics : 0.0;
218
+ buffer_json_add_array_item_double(wb, percentage_of_online_metrics);
219
+
220
+ double percentage_of_offline_metrics = (all.offline_metrics) ? ((double)cnt->offline_metrics * 100.0) / (double)all.offline_metrics : 0.0;
221
+ buffer_json_add_array_item_double(wb, percentage_of_offline_metrics);
222
}
223
buffer_json_array_close(wb);
224
@@ -140,33 +249,45 @@ int function_metrics_cardinality(BUFFER *wb, const char *function __maybe_unused
249
{
250
size_t field_id = 0;
251
143
- buffer_rrdf_table_add_field(wb, field_id++, "Context", "Context Name",
144
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
145
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
146
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
147
- RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_UNIQUE_KEY | RRDF_FIELD_OPTS_VISIBLE,
148
- NULL);
149
-
150
- buffer_rrdf_table_add_field(wb, field_id++, "All Nodes", "Number of Nodes",
151
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
152
- 0, "nodes", (double)max.nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
153
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
154
- RRDF_FIELD_OPTS_NONE,
155
- NULL);
156
-
157
- buffer_rrdf_table_add_field(wb, field_id++, "Curr. Nodes", "Number of Online Nodes",
158
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
159
- 0, "nodes", (double)max.online_nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
160
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
161
- RRDF_FIELD_OPTS_VISIBLE,
162
- NULL);
252
+ if(by_node) {
253
+ buffer_rrdf_table_add_field(wb, field_id++, "Hostname", "Hostname",
254
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
255
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
256
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
257
+ RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_UNIQUE_KEY | RRDF_FIELD_OPTS_VISIBLE,
258
+ NULL);
259
+ }
260
+ else {
261
+ buffer_rrdf_table_add_field(wb, field_id++, "Context", "Context Name",
262
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
263
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
264
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
265
+ RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_UNIQUE_KEY | RRDF_FIELD_OPTS_VISIBLE,
266
+ NULL);
267
+ }
268
164
- buffer_rrdf_table_add_field(wb, field_id++, "Old Nodes", "Number of Offline Nodes",
165
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
166
- 0, "nodes", (double)max.offline_nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
167
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
168
- RRDF_FIELD_OPTS_VISIBLE,
169
- NULL);
269
+ if(!by_node) {
270
+ buffer_rrdf_table_add_field(wb, field_id++, "All Nodes", "Number of Nodes",
271
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
272
+ 0, "nodes", (double)max.nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
273
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
274
+ RRDF_FIELD_OPTS_NONE,
275
+ NULL);
276
+
277
+ buffer_rrdf_table_add_field(wb, field_id++, "Curr. Nodes", "Number of Online Nodes",
278
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
279
+ 0, "nodes", (double)max.online_nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
280
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
281
+ RRDF_FIELD_OPTS_VISIBLE,
282
+ NULL);
283
+
284
+ buffer_rrdf_table_add_field(wb, field_id++, "Old Nodes", "Number of Offline Nodes",
285
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
286
+ 0, "nodes", (double)max.offline_nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
287
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
288
+ RRDF_FIELD_OPTS_VISIBLE,
289
+ NULL);
290
+ }
291
292
buffer_rrdf_table_add_field(wb, field_id++, "All Instances", "Total Number of Instances",
293
RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
@@ -210,14 +331,56 @@ int function_metrics_cardinality(BUFFER *wb, const char *function __maybe_unused
331
RRDF_FIELD_OPTS_NONE,
332
NULL);
333
213
- buffer_rrdf_table_add_field(wb, field_id++, "Ephemeral Instances", "Percentage of Archived Instances",
334
+ buffer_rrdf_table_add_field(wb, field_id++, "Ephemeral Instances", "Percentage of Archived Instances vs All Instances of the row",
335
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
336
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
337
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
338
+ RRDF_FIELD_OPTS_VISIBLE,
339
+ NULL);
340
+
341
+ buffer_rrdf_table_add_field(wb, field_id++, "Ephemeral Dimensions", "Percentage of Archived Time-Series vs All Time-Series of the row",
342
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
343
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
344
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
345
+ RRDF_FIELD_OPTS_NONE,
346
+ NULL);
347
+
348
+ buffer_rrdf_table_add_field(wb, field_id++, "All Instances %", "Percentage of All Instances of row vs the sum of All Instances across all rows",
349
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
350
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
351
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
352
+ RRDF_FIELD_OPTS_VISIBLE,
353
+ NULL);
354
+
355
+ buffer_rrdf_table_add_field(wb, field_id++, "Curr. Instances %", "Percentage of Currently Collected Instances of row vs the sum of Currently Collected Instances across all rows",
356
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
357
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
358
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
359
+ RRDF_FIELD_OPTS_NONE,
360
+ NULL);
361
+
362
+ buffer_rrdf_table_add_field(wb, field_id++, "Old Instances %", "Percentage of Old Instances of row vs the sum of Old Instances across all rows",
363
RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
364
2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
365
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
366
RRDF_FIELD_OPTS_VISIBLE,
367
NULL);
368
220
- buffer_rrdf_table_add_field(wb, field_id++, "Ephemeral Dimensions", "Percentage of Archived Time-Series",
369
+ buffer_rrdf_table_add_field(wb, field_id++, "All Dimensions %", "Percentage of All Time-Series of row vs the sum of All Time-Series across all rows",
370
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
371
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
372
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
373
+ RRDF_FIELD_OPTS_NONE,
374
+ NULL);
375
+
376
+ buffer_rrdf_table_add_field(wb, field_id++, "Curr. Dimensions %", "Percentage of Currently Collected Time-Series of row vs the sum of Currently Collected Time-Series across all rows",
377
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
378
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
379
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
380
+ RRDF_FIELD_OPTS_NONE,
381
+ NULL);
382
+
383
+ buffer_rrdf_table_add_field(wb, field_id++, "Old Dimensions %", "Percentage of Archived Time-Series of row vs the sum of Archived Time-Series across all rows",
384
RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
385
2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
386
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,