metrics-cardinality function (#19362)
Costa Tsaousis committed
Jan 9, 2025 at 23:43 UTC
098ddc951b60bb1035eb8bc9dbc1b8d960389b88
5 files changed
+245
CMakeLists.txt
+2
@@ -1599,6 +1599,8 @@ set(WEB_PLUGIN_FILES
1599
src/web/api/v3/api_v3_stream_path.c
1600
src/web/api/queries/backfill.c
1601
src/web/api/queries/backfill.h
1602
+ src/web/api/functions/function-metrics-cardinality.c
1603
+ src/web/api/functions/function-metrics-cardinality.h
1604
)
1605
1606
set(CLAIM_PLUGIN_FILES
src/web/api/functions/function-metrics-cardinality.c
new
+218
@@ -0,0 +1,218 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "function-metrics-cardinality.h"
4
+#include "database/contexts/internal.h"
5
+
6
+struct context_counts {
7
+ size_t nodes;
8
+ size_t instances;
9
+ size_t metrics;
10
+
11
+ size_t online_nodes;
12
+ size_t online_instances;
13
+ size_t online_metrics;
14
+
15
+ size_t offline_nodes;
16
+ size_t offline_instances;
17
+ size_t offline_metrics;
18
+};
19
+
20
+int function_metrics_cardinality(BUFFER *wb, const char *function __maybe_unused, BUFFER *payload __maybe_unused, const char *source __maybe_unused) {
21
+ buffer_flush(wb);
22
+ wb->content_type = CT_APPLICATION_JSON;
23
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
24
+
25
+ buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
26
+ buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
27
+ buffer_json_member_add_string(wb, "type", "table");
28
+ buffer_json_member_add_time_t(wb, "update_every", 1);
29
+ buffer_json_member_add_boolean(wb, "has_history", false);
30
+ buffer_json_member_add_string(wb, "help", RRDFUNCTIONS_METRICS_CARDINALITY_HELP);
31
+ buffer_json_member_add_array(wb, "data");
32
+
33
+ DICTIONARY *contexts_dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
34
+
35
+ // Collect stats for each context across all nodes
36
+ RRDHOST *host;
37
+ dfe_start_read(rrdhost_root_index, host) {
38
+ if(!host->rrdctx.contexts) continue;
39
+
40
+ bool host_online = rrdhost_is_online(host);
41
+
42
+ RRDCONTEXT *rc;
43
+ dfe_start_read(host->rrdctx.contexts, rc) {
44
+ const char *context_id = string2str(rc->id);
45
+
46
+ struct context_counts tmp = { 0 };
47
+ struct context_counts *cnt = dictionary_set(contexts_dict, context_id, &tmp, sizeof(tmp));
48
+
49
+ cnt->nodes++;
50
+
51
+ if(host_online)
52
+ cnt->online_nodes++;
53
+ else
54
+ cnt->offline_nodes++;
55
+
56
+ RRDINSTANCE *ri;
57
+ dfe_start_read(rc->rrdinstances, ri) {
58
+ cnt->instances++;
59
+
60
+ if(rrd_flag_check(ri, RRD_FLAG_COLLECTED))
61
+ cnt->online_instances++;
62
+ else
63
+ cnt->offline_instances++;
64
+
65
+ RRDMETRIC *rm;
66
+ dfe_start_read(ri->rrdmetrics, rm) {
67
+ cnt->metrics++;
68
+
69
+ if(rrd_flag_check(rm, RRD_FLAG_COLLECTED))
70
+ cnt->online_metrics++;
71
+ else
72
+ cnt->offline_metrics++;
73
+ }
74
+ dfe_done(rm);
75
+ }
76
+ dfe_done(ri);
77
+ }
78
+ dfe_done(rc);
79
+ }
80
+ dfe_done(host);
81
+
82
+ struct context_counts max = { 0 };
83
+
84
+ // Output collected stats
85
+ struct context_counts *cnt;
86
+ dfe_start_read(contexts_dict, cnt) {
87
+ buffer_json_add_array_item_array(wb);
88
+ buffer_json_add_array_item_string(wb, cnt_dfe.name);
89
+ buffer_json_add_array_item_uint64(wb, cnt->nodes);
90
+ buffer_json_add_array_item_uint64(wb, cnt->instances);
91
+ buffer_json_add_array_item_uint64(wb, cnt->metrics);
92
+ buffer_json_add_array_item_uint64(wb, cnt->online_nodes);
93
+ buffer_json_add_array_item_uint64(wb, cnt->online_instances);
94
+ buffer_json_add_array_item_uint64(wb, cnt->online_metrics);
95
+ buffer_json_add_array_item_uint64(wb, cnt->offline_nodes);
96
+ buffer_json_add_array_item_uint64(wb, cnt->offline_instances);
97
+ buffer_json_add_array_item_uint64(wb, cnt->offline_metrics);
98
+
99
+ double ephemerality = (cnt->metrics > 0) ? ((double)cnt->offline_metrics * 100.0) / (double)cnt->metrics : 0.0;
100
+ buffer_json_add_array_item_double(wb, ephemerality);
101
+
102
+ buffer_json_array_close(wb);
103
+
104
+ if(cnt->nodes > max.nodes)
105
+ max.nodes = cnt->nodes;
106
+ if(cnt->instances > max.instances)
107
+ max.instances = cnt->instances;
108
+ if(cnt->metrics > max.metrics)
109
+ max.metrics = cnt->metrics;
110
+ if(cnt->online_nodes > max.online_nodes)
111
+ max.online_nodes = cnt->online_nodes;
112
+ if(cnt->online_instances > max.online_instances)
113
+ max.online_instances = cnt->online_instances;
114
+ if(cnt->online_metrics > max.online_metrics)
115
+ max.online_metrics = cnt->online_metrics;
116
+ if(cnt->offline_nodes > max.offline_nodes)
117
+ max.offline_nodes = cnt->offline_nodes;
118
+ if(cnt->offline_instances > max.offline_instances)
119
+ max.offline_instances = cnt->offline_instances;
120
+ if(cnt->offline_metrics > max.offline_metrics)
121
+ max.offline_metrics = cnt->offline_metrics;
122
+ }
123
+ dfe_done(cnt);
124
+
125
+ buffer_json_array_close(wb); // data
126
+
127
+ buffer_json_member_add_object(wb, "columns");
128
+ {
129
+ size_t field_id = 0;
130
+
131
+ buffer_rrdf_table_add_field(wb, field_id++, "Context", "Context Name",
132
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
133
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
134
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
135
+ RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_UNIQUE_KEY | RRDF_FIELD_OPTS_VISIBLE,
136
+ NULL);
137
+
138
+ buffer_rrdf_table_add_field(wb, field_id++, "All Nodes", "Number of Nodes",
139
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
140
+ 0, "nodes", (double)max.nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
141
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
142
+ RRDF_FIELD_OPTS_NONE,
143
+ NULL);
144
+
145
+ buffer_rrdf_table_add_field(wb, field_id++, "All Instances", "Total Number of Instances",
146
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
147
+ 0, "instances", (double)max.instances, RRDF_FIELD_SORT_DESCENDING, NULL,
148
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
149
+ RRDF_FIELD_OPTS_NONE,
150
+ NULL);
151
+
152
+ buffer_rrdf_table_add_field(wb, field_id++, "All Metrics", "Total Number of Metrics",
153
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
154
+ 0, "metrics", (double)max.metrics, RRDF_FIELD_SORT_DESCENDING, NULL,
155
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
156
+ RRDF_FIELD_OPTS_NONE,
157
+ NULL);
158
+
159
+ buffer_rrdf_table_add_field(wb, field_id++, "Curr. Nodes", "Number of Online Nodes",
160
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
161
+ 0, "nodes", (double)max.online_nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
162
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
163
+ RRDF_FIELD_OPTS_VISIBLE,
164
+ NULL);
165
+
166
+ buffer_rrdf_table_add_field(wb, field_id++, "Curr. Instances", "Total Number of Online Instances",
167
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
168
+ 0, "instances", (double)max.online_instances, RRDF_FIELD_SORT_DESCENDING, NULL,
169
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
170
+ RRDF_FIELD_OPTS_VISIBLE,
171
+ NULL);
172
+
173
+ buffer_rrdf_table_add_field(wb, field_id++, "Curr. Metrics", "Total Number of Online Metrics",
174
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
175
+ 0, "metrics", (double)max.online_metrics, RRDF_FIELD_SORT_DESCENDING, NULL,
176
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
177
+ RRDF_FIELD_OPTS_VISIBLE,
178
+ NULL);
179
+
180
+ buffer_rrdf_table_add_field(wb, field_id++, "Old Nodes", "Number of Offline Nodes",
181
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
182
+ 0, "nodes", (double)max.offline_nodes, RRDF_FIELD_SORT_DESCENDING, NULL,
183
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
184
+ RRDF_FIELD_OPTS_VISIBLE,
185
+ NULL);
186
+
187
+ buffer_rrdf_table_add_field(wb, field_id++, "Old Instances", "Total Number of Offline Instances",
188
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
189
+ 0, "instances", (double)max.offline_instances, RRDF_FIELD_SORT_DESCENDING, NULL,
190
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
191
+ RRDF_FIELD_OPTS_VISIBLE,
192
+ NULL);
193
+
194
+ buffer_rrdf_table_add_field(wb, field_id++, "Old Metrics", "Total Number of Offline Metrics",
195
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
196
+ 0, "metrics", (double)max.offline_metrics, RRDF_FIELD_SORT_DESCENDING, NULL,
197
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
198
+ RRDF_FIELD_OPTS_VISIBLE,
199
+ NULL);
200
+
201
+ buffer_rrdf_table_add_field(wb, field_id++, "Ephemerality", "Percentage of Metrics that are Offline",
202
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
203
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
204
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
205
+ RRDF_FIELD_OPTS_VISIBLE,
206
+ NULL);
207
+ }
208
+ buffer_json_object_close(wb); // columns
209
+
210
+ buffer_json_member_add_string(wb, "default_sort_column", "Old Metrics");
211
+
212
+ buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
213
+ buffer_json_finalize(wb);
214
+
215
+ dictionary_destroy(contexts_dict);
216
+
217
+ return HTTP_RESP_OK;
218
+}
src/web/api/functions/function-metrics-cardinality.h
new
+12
@@ -0,0 +1,12 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_FUNCTION_METRICS_CARDINALITY_H
4
+#define NETDATA_FUNCTION_METRICS_CARDINALITY_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+
8
+#define RRDFUNCTIONS_METRICS_CARDINALITY_HELP "Shows metrics cardinality per context across all nodes."
9
+
10
+int function_metrics_cardinality(BUFFER *wb, const char *function, BUFFER *payload, const char *source);
11
+
12
+#endif //NETDATA_FUNCTION_METRICS_CARDINALITY_H
src/web/api/functions/functions.c
+12
@@ -40,4 +40,16 @@ void global_functions_add(void) {
40
RRDFUNCTIONS_TAG_HIDDEN,
41
HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA,
42
function_bearer_get_token);
43
+
44
+ rrd_function_add_inline(
45
+ localhost,
46
+ NULL,
47
+ "metrics-cardinality",
48
+ 10,
49
+ RRDFUNCTIONS_PRIORITY_DEFAULT,
50
+ RRDFUNCTIONS_VERSION_DEFAULT,
51
+ RRDFUNCTIONS_METRICS_CARDINALITY_HELP,
52
+ "top",
53
+ HTTP_ACCESS_ANONYMOUS_DATA,
54
+ function_metrics_cardinality);
55
}
src/web/api/functions/functions.h
+1
@@ -5,6 +5,7 @@
5
6
#include "daemon/common.h"
7
8
+#include "function-metrics-cardinality.h"
9
#include "function-streaming.h"
10
#include "function-progress.h"
11
#include "function-bearer_get_token.h"