@cryptotaxi247 / netdata-1 / commits / ce75313de

systemd-journal plugin (#15363)

Costa Tsaousis committed Aug 3, 2023 at 15:42 UTC ce75313de06ee1e8aa5a59d57625ca25d467f0f8
40 files changed +1879 -224
Makefile.am
+16
@@ -154,6 +154,8 @@ LIBNETDATA_FILES = \
154 libnetdata/dictionary/dictionary.h \
155 libnetdata/eval/eval.c \
156 libnetdata/eval/eval.h \
157 + libnetdata/facets/facets.c \
158 + libnetdata/facets/facets.h \
159 libnetdata/gorilla/gorilla.h \
160 libnetdata/gorilla/gorilla.cc \
161 libnetdata/inlined.h \
@@ -302,6 +304,11 @@ FREEIPMI_PLUGIN_FILES = \
304 $(LIBNETDATA_FILES) \
305 $(NULL)
306
307 +SYSTEMD_JOURNAL_PLUGIN_FILES = \
308 + collectors/systemd-journal.plugin/systemd-journal.c \
309 + $(LIBNETDATA_FILES) \
310 + $(NULL)
311 +
312 CUPS_PLUGIN_FILES = \
313 collectors/cups.plugin/cups_plugin.c \
314 $(LIBNETDATA_FILES) \
@@ -1232,6 +1239,15 @@ if ENABLE_PLUGIN_FREEIPMI
1239 $(NULL)
1240 endif
1241
1242 +if ENABLE_PLUGIN_SYSTEMD_JOURNAL
1243 + plugins_PROGRAMS += systemd-journal.plugin
1244 + systemd_journal_plugin_SOURCES = $(SYSTEMD_JOURNAL_PLUGIN_FILES)
1245 + systemd_journal_plugin_LDADD = \
1246 + $(NETDATA_COMMON_LIBS) \
1247 + $(OPTIONAL_SYSTEMD_LIBS) \
1248 + $(NULL)
1249 +endif
1250 +
1251 if ENABLE_PLUGIN_EBPF
1252 plugins_PROGRAMS += ebpf.plugin
1253 ebpf_plugin_SOURCES = $(EBPF_PLUGIN_FILES)
claim/claim.c
+2 -2
@@ -356,7 +356,7 @@ int api_v2_claim(struct web_client *w, char *url) {
356
357 BUFFER *wb = w->response.data;
358 buffer_flush(wb);
359 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
359 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
360
361 time_t now_s = now_realtime_sec();
362 CLOUD_STATUS status = buffer_json_cloud_status(wb, now_s);
@@ -462,7 +462,7 @@ int api_v2_claim(struct web_client *w, char *url) {
462 // our status may have changed
463 // refresh the status in our output
464 buffer_flush(wb);
465 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
465 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
466 now_s = now_realtime_sec();
467 buffer_json_cloud_status(wb, now_s);
468
collectors/Makefile.am
+1
@@ -25,6 +25,7 @@ SUBDIRS = \
25 statsd.plugin \
26 ebpf.plugin \
27 tc.plugin \
28 + systemd-journal.plugin \
29 $(NULL)
30
31 usercustompluginsconfigdir=$(configdir)/custom-plugins.d
collectors/apps.plugin/apps_plugin.c
+15 -22
@@ -13,7 +13,7 @@
13 #define APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION "Detailed information on the currently running processes."
14
15 #define APPS_PLUGIN_FUNCTIONS() do { \
16 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" 10 \"%s\"\n", APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
16 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" %d \"%s\"\n", PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
17 } while(0)
18
19
@@ -4572,7 +4572,7 @@ static int check_capabilities() {
4572 }
4573 #endif
4574
4575 -netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
4575 +static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
4576
4577 #define PROCESS_FILTER_CATEGORY "category:"
4578 #define PROCESS_FILTER_USER "user:"
@@ -4625,15 +4625,6 @@ static void get_MemTotal(void) {
4625 #endif
4626 }
4627
4628 -static void apps_plugin_function_error(const char *transaction, int code, const char *msg) {
4629 - char buffer[PLUGINSD_LINE_MAX + 1];
4630 - json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
4631 -
4632 - pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
4633 - fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
4634 - pluginsd_function_result_end_to_stdout();
4635 -}
4636 -
4628 static void apps_plugin_function_processes_help(const char *transaction) {
4629 pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);
4630 fprintf(stdout, "%s",
@@ -4681,7 +4672,7 @@ static void apps_plugin_function_processes_help(const char *transaction) {
4672 buffer_json_add_array_item_double(wb, _tmp); \
4673 } while(0)
4674
4684 -static void apps_plugin_function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
4675 +static void function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
4676 struct pid_stat *p;
4677
4678 char *words[PLUGINSD_MAX_WORDS] = { NULL };
@@ -4702,21 +4693,21 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
4693 if(!category && strncmp(keyword, PROCESS_FILTER_CATEGORY, strlen(PROCESS_FILTER_CATEGORY)) == 0) {
4694 category = find_target_by_name(apps_groups_root_target, &keyword[strlen(PROCESS_FILTER_CATEGORY)]);
4695 if(!category) {
4705 - apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
4696 + pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
4697 return;
4698 }
4699 }
4700 else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {
4701 user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);
4702 if(!user) {
4712 - apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
4703 + pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
4704 return;
4705 }
4706 }
4707 else if(strncmp(keyword, PROCESS_FILTER_GROUP, strlen(PROCESS_FILTER_GROUP)) == 0) {
4708 group = find_target_by_name(groups_root_target, &keyword[strlen(PROCESS_FILTER_GROUP)]);
4709 if(!group) {
4719 - apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
4710 + pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
4711 return;
4712 }
4713 }
@@ -4742,7 +4733,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
4733 else {
4734 char msg[PLUGINSD_LINE_MAX];
4735 snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);
4745 - apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
4736 + pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
4737 return;
4738 }
4739 }
@@ -4755,7 +4746,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
4746 unsigned int io_divisor = 1024 * RATES_DETAIL;
4747
4748 BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
4758 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
4749 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
4750 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
4751 buffer_json_member_add_string(wb, "type", "table");
4752 buffer_json_member_add_time_t(wb, "update_every", update_every);
@@ -5232,7 +5223,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
5223 RRDF_FIELD_FILTER_RANGE,
5224 RRDF_FIELD_OPTS_VISIBLE, NULL);
5225 buffer_rrdf_table_add_field(wb, field_id++, "Uptime", "Uptime in seconds", RRDF_FIELD_TYPE_DURATION,
5235 - RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_DURATION, 2,
5226 + RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_DURATION_S, 2,
5227 "seconds", Uptime_max, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_MAX,
5228 RRDF_FIELD_FILTER_RANGE,
5229 RRDF_FIELD_OPTS_VISIBLE, NULL);
@@ -5532,9 +5523,9 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
5523 pluginsd_function_result_end_to_stdout();
5524 }
5525
5535 -bool apps_plugin_exit = false;
5526 +static bool apps_plugin_exit = false;
5527
5537 -void *reader_main(void *arg __maybe_unused) {
5528 +static void *reader_main(void *arg __maybe_unused) {
5529 char buffer[PLUGINSD_LINE_MAX + 1];
5530
5531 char *s = NULL;
@@ -5566,9 +5557,9 @@ void *reader_main(void *arg __maybe_unused) {
5557 netdata_mutex_lock(&mutex);
5558
5559 if(strncmp(function, "processes", strlen("processes")) == 0)
5569 - apps_plugin_function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
5560 + function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
5561 else
5571 - apps_plugin_function_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
5562 + pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
5563
5564 fflush(stdout);
5565 netdata_mutex_unlock(&mutex);
@@ -5696,6 +5687,8 @@ int main(int argc, char **argv) {
5687 netdata_thread_create(&reader_thread, "APPS_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);
5688 netdata_mutex_lock(&mutex);
5689
5690 + APPS_PLUGIN_FUNCTIONS();
5691 +
5692 usec_t step = update_every * USEC_PER_SEC;
5693 global_iterations_counter = 1;
5694 heartbeat_t hb;
collectors/ebpf.plugin/ebpf_functions.c
+1 -1
@@ -206,7 +206,7 @@ static void ebpf_function_thread_manipulation(const char *transaction,
206 time_t expires = now_realtime_sec() + em->update_every;
207
208 BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
209 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
209 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
210 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
211 buffer_json_member_add_string(wb, "type", "table");
212 buffer_json_member_add_time_t(wb, "update_every", em->update_every);
collectors/plugins.d/plugins_d.h
+9 -2
@@ -99,8 +99,6 @@ void pluginsd_process_thread_cleanup(void *ptr);
99
100 size_t pluginsd_initialize_plugin_directories();
101
102 -
103 -
102 #define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \
103 buffer_sprintf(wb \
104 , PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \
@@ -125,4 +123,13 @@ size_t pluginsd_initialize_plugin_directories();
123 #define pluginsd_function_result_end_to_stdout() \
124 fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
125
126 +static inline void pluginsd_function_json_error(const char *transaction, int code, const char *msg) {
127 + char buffer[PLUGINSD_LINE_MAX + 1];
128 + json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
129 +
130 + pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
131 + fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
132 + pluginsd_function_result_end_to_stdout();
133 +}
134 +
135 #endif /* NETDATA_PLUGINS_D_H */
collectors/systemd-journal.plugin/Makefile.am
collectors/systemd-journal.plugin/README.md
collectors/systemd-journal.plugin/systemd-journal.c new
+578
@@ -0,0 +1,578 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +/*
3 + * netdata systemd-journal.plugin
4 + * Copyright (C) 2023 Netdata Inc.
5 + * GPL v3+
6 + */
7 +
8 +// TODO - 1) MARKDOC
9 +
10 +#include "collectors/all.h"
11 +#include "libnetdata/libnetdata.h"
12 +#include "libnetdata/required_dummies.h"
13 +
14 +#include <systemd/sd-journal.h>
15 +#include <syslog.h>
16 +
17 +#define FACET_MAX_VALUE_LENGTH 8192
18 +
19 +#define SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
20 +#define SYSTEMD_JOURNAL_FUNCTION_NAME "systemd-journal"
21 +#define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 30
22 +#define SYSTEMD_JOURNAL_MAX_PARAMS 100
23 +#define SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION (3 * 3600)
24 +#define SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY 200
25 +
26 +#define JOURNAL_PARAMETER_HELP "help"
27 +#define JOURNAL_PARAMETER_AFTER "after"
28 +#define JOURNAL_PARAMETER_BEFORE "before"
29 +#define JOURNAL_PARAMETER_ANCHOR "anchor"
30 +#define JOURNAL_PARAMETER_LAST "last"
31 +#define JOURNAL_PARAMETER_QUERY "query"
32 +
33 +#define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL
34 +#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS NULL
35 +#define SYSTEMD_KEYS_INCLUDED_IN_FACETS \
36 + "_TRANSPORT" \
37 + "|SYSLOG_IDENTIFIER" \
38 + "|SYSLOG_FACILITY" \
39 + "|PRIORITY" \
40 + "|_HOSTNAME" \
41 + "|_RUNTIME_SCOPE" \
42 + "|_PID" \
43 + "|_UID" \
44 + "|_GID" \
45 + "|_SYSTEMD_UNIT" \
46 + "|_SYSTEMD_SLICE" \
47 + "|_SYSTEMD_USER_SLICE" \
48 + "|_COMM" \
49 + "|_EXE" \
50 + "|_SYSTEMD_CGROUP" \
51 + "|_SYSTEMD_USER_UNIT" \
52 + "|USER_UNIT" \
53 + "|UNIT" \
54 + ""
55 +
56 +static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
57 +static bool plugin_should_exit = false;
58 +
59 +DICTIONARY *uids = NULL;
60 +DICTIONARY *gids = NULL;
61 +
62 +
63 +// ----------------------------------------------------------------------------
64 +
65 +int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t stop_monotonic_ut) {
66 + sd_journal *j;
67 + int r;
68 +
69 + // Open the system journal for reading
70 + r = sd_journal_open(&j, SD_JOURNAL_ALL_NAMESPACES);
71 + if (r < 0)
72 + return HTTP_RESP_INTERNAL_SERVER_ERROR;
73 +
74 + facets_rows_begin(facets);
75 +
76 + bool timed_out = false;
77 + size_t row_counter = 0;
78 + sd_journal_seek_realtime_usec(j, before_ut);
79 + SD_JOURNAL_FOREACH_BACKWARDS(j) {
80 + row_counter++;
81 +
82 + uint64_t msg_ut;
83 + sd_journal_get_realtime_usec(j, &msg_ut);
84 + if (msg_ut < after_ut)
85 + break;
86 +
87 + const void *data;
88 + size_t length;
89 + SD_JOURNAL_FOREACH_DATA(j, data, length) {
90 + const char *key = data;
91 + const char *equal = strchr(key, '=');
92 + if(unlikely(!equal))
93 + continue;
94 +
95 + const char *value = ++equal;
96 + size_t key_length = value - key; // including '\0'
97 +
98 + char key_copy[key_length];
99 + memcpy(key_copy, key, key_length - 1);
100 + key_copy[key_length - 1] = '\0';
101 +
102 + size_t value_length = length - key_length; // without '\0'
103 + facets_add_key_value_length(facets, key_copy, value, value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH);
104 + }
105 +
106 + facets_row_finished(facets, msg_ut);
107 +
108 + if((row_counter % 100) == 0 && now_monotonic_usec() > stop_monotonic_ut) {
109 + timed_out = true;
110 + break;
111 + }
112 + }
113 +
114 + sd_journal_close(j);
115 +
116 + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
117 + buffer_json_member_add_boolean(wb, "partial", timed_out);
118 + buffer_json_member_add_string(wb, "type", "table");
119 + buffer_json_member_add_time_t(wb, "update_every", 1);
120 + buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
121 +
122 + facets_report(facets, wb);
123 +
124 + buffer_json_member_add_time_t(wb, "expires", now_realtime_sec());
125 + buffer_json_finalize(wb);
126 +
127 + return HTTP_RESP_OK;
128 +}
129 +
130 +static void systemd_journal_function_help(const char *transaction) {
131 + pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);
132 + fprintf(stdout,
133 + "%s / %s\n"
134 + "\n"
135 + "%s\n"
136 + "\n"
137 + "The following filters are supported:\n"
138 + "\n"
139 + " help\n"
140 + " Shows this help message.\n"
141 + "\n"
142 + " before:TIMESTAMP\n"
143 + " Absolute or relative (to now) timestamp in seconds, to start the query.\n"
144 + " The query is always executed from the most recent to the oldest log entry.\n"
145 + " If not given the default is: now.\n"
146 + "\n"
147 + " after:TIMESTAMP\n"
148 + " Absolute or relative (to `before`) timestamp in seconds, to end the query.\n"
149 + " If not given, the default is %d.\n"
150 + "\n"
151 + " last:ITEMS\n"
152 + " The number of items to return.\n"
153 + " The default is %d.\n"
154 + "\n"
155 + " anchor:NUMBER\n"
156 + " The `timestamp` of the item last received, to return log entries after that.\n"
157 + " If not given, the query will return the top `ITEMS` from the most recent.\n"
158 + "\n"
159 + " facet_id:value_id1,value_id2,value_id3,...\n"
160 + " Apply filters to the query, based on the facet IDs returned.\n"
161 + " Each `facet_id` can be given once, but multiple `facet_ids` can be given.\n"
162 + "\n"
163 + "Filters can be combined. Each filter can be given only one time.\n"
164 + , program_name
165 + , SYSTEMD_JOURNAL_FUNCTION_NAME
166 + , SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION
167 + , -SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION
168 + , SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY
169 + );
170 + pluginsd_function_result_end_to_stdout();
171 +}
172 +
173 +static const char *syslog_facility_to_name(int facility) {
174 + switch (facility) {
175 + case LOG_FAC(LOG_KERN): return "kern";
176 + case LOG_FAC(LOG_USER): return "user";
177 + case LOG_FAC(LOG_MAIL): return "mail";
178 + case LOG_FAC(LOG_DAEMON): return "daemon";
179 + case LOG_FAC(LOG_AUTH): return "auth";
180 + case LOG_FAC(LOG_SYSLOG): return "syslog";
181 + case LOG_FAC(LOG_LPR): return "lpr";
182 + case LOG_FAC(LOG_NEWS): return "news";
183 + case LOG_FAC(LOG_UUCP): return "uucp";
184 + case LOG_FAC(LOG_CRON): return "cron";
185 + case LOG_FAC(LOG_AUTHPRIV): return "authpriv";
186 + case LOG_FAC(LOG_FTP): return "ftp";
187 + case LOG_FAC(LOG_LOCAL0): return "local0";
188 + case LOG_FAC(LOG_LOCAL1): return "local1";
189 + case LOG_FAC(LOG_LOCAL2): return "local2";
190 + case LOG_FAC(LOG_LOCAL3): return "local3";
191 + case LOG_FAC(LOG_LOCAL4): return "local4";
192 + case LOG_FAC(LOG_LOCAL5): return "local5";
193 + case LOG_FAC(LOG_LOCAL6): return "local6";
194 + case LOG_FAC(LOG_LOCAL7): return "local7";
195 + default: return NULL;
196 + }
197 +}
198 +
199 +static const char *syslog_priority_to_name(int priority) {
200 + switch (priority) {
201 + case LOG_ALERT: return "alert";
202 + case LOG_CRIT: return "critical";
203 + case LOG_DEBUG: return "debug";
204 + case LOG_EMERG: return "panic";
205 + case LOG_ERR: return "error";
206 + case LOG_INFO: return "info";
207 + case LOG_NOTICE: return "notice";
208 + case LOG_WARNING: return "warning";
209 + default: return NULL;
210 + }
211 +}
212 +
213 +static char *uid_to_username(uid_t uid, char *buffer, size_t buffer_size) {
214 + struct passwd pw, *result;
215 + char tmp[1024 + 1];
216 +
217 + if (getpwuid_r(uid, &pw, tmp, 1024, &result) != 0 || result == NULL)
218 + return NULL;
219 +
220 + strncpy(buffer, pw.pw_name, buffer_size - 1);
221 + buffer[buffer_size - 1] = '\0'; // Null-terminate just in case
222 + return buffer;
223 +}
224 +
225 +static char *gid_to_groupname(gid_t gid, char* buffer, size_t buffer_size) {
226 + struct group grp, *result;
227 + char tmp[1024 + 1];
228 +
229 + if (getgrgid_r(gid, &grp, tmp, 1024, &result) != 0 || result == NULL)
230 + return NULL;
231 +
232 + strncpy(buffer, grp.gr_name, buffer_size - 1);
233 + buffer[buffer_size - 1] = '\0'; // Null-terminate just in case
234 + return buffer;
235 +}
236 +
237 +static void systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {
238 + const char *v = buffer_tostring(wb);
239 + if(*v && isdigit(*v)) {
240 + int facility = str2i(buffer_tostring(wb));
241 + const char *name = syslog_facility_to_name(facility);
242 + if (name) {
243 + buffer_flush(wb);
244 + buffer_json_add_array_item_string(wb, name);
245 + }
246 + }
247 +}
248 +
249 +static void systemd_journal_transform_priority(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {
250 + const char *v = buffer_tostring(wb);
251 + if(*v && isdigit(*v)) {
252 + int priority = str2i(buffer_tostring(wb));
253 + const char *name = syslog_priority_to_name(priority);
254 + if (name) {
255 + buffer_flush(wb);
256 + buffer_json_add_array_item_string(wb, name);
257 + }
258 + }
259 +}
260 +
261 +static void systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {
262 + DICTIONARY *cache = data;
263 + const char *v = buffer_tostring(wb);
264 + if(*v && isdigit(*v)) {
265 + const char *sv = dictionary_get(cache, v);
266 + if(!sv) {
267 + char buf[1024 + 1];
268 + int uid = str2i(buffer_tostring(wb));
269 + const char *name = uid_to_username(uid, buf, 1024);
270 + if (!name)
271 + name = v;
272 +
273 + sv = dictionary_set(cache, v, (void *)name, strlen(name) + 1);
274 + }
275 +
276 + buffer_flush(wb);
277 + buffer_strcat(wb, sv);
278 + }
279 +}
280 +
281 +static void systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {
282 + DICTIONARY *cache = data;
283 + const char *v = buffer_tostring(wb);
284 + if(*v && isdigit(*v)) {
285 + const char *sv = dictionary_get(cache, v);
286 + if(!sv) {
287 + char buf[1024 + 1];
288 + int gid = str2i(buffer_tostring(wb));
289 + const char *name = gid_to_groupname(gid, buf, 1024);
290 + if (!name)
291 + name = v;
292 +
293 + sv = dictionary_set(cache, v, (void *)name, strlen(name) + 1);
294 + }
295 +
296 + buffer_flush(wb);
297 + buffer_strcat(wb, sv);
298 + }
299 +}
300 +
301 +static void systemd_journal_dynamic_row_id(FACETS *facets __maybe_unused, BUFFER *wb, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data __maybe_unused) {
302 + FACET_ROW_KEY_VALUE *syslog_identifier_rkv = dictionary_get(row->dict, "SYSLOG_IDENTIFIER");
303 + FACET_ROW_KEY_VALUE *pid_rkv = dictionary_get(row->dict, "_PID");
304 +
305 + const char *identifier = syslog_identifier_rkv ? buffer_tostring(syslog_identifier_rkv->wb) : "UNKNOWN";
306 + const char *pid = pid_rkv ? buffer_tostring(pid_rkv->wb) : "UNKNOWN";
307 +
308 + buffer_flush(rkv->wb);
309 + buffer_sprintf(rkv->wb, "%s[%s]", identifier, pid);
310 +
311 + buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb));
312 +}
313 +
314 +static void function_systemd_journal(const char *transaction, char *function, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
315 + char *words[SYSTEMD_JOURNAL_MAX_PARAMS] = { NULL };
316 + size_t num_words = quoted_strings_splitter_pluginsd(function, words, SYSTEMD_JOURNAL_MAX_PARAMS);
317 +
318 + BUFFER *wb = buffer_create(0, NULL);
319 + buffer_flush(wb);
320 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
321 +
322 + FACETS *facets = facets_create(50, 0, FACETS_OPTION_ALL_KEYS_FTS,
323 + SYSTEMD_ALWAYS_VISIBLE_KEYS,
324 + SYSTEMD_KEYS_INCLUDED_IN_FACETS,
325 + SYSTEMD_KEYS_EXCLUDED_FROM_FACETS);
326 +
327 + facets_accepted_param(facets, JOURNAL_PARAMETER_AFTER);
328 + facets_accepted_param(facets, JOURNAL_PARAMETER_BEFORE);
329 + facets_accepted_param(facets, JOURNAL_PARAMETER_ANCHOR);
330 + facets_accepted_param(facets, JOURNAL_PARAMETER_LAST);
331 + facets_accepted_param(facets, JOURNAL_PARAMETER_QUERY);
332 +
333 + // register the fields in the order you want them on the dashboard
334 +
335 + facets_register_dynamic_key(facets, "ND_JOURNAL_PROCESS", FACET_KEY_OPTION_NO_FACET|FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_FTS,
336 + systemd_journal_dynamic_row_id, NULL);
337 +
338 + facets_register_key(facets, "MESSAGE",
339 + FACET_KEY_OPTION_NO_FACET|FACET_KEY_OPTION_MAIN_TEXT|FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_FTS);
340 +
341 + facets_register_key_transformation(facets, "PRIORITY", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
342 + systemd_journal_transform_priority, NULL);
343 +
344 + facets_register_key_transformation(facets, "SYSLOG_FACILITY", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
345 + systemd_journal_transform_syslog_facility, NULL);
346 +
347 + facets_register_key(facets, "SYSLOG_IDENTIFIER", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
348 + facets_register_key(facets, "UNIT", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
349 + facets_register_key(facets, "USER_UNIT", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
350 +
351 + facets_register_key_transformation(facets, "_UID", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
352 + systemd_journal_transform_uid, uids);
353 +
354 + facets_register_key_transformation(facets, "_GID", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
355 + systemd_journal_transform_gid, gids);
356 +
357 + time_t after_s = 0, before_s = 0;
358 + usec_t anchor = 0;
359 + size_t last = 0;
360 + const char *query = NULL;
361 +
362 + buffer_json_member_add_object(wb, "request");
363 + buffer_json_member_add_object(wb, "filters");
364 +
365 + for(int i = 1; i < SYSTEMD_JOURNAL_MAX_PARAMS ;i++) {
366 + const char *keyword = get_word(words, num_words, i);
367 + if(!keyword) break;
368 +
369 + if(strcmp(keyword, JOURNAL_PARAMETER_HELP) == 0) {
370 + systemd_journal_function_help(transaction);
371 + goto cleanup;
372 + }
373 + else if(strncmp(keyword, JOURNAL_PARAMETER_AFTER ":", strlen(JOURNAL_PARAMETER_AFTER ":")) == 0) {
374 + after_s = str2l(&keyword[strlen(JOURNAL_PARAMETER_AFTER ":")]);
375 + }
376 + else if(strncmp(keyword, JOURNAL_PARAMETER_BEFORE ":", strlen(JOURNAL_PARAMETER_BEFORE ":")) == 0) {
377 + before_s = str2l(&keyword[strlen(JOURNAL_PARAMETER_BEFORE ":")]);
378 + }
379 + else if(strncmp(keyword, JOURNAL_PARAMETER_ANCHOR ":", strlen(JOURNAL_PARAMETER_ANCHOR ":")) == 0) {
380 + anchor = str2ull(&keyword[strlen(JOURNAL_PARAMETER_ANCHOR ":")], NULL);
381 + }
382 + else if(strncmp(keyword, JOURNAL_PARAMETER_LAST ":", strlen(JOURNAL_PARAMETER_LAST ":")) == 0) {
383 + last = str2ul(&keyword[strlen(JOURNAL_PARAMETER_LAST ":")]);
384 + }
385 + else if(strncmp(keyword, JOURNAL_PARAMETER_QUERY ":", strlen(JOURNAL_PARAMETER_QUERY ":")) == 0) {
386 + query= &keyword[strlen(JOURNAL_PARAMETER_QUERY ":")];
387 + }
388 + else {
389 + char *value = strchr(keyword, ':');
390 + if(value) {
391 + *value++ = '\0';
392 +
393 + buffer_json_member_add_array(wb, keyword);
394 +
395 + while(value) {
396 + char *sep = strchr(value, ',');
397 + if(sep)
398 + *sep++ = '\0';
399 +
400 + facets_register_facet_filter(facets, keyword, value, FACET_KEY_OPTION_REORDER);
401 + buffer_json_add_array_item_string(wb, value);
402 +
403 + value = sep;
404 + }
405 +
406 + buffer_json_array_close(wb); // keyword
407 + }
408 + }
409 + }
410 +
411 + buffer_json_object_close(wb); // filters
412 +
413 + time_t expires = now_realtime_sec() + 1;
414 + time_t now_s;
415 +
416 + if(!after_s && !before_s) {
417 + now_s = now_realtime_sec();
418 + before_s = now_s;
419 + after_s = before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
420 + }
421 + else
422 + rrdr_relative_window_to_absolute(&after_s, &before_s, &now_s, false);
423 +
424 + if(after_s > before_s) {
425 + time_t tmp = after_s;
426 + after_s = before_s;
427 + before_s = tmp;
428 + }
429 +
430 + if(after_s == before_s)
431 + after_s = before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
432 +
433 + if(!last)
434 + last = SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY;
435 +
436 + buffer_json_member_add_time_t(wb, "after", after_s);
437 + buffer_json_member_add_time_t(wb, "before", before_s);
438 + buffer_json_member_add_uint64(wb, "anchor", anchor);
439 + buffer_json_member_add_uint64(wb, "last", last);
440 + buffer_json_member_add_string(wb, "query", query);
441 + buffer_json_member_add_time_t(wb, "timeout", timeout);
442 + buffer_json_object_close(wb); // request
443 +
444 + facets_set_items(facets, last);
445 + facets_set_anchor(facets, anchor);
446 + facets_set_query(facets, query);
447 + int response = systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,
448 + now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);
449 +
450 + if(response != HTTP_RESP_OK) {
451 + pluginsd_function_json_error(transaction, response, "failed");
452 + goto cleanup;
453 + }
454 +
455 + pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);
456 + fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
457 +
458 + pluginsd_function_result_end_to_stdout();
459 +
460 +cleanup:
461 + facets_destroy(facets);
462 + buffer_free(wb);
463 +}
464 +
465 +static void *reader_main(void *arg __maybe_unused) {
466 + char buffer[PLUGINSD_LINE_MAX + 1];
467 +
468 + char *s = NULL;
469 + while(!plugin_should_exit && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) {
470 +
471 + char *words[PLUGINSD_MAX_WORDS] = { NULL };
472 + size_t num_words = quoted_strings_splitter_pluginsd(buffer, words, PLUGINSD_MAX_WORDS);
473 +
474 + const char *keyword = get_word(words, num_words, 0);
475 +
476 + if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) {
477 + char *transaction = get_word(words, num_words, 1);
478 + char *timeout_s = get_word(words, num_words, 2);
479 + char *function = get_word(words, num_words, 3);
480 +
481 + if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
482 + netdata_log_error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
483 + keyword,
484 + transaction?transaction:"(unset)",
485 + timeout_s?timeout_s:"(unset)",
486 + function?function:"(unset)");
487 + }
488 + else {
489 + int timeout = str2i(timeout_s);
490 + if(timeout <= 0) timeout = SYSTEMD_JOURNAL_DEFAULT_TIMEOUT;
491 +
492 + netdata_mutex_lock(&mutex);
493 +
494 + if(strncmp(function, SYSTEMD_JOURNAL_FUNCTION_NAME, strlen(SYSTEMD_JOURNAL_FUNCTION_NAME)) == 0)
495 + function_systemd_journal(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
496 + else
497 + pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in systemd-journal.plugin.");
498 +
499 + fflush(stdout);
500 + netdata_mutex_unlock(&mutex);
501 + }
502 + }
503 + else
504 + netdata_log_error("Received unknown command: %s", keyword?keyword:"(unset)");
505 + }
506 +
507 + if(!s || feof(stdin) || ferror(stdin)) {
508 + plugin_should_exit = true;
509 + netdata_log_error("Received error on stdin.");
510 + }
511 +
512 + exit(1);
513 +}
514 +
515 +int main(int argc __maybe_unused, char **argv __maybe_unused) {
516 + stderror = stderr;
517 + clocks_init();
518 +
519 + program_name = "systemd-journal.plugin";
520 +
521 + // disable syslog
522 + error_log_syslog = 0;
523 +
524 + // set errors flood protection to 100 logs per hour
525 + error_log_errors_per_period = 100;
526 + error_log_throttle_period = 3600;
527 +
528 + uids = dictionary_create(0);
529 + gids = dictionary_create(0);
530 +
531 + // ------------------------------------------------------------------------
532 + // debug
533 +
534 + if(argc == 2 && strcmp(argv[1], "debug") == 0) {
535 + char buf[] = "systemd-journal after:-86400 before:0 last:500";
536 + function_systemd_journal("123", buf, "", 0, 30);
537 + exit(1);
538 + }
539 +
540 + // ------------------------------------------------------------------------
541 +
542 + netdata_thread_t reader_thread;
543 + netdata_thread_create(&reader_thread, "SDJ_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);
544 +
545 + // ------------------------------------------------------------------------
546 +
547 + time_t started_t = now_monotonic_sec();
548 +
549 + size_t iteration;
550 + usec_t step = 1000 * USEC_PER_MS;
551 + bool tty = isatty(fileno(stderr)) == 1;
552 +
553 + netdata_mutex_lock(&mutex);
554 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\"\n",
555 + SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
556 +
557 + heartbeat_t hb;
558 + heartbeat_init(&hb);
559 + for(iteration = 0; 1 ; iteration++) {
560 + netdata_mutex_unlock(&mutex);
561 + heartbeat_next(&hb, step);
562 + netdata_mutex_lock(&mutex);
563 +
564 + if(!tty)
565 + fprintf(stdout, "\n");
566 +
567 + fflush(stdout);
568 +
569 + time_t now = now_monotonic_sec();
570 + if(now - started_t > 86400)
571 + break;
572 + }
573 +
574 + dictionary_destroy(uids);
575 + dictionary_destroy(gids);
576 +
577 + exit(0);
578 +}
configure.ac
+42
@@ -69,6 +69,12 @@ AC_ARG_ENABLE(
69 ,
70 [enable_plugin_freeipmi="detect"]
71 )
72 +AC_ARG_ENABLE(
73 + [plugin-systemd-journal],
74 + [AS_HELP_STRING([--enable-plugin-systemd-journal], [enable systemd-journal plugin @<:@default autodetect@:>@])],
75 + ,
76 + [enable_plugin_systemd_journal="detect"]
77 +)
78 AC_ARG_ENABLE(
79 [plugin-cups],
80 [AS_HELP_STRING([--enable-plugin-cups], [enable cups plugin @<:@default autodetect@:>@])],
@@ -1106,6 +1112,39 @@ AC_MSG_RESULT([${enable_plugin_freeipmi}])
1112 AM_CONDITIONAL([ENABLE_PLUGIN_FREEIPMI], [test "${enable_plugin_freeipmi}" = "yes"])
1113
1114
1115 +# -----------------------------------------------------------------------------
1116 +# systemd-journal.plugin - systemd
1117 +
1118 +LIBS_BAK="${LIBS}"
1119 +
1120 +AC_CHECK_LIB([systemd], [sd_journal_open], [have_systemd_libs=yes], [have_systemd_libs=no])
1121 +AC_CHECK_HEADERS([systemd/sd-journal.h], [have_systemd_journal_header=yes], [have_systemd_journal_header=no])
1122 +
1123 +if test "${have_systemd_libs}" = "yes" -a "${have_systemd_journal_header}" = "yes"; then
1124 + have_systemd="yes"
1125 +else
1126 + have_systemd="no"
1127 +fi
1128 +
1129 +test "${enable_plugin_systemd_journal}" = "yes" -a "${have_systemd}" != "yes" && \
1130 + AC_MSG_ERROR([systemd is required but not found. Try installing 'libsystemd-dev' or 'libsystemd-devel'])
1131 +
1132 +AC_MSG_CHECKING([if systemd-journal.plugin should be enabled])
1133 +if test "${enable_plugin_systemd_journal}" != "no" -a "${have_systemd}" = "yes"; then
1134 + enable_plugin_systemd_journal="yes"
1135 + AC_DEFINE([HAVE_SYSTEMD], [1], [systemd usability])
1136 + OPTIONAL_SYSTEMD_CFLAGS="-I/usr/include"
1137 + OPTIONAL_SYSTEMD_LIBS="-lsystemd"
1138 +else
1139 + enable_plugin_systemd_journal="no"
1140 +fi
1141 +AC_MSG_RESULT([${enable_plugin_systemd_journal}])
1142 +AM_CONDITIONAL([ENABLE_PLUGIN_SYSTEMD_JOURNAL], [test "${enable_plugin_systemd_journal}" = "yes"])
1143 +
1144 +AC_MSG_NOTICE([OPTIONAL_SYSTEMD_LIBS is set to: ${OPTIONAL_SYSTEMD_LIBS}])
1145 +
1146 +LIBS="${LIBS_BAK}"
1147 +
1148 # -----------------------------------------------------------------------------
1149 # cups.plugin - libcups
1150
@@ -1874,6 +1913,7 @@ AC_SUBST([OPTIONAL_GTEST_CFLAGS])
1913 AC_SUBST([OPTIONAL_GTEST_LIBS])
1914 AC_SUBST([OPTIONAL_ML_CFLAGS])
1915 AC_SUBST([OPTIONAL_ML_LIBS])
1916 +AC_SUBST(OPTIONAL_SYSTEMD_LIBS)
1917
1918 # -----------------------------------------------------------------------------
1919 # Check if cmocka is available - needed for unit testing
@@ -1937,6 +1977,7 @@ AC_CONFIG_FILES([
1977 collectors/tc.plugin/Makefile
1978 collectors/xenstat.plugin/Makefile
1979 collectors/perf.plugin/Makefile
1980 + collectors/systemd-journal.plugin/Makefile
1981 daemon/Makefile
1982 database/Makefile
1983 database/contexts/Makefile
@@ -1968,6 +2009,7 @@ AC_CONFIG_FILES([
2009 libnetdata/dictionary/Makefile
2010 libnetdata/ebpf/Makefile
2011 libnetdata/eval/Makefile
2012 + libnetdata/facets/Makefile
2013 libnetdata/july/Makefile
2014 libnetdata/locks/Makefile
2015 libnetdata/log/Makefile
contrib/debian/rules
+3
@@ -213,6 +213,9 @@ override_dh_fixperms:
213 # local-listeners
214 chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/local-listeners
215
216 + # systemd-journal
217 + # chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/systemd-journal.plugin
218 +
219 override_dh_installlogrotate:
220 cp system/logrotate/netdata debian/netdata.logrotate
221 dh_installlogrotate
daemon/buildinfo.c
+1 -1
@@ -1469,7 +1469,7 @@ void print_build_info_json(void) {
1469 populate_directories();
1470
1471 BUFFER *b = buffer_create(0, NULL);
1472 - buffer_json_initialize(b, "\"", "\"", 0, true, false);
1472 + buffer_json_initialize(b, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
1473
1474 build_info_to_json_object(b);
1475
daemon/main.c
+1
@@ -1332,6 +1332,7 @@ int mrg_unittest(void);
1332 int julytest(void);
1333 int pluginsd_parser_unittest(void);
1334 void replication_initialize(void);
1335 +void bearer_tokens_init(void);
1336
1337 int main(int argc, char **argv) {
1338 // initialize the system clocks
database/contexts/api_v1.c
+6 -6
@@ -148,7 +148,7 @@ static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void
148 if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) {
149
150 wb_metrics = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
151 - buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, false);
151 + buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
152
153 struct rrdcontext_to_json t_metrics = {
154 .wb = wb_metrics,
@@ -268,7 +268,7 @@ static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void
268 || t_parent->chart_dimensions) {
269
270 wb_instances = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
271 - buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, false);
271 + buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
272
273 struct rrdcontext_to_json t_instances = {
274 .wb = wb_instances,
@@ -366,9 +366,9 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
366 RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
367
368 if(after != 0 && before != 0)
369 - rrdr_relative_window_to_absolute(&after, &before, NULL);
369 + rrdr_relative_window_to_absolute(&after, &before, NULL, false);
370
371 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
371 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
372 struct rrdcontext_to_json t_contexts = {
373 .wb = wb,
374 .options = options|RRDCONTEXT_OPTION_SKIP_ID,
@@ -403,9 +403,9 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
403 uuid_unparse(*host->node_id, node_uuid);
404
405 if(after != 0 && before != 0)
406 - rrdr_relative_window_to_absolute(&after, &before, NULL);
406 + rrdr_relative_window_to_absolute(&after, &before, NULL, false);
407
408 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
408 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
409 buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
410 buffer_json_member_add_string(wb, "machine_guid", host->machine_guid);
411 buffer_json_member_add_string(wb, "node_id", node_uuid);
database/contexts/api_v2.c
+4 -4
@@ -1298,7 +1298,7 @@ int contexts_v2_alert_config_to_json(struct web_client *w, const char *config_ha
1298
1299 buffer_flush(w->response.data);
1300
1301 - buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
1301 + buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
1302
1303 int added = sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, false);
1304 buffer_json_finalize(w->response.data);
@@ -1934,14 +1934,14 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
1934 }
1935
1936 if(req->after || req->before) {
1937 - ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now);
1937 + ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now, false);
1938 ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS);
1939 }
1940 else
1941 ctl.now = now_realtime_sec();
1942
1943 - buffer_json_initialize(wb, "\"", "\"", 0,
1944 - true, (req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG));
1943 + buffer_json_initialize(wb, "\"", "\"", 0, true,
1944 + ((req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
1945
1946 buffer_json_member_add_uint64(wb, "api", 2);
1947
database/contexts/query_target.c
+2 -1
@@ -1052,7 +1052,8 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
1052 if(query_target_has_percentage_of_group(qt))
1053 qt->window.options &= ~RRDR_OPTION_PERCENTAGE;
1054
1055 - qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now);
1055 + qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now,
1056 + unittest_running);
1057
1058 // prepare our local variables - we need these across all these functions
1059 QUERY_TARGET_LOCALS qtl = {
database/rrdfunctions.c
+16 -16
@@ -270,7 +270,7 @@ static inline size_t sanitize_function_text(char *dst, const char *src, size_t d
270 // we keep a dictionary per RRDSET with these functions
271 // the dictionary is created on demand (only when a function is added to an RRDSET)
272
273 -typedef enum {
273 +typedef enum __attribute__((packed)) {
274 RRD_FUNCTION_LOCAL = (1 << 0),
275 RRD_FUNCTION_GLOBAL = (1 << 1),
276
@@ -279,7 +279,7 @@ typedef enum {
279
280 struct rrd_collector_function {
281 bool sync; // when true, the function is called synchronously
282 - uint8_t options; // RRD_FUNCTION_OPTIONS
282 + RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS
283 STRING *help;
284 int timeout; // the default timeout of the function
285
@@ -814,7 +814,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
814
815 buffer_flush(wb);
816 wb->content_type = CT_APPLICATION_JSON;
817 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
817 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
818
819 buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
820 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
@@ -858,8 +858,8 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
858
859 // retention
860 buffer_json_add_array_item_string(wb, rrdhost_hostname(s.host)); // Node
861 - buffer_json_add_array_item_uint64(wb, s.db.first_time_s * 1000); // dbFrom
862 - buffer_json_add_array_item_uint64(wb, s.db.last_time_s * 1000); // dbTo
861 + buffer_json_add_array_item_uint64(wb, s.db.first_time_s * MSEC_PER_SEC); // dbFrom
862 + buffer_json_add_array_item_uint64(wb, s.db.last_time_s * MSEC_PER_SEC); // dbTo
863
864 if(s.db.first_time_s && s.db.last_time_s && s.db.last_time_s > s.db.first_time_s)
865 buffer_json_add_array_item_uint64(wb, s.db.last_time_s - s.db.first_time_s); // dbDuration
@@ -877,7 +877,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
877
878 // collection
879 if(s.ingest.since) {
880 - buffer_json_add_array_item_uint64(wb, s.ingest.since * 1000); // InSince
880 + buffer_json_add_array_item_uint64(wb, s.ingest.since * MSEC_PER_SEC); // InSince
881 buffer_json_add_array_item_time_t(wb, s.now - s.ingest.since); // InAge
882 }
883 else {
@@ -897,7 +897,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
897
898 // streaming
899 if(s.stream.since) {
900 - buffer_json_add_array_item_uint64(wb, s.stream.since * 1000); // OutSince
900 + buffer_json_add_array_item_uint64(wb, s.stream.since * MSEC_PER_SEC); // OutSince
901 buffer_json_add_array_item_time_t(wb, s.now - s.stream.since); // OutAge
902 }
903 else {
@@ -990,19 +990,19 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
990 NULL);
991
992 buffer_rrdf_table_add_field(wb, field_id++, "dbFrom", "DB Data Retention From",
993 - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
993 + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
994 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
995 RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
996 RRDF_FIELD_OPTS_VISIBLE, NULL);
997
998 buffer_rrdf_table_add_field(wb, field_id++, "dbTo", "DB Data Retention To",
999 - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
999 + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1000 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1001 RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1002 RRDF_FIELD_OPTS_VISIBLE, NULL);
1003
1004 buffer_rrdf_table_add_field(wb, field_id++, "dbDuration", "DB Data Retention Duration",
1005 - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
1005 + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1006 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1007 RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1008 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1049,13 +1049,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1049 // --- collection ---
1050
1051 buffer_rrdf_table_add_field(wb, field_id++, "InSince", "Last Data Collection Status Change",
1052 - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
1052 + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1053 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
1054 RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1055 RRDF_FIELD_OPTS_NONE, NULL);
1056
1057 buffer_rrdf_table_add_field(wb, field_id++, "InAge", "Last Data Collection Online Status Change Age",
1058 - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
1058 + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1059 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1060 RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1061 RRDF_FIELD_OPTS_VISIBLE, NULL);
@@ -1124,13 +1124,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1124 // --- streaming ---
1125
1126 buffer_rrdf_table_add_field(wb, field_id++, "OutSince", "Last Streaming Status Change",
1127 - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
1127 + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1128 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
1129 RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1130 RRDF_FIELD_OPTS_NONE, NULL);
1131
1132 buffer_rrdf_table_add_field(wb, field_id++, "OutAge", "Last Streaming Status Change Age",
1133 - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
1133 + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1134 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1135 RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1136 RRDF_FIELD_OPTS_VISIBLE, NULL);
@@ -1242,14 +1242,14 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1242
1243 buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptSince",
1244 "Last Outbound Connection Attempt Status Change Time",
1245 - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
1245 + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1246 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
1247 RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1248 RRDF_FIELD_OPTS_NONE, NULL);
1249
1250 buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptAge",
1251 "Last Outbound Connection Attempt Status Change Age",
1252 - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
1252 + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1253 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1254 RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1255 RRDF_FIELD_OPTS_VISIBLE, NULL);
libnetdata/Makefile.am
+1
@@ -14,6 +14,7 @@ SUBDIRS = \
14 dictionary \
15 ebpf \
16 eval \
17 + facets \
18 json \
19 july \
20 health \
libnetdata/buffer/buffer.c
+5 -5
@@ -307,11 +307,11 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
307 // ----------------------------------------------------------------------------
308
309 void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
310 - bool add_anonymous_object, bool minify) {
310 + bool add_anonymous_object, BUFFER_JSON_OPTIONS options) {
311 strncpyz(wb->json.key_quote, key_quote, BUFFER_QUOTE_MAX_SIZE);
312 strncpyz(wb->json.value_quote, value_quote, BUFFER_QUOTE_MAX_SIZE);
313
314 - wb->json.minify = minify;
314 + wb->json.options = options;
315 wb->json.depth = (int8_t)(depth - 1);
316 _buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
317
@@ -339,7 +339,7 @@ void buffer_json_finalize(BUFFER *wb) {
339 }
340 }
341
342 - if(!wb->json.minify)
342 + if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY))
343 buffer_fast_strcat(wb, "\n", 1);
344 }
345
@@ -490,13 +490,13 @@ int buffer_unittest(void) {
490
491 buffer_flush(wb);
492
493 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
493 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
494 buffer_json_finalize(wb);
495 errors += buffer_expect(wb, "{\n}\n");
496
497 buffer_flush(wb);
498
499 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
499 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
500 buffer_json_member_add_string(wb, "hello", "world");
501 buffer_json_member_add_string(wb, "alpha", "this: \" is a double quote");
502 buffer_json_member_add_object(wb, "object1");
libnetdata/buffer/buffer.h
+52 -27
@@ -68,6 +68,12 @@ typedef enum __attribute__ ((__packed__)) {
68 CT_APPLICATION_ZIP,
69 } HTTP_CONTENT_TYPE;
70
71 +typedef enum __attribute__ ((__packed__)) {
72 + BUFFER_JSON_OPTIONS_DEFAULT = 0,
73 + BUFFER_JSON_OPTIONS_MINIFY = (1 << 0),
74 + BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS = (1 << 1),
75 +} BUFFER_JSON_OPTIONS;
76 +
77 typedef struct web_buffer {
78 size_t size; // allocation size of buffer, in bytes
79 size_t len; // current data length in buffer, in bytes
@@ -82,7 +88,7 @@ typedef struct web_buffer {
88 char key_quote[BUFFER_QUOTE_MAX_SIZE + 1];
89 char value_quote[BUFFER_QUOTE_MAX_SIZE + 1];
90 int8_t depth;
85 - bool minify;
91 + BUFFER_JSON_OPTIONS options;
92 BUFFER_JSON_NODE stack[BUFFER_JSON_MAX_DEPTH];
93 } json;
94 } BUFFER;
@@ -148,7 +154,7 @@ static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
154 }
155
156 void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
151 - bool add_anonymous_object, bool minify);
157 + bool add_anonymous_object, BUFFER_JSON_OPTIONS options);
158
159 void buffer_json_finalize(BUFFER *wb);
160
@@ -668,7 +674,8 @@ static inline void buffer_print_json_comma_newline_spacing(BUFFER *wb) {
674 if(wb->json.stack[wb->json.depth].count)
675 buffer_fast_strcat(wb, ",", 1);
676
671 - if(wb->json.minify)
677 + if((wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) ||
678 + (wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && !(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS)))
679 return;
680
681 buffer_fast_strcat(wb, "\n", 1);
@@ -715,7 +722,7 @@ static inline void buffer_json_object_close(BUFFER *wb) {
722 assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
723 assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_OBJECT && "BUFFER JSON: an object is not open to close it");
724 #endif
718 - if(!wb->json.minify) {
725 + if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY)) {
726 buffer_fast_strcat(wb, "\n", 1);
727 buffer_print_spaces(wb, wb->json.depth);
728 }
@@ -801,48 +808,42 @@ static inline void buffer_json_add_array_item_array(BUFFER *wb) {
808 }
809
810 static inline void buffer_json_add_array_item_string(BUFFER *wb, const char *value) {
804 - if(wb->json.stack[wb->json.depth].count)
805 - buffer_fast_strcat(wb, ",", 1);
811 + buffer_print_json_comma_newline_spacing(wb);
812
813 buffer_json_add_string_value(wb, value);
814 wb->json.stack[wb->json.depth].count++;
815 }
816
817 static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE value) {
812 - if(wb->json.stack[wb->json.depth].count)
813 - buffer_fast_strcat(wb, ",", 1);
818 + buffer_print_json_comma_newline_spacing(wb);
819
820 buffer_print_netdata_double(wb, value);
821 wb->json.stack[wb->json.depth].count++;
822 }
823
824 static inline void buffer_json_add_array_item_int64(BUFFER *wb, int64_t value) {
820 - if(wb->json.stack[wb->json.depth].count)
821 - buffer_fast_strcat(wb, ",", 1);
825 + buffer_print_json_comma_newline_spacing(wb);
826
827 buffer_print_int64(wb, value);
828 wb->json.stack[wb->json.depth].count++;
829 }
830
831 static inline void buffer_json_add_array_item_uint64(BUFFER *wb, uint64_t value) {
828 - if(wb->json.stack[wb->json.depth].count)
829 - buffer_fast_strcat(wb, ",", 1);
832 + buffer_print_json_comma_newline_spacing(wb);
833
834 buffer_print_uint64(wb, value);
835 wb->json.stack[wb->json.depth].count++;
836 }
837
838 static inline void buffer_json_add_array_item_time_t(BUFFER *wb, time_t value) {
836 - if(wb->json.stack[wb->json.depth].count)
837 - buffer_fast_strcat(wb, ",", 1);
839 + buffer_print_json_comma_newline_spacing(wb);
840
841 buffer_print_int64(wb, value);
842 wb->json.stack[wb->json.depth].count++;
843 }
844
845 static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value) {
844 - if(wb->json.stack[wb->json.depth].count)
845 - buffer_fast_strcat(wb, ",", 1);
846 + buffer_print_json_comma_newline_spacing(wb);
847
848 buffer_print_int64(wb, value);
849 buffer_fast_strcat(wb, "000", 3);
@@ -850,8 +851,7 @@ static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value)
851 }
852
853 static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value) {
853 - if(wb->json.stack[wb->json.depth].count)
854 - buffer_fast_strcat(wb, ",", 1);
854 + buffer_print_json_comma_newline_spacing(wb);
855
856 buffer_print_int64(wb, value);
857 buffer_fast_strcat(wb, "000", 3);
@@ -859,8 +859,7 @@ static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value
859 }
860
861 static inline void buffer_json_add_array_item_object(BUFFER *wb) {
862 - if(wb->json.stack[wb->json.depth].count)
863 - buffer_fast_strcat(wb, ",", 1);
862 + buffer_print_json_comma_newline_spacing(wb);
863
864 buffer_fast_strcat(wb, "{", 1);
865 wb->json.stack[wb->json.depth].count++;
@@ -919,6 +918,11 @@ static inline void buffer_json_array_close(BUFFER *wb) {
918 assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
919 assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && "BUFFER JSON: an array is not open to close it");
920 #endif
921 + if(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS) {
922 + buffer_fast_strcat(wb, "\n", 1);
923 + buffer_print_spaces(wb, wb->json.depth);
924 + }
925 +
926 buffer_fast_strcat(wb, "]", 1);
927 _buffer_json_depth_pop(wb);
928 }
@@ -928,6 +932,8 @@ typedef enum __attribute__((packed)) {
932 RRDF_FIELD_OPTS_UNIQUE_KEY = (1 << 0), // the field is the unique key of the row
933 RRDF_FIELD_OPTS_VISIBLE = (1 << 1), // the field should be visible by default
934 RRDF_FIELD_OPTS_STICKY = (1 << 2), // the field should be sticky
935 + RRDF_FIELD_OPTS_FULL_WIDTH = (1 << 3), // the field should get full width
936 + RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should get full width
937 } RRDF_FIELD_OPTIONS;
938
939 typedef enum __attribute__((packed)) {
@@ -969,7 +975,8 @@ static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) {
975 typedef enum __attribute__((packed)) {
976 RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation
977 RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100%
972 - RRDF_FIELD_VISUAL_PILL, // array of values (transformation is respected)
978 + RRDF_FIELD_VISUAL_PILL, //
979 + RRDF_FIELD_VISUAL_MARKDOC, //
980 } RRDF_FIELD_VISUAL;
981
982 static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) {
@@ -983,14 +990,18 @@ static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual)
990
991 case RRDF_FIELD_VISUAL_PILL:
992 return "pill";
993 +
994 + case RRDF_FIELD_VISUAL_MARKDOC:
995 + return "markdoc";
996 }
997 }
998
999 typedef enum __attribute__((packed)) {
1000 RRDF_FIELD_TRANSFORM_NONE, // show the value as-is
991 - RRDF_FIELD_TRANSFORM_NUMBER, // show the value repsecting the decimal_points
992 - RRDF_FIELD_TRANSFORM_DURATION, // transform as duration in second to a human readable duration
993 - RRDF_FIELD_TRANSFORM_DATETIME, // UNIX epoch timestamp in ms
1001 + RRDF_FIELD_TRANSFORM_NUMBER, // show the value respecting the decimal_points
1002 + RRDF_FIELD_TRANSFORM_DURATION_S, // transform as duration in second to a human-readable duration
1003 + RRDF_FIELD_TRANSFORM_DATETIME_MS, // UNIX epoch timestamp in ms
1004 + RRDF_FIELD_TRANSFORM_DATETIME_USEC, // UNIX epoch timestamp in usec
1005 } RRDF_FIELD_TRANSFORM;
1006
1007 static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM transform) {
@@ -1002,11 +1013,14 @@ static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM tr
1013 case RRDF_FIELD_TRANSFORM_NUMBER:
1014 return "number";
1015
1005 - case RRDF_FIELD_TRANSFORM_DURATION:
1016 + case RRDF_FIELD_TRANSFORM_DURATION_S:
1017 return "duration";
1018
1008 - case RRDF_FIELD_TRANSFORM_DATETIME:
1019 + case RRDF_FIELD_TRANSFORM_DATETIME_MS:
1020 return "datetime";
1021 +
1022 + case RRDF_FIELD_TRANSFORM_DATETIME_USEC:
1023 + return "datetime_usec";
1024 }
1025 }
1026
@@ -1064,18 +1078,26 @@ static inline const char *rrdf_field_summary_to_string(RRDF_FIELD_SUMMARY summar
1078 }
1079
1080 typedef enum __attribute__((packed)) {
1081 + RRDF_FIELD_FILTER_NONE,
1082 RRDF_FIELD_FILTER_RANGE,
1083 RRDF_FIELD_FILTER_MULTISELECT,
1084 + RRDF_FIELD_FILTER_FACET,
1085 } RRDF_FIELD_FILTER;
1086
1087 static inline const char *rrdf_field_filter_to_string(RRDF_FIELD_FILTER filter) {
1088 switch(filter) {
1073 - default:
1089 case RRDF_FIELD_FILTER_RANGE:
1090 return "range";
1091
1092 case RRDF_FIELD_FILTER_MULTISELECT:
1093 return "multiselect";
1094 +
1095 + case RRDF_FIELD_FILTER_FACET:
1096 + return "facet";
1097 +
1098 + default:
1099 + case RRDF_FIELD_FILTER_NONE:
1100 + return "none";
1101 }
1102 }
1103
@@ -1114,6 +1136,9 @@ buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const
1136 buffer_json_member_add_boolean(wb, "sticky", options & RRDF_FIELD_OPTS_STICKY);
1137 buffer_json_member_add_string(wb, "summary", rrdf_field_summary_to_string(summary));
1138 buffer_json_member_add_string(wb, "filter", rrdf_field_filter_to_string(filter));
1139 +
1140 + buffer_json_member_add_boolean(wb, "full_width", options & RRDF_FIELD_OPTS_FULL_WIDTH);
1141 + buffer_json_member_add_boolean(wb, "wrap", options & RRDF_FIELD_OPTS_WRAP);
1142 }
1143 buffer_json_object_close(wb);
1144 }
libnetdata/facets/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
libnetdata/facets/README.md
libnetdata/facets/facets.c new
+827
@@ -0,0 +1,827 @@
1 +#include "facets.h"
2 +
3 +#define FACET_VALUE_UNSET "-"
4 +#define HISTOGRAM_COLUMNS 60
5 +
6 +static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);
7 +
8 +// ----------------------------------------------------------------------------
9 +
10 +time_t calculate_bar_width(time_t before, time_t after) {
11 + // Array of valid durations in seconds
12 + static time_t valid_durations[] = {
13 + 1,
14 + 15,
15 + 30,
16 + 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes
17 + 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours
18 + 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days
19 + 1 * (30*86400) // months
20 + };
21 + static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]);
22 +
23 + time_t duration = before - after;
24 + time_t bar_width = 1;
25 +
26 + for (int i = array_size - 1; i >= 0; --i) {
27 + if (duration / valid_durations[i] >= HISTOGRAM_COLUMNS) {
28 + bar_width = valid_durations[i];
29 + break;
30 + }
31 + }
32 +
33 + return bar_width;
34 +}
35 +
36 +// ----------------------------------------------------------------------------
37 +
38 +static inline void uint32_to_char(uint32_t num, char *out) {
39 + static char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
40 +
41 + int i;
42 + for(i = 5; i >= 0; --i) {
43 + out[i] = id_encoding_characters[num & 63];
44 + num >>= 6;
45 + }
46 + out[6] = '\0';
47 +}
48 +
49 +inline void facets_string_hash(const char *src, char *out) {
50 + uint32_t hash1 = fnv1a_hash32(src);
51 + uint32_t hash2 = djb2_hash32(src);
52 + uint32_t hash3 = larson_hash32(src);
53 +
54 + uint32_to_char(hash1, out);
55 + uint32_to_char(hash2, &out[6]);
56 + uint32_to_char(hash3, &out[12]);
57 +
58 + out[18] = '\0';
59 +}
60 +
61 +// ----------------------------------------------------------------------------
62 +
63 +typedef struct facet_value {
64 + const char *name;
65 +
66 + bool selected;
67 +
68 + uint32_t rows_matching_facet_value;
69 + uint32_t final_facet_value_counter;
70 +} FACET_VALUE;
71 +
72 +struct facet_key {
73 + const char *name;
74 +
75 + DICTIONARY *values;
76 +
77 + FACET_KEY_OPTIONS options;
78 +
79 + bool default_selected_for_values; // the default "selected" for all values in the dictionary
80 +
81 + // members about the current row
82 + uint32_t key_found_in_row;
83 + uint32_t key_values_selected_in_row;
84 + BUFFER *current_value;
85 +
86 + uint32_t order;
87 +
88 + struct {
89 + facet_dynamic_row_t cb;
90 + void *data;
91 + } dynamic;
92 +
93 + struct {
94 + facets_key_transformer_t cb;
95 + void *data;
96 +
97 + } transform;
98 +};
99 +
100 +struct facets {
101 + SIMPLE_PATTERN *visible_keys;
102 + SIMPLE_PATTERN *excluded_keys;
103 + SIMPLE_PATTERN *included_keys;
104 +
105 + FACETS_OPTIONS options;
106 + usec_t anchor;
107 +
108 + SIMPLE_PATTERN *query; // the full text search pattern
109 + size_t keys_filtered_by_query; // the number of fields we do full text search (constant)
110 + size_t keys_matched_by_query; // the number of fields matched the full text search (per row)
111 +
112 + DICTIONARY *accepted_params;
113 +
114 + DICTIONARY *keys;
115 + FACET_ROW *base; // double linked list of the selected facets rows
116 +
117 + uint32_t items_to_return;
118 + uint32_t max_items_to_return;
119 + uint32_t order;
120 +
121 + struct {
122 + FACET_ROW *last_added;
123 +
124 + size_t evaluated;
125 + size_t matched;
126 +
127 + size_t first;
128 + size_t forwards;
129 + size_t backwards;
130 + size_t skips_before;
131 + size_t skips_after;
132 + size_t prepends;
133 + size_t appends;
134 + size_t shifts;
135 + } operations;
136 +};
137 +
138 +// ----------------------------------------------------------------------------
139 +
140 +static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) {
141 + if(!k->key_found_in_row)
142 + v->rows_matching_facet_value++;
143 +
144 + k->key_found_in_row++;
145 +
146 + if(v->selected)
147 + k->key_values_selected_in_row++;
148 +}
149 +
150 +static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
151 + bool included = true, excluded = false;
152 +
153 + if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET)) {
154 + if(k->options & FACET_KEY_OPTION_FACET) {
155 + included = true;
156 + excluded = false;
157 + }
158 + else if(k->options & FACET_KEY_OPTION_NO_FACET) {
159 + included = false;
160 + excluded = true;
161 + }
162 + }
163 + else {
164 + if (facets->included_keys) {
165 + if (!simple_pattern_matches(facets->included_keys, k->name))
166 + included = false;
167 + }
168 +
169 + if (facets->excluded_keys) {
170 + if (simple_pattern_matches(facets->excluded_keys, k->name))
171 + excluded = true;
172 + }
173 + }
174 +
175 + if(included && !excluded) {
176 + k->options |= FACET_KEY_OPTION_FACET;
177 + k->options &= ~FACET_KEY_OPTION_NO_FACET;
178 + return true;
179 + }
180 +
181 + k->options |= FACET_KEY_OPTION_NO_FACET;
182 + k->options &= ~FACET_KEY_OPTION_FACET;
183 + return false;
184 +}
185 +
186 +// ----------------------------------------------------------------------------
187 +// FACET_VALUE dictionary hooks
188 +
189 +static void facet_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
190 + FACET_VALUE *v = value;
191 + FACET_KEY *k = data;
192 +
193 + if(v->name) {
194 + // an actual value, not a filter
195 + v->name = strdupz(v->name);
196 + facet_value_is_used(k, v);
197 + }
198 +
199 + if(!v->selected)
200 + v->selected = k->default_selected_for_values;
201 +}
202 +
203 +static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
204 + FACET_VALUE *v = old_value;
205 + FACET_VALUE *nv = new_value;
206 + FACET_KEY *k = data;
207 +
208 + if(!v->name && nv->name)
209 + // an actual value, not a filter
210 + v->name = strdupz(nv->name);
211 +
212 + if(v->name)
213 + facet_value_is_used(k, v);
214 +
215 + return false;
216 +}
217 +
218 +static void facet_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
219 + FACET_VALUE *v = value;
220 + freez((char *)v->name);
221 +}
222 +
223 +// ----------------------------------------------------------------------------
224 +// FACET_KEY dictionary hooks
225 +
226 +static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) {
227 + if(k->values)
228 + return;
229 +
230 + if(facets_key_is_facet(facets, k)) {
231 + k->values = dictionary_create_advanced(
232 + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
233 + NULL, sizeof(FACET_VALUE));
234 + dictionary_register_insert_callback(k->values, facet_value_insert_callback, k);
235 + dictionary_register_conflict_callback(k->values, facet_value_conflict_callback, k);
236 + dictionary_register_delete_callback(k->values, facet_value_delete_callback, k);
237 + }
238 +}
239 +
240 +static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
241 + FACET_KEY *k = value;
242 + FACETS *facets = data;
243 +
244 + if(!(k->options & FACET_KEY_OPTION_REORDER))
245 + k->order = facets->order++;
246 +
247 + if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS))
248 + facets->keys_filtered_by_query++;
249 +
250 + if(k->name) {
251 + // an actual value, not a filter
252 + k->name = strdupz(k->name);
253 + facet_key_late_init(facets, k);
254 + }
255 +
256 + k->current_value = buffer_create(0, NULL);
257 +}
258 +
259 +static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
260 + FACET_KEY *k = old_value;
261 + FACET_KEY *nk = new_value;
262 + FACETS *facets = data;
263 +
264 + if(!k->name && nk->name) {
265 + // an actual value, not a filter
266 + k->name = strdupz(nk->name);
267 + facet_key_late_init(facets, k);
268 + }
269 +
270 + if(k->options & FACET_KEY_OPTION_REORDER) {
271 + k->order = facets->order++;
272 + k->options &= ~FACET_KEY_OPTION_REORDER;
273 + }
274 +
275 + return false;
276 +}
277 +
278 +static void facet_key_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
279 + FACET_KEY *k = value;
280 + dictionary_destroy(k->values);
281 + buffer_free(k->current_value);
282 + freez((char *)k->name);
283 +}
284 +
285 +// ----------------------------------------------------------------------------
286 +
287 +FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
288 + FACETS *facets = callocz(1, sizeof(FACETS));
289 + facets->options = options;
290 + facets->keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_KEY));
291 + dictionary_register_insert_callback(facets->keys, facet_key_insert_callback, facets);
292 + dictionary_register_conflict_callback(facets->keys, facet_key_conflict_callback, facets);
293 + dictionary_register_delete_callback(facets->keys, facet_key_delete_callback, facets);
294 +
295 + if(facet_keys && *facet_keys)
296 + facets->included_keys = simple_pattern_create(facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
297 +
298 + if(non_facet_keys && *non_facet_keys)
299 + facets->excluded_keys = simple_pattern_create(non_facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
300 +
301 + if(visible_keys && *visible_keys)
302 + facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true);
303 +
304 + facets->max_items_to_return = items_to_return;
305 + facets->anchor = anchor;
306 + facets->order = 1;
307 +
308 + return facets;
309 +}
310 +
311 +void facets_destroy(FACETS *facets) {
312 + dictionary_destroy(facets->accepted_params);
313 + dictionary_destroy(facets->keys);
314 + simple_pattern_free(facets->visible_keys);
315 + simple_pattern_free(facets->included_keys);
316 + simple_pattern_free(facets->excluded_keys);
317 +
318 + while(facets->base) {
319 + FACET_ROW *r = facets->base;
320 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, r, prev, next);
321 +
322 + facets_row_free(facets, r);
323 + }
324 +
325 + freez(facets);
326 +}
327 +
328 +void facets_accepted_param(FACETS *facets, const char *param) {
329 + if(!facets->accepted_params)
330 + facets->accepted_params = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
331 +
332 + dictionary_set(facets->accepted_params, param, NULL, 0);
333 +}
334 +
335 +inline FACET_KEY *facets_register_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) {
336 + FACET_KEY tk = {
337 + .name = key,
338 + .options = options,
339 + .default_selected_for_values = true,
340 + };
341 + char hash[FACET_STRING_HASH_SIZE];
342 + facets_string_hash(tk.name, hash);
343 + return dictionary_set(facets->keys, hash, &tk, sizeof(tk));
344 +}
345 +
346 +inline FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) {
347 + FACET_KEY *k = facets_register_key(facets, key, options);
348 + k->transform.cb = cb;
349 + k->transform.data = data;
350 + return k;
351 +}
352 +
353 +inline FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) {
354 + FACET_KEY *k = facets_register_key(facets, key, options);
355 + k->dynamic.cb = cb;
356 + k->dynamic.data = data;
357 + return k;
358 +}
359 +
360 +void facets_set_query(FACETS *facets, const char *query) {
361 + if(!query)
362 + return;
363 +
364 + facets->query = simple_pattern_create(query, " \t", SIMPLE_PATTERN_SUBSTRING, false);
365 +}
366 +
367 +void facets_set_items(FACETS *facets, uint32_t items) {
368 + facets->max_items_to_return = items;
369 +}
370 +
371 +void facets_set_anchor(FACETS *facets, usec_t anchor) {
372 + facets->anchor = anchor;
373 +}
374 +
375 +void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options) {
376 + FACET_KEY tk = {
377 + .options = options,
378 + };
379 + FACET_KEY *k = dictionary_set(facets->keys, key_id, &tk, sizeof(tk));
380 +
381 + k->default_selected_for_values = false;
382 + k->options |= FACET_KEY_OPTION_FACET;
383 + k->options &= ~FACET_KEY_OPTION_NO_FACET;
384 + facet_key_late_init(facets, k);
385 +
386 + FACET_VALUE tv = {
387 + .selected = true,
388 + };
389 + dictionary_set(k->values, value_ids, &tv, sizeof(tv));
390 +}
391 +
392 +// ----------------------------------------------------------------------------
393 +
394 +static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *k) {
395 + if(k->transform.cb)
396 + k->transform.cb(facets, k->current_value, k->transform.data);
397 +
398 + if(buffer_strlen(k->current_value) == 0)
399 + buffer_strcat(k->current_value, FACET_VALUE_UNSET);
400 +
401 +// bool found = false;
402 +// if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL)
403 +// found = true;
404 +
405 + if(facets->query && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) {
406 + if(simple_pattern_matches(facets->query, buffer_tostring(k->current_value)))
407 + facets->keys_matched_by_query++;
408 + }
409 +
410 + if(k->values) {
411 + FACET_VALUE tk = {
412 + .name = buffer_tostring(k->current_value),
413 + };
414 + char hash[FACET_STRING_HASH_SIZE];
415 + facets_string_hash(tk.name, hash);
416 + dictionary_set(k->values, hash, &tk, sizeof(tk));
417 + }
418 + else {
419 + k->key_found_in_row++;
420 + k->key_values_selected_in_row++;
421 + }
422 +}
423 +
424 +void facets_add_key_value(FACETS *facets, const char *key, const char *value) {
425 + FACET_KEY *k = facets_register_key(facets, key, 0);
426 + buffer_flush(k->current_value);
427 + buffer_strcat(k->current_value, value);
428 +
429 + facets_check_value(facets, k);
430 +}
431 +
432 +void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len) {
433 + FACET_KEY *k = facets_register_key(facets, key, 0);
434 + buffer_flush(k->current_value);
435 + buffer_strncat(k->current_value, value, value_len);
436 +
437 + facets_check_value(facets, k);
438 +}
439 +
440 +// ----------------------------------------------------------------------------
441 +// FACET_ROW dictionary hooks
442 +
443 +static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
444 + FACET_ROW_KEY_VALUE *rkv = value;
445 + FACET_ROW *row = data; (void)row;
446 +
447 + rkv->wb = buffer_create(0, NULL);
448 + buffer_strcat(rkv->wb, rkv->tmp && *rkv->tmp ? rkv->tmp : FACET_VALUE_UNSET);
449 +}
450 +
451 +static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
452 + FACET_ROW_KEY_VALUE *rkv = old_value;
453 + FACET_ROW_KEY_VALUE *n_rkv = new_value;
454 + FACET_ROW *row = data; (void)row;
455 +
456 + buffer_flush(rkv->wb);
457 + buffer_strcat(rkv->wb, n_rkv->tmp && *n_rkv->tmp ? n_rkv->tmp : FACET_VALUE_UNSET);
458 +
459 + return false;
460 +}
461 +
462 +static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
463 + FACET_ROW_KEY_VALUE *rkv = value;
464 + FACET_ROW *row = data; (void)row;
465 +
466 + buffer_free(rkv->wb);
467 +}
468 +
469 +// ----------------------------------------------------------------------------
470 +// FACET_ROW management
471 +
472 +static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) {
473 + dictionary_destroy(row->dict);
474 + freez(row);
475 +}
476 +
477 +static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) {
478 + FACET_ROW *row;
479 +
480 + if(into)
481 + row = into;
482 + else {
483 + row = callocz(1, sizeof(FACET_ROW));
484 + row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE));
485 + dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row);
486 + dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row);
487 + dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row);
488 + }
489 +
490 + row->usec = usec;
491 +
492 + FACET_KEY *k;
493 + dfe_start_read(facets->keys, k) {
494 + FACET_ROW_KEY_VALUE t = {
495 + .tmp = buffer_strlen(k->current_value) ? buffer_tostring(k->current_value) : FACET_VALUE_UNSET,
496 + .wb = NULL,
497 + };
498 + dictionary_set(row->dict, k->name, &t, sizeof(t));
499 + }
500 + dfe_done(k);
501 +
502 + return row;
503 +}
504 +
505 +// ----------------------------------------------------------------------------
506 +
507 +static void facets_row_keep(FACETS *facets, usec_t usec) {
508 + facets->operations.matched++;
509 +
510 + if(usec < facets->anchor) {
511 + facets->operations.skips_before++;
512 + return;
513 + }
514 +
515 + if(unlikely(!facets->base)) {
516 + facets->operations.last_added = facets_row_create(facets, usec, NULL);
517 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
518 + facets->items_to_return++;
519 + facets->operations.first++;
520 + return;
521 + }
522 +
523 + if(likely(usec > facets->base->prev->usec))
524 + facets->operations.last_added = facets->base->prev;
525 +
526 + FACET_ROW *last = facets->operations.last_added;
527 + while(last->prev != facets->base->prev && usec > last->prev->usec) {
528 + last = last->prev;
529 + facets->operations.backwards++;
530 + }
531 +
532 + while(last->next && usec < last->next->usec) {
533 + last = last->next;
534 + facets->operations.forwards++;
535 + }
536 +
537 + if(facets->items_to_return >= facets->max_items_to_return) {
538 + if(last == facets->base->prev && usec < last->usec) {
539 + facets->operations.skips_after++;
540 + return;
541 + }
542 + }
543 +
544 + facets->items_to_return++;
545 +
546 + if(usec > last->usec) {
547 + if(facets->items_to_return > facets->max_items_to_return) {
548 + facets->items_to_return--;
549 + facets->operations.shifts++;
550 + facets->operations.last_added = facets->base->prev;
551 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
552 + facets->operations.last_added = facets_row_create(facets, usec, facets->operations.last_added);
553 + }
554 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
555 + facets->operations.prepends++;
556 + }
557 + else {
558 + facets->operations.last_added = facets_row_create(facets, usec, NULL);
559 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
560 + facets->operations.appends++;
561 + }
562 +
563 + while(facets->items_to_return > facets->max_items_to_return) {
564 + // we have to remove something
565 +
566 + FACET_ROW *tmp = facets->base->prev;
567 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, tmp, prev, next);
568 + facets->items_to_return--;
569 +
570 + if(unlikely(facets->operations.last_added == tmp))
571 + facets->operations.last_added = facets->base->prev;
572 +
573 + facets_row_free(facets, tmp);
574 + facets->operations.shifts++;
575 + }
576 +}
577 +
578 +void facets_rows_begin(FACETS *facets) {
579 + FACET_KEY *k;
580 + dfe_start_read(facets->keys, k) {
581 + k->key_found_in_row = 0;
582 + k->key_values_selected_in_row = 0;
583 + buffer_flush(k->current_value);
584 + }
585 + dfe_done(k);
586 +
587 + facets->keys_matched_by_query = 0;
588 +}
589 +
590 +void facets_row_finished(FACETS *facets, usec_t usec) {
591 + if(facets->query && facets->keys_filtered_by_query && !facets->keys_matched_by_query)
592 + goto cleanup;
593 +
594 + facets->operations.evaluated++;
595 +
596 + uint32_t total_keys = 0;
597 + uint32_t selected_by = 0;
598 +
599 + FACET_KEY *k;
600 + dfe_start_read(facets->keys, k) {
601 + if(!k->key_found_in_row) {
602 + internal_fatal(buffer_strlen(k->current_value), "key is not found in row but it has a current value");
603 + // put the FACET_VALUE_UNSET value into it
604 + facets_check_value(facets, k);
605 + }
606 +
607 + internal_fatal(!k->key_found_in_row, "all keys should be found in the row at this point");
608 + internal_fatal(k->key_found_in_row != 1, "all keys should be matched exactly once at this point");
609 + internal_fatal(k->key_values_selected_in_row > 1, "key values are selected in row more than once");
610 +
611 + k->key_found_in_row = 1;
612 +
613 + total_keys += k->key_found_in_row;
614 + selected_by += (k->key_values_selected_in_row) ? 1 : 0;
615 + }
616 + dfe_done(k);
617 +
618 + if(selected_by >= total_keys - 1) {
619 + uint32_t found = 0;
620 +
621 + dfe_start_read(facets->keys, k){
622 + uint32_t counted_by = selected_by;
623 +
624 + if (counted_by != total_keys && !k->key_values_selected_in_row)
625 + counted_by++;
626 +
627 + if(counted_by == total_keys) {
628 + if(k->values) {
629 + char hash[FACET_STRING_HASH_SIZE];
630 + facets_string_hash(buffer_tostring(k->current_value), hash);
631 + FACET_VALUE *v = dictionary_get(k->values, hash);
632 + v->final_facet_value_counter++;
633 + }
634 +
635 + found++;
636 + }
637 + }
638 + dfe_done(k);
639 +
640 + internal_fatal(!found, "We should find at least one facet to count this row");
641 + (void)found;
642 + }
643 +
644 + if(selected_by == total_keys)
645 + facets_row_keep(facets, usec);
646 +
647 +cleanup:
648 + facets_rows_begin(facets);
649 +}
650 +
651 +// ----------------------------------------------------------------------------
652 +// output
653 +
654 +void facets_report(FACETS *facets, BUFFER *wb) {
655 + buffer_json_member_add_boolean(wb, "show_ids", false);
656 + buffer_json_member_add_boolean(wb, "has_history", true);
657 +
658 + buffer_json_member_add_object(wb, "pagination");
659 + buffer_json_member_add_boolean(wb, "enabled", true);
660 + buffer_json_member_add_string(wb, "key", "anchor");
661 + buffer_json_member_add_string(wb, "column", "timestamp");
662 + buffer_json_object_close(wb);
663 +
664 + buffer_json_member_add_array(wb, "accepted_params");
665 + {
666 + if(facets->accepted_params) {
667 + void *t;
668 + dfe_start_read(facets->accepted_params, t) {
669 + buffer_json_add_array_item_string(wb, t_dfe.name);
670 + }
671 + dfe_done(t);
672 + }
673 +
674 + FACET_KEY *k;
675 + dfe_start_read(facets->keys, k) {
676 + if(!k->values)
677 + continue;
678 +
679 + buffer_json_add_array_item_string(wb, k_dfe.name);
680 + }
681 + dfe_done(k);
682 + }
683 + buffer_json_array_close(wb); // accepted_params
684 +
685 + buffer_json_member_add_array(wb, "facets");
686 + {
687 + FACET_KEY *k;
688 + dfe_start_read(facets->keys, k) {
689 + if(!k->values)
690 + continue;
691 +
692 + buffer_json_add_array_item_object(wb); // key
693 + {
694 + buffer_json_member_add_string(wb, "id", k_dfe.name);
695 + buffer_json_member_add_string(wb, "name", k->name);
696 +
697 + if(!k->order)
698 + k->order = facets->order++;
699 +
700 + buffer_json_member_add_uint64(wb, "order", k->order);
701 + buffer_json_member_add_array(wb, "options");
702 + {
703 + FACET_VALUE *v;
704 + dfe_start_read(k->values, v) {
705 + buffer_json_add_array_item_object(wb);
706 + {
707 + buffer_json_member_add_string(wb, "id", v_dfe.name);
708 + buffer_json_member_add_string(wb, "name", v->name);
709 + buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);
710 + }
711 + buffer_json_object_close(wb);
712 + }
713 + dfe_done(v);
714 + }
715 + buffer_json_array_close(wb); // options
716 + }
717 + buffer_json_object_close(wb); // key
718 + }
719 + dfe_done(k);
720 + }
721 + buffer_json_array_close(wb); // facets
722 +
723 + buffer_json_member_add_object(wb, "columns");
724 + {
725 + size_t field_id = 0;
726 + buffer_rrdf_table_add_field(
727 + wb, field_id++,
728 + "timestamp", "Timestamp",
729 + RRDF_FIELD_TYPE_TIMESTAMP,
730 + RRDF_FIELD_VISUAL_VALUE,
731 + RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,
732 + RRDF_FIELD_SORT_DESCENDING,
733 + NULL,
734 + RRDF_FIELD_SUMMARY_COUNT,
735 + RRDF_FIELD_FILTER_RANGE,
736 + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY,
737 + NULL);
738 +
739 + FACET_KEY *k;
740 + dfe_start_read(facets->keys, k) {
741 + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE;
742 + bool visible = k->options & (FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_STICKY);
743 +
744 + if((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values))
745 + visible = true;
746 +
747 + if(!visible)
748 + visible = simple_pattern_matches(facets->visible_keys, k->name);
749 +
750 + if(visible)
751 + options |= RRDF_FIELD_OPTS_VISIBLE;
752 +
753 + if(k->options & FACET_KEY_OPTION_MAIN_TEXT)
754 + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
755 +
756 + buffer_rrdf_table_add_field(
757 + wb, field_id++,
758 + k_dfe.name, k->name ? k->name : k_dfe.name,
759 + RRDF_FIELD_TYPE_STRING,
760 + RRDF_FIELD_VISUAL_VALUE,
761 + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
762 + RRDF_FIELD_SORT_ASCENDING,
763 + NULL,
764 + RRDF_FIELD_SUMMARY_COUNT,
765 + k->values ? RRDF_FIELD_FILTER_FACET : RRDF_FIELD_FILTER_NONE,
766 + options,
767 + FACET_VALUE_UNSET);
768 + }
769 + dfe_done(k);
770 + }
771 + buffer_json_object_close(wb); // columns
772 +
773 + buffer_json_member_add_array(wb, "data");
774 + {
775 + for(FACET_ROW *row = facets->base ; row ;row = row->next) {
776 + buffer_json_add_array_item_array(wb); // each row
777 + buffer_json_add_array_item_uint64(wb, row->usec);
778 +
779 + FACET_KEY *k;
780 + dfe_start_read(facets->keys, k)
781 + {
782 + FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
783 +
784 + if(unlikely(k->dynamic.cb)) {
785 + if(unlikely(!rkv))
786 + rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv));
787 +
788 + k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data);
789 + }
790 + else
791 + buffer_json_add_array_item_string(wb, rkv ? buffer_tostring(rkv->wb) : FACET_VALUE_UNSET);
792 + }
793 + dfe_done(k);
794 + buffer_json_array_close(wb); // each row
795 + }
796 + }
797 + buffer_json_array_close(wb); // data
798 +
799 + buffer_json_member_add_string(wb, "default_sort_column", "timestamp");
800 + buffer_json_member_add_array(wb, "default_charts");
801 + buffer_json_array_close(wb);
802 +
803 + buffer_json_member_add_object(wb, "items");
804 + {
805 + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.evaluated);
806 + buffer_json_member_add_uint64(wb, "matched", facets->operations.matched);
807 + buffer_json_member_add_uint64(wb, "returned", facets->items_to_return);
808 + buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return);
809 + buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before);
810 + buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts);
811 + }
812 + buffer_json_object_close(wb); // items
813 +
814 + buffer_json_member_add_object(wb, "stats");
815 + {
816 + buffer_json_member_add_uint64(wb, "first", facets->operations.first);
817 + buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards);
818 + buffer_json_member_add_uint64(wb, "backwards", facets->operations.backwards);
819 + buffer_json_member_add_uint64(wb, "skips_before", facets->operations.skips_before);
820 + buffer_json_member_add_uint64(wb, "skips_after", facets->operations.skips_after);
821 + buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends);
822 + buffer_json_member_add_uint64(wb, "appends", facets->operations.appends);
823 + buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts);
824 + }
825 + buffer_json_object_close(wb); // items
826 +
827 +}
libnetdata/facets/facets.h new
+62
@@ -0,0 +1,62 @@
1 +#ifndef FACETS_H
2 +#define FACETS_H 1
3 +
4 +#include "../libnetdata.h"
5 +
6 +typedef enum __attribute__((packed)) {
7 + FACET_KEY_OPTION_FACET = (1 << 0), // filterable values
8 + FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value
9 + FACET_KEY_OPTION_STICKY = (1 << 2), // should be sticky in the table
10 + FACET_KEY_OPTION_VISIBLE = (1 << 3), // should be in the default table
11 + FACET_KEY_OPTION_FTS = (1 << 4), // the key is filterable by full text search (FTS)
12 + FACET_KEY_OPTION_MAIN_TEXT = (1 << 5), // full width and wrap
13 + FACET_KEY_OPTION_REORDER = (1 << 6), // give the key a new order id on first encounter
14 +} FACET_KEY_OPTIONS;
15 +
16 +typedef struct facet_row_key_value {
17 + const char *tmp;
18 + BUFFER *wb;
19 +} FACET_ROW_KEY_VALUE;
20 +
21 +typedef struct facet_row {
22 + usec_t usec;
23 + DICTIONARY *dict;
24 + struct facet_row *prev, *next;
25 +} FACET_ROW;
26 +
27 +typedef struct facets FACETS;
28 +typedef struct facet_key FACET_KEY;
29 +
30 +#define FACET_STRING_HASH_SIZE 19
31 +void facets_string_hash(const char *src, char *out);
32 +
33 +typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data);
34 +typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *wb, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data);
35 +FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data);
36 +FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data);
37 +
38 +typedef enum __attribute__((packed)) {
39 + FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table
40 + FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search
41 +} FACETS_OPTIONS;
42 +
43 +FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);
44 +void facets_destroy(FACETS *facets);
45 +
46 +void facets_accepted_param(FACETS *facets, const char *param);
47 +
48 +void facets_rows_begin(FACETS *facets);
49 +void facets_row_finished(FACETS *facets, usec_t usec);
50 +
51 +FACET_KEY *facets_register_key(FACETS *facets, const char *param, FACET_KEY_OPTIONS options);
52 +void facets_set_query(FACETS *facets, const char *query);
53 +void facets_set_items(FACETS *facets, uint32_t items);
54 +void facets_set_anchor(FACETS *facets, usec_t anchor);
55 +void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options);
56 +
57 +void facets_add_key_value(FACETS *facets, const char *key, const char *value);
58 +void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len);
59 +
60 +void facets_report(FACETS *facets, BUFFER *wb);
61 +
62 +#endif
libnetdata/libnetdata.c
+132
@@ -1966,3 +1966,135 @@ int hash256_string(const unsigned char *string, size_t size, char *hash) {
1966 EVP_MD_CTX_destroy(ctx);
1967 return 1;
1968 }
1969 +
1970 +// Returns 1 if an absolute period was requested or 0 if it was a relative period
1971 +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running) {
1972 + time_t now = now_realtime_sec() - 1;
1973 +
1974 + if(now_ptr)
1975 + *now_ptr = now;
1976 +
1977 + int absolute_period_requested = -1;
1978 + long long after_requested, before_requested;
1979 +
1980 + before_requested = *before;
1981 + after_requested = *after;
1982 +
1983 + // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
1984 + if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
1985 + // if the user asked for a positive relative time,
1986 + // flip it to a negative
1987 + if(before_requested > 0)
1988 + before_requested = -before_requested;
1989 +
1990 + before_requested = now + before_requested;
1991 + absolute_period_requested = 0;
1992 + }
1993 +
1994 + // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
1995 + if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
1996 + if(after_requested > 0)
1997 + after_requested = -after_requested;
1998 +
1999 + // if the user didn't give an after, use the number of points
2000 + // to give a sane default
2001 + if(after_requested == 0)
2002 + after_requested = -600;
2003 +
2004 + // since the query engine now returns inclusive timestamps
2005 + // it is awkward to return 6 points when after=-5 is given
2006 + // so for relative queries we add 1 second, to give
2007 + // more predictable results to users.
2008 + after_requested = before_requested + after_requested + 1;
2009 + absolute_period_requested = 0;
2010 + }
2011 +
2012 + if(absolute_period_requested == -1)
2013 + absolute_period_requested = 1;
2014 +
2015 + // check if the parameters are flipped
2016 + if(after_requested > before_requested) {
2017 + long long t = before_requested;
2018 + before_requested = after_requested;
2019 + after_requested = t;
2020 + }
2021 +
2022 + // if the query requests future data
2023 + // shift the query back to be in the present time
2024 + // (this may also happen because of the rules above)
2025 + if(before_requested > now) {
2026 + long long delta = before_requested - now;
2027 + before_requested -= delta;
2028 + after_requested -= delta;
2029 + }
2030 +
2031 + time_t absolute_minimum_time = now - (10 * 365 * 86400);
2032 + time_t absolute_maximum_time = now + (1 * 365 * 86400);
2033 +
2034 + if (after_requested < absolute_minimum_time && !unittest_running)
2035 + after_requested = absolute_minimum_time;
2036 +
2037 + if (after_requested > absolute_maximum_time && !unittest_running)
2038 + after_requested = absolute_maximum_time;
2039 +
2040 + if (before_requested < absolute_minimum_time && !unittest_running)
2041 + before_requested = absolute_minimum_time;
2042 +
2043 + if (before_requested > absolute_maximum_time && !unittest_running)
2044 + before_requested = absolute_maximum_time;
2045 +
2046 + *before = before_requested;
2047 + *after = after_requested;
2048 +
2049 + return (absolute_period_requested != 1);
2050 +}
2051 +
2052 +int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size) {
2053 + static const unsigned char base64_table[256] = {
2054 + ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7,
2055 + ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
2056 + ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
2057 + ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31,
2058 + ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39,
2059 + ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47,
2060 + ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55,
2061 + ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63,
2062 + [0 ... '+' - 1] = 255,
2063 + ['+' + 1 ... '/' - 1] = 255,
2064 + ['9' + 1 ... 'A' - 1] = 255,
2065 + ['Z' + 1 ... 'a' - 1] = 255,
2066 + ['z' + 1 ... 255] = 255
2067 + };
2068 +
2069 + size_t count = 0;
2070 + unsigned int tmp = 0;
2071 + int i, bit;
2072 +
2073 + if (decoded_size < 1)
2074 + return 0; // Buffer size must be at least 1 for null termination
2075 +
2076 + for (i = 0, bit = 0; encoded[i]; i++) {
2077 + unsigned char value = base64_table[(unsigned char)encoded[i]];
2078 + if (value > 63)
2079 + return -1; // Invalid character in input
2080 +
2081 + tmp = tmp << 6 | value;
2082 + if (++bit == 4) {
2083 + if (count + 3 >= decoded_size) break; // Stop decoding if buffer is full
2084 + decoded[count++] = (tmp >> 16) & 0xFF;
2085 + decoded[count++] = (tmp >> 8) & 0xFF;
2086 + decoded[count++] = tmp & 0xFF;
2087 + tmp = 0;
2088 + bit = 0;
2089 + }
2090 + }
2091 +
2092 + if (bit > 0 && count + 1 < decoded_size) {
2093 + tmp <<= 6 * (4 - bit);
2094 + if (bit > 2 && count + 1 < decoded_size) decoded[count++] = (tmp >> 16) & 0xFF;
2095 + if (bit > 3 && count + 1 < decoded_size) decoded[count++] = (tmp >> 8) & 0xFF;
2096 + }
2097 +
2098 + decoded[count] = '\0'; // Null terminate the output string
2099 + return count;
2100 +}
libnetdata/libnetdata.h
+9
@@ -836,6 +836,7 @@ extern char *netdata_configured_host_prefix;
836 #include "yaml.h"
837 #include "http/http_defs.h"
838 #include "gorilla/gorilla.h"
839 +#include "facets/facets.h"
840 #include "dyn_conf/dyn_conf.h"
841
842 // BEWARE: this exists in alarm-notify.sh
@@ -979,6 +980,14 @@ typedef enum {
980 void timing_action(TIMING_ACTION action, TIMING_STEP step);
981
982 int hash256_string(const unsigned char *string, size_t size, char *hash);
983 +
984 +extern bool unittest_running;
985 +#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
986 +
987 +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running);
988 +
989 +int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size);
990 +
991 # ifdef __cplusplus
992 }
993 # endif
libnetdata/socket/socket.h
+15 -15
@@ -11,27 +11,27 @@
11
12 typedef enum web_client_acl {
13 WEB_CLIENT_ACL_NONE = (0),
14 - WEB_CLIENT_ACL_NOCHECK = (0), // Don't check anything - this should work on all channels
15 - WEB_CLIENT_ACL_DASHBOARD = (1 << 0),
16 - WEB_CLIENT_ACL_REGISTRY = (1 << 1),
17 - WEB_CLIENT_ACL_BADGE = (1 << 2),
18 - WEB_CLIENT_ACL_MGMT = (1 << 3),
19 - WEB_CLIENT_ACL_STREAMING = (1 << 4),
20 - WEB_CLIENT_ACL_NETDATACONF = (1 << 5),
21 - WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 6),
22 - WEB_CLIENT_ACL_SSL_FORCE = (1 << 7),
23 - WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 8),
24 - WEB_CLIENT_ACL_ACLK = (1 << 9),
25 - WEB_CLIENT_ACL_WEBRTC = (1 << 10),
26 - WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 11), // allow unprotected access if bearer is not enabled in netdata
27 - WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 12), // allow access only if a valid bearer is used
14 + WEB_CLIENT_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels
15 + WEB_CLIENT_ACL_DASHBOARD = (1 << 1),
16 + WEB_CLIENT_ACL_REGISTRY = (1 << 2),
17 + WEB_CLIENT_ACL_BADGE = (1 << 3),
18 + WEB_CLIENT_ACL_MGMT = (1 << 4),
19 + WEB_CLIENT_ACL_STREAMING = (1 << 5),
20 + WEB_CLIENT_ACL_NETDATACONF = (1 << 6),
21 + WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 7),
22 + WEB_CLIENT_ACL_SSL_FORCE = (1 << 8),
23 + WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 9),
24 + WEB_CLIENT_ACL_ACLK = (1 << 10),
25 + WEB_CLIENT_ACL_WEBRTC = (1 << 11),
26 + WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 12), // allow unprotected access if bearer is not enabled in netdata
27 + WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used
28 } WEB_CLIENT_ACL;
29
30 #define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL)
31 #define WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_REQUIRED)
32
33 #ifdef NETDATA_DEV_MODE
34 -#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_DASHBOARD
34 +#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_NOCHECK
35 #else
36 #define ACL_DEV_OPEN_ACCESS 0
37 #endif
libnetdata/worker_utilization/worker_utilization.h
+1 -1
@@ -7,7 +7,7 @@
7
8 #define WORKER_UTILIZATION_MAX_JOB_TYPES 50
9
10 -typedef enum {
10 +typedef enum __attribute__((packed)) {
11 WORKER_METRIC_EMPTY = 0,
12 WORKER_METRIC_IDLE_BUSY = 1,
13 WORKER_METRIC_ABSOLUTE = 2,
netdata-installer.sh
+5
@@ -1290,6 +1290,11 @@ if [ "$(id -u)" -eq 0 ]; then
1290 run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/local-listeners"
1291 run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/local-listeners"
1292 fi
1293 +
1294 + if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin" ]; then
1295 + run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
1296 + run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
1297 + fi
1298 else
1299 # non-privileged user installation
1300 run chown "${NETDATA_USER}:${NETDATA_GROUP}" "${NETDATA_LOG_DIR}"
registry/registry.c
+1 -1
@@ -58,7 +58,7 @@ static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PER
58 static inline void registry_json_header(RRDHOST *host, struct web_client *w, const char *action, const char *status) {
59 buffer_flush(w->response.data);
60 w->response.data->content_type = CT_APPLICATION_JSON;
61 - buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
61 + buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
62 buffer_json_member_add_string(w->response.data, "action", action);
63 buffer_json_member_add_string(w->response.data, "status", status);
64 buffer_json_member_add_string(w->response.data, "hostname", rrdhost_registry_hostname(host));
web/api/formatters/json_wrapper.c
+2 -2
@@ -874,7 +874,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
874 sq[0] = '"';
875 }
876
877 - buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
877 + buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
878
879 buffer_json_member_add_uint64(wb, "api", 1);
880 buffer_json_member_add_string(wb, "id", qt->id);
@@ -1289,7 +1289,7 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
1289 sq[0] = '\'';
1290 }
1291
1292 - buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
1292 + buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
1293 buffer_json_member_add_uint64(wb, "api", 2);
1294
1295 if(options & RRDR_OPTION_DEBUG) {
web/api/formatters/rrd2json.h
-2
@@ -38,8 +38,6 @@ typedef enum {
38
39 #define HOSTNAME_MAX 1024
40
41 -#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
42 -
41 #define DATASOURCE_FORMAT_JSON "json"
42 #define DATASOURCE_FORMAT_JSON2 "json2"
43 #define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
web/api/queries/query.c
+1 -83
@@ -2075,88 +2075,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
2075 }
2076 #endif // NETDATA_INTERNAL_CHECKS
2077
2078 -// Returns 1 if an absolute period was requested or 0 if it was a relative period
2079 -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr) {
2080 - time_t now = now_realtime_sec() - 1;
2081 -
2082 - if(now_ptr)
2083 - *now_ptr = now;
2084 -
2085 - int absolute_period_requested = -1;
2086 - long long after_requested, before_requested;
2087 -
2088 - before_requested = *before;
2089 - after_requested = *after;
2090 -
2091 - // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
2092 - if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
2093 - // if the user asked for a positive relative time,
2094 - // flip it to a negative
2095 - if(before_requested > 0)
2096 - before_requested = -before_requested;
2097 -
2098 - before_requested = now + before_requested;
2099 - absolute_period_requested = 0;
2100 - }
2101 -
2102 - // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
2103 - if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
2104 - if(after_requested > 0)
2105 - after_requested = -after_requested;
2106 -
2107 - // if the user didn't give an after, use the number of points
2108 - // to give a sane default
2109 - if(after_requested == 0)
2110 - after_requested = -600;
2111 -
2112 - // since the query engine now returns inclusive timestamps
2113 - // it is awkward to return 6 points when after=-5 is given
2114 - // so for relative queries we add 1 second, to give
2115 - // more predictable results to users.
2116 - after_requested = before_requested + after_requested + 1;
2117 - absolute_period_requested = 0;
2118 - }
2119 -
2120 - if(absolute_period_requested == -1)
2121 - absolute_period_requested = 1;
2122 -
2123 - // check if the parameters are flipped
2124 - if(after_requested > before_requested) {
2125 - long long t = before_requested;
2126 - before_requested = after_requested;
2127 - after_requested = t;
2128 - }
2129 -
2130 - // if the query requests future data
2131 - // shift the query back to be in the present time
2132 - // (this may also happen because of the rules above)
2133 - if(before_requested > now) {
2134 - long long delta = before_requested - now;
2135 - before_requested -= delta;
2136 - after_requested -= delta;
2137 - }
2138 -
2139 - time_t absolute_minimum_time = now - (10 * 365 * 86400);
2140 - time_t absolute_maximum_time = now + (1 * 365 * 86400);
2141 -
2142 - if (after_requested < absolute_minimum_time && !unittest_running)
2143 - after_requested = absolute_minimum_time;
2144 -
2145 - if (after_requested > absolute_maximum_time && !unittest_running)
2146 - after_requested = absolute_maximum_time;
2147 -
2148 - if (before_requested < absolute_minimum_time && !unittest_running)
2149 - before_requested = absolute_minimum_time;
2150 -
2151 - if (before_requested > absolute_maximum_time && !unittest_running)
2152 - before_requested = absolute_maximum_time;
2153 -
2154 - *before = before_requested;
2155 - *after = after_requested;
2156 -
2157 - return (absolute_period_requested != 1);
2158 -}
2159 -
2078 // #define DEBUG_QUERY_LOGIC 1
2079
2080 #ifdef DEBUG_QUERY_LOGIC
@@ -2283,7 +2201,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
2201 }
2202
2203 // convert our before_wanted and after_wanted to absolute
2286 - rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL);
2204 + rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL, unittest_running);
2205 query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
2206
2207 if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
web/api/queries/rrdr.h
-2
@@ -206,8 +206,6 @@ RRDR *rrd2rrdr_legacy(
206 RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
207 bool query_target_calculate_window(struct query_target *qt);
208
209 -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr);
210 -
209 #ifdef __cplusplus
210 }
211 #endif
web/api/queries/weights.c
+6 -6
@@ -169,7 +169,7 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
169 size_t examined_dimensions, usec_t duration,
170 WEIGHTS_STATS *stats) {
171
172 - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
172 + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
173
174 results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
175 points, method, group, options, shifts, examined_dimensions, duration, stats);
@@ -221,7 +221,7 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w
221 size_t examined_dimensions, usec_t duration,
222 WEIGHTS_STATS *stats) {
223
224 - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
224 + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
225
226 results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
227 points, method, group, options, shifts, examined_dimensions, duration, stats);
@@ -739,7 +739,7 @@ static size_t registered_results_to_json_multinode_no_group_by(
739 size_t examined_dimensions, struct query_weights_data *qwd,
740 WEIGHTS_STATS *stats,
741 struct query_versions *versions) {
742 - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
742 + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
743 buffer_json_member_add_uint64(wb, "api", 2);
744
745 results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
@@ -958,7 +958,7 @@ static size_t registered_results_to_json_multinode_group_by(
958 size_t examined_dimensions, struct query_weights_data *qwd,
959 WEIGHTS_STATS *stats,
960 struct query_versions *versions) {
961 - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
961 + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
962 buffer_json_member_add_uint64(wb, "api", 2);
963
964 results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
@@ -1806,7 +1806,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1806 }
1807 };
1808
1809 - if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL))
1809 + if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL, false))
1810 buffer_no_cacheable(wb);
1811 else
1812 buffer_cacheable(wb);
@@ -1823,7 +1823,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1823 if(qwr->baseline_before <= API_RELATIVE_TIME_MAX)
1824 qwr->baseline_before += qwr->after;
1825
1826 - rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL);
1826 + rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL, false);
1827
1828 if (qwr->baseline_before <= qwr->baseline_after) {
1829 resp = HTTP_RESP_BAD_REQUEST;
web/api/web_api.c
+17 -4
@@ -6,7 +6,7 @@ bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at
6 DICTIONARY *netdata_authorized_bearers = NULL;
7
8 static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL endpoint_acl) {
9 - if(endpoint_acl == WEB_CLIENT_ACL_NOCHECK)
9 + if(endpoint_acl == WEB_CLIENT_ACL_NONE || (endpoint_acl & WEB_CLIENT_ACL_NOCHECK))
10 // the endpoint is totally public
11 return true;
12
@@ -23,11 +23,24 @@ static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL
23 // endpoint does not require a bearer
24 return true;
25
26 - if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)) || api_check_bearer_token(w))
26 + if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)))
27 // the request is coming from ACLK or WEBRTC (authorized already),
28 - // or we have a valid bearer on the request
28 return true;
29
30 + // at this point we need a bearer to serve the request
31 + // either because:
32 + //
33 + // 1. WEB_CLIENT_ACL_BEARER_REQUIRED, or
34 + // 2. netdata_is_protected_by_bearer == true
35 + //
36 +
37 + BEARER_STATUS t = api_check_bearer_token(w);
38 + if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED)
39 + // we have a valid bearer on the request
40 + return true;
41 +
42 + netdata_log_info("BEARER: bearer is required for request: code %d", t);
43 +
44 return false;
45 }
46
@@ -60,7 +73,7 @@ int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_pat
73 return HTTP_RESP_BAD_REQUEST;
74 }
75
63 - if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))
76 + if(unlikely(!web_client_check_acl_and_bearer(w, api_commands[i].acl)))
77 return web_client_permission_denied(w);
78
79 char *query_string = (char *)buffer_tostring(w->url_query_string_decoded);
web/api/web_api.h
+13 -3
@@ -11,9 +11,19 @@
11
12 extern bool netdata_is_protected_by_bearer;
13 extern DICTIONARY *netdata_authorized_bearers;
14 -bool api_check_bearer_token(struct web_client *w);
15 -bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
16 -void bearer_tokens_init(void);
14 +typedef enum __attribute__((packed)) {
15 + BEARER_STATUS_NO_BEARER_IN_HEADERS,
16 + BEARER_STATUS_BEARER_DOES_NOT_FIT,
17 + BEARER_STATUS_NOT_PARSABLE,
18 + BEARER_STATUS_EXTRACTED_FROM_HEADER,
19 + BEARER_STATUS_NO_BEARERS_DICTIONARY,
20 + BEARER_STATUS_NOT_FOUND_IN_DICTIONARY,
21 + BEARER_STATUS_EXPIRED,
22 + BEARER_STATUS_AVAILABLE_AND_VALIDATED,
23 +} BEARER_STATUS;
24 +
25 +BEARER_STATUS api_check_bearer_token(struct web_client *w);
26 +BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
27
28 struct web_api_command {
29 const char *command;
web/api/web_api_v1.c
+4 -4
@@ -941,7 +941,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
941 char *cookie = strstr(w->response.data->buffer, NETDATA_REGISTRY_COOKIE_NAME "=");
942 if(cookie)
943 strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME)], UUID_STR_LEN - 1);
944 - else if(!extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)))
944 + else if(extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)) != BEARER_STATUS_EXTRACTED_FROM_HEADER)
945 person_guid[0] = '\0';
946
947 char action = '\0';
@@ -1199,7 +1199,7 @@ static void host_collectors(RRDHOST *host, BUFFER *wb) {
1199
1200 extern int aclk_connected;
1201 inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) {
1202 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
1202 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
1203
1204 buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));
1205 buffer_json_member_add_string(wb, "uid", host->machine_guid);
@@ -1319,7 +1319,7 @@ int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char
1319 buffer_flush(wb);
1320 wb->content_type = CT_APPLICATION_JSON;
1321
1322 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
1322 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
1323 ml_host_get_detection_info(host, wb);
1324 buffer_json_finalize(wb);
1325
@@ -1424,7 +1424,7 @@ int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, cha
1424 wb->content_type = CT_APPLICATION_JSON;
1425 buffer_no_cacheable(wb);
1426
1427 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
1427 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
1428 host_functions2json(host, wb);
1429 buffer_json_finalize(wb);
1430
web/api/web_api_v2.c
+19 -12
@@ -54,13 +54,13 @@ static time_t bearer_get_token(uuid_t *uuid) {
54
55 #define HTTP_REQUEST_AUTHORIZATION_BEARER "\r\nAuthorization: Bearer "
56
57 -bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
57 +BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
58 const char *req = buffer_tostring(w->response.data);
59 size_t req_len = buffer_strlen(w->response.data);
60 const char *bearer = strcasestr(req, HTTP_REQUEST_AUTHORIZATION_BEARER);
61
62 if(!bearer)
63 - return false;
63 + return BEARER_STATUS_NO_BEARER_IN_HEADERS;
64
65 const char *token_start = bearer + sizeof(HTTP_REQUEST_AUTHORIZATION_BEARER) - 1;
66
@@ -69,26 +69,33 @@ bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t d
69
70 const char *token_end = token_start + UUID_STR_LEN - 1 + 2;
71 if (token_end > req + req_len)
72 - return false;
72 + return BEARER_STATUS_BEARER_DOES_NOT_FIT;
73
74 strncpyz(dst, token_start, dst_len - 1);
75 uuid_t uuid;
76 if (uuid_parse(dst, uuid) != 0)
77 - return false;
77 + return BEARER_STATUS_NOT_PARSABLE;
78
79 - return true;
79 + return BEARER_STATUS_EXTRACTED_FROM_HEADER;
80 }
81
82 -bool api_check_bearer_token(struct web_client *w) {
82 +BEARER_STATUS api_check_bearer_token(struct web_client *w) {
83 if(!netdata_authorized_bearers)
84 - return false;
84 + return BEARER_STATUS_NO_BEARERS_DICTIONARY;
85
86 char token[UUID_STR_LEN];
87 - if(!extract_bearer_token_from_request(w, token, sizeof(token)))
88 - return false;
87 + BEARER_STATUS t = extract_bearer_token_from_request(w, token, sizeof(token));
88 + if(t != BEARER_STATUS_EXTRACTED_FROM_HEADER)
89 + return t;
90
91 struct bearer_token *z = dictionary_get(netdata_authorized_bearers, token);
91 - return z && z->expires_s > now_monotonic_sec();
92 + if(!z)
93 + return BEARER_STATUS_NOT_FOUND_IN_DICTIONARY;
94 +
95 + if(z->expires_s < now_monotonic_sec())
96 + return BEARER_STATUS_EXPIRED;
97 +
98 + return BEARER_STATUS_AVAILABLE_AND_VALIDATED;
99 }
100
101 static bool verify_agent_uuids(const char *machine_guid, const char *node_id, const char *claim_id) {
@@ -153,7 +160,7 @@ int api_v2_bearer_protection(RRDHOST *host __maybe_unused, struct web_client *w
160
161 BUFFER *wb = w->response.data;
162 buffer_flush(wb);
156 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
163 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
164 buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
165 buffer_json_finalize(wb);
166
@@ -192,7 +199,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may
199
200 BUFFER *wb = w->response.data;
201 buffer_flush(wb);
195 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
202 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
203 buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
204 buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
205 buffer_json_member_add_uuid(wb, "token", &uuid);
web/rtc/webrtc.c
+2 -2
@@ -304,7 +304,7 @@ static void webrtc_execute_api_request(WEBRTC_DC *chan, const char *request, siz
304 web_client_timeout_checkpoint_set(w, 0);
305 web_client_decode_path_and_query_string(w, path);
306 path = (char *)buffer_tostring(w->url_path_decoded);
307 - w->response.code = (short)web_client_api_request_with_node_selection(rrdb.localhost, w, path);
307 + w->response.code = (short)web_client_api_request_with_node_selection(localhost, w, path);
308 web_client_timeout_checkpoint_response_ready(w, NULL);
309
310 size_t sent_bytes = 0;
@@ -643,7 +643,7 @@ int webrtc_new_connection(const char *sdp, BUFFER *wb) {
643 }
644
645 buffer_flush(wb);
646 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);
646 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
647 wb->content_type = CT_APPLICATION_JSON;
648
649 WEBRTC_CONN *conn = webrtc_create_connection();