[WIP] Windows-Events Logs Explorer (#18483)
* preparations * work on systemd-journal to abstract the components we need to use in windows events logs * more code reorg to reuse systemd journal * abstracted logs query parsing and preparation * fix help printing * moved non-LQS fields outside logs_query_status.h * code re-organization and cleanup * move lqs cleanup to lqs * code re-organization and cleanup * support accurate and approximate data sizes printing * support for printing durations with spaces, for clarity * progress on windows events sources * windows events sources, with a count of logs * windows events sources, with duration * windows events sources, with file size * support for converting between unix and windows timestamps * fixed conditions * internal api for querying windows event logs * extract basic fields from windows events * fix sid and guid * users lookup with caching * basic structure for the netdata plugin * decode more message fields * event log sources are now periodically scanned * progress on building the query for windows events * first working version * support external plugins with .plugin.exe suffix * fix compilation warnings * fix thread safety * fixes in severity and buffers used * match all rows and do not pass empty data to facets * keyword should be hex * strict type checking and more fields, including the full XML view * restructure the code to keep caches between channel queries * format a message using the XML event data * message is now formatted as a dynamic field at presentation time * extracted UserData and applied basic XML formatting on the message * properly intent nested nodes
Costa Tsaousis committed
Sep 10, 2024 at 20:03 UTC
2c9bca448e4f4fc63e24d58e57495aa84f5c91d5
43 files changed
+4648
-1483
CMakeLists.txt
+28
@@ -789,6 +789,9 @@ set(LIBNETDATA_FILES
789
src/libnetdata/config/appconfig_api_text.h
790
src/libnetdata/config/appconfig_api_boolean.c
791
src/libnetdata/config/appconfig_api_boolean.h
792
+ src/libnetdata/facets/logs_query_status.h
793
+ src/libnetdata/os/timestamps.c
794
+ src/libnetdata/os/timestamps.h
795
)
796
797
if(ENABLE_PLUGIN_EBPF)
@@ -1271,6 +1274,7 @@ set(SYSTEMD_JOURNAL_PLUGIN_FILES
1274
src/libnetdata/maps/system-users.h
1275
src/libnetdata/maps/system-groups.h
1276
src/libnetdata/maps/system-services.h
1277
+ src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
1278
)
1279
1280
set(STREAMING_PLUGIN_FILES
@@ -1393,6 +1397,21 @@ set(FREEBSD_PLUGIN_FILES
1397
src/collectors/proc.plugin/zfs_common.h
1398
)
1399
1400
+set(WINDOWS_EVENTS_PLUGIN_FILES
1401
+ src/collectors/windows-events.plugin/windows-events.c
1402
+ src/collectors/windows-events.plugin/windows-events.h
1403
+ src/collectors/windows-events.plugin/windows-events-query.h
1404
+ src/collectors/windows-events.plugin/windows-events-query.c
1405
+ src/collectors/windows-events.plugin/windows-events-sources.c
1406
+ src/collectors/windows-events.plugin/windows-events-sources.h
1407
+ src/collectors/windows-events.plugin/windows-events-sid.c
1408
+ src/collectors/windows-events.plugin/windows-events-sid.h
1409
+ src/collectors/windows-events.plugin/windows-events-unicode.c
1410
+ src/collectors/windows-events.plugin/windows-events-unicode.h
1411
+ src/collectors/windows-events.plugin/windows-events-xml.c
1412
+ src/collectors/windows-events.plugin/windows-events-xml.h
1413
+)
1414
+
1415
set(WINDOWS_PLUGIN_FILES
1416
src/collectors/windows.plugin/windows_plugin.c
1417
src/collectors/windows.plugin/windows_plugin.h
@@ -2081,6 +2100,15 @@ if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
2100
endif()
2101
endif()
2102
2103
+if(OS_WINDOWS)
2104
+ add_executable(windows-events.plugin ${WINDOWS_EVENTS_PLUGIN_FILES})
2105
+ target_link_libraries(windows-events.plugin libnetdata wevtapi)
2106
+
2107
+ install(TARGETS windows-events.plugin
2108
+ COMPONENT plugin-windows-events
2109
+ DESTINATION usr/libexec/netdata/plugins.d)
2110
+endif()
2111
+
2112
if(ENABLE_PLUGIN_EBPF)
2113
set(EBPF_PLUGIN_FILES
2114
src/collectors/ebpf.plugin/ebpf.c
packaging/utils/compile-on-windows.sh
+25
-20
@@ -51,28 +51,33 @@ fi
51
52
set -exu -o pipefail
53
54
-if [ -d "${build}" ]
54
+if [ ! -d "${build}" ]
55
then
56
- rm -rf "${build}"
56
+ /usr/bin/cmake -S "${WT_ROOT}" -B "${build}" \
57
+ -G Ninja \
58
+ -DCMAKE_INSTALL_PREFIX="/opt/netdata" \
59
+ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
60
+ -DCMAKE_C_FLAGS="-fstack-protector-all -O0 -ggdb -Wall -Wextra -Wno-char-subscripts -Wa,-mbig-obj -pipe -DNETDATA_INTERNAL_CHECKS=1 -D_FILE_OFFSET_BITS=64 -D__USE_MINGW_ANSI_STDIO=1" \
61
+ -DBUILD_FOR_PACKAGING=${BUILD_FOR_PACKAGING} \
62
+ -DUSE_MOLD=Off \
63
+ -DNETDATA_USER="${USER}" \
64
+ -DDEFAULT_FEATURE_STATE=Off \
65
+ -DENABLE_H2O=Off \
66
+ -DENABLE_ML=On \
67
+ -DENABLE_BUNDLED_JSONC=On \
68
+ -DENABLE_BUNDLED_PROTOBUF=Off \
69
+ ${NULL}
70
fi
71
59
-/usr/bin/cmake -S "${WT_ROOT}" -B "${build}" \
60
- -G Ninja \
61
- -DCMAKE_INSTALL_PREFIX="/opt/netdata" \
62
- -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
63
- -DCMAKE_C_FLAGS="-fstack-protector-all -O0 -ggdb -Wall -Wextra -Wno-char-subscripts -Wa,-mbig-obj -pipe -DNETDATA_INTERNAL_CHECKS=1 -D_FILE_OFFSET_BITS=64 -D__USE_MINGW_ANSI_STDIO=1" \
64
- -DBUILD_FOR_PACKAGING=${BUILD_FOR_PACKAGING} \
65
- -DUSE_MOLD=Off \
66
- -DNETDATA_USER="${USER}" \
67
- -DDEFAULT_FEATURE_STATE=Off \
68
- -DENABLE_H2O=Off \
69
- -DENABLE_ML=On \
70
- -DENABLE_BUNDLED_JSONC=On \
71
- -DENABLE_BUNDLED_PROTOBUF=Off \
72
- ${NULL}
73
-
72
ninja -v -C "${build}" install || ninja -v -C "${build}" -j 1
73
76
-echo
77
-echo "Compile with:"
78
-echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"
74
+#echo
75
+#echo "Compile with:"
76
+#echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"
77
+
78
+echo "starting netdata..."
79
+# enable JIT debug with gdb
80
+export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"
81
+
82
+rm -rf /opt/netdata/var/log/netdata/*.log || echo
83
+/opt/netdata/usr/bin/netdata -D
src/collectors/apps.plugin/apps_functions.c
+8
-2
@@ -45,7 +45,10 @@ static void apps_plugin_function_processes_help(const char *transaction) {
45
"Filters can be combined. Each filter can be given only one time.\n"
46
);
47
48
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);
48
+ wb->response_code = HTTP_RESP_OK;
49
+ wb->content_type = CT_TEXT_PLAIN;
50
+ wb->expires = now_realtime_sec() + 3600;
51
+ pluginsd_function_result_to_stdout(transaction, wb);
52
buffer_free(wb);
53
}
54
@@ -922,7 +925,10 @@ close_and_send:
925
buffer_json_member_add_time_t(wb, "expires", now_s + update_every);
926
buffer_json_finalize(wb);
927
925
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", now_s + update_every, wb);
928
+ wb->response_code = HTTP_RESP_OK;
929
+ wb->content_type = CT_APPLICATION_JSON;
930
+ wb->expires = now_s + update_every;
931
+ pluginsd_function_result_to_stdout(transaction, wb);
932
933
buffer_free(wb);
934
}
src/collectors/freeipmi.plugin/freeipmi_plugin.c
+6
-2
@@ -1629,7 +1629,10 @@ close_and_send:
1629
buffer_json_member_add_time_t(wb, "expires", now_s + update_every);
1630
buffer_json_finalize(wb);
1631
1632
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", now_s + update_every, wb);
1632
+ wb->response_code = HTTP_RESP_OK;
1633
+ wb->content_type = CT_APPLICATION_JSON;
1634
+ wb->expires = now_s + update_every;
1635
+ pluginsd_function_result_to_stdout(transaction, wb);
1636
1637
buffer_free(wb);
1638
}
@@ -1637,7 +1640,6 @@ close_and_send:
1640
// ----------------------------------------------------------------------------
1641
// main, command line arguments parsing
1642
1640
-static void plugin_exit(int code) NORETURN;
1643
static void plugin_exit(int code) {
1644
fflush(stdout);
1645
function_plugin_should_exit = true;
@@ -2042,11 +2044,13 @@ int main (int argc, char **argv) {
2044
collector_error("%s(): sensors failed to initialize. Calling DISABLE.", __FUNCTION__);
2045
fprintf(stdout, "DISABLE\n");
2046
plugin_exit(0);
2047
+ break;
2048
2049
case ICS_FAILED:
2050
collector_error("%s(): sensors fails repeatedly to collect metrics. Exiting to restart.", __FUNCTION__);
2051
fprintf(stdout, "EXIT\n");
2052
plugin_exit(0);
2053
+ break;
2054
}
2055
2056
if(netdata_do_sel) {
src/collectors/network-viewer.plugin/network-viewer.c
+4
-1
@@ -945,7 +945,10 @@ close_and_send:
945
buffer_json_finalize(wb);
946
947
netdata_mutex_lock(&stdout_mutex);
948
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", now_s + 1, wb);
948
+ wb->response_code = HTTP_RESP_OK;
949
+ wb->content_type = CT_APPLICATION_JSON;
950
+ wb->expires = now_s + 1;
951
+ pluginsd_function_result_to_stdout(transaction, wb);
952
netdata_mutex_unlock(&stdout_mutex);
953
}
954
src/collectors/plugins.d/plugins_d.c
+32
-10
@@ -186,8 +186,8 @@ static void *pluginsd_worker_thread(void *arg) {
186
spawn_popen_write_fd(cd->unsafe.pi),
187
0);
188
189
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
190
- "PLUGINSD: 'host:%s', '%s' (pid %d) disconnected after %zu successful data collections (ENDs).",
189
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING,
190
+ "PLUGINSD: 'host:%s', '%s' (pid %d) disconnected after %zu successful data collections.",
191
rrdhost_hostname(cd->host), cd->fullfilename, cd->unsafe.pid, count);
192
193
int worker_ret_code = spawn_popen_kill(cd->unsafe.pi);
@@ -231,6 +231,33 @@ static void pluginsd_main_cleanup(void *pptr) {
231
worker_unregister();
232
}
233
234
+static bool is_plugin(char *dst, size_t dst_size, const char *filename) {
235
+ size_t len = strlen(filename);
236
+
237
+ const char *suffix;
238
+ size_t suffix_len;
239
+
240
+ suffix = ".plugin";
241
+ suffix_len = strlen(suffix);
242
+ if (len > suffix_len &&
243
+ strcmp(suffix, &filename[len - suffix_len]) == 0) {
244
+ snprintfz(dst, dst_size, "%.*s", (int)(len - suffix_len), filename);
245
+ return true;
246
+ }
247
+
248
+#if defined(OS_WINDOWS)
249
+ suffix = ".plugin.exe";
250
+ suffix_len = strlen(suffix);
251
+ if (len > suffix_len &&
252
+ strcmp(suffix, &filename[len - suffix_len]) == 0) {
253
+ snprintfz(dst, dst_size, "%.*s", (int)(len - suffix_len), filename);
254
+ return true;
255
+ }
256
+#endif
257
+
258
+ return false;
259
+}
260
+
261
void *pluginsd_main(void *ptr) {
262
CLEANUP_FUNCTION_REGISTER(pluginsd_main_cleanup) cleanup_ptr = ptr;
263
@@ -279,18 +306,13 @@ void *pluginsd_main(void *ptr) {
306
if (unlikely(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0))
307
continue;
308
282
- int len = (int)strlen(file->d_name);
283
- if (unlikely(len <= (int)PLUGINSD_FILE_SUFFIX_LEN))
284
- continue;
285
- if (unlikely(strcmp(PLUGINSD_FILE_SUFFIX, &file->d_name[len - (int)PLUGINSD_FILE_SUFFIX_LEN]) != 0)) {
286
- netdata_log_debug(D_PLUGINSD, "file '%s' does not end in '%s'", file->d_name, PLUGINSD_FILE_SUFFIX);
309
+ char pluginname[CONFIG_MAX_NAME + 1];
310
+ if(!is_plugin(pluginname, sizeof(pluginname), file->d_name)) {
311
+ netdata_log_debug(D_PLUGINSD, "file '%s' does not look like a plugin", file->d_name);
312
continue;
313
}
314
290
- char pluginname[CONFIG_MAX_NAME + 1];
291
- snprintfz(pluginname, CONFIG_MAX_NAME, "%.*s", (int)(len - PLUGINSD_FILE_SUFFIX_LEN), file->d_name);
315
int enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, pluginname, automatic_run);
293
-
316
if (unlikely(!enabled)) {
317
netdata_log_debug(D_PLUGINSD, "plugin '%s' is not enabled", file->d_name);
318
continue;
src/collectors/plugins.d/plugins_d.h
+1
-3
@@ -5,9 +5,7 @@
5
6
#include "daemon/common.h"
7
8
-#define PLUGINSD_FILE_SUFFIX ".plugin"
9
-#define PLUGINSD_FILE_SUFFIX_LEN strlen(PLUGINSD_FILE_SUFFIX)
10
-#define PLUGINSD_CMD_MAX (FILENAME_MAX*2)
8
+ #define PLUGINSD_CMD_MAX (FILENAME_MAX*2)
9
#define PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH 0
10
11
#define PLUGINSD_MAX_DIRECTORIES 20
src/collectors/plugins.d/pluginsd_parser.c
+3
-1
@@ -1215,8 +1215,10 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, int fd_input,
1215
&parser->reader, parser->fd_input,
1216
2 * 60 * MSEC_PER_SEC, true);
1217
1218
- if(unlikely(ret != BUFFERED_READER_READ_OK))
1218
+ if(unlikely(ret != BUFFERED_READER_READ_OK)) {
1219
+ nd_log(NDLS_COLLECTORS, NDLP_INFO, "Buffered reader not OK");
1220
break;
1221
+ }
1222
1223
continue;
1224
}
src/collectors/systemd-journal.plugin/systemd-journal-files.c
+6
-34
@@ -475,19 +475,6 @@ struct journal_file_source {
475
uint64_t size;
476
};
477
478
-static void human_readable_size_ib(uint64_t size, char *dst, size_t dst_len) {
479
- if(size > 1024ULL * 1024 * 1024 * 1024)
480
- snprintfz(dst, dst_len, "%0.2f TiB", (double)size / 1024.0 / 1024.0 / 1024.0 / 1024.0);
481
- else if(size > 1024ULL * 1024 * 1024)
482
- snprintfz(dst, dst_len, "%0.2f GiB", (double)size / 1024.0 / 1024.0 / 1024.0);
483
- else if(size > 1024ULL * 1024)
484
- snprintfz(dst, dst_len, "%0.2f MiB", (double)size / 1024.0 / 1024.0);
485
- else if(size > 1024ULL)
486
- snprintfz(dst, dst_len, "%0.2f KiB", (double)size / 1024.0);
487
- else
488
- snprintfz(dst, dst_len, "%"PRIu64" B", size);
489
-}
490
-
478
#define print_duration(dst, dst_len, pos, remaining, duration, one, many, printed) do { \
479
if((remaining) > (duration)) { \
480
uint64_t _count = (remaining) / (duration); \
@@ -498,22 +485,6 @@ static void human_readable_size_ib(uint64_t size, char *dst, size_t dst_len) {
485
} \
486
} while(0)
487
501
-static void human_readable_duration_s(time_t duration_s, char *dst, size_t dst_len) {
502
- if(duration_s < 0)
503
- duration_s = -duration_s;
504
-
505
- size_t pos = 0;
506
- dst[0] = 0 ;
507
-
508
- bool printed = false;
509
- print_duration(dst, dst_len, pos, duration_s, 86400 * 365, "year", "years", printed);
510
- print_duration(dst, dst_len, pos, duration_s, 86400 * 30, "month", "months", printed);
511
- print_duration(dst, dst_len, pos, duration_s, 86400 * 1, "day", "days", printed);
512
- print_duration(dst, dst_len, pos, duration_s, 3600 * 1, "hour", "hours", printed);
513
- print_duration(dst, dst_len, pos, duration_s, 60 * 1, "min", "mins", printed);
514
- print_duration(dst, dst_len, pos, duration_s, 1, "sec", "secs", printed);
515
-}
516
-
488
static int journal_file_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) {
489
struct journal_file_source *jfs = entry;
490
BUFFER *wb = data;
@@ -522,12 +493,12 @@ static int journal_file_to_json_array_cb(const DICTIONARY_ITEM *item, void *entr
493
494
buffer_json_add_array_item_object(wb);
495
{
525
- char size_for_humans[100];
526
- human_readable_size_ib(jfs->size, size_for_humans, sizeof(size_for_humans));
496
+ char size_for_humans[128];
497
+ size_snprintf(size_for_humans, sizeof(size_for_humans), jfs->size, "B", false);
498
528
- char duration_for_humans[1024];
529
- human_readable_duration_s((time_t)((jfs->last_ut - jfs->first_ut) / USEC_PER_SEC),
530
- duration_for_humans, sizeof(duration_for_humans));
499
+ char duration_for_humans[128];
500
+ duration_snprintf(duration_for_humans, sizeof(duration_for_humans),
501
+ (time_t)((jfs->last_ut - jfs->first_ut) / USEC_PER_SEC), "s", true);
502
503
char info[1024];
504
snprintfz(info, sizeof(info), "%zu files, with a total size of %s, covering %s",
@@ -756,6 +727,7 @@ void journal_files_registry_update(void) {
727
dictionary_del(journal_files_registry, jf_dfe.name);
728
}
729
dfe_done(jf);
730
+ dictionary_garbage_collect(journal_files_registry);
731
732
journal_files_scans++;
733
spinlock_unlock(&spinlock);
src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
new
+378
@@ -0,0 +1,378 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_SYSTEMD_JOURNAL_SAMPLING_H
4
+#define NETDATA_SYSTEMD_JOURNAL_SAMPLING_H
5
+
6
+// ----------------------------------------------------------------------------
7
+// sampling support
8
+
9
+static inline void sampling_query_init(LOGS_QUERY_STATUS *lqs, FACETS *facets) {
10
+ if(!lqs->rq.sampling)
11
+ return;
12
+
13
+ if(!lqs->rq.slice) {
14
+ // the user is doing a full data query
15
+ // disable sampling
16
+ lqs->rq.sampling = 0;
17
+ return;
18
+ }
19
+
20
+ if(lqs->rq.data_only) {
21
+ // the user is doing a data query
22
+ // disable sampling
23
+ lqs->rq.sampling = 0;
24
+ return;
25
+ }
26
+
27
+ if(!lqs->c.files_matched) {
28
+ // no files have been matched
29
+ // disable sampling
30
+ lqs->rq.sampling = 0;
31
+ return;
32
+ }
33
+
34
+ lqs->c.samples.slots = facets_histogram_slots(facets);
35
+ if(lqs->c.samples.slots < 2)
36
+ lqs->c.samples.slots = 2;
37
+ if(lqs->c.samples.slots > SYSTEMD_JOURNAL_SAMPLING_SLOTS)
38
+ lqs->c.samples.slots = SYSTEMD_JOURNAL_SAMPLING_SLOTS;
39
+
40
+ if(!lqs->rq.after_ut || !lqs->rq.before_ut || lqs->rq.after_ut >= lqs->rq.before_ut) {
41
+ // we don't have enough information for sampling
42
+ lqs->rq.sampling = 0;
43
+ return;
44
+ }
45
+
46
+ usec_t delta = lqs->rq.before_ut - lqs->rq.after_ut;
47
+ usec_t step = delta / facets_histogram_slots(facets) - 1;
48
+ if(step < 1) step = 1;
49
+
50
+ lqs->c.samples_per_time_slot.start_ut = lqs->rq.after_ut;
51
+ lqs->c.samples_per_time_slot.end_ut = lqs->rq.before_ut;
52
+ lqs->c.samples_per_time_slot.step_ut = step;
53
+
54
+ // the minimum number of rows to enable sampling
55
+ lqs->c.samples.enable_after_samples = lqs->rq.sampling / 2;
56
+
57
+ size_t files_matched = lqs->c.files_matched;
58
+ if(!files_matched)
59
+ files_matched = 1;
60
+
61
+ // the minimum number of rows per file to enable sampling
62
+ lqs->c.samples_per_file.enable_after_samples = (lqs->rq.sampling / 4) / files_matched;
63
+ if(lqs->c.samples_per_file.enable_after_samples < lqs->rq.entries)
64
+ lqs->c.samples_per_file.enable_after_samples = lqs->rq.entries;
65
+
66
+ // the minimum number of rows per time slot to enable sampling
67
+ lqs->c.samples_per_time_slot.enable_after_samples = (lqs->rq.sampling / 4) / lqs->c.samples.slots;
68
+ if(lqs->c.samples_per_time_slot.enable_after_samples < lqs->rq.entries)
69
+ lqs->c.samples_per_time_slot.enable_after_samples = lqs->rq.entries;
70
+}
71
+
72
+static inline void sampling_file_init(LOGS_QUERY_STATUS *lqs, struct journal_file *jf __maybe_unused) {
73
+ lqs->c.samples_per_file.sampled = 0;
74
+ lqs->c.samples_per_file.unsampled = 0;
75
+ lqs->c.samples_per_file.estimated = 0;
76
+ lqs->c.samples_per_file.every = 0;
77
+ lqs->c.samples_per_file.skipped = 0;
78
+ lqs->c.samples_per_file.recalibrate = 0;
79
+}
80
+
81
+static inline size_t sampling_file_lines_scanned_so_far(LOGS_QUERY_STATUS *lqs) {
82
+ size_t sampled = lqs->c.samples_per_file.sampled + lqs->c.samples_per_file.unsampled;
83
+ if(!sampled) sampled = 1;
84
+ return sampled;
85
+}
86
+
87
+static inline void sampling_running_file_query_overlapping_timeframe_ut(
88
+ LOGS_QUERY_STATUS *lqs, struct journal_file *jf, FACETS_ANCHOR_DIRECTION direction,
89
+ usec_t msg_ut, usec_t *after_ut, usec_t *before_ut) {
90
+
91
+ // find the overlap of the query and file timeframes
92
+ // taking into account the first message we encountered
93
+
94
+ usec_t oldest_ut, newest_ut;
95
+ if(direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
96
+ // the first message we know (oldest)
97
+ oldest_ut = lqs->c.query_file.first_msg_ut ? lqs->c.query_file.first_msg_ut : jf->msg_first_ut;
98
+ if(!oldest_ut) oldest_ut = lqs->c.query_file.start_ut;
99
+
100
+ if(jf->msg_last_ut)
101
+ newest_ut = MIN(lqs->c.query_file.stop_ut, jf->msg_last_ut);
102
+ else if(jf->file_last_modified_ut)
103
+ newest_ut = MIN(lqs->c.query_file.stop_ut, jf->file_last_modified_ut);
104
+ else
105
+ newest_ut = lqs->c.query_file.stop_ut;
106
+
107
+ if(msg_ut < oldest_ut)
108
+ oldest_ut = msg_ut - 1;
109
+ }
110
+ else /* BACKWARD */ {
111
+ // the latest message we know (newest)
112
+ newest_ut = lqs->c.query_file.first_msg_ut ? lqs->c.query_file.first_msg_ut : jf->msg_last_ut;
113
+ if(!newest_ut) newest_ut = lqs->c.query_file.start_ut;
114
+
115
+ if(jf->msg_first_ut)
116
+ oldest_ut = MAX(lqs->c.query_file.stop_ut, jf->msg_first_ut);
117
+ else
118
+ oldest_ut = lqs->c.query_file.stop_ut;
119
+
120
+ if(newest_ut < msg_ut)
121
+ newest_ut = msg_ut + 1;
122
+ }
123
+
124
+ *after_ut = oldest_ut;
125
+ *before_ut = newest_ut;
126
+}
127
+
128
+static inline double sampling_running_file_query_progress_by_time(
129
+ LOGS_QUERY_STATUS *lqs, struct journal_file *jf,
130
+ FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut) {
131
+
132
+ usec_t after_ut, before_ut, elapsed_ut;
133
+ sampling_running_file_query_overlapping_timeframe_ut(lqs, jf, direction, msg_ut, &after_ut, &before_ut);
134
+
135
+ if(direction == FACETS_ANCHOR_DIRECTION_FORWARD)
136
+ elapsed_ut = msg_ut - after_ut;
137
+ else
138
+ elapsed_ut = before_ut - msg_ut;
139
+
140
+ usec_t total_ut = before_ut - after_ut;
141
+ double progress = (double)elapsed_ut / (double)total_ut;
142
+
143
+ return progress;
144
+}
145
+
146
+static inline usec_t sampling_running_file_query_remaining_time(
147
+ LOGS_QUERY_STATUS *lqs, struct journal_file *jf,
148
+ FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut,
149
+ usec_t *total_time_ut, usec_t *remaining_start_ut,
150
+ usec_t *remaining_end_ut) {
151
+ usec_t after_ut, before_ut;
152
+ sampling_running_file_query_overlapping_timeframe_ut(lqs, jf, direction, msg_ut, &after_ut, &before_ut);
153
+
154
+ // since we have a timestamp in msg_ut
155
+ // this timestamp can extend the overlap
156
+ if(msg_ut <= after_ut)
157
+ after_ut = msg_ut - 1;
158
+
159
+ if(msg_ut >= before_ut)
160
+ before_ut = msg_ut + 1;
161
+
162
+ // return the remaining duration
163
+ usec_t remaining_from_ut, remaining_to_ut;
164
+ if(direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
165
+ remaining_from_ut = msg_ut;
166
+ remaining_to_ut = before_ut;
167
+ }
168
+ else {
169
+ remaining_from_ut = after_ut;
170
+ remaining_to_ut = msg_ut;
171
+ }
172
+
173
+ usec_t remaining_ut = remaining_to_ut - remaining_from_ut;
174
+
175
+ if(total_time_ut)
176
+ *total_time_ut = (before_ut > after_ut) ? before_ut - after_ut : 1;
177
+
178
+ if(remaining_start_ut)
179
+ *remaining_start_ut = remaining_from_ut;
180
+
181
+ if(remaining_end_ut)
182
+ *remaining_end_ut = remaining_to_ut;
183
+
184
+ return remaining_ut;
185
+}
186
+
187
+static inline size_t sampling_running_file_query_estimate_remaining_lines_by_time(
188
+ LOGS_QUERY_STATUS *lqs,
189
+ struct journal_file *jf,
190
+ FACETS_ANCHOR_DIRECTION direction,
191
+ usec_t msg_ut) {
192
+ size_t scanned_lines = sampling_file_lines_scanned_so_far(lqs);
193
+
194
+ // Calculate the proportion of time covered
195
+ usec_t total_time_ut, remaining_start_ut, remaining_end_ut;
196
+ usec_t remaining_time_ut = sampling_running_file_query_remaining_time(
197
+ lqs, jf, direction, msg_ut, &total_time_ut, &remaining_start_ut, &remaining_end_ut);
198
+ if (total_time_ut == 0) total_time_ut = 1;
199
+
200
+ double proportion_by_time = (double) (total_time_ut - remaining_time_ut) / (double) total_time_ut;
201
+
202
+ if (proportion_by_time == 0 || proportion_by_time > 1.0 || !isfinite(proportion_by_time))
203
+ proportion_by_time = 1.0;
204
+
205
+ // Estimate the total number of lines in the file
206
+ size_t expected_matching_logs_by_time = (size_t)((double)scanned_lines / proportion_by_time);
207
+
208
+ if(jf->messages_in_file && expected_matching_logs_by_time > jf->messages_in_file)
209
+ expected_matching_logs_by_time = jf->messages_in_file;
210
+
211
+ // Calculate the estimated number of remaining lines
212
+ size_t remaining_logs_by_time = expected_matching_logs_by_time - scanned_lines;
213
+ if (remaining_logs_by_time < 1) remaining_logs_by_time = 1;
214
+
215
+ // nd_log(NDLS_COLLECTORS, NDLP_INFO,
216
+ // "JOURNAL ESTIMATION: '%s' "
217
+ // "scanned_lines=%zu [sampled=%zu, unsampled=%zu, estimated=%zu], "
218
+ // "file [%"PRIu64" - %"PRIu64", duration %"PRId64", known lines in file %zu], "
219
+ // "query [%"PRIu64" - %"PRIu64", duration %"PRId64"], "
220
+ // "first message read from the file at %"PRIu64", current message at %"PRIu64", "
221
+ // "proportion of time %.2f %%, "
222
+ // "expected total lines in file %zu, "
223
+ // "remaining lines %zu, "
224
+ // "remaining time %"PRIu64" [%"PRIu64" - %"PRIu64", duration %"PRId64"]"
225
+ // , jf->filename
226
+ // , scanned_lines, fqs->samples_per_file.sampled, fqs->samples_per_file.unsampled, fqs->samples_per_file.estimated
227
+ // , jf->msg_first_ut, jf->msg_last_ut, jf->msg_last_ut - jf->msg_first_ut, jf->messages_in_file
228
+ // , fqs->query_file.start_ut, fqs->query_file.stop_ut, fqs->query_file.stop_ut - fqs->query_file.start_ut
229
+ // , fqs->query_file.first_msg_ut, msg_ut
230
+ // , proportion_by_time * 100.0
231
+ // , expected_matching_logs_by_time
232
+ // , remaining_logs_by_time
233
+ // , remaining_time_ut, remaining_start_ut, remaining_end_ut, remaining_end_ut - remaining_start_ut
234
+ // );
235
+
236
+ return remaining_logs_by_time;
237
+}
238
+
239
+static inline size_t sampling_running_file_query_estimate_remaining_lines(
240
+ sd_journal *j __maybe_unused, LOGS_QUERY_STATUS *lqs, struct journal_file *jf,
241
+ FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut) {
242
+ size_t remaining_logs_by_seqnum = 0;
243
+
244
+#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
245
+ size_t expected_matching_logs_by_seqnum = 0;
246
+ double proportion_by_seqnum = 0.0;
247
+ uint64_t current_msg_seqnum;
248
+ sd_id128_t current_msg_writer;
249
+ if(!lqs->c.query_file.first_msg_seqnum || sd_journal_get_seqnum(j, ¤t_msg_seqnum, ¤t_msg_writer) < 0) {
250
+ lqs->c.query_file.first_msg_seqnum = 0;
251
+ lqs->c.query_file.first_msg_writer = SD_ID128_NULL;
252
+ }
253
+ else if(jf->messages_in_file) {
254
+ size_t scanned_lines = sampling_file_lines_scanned_so_far(lqs);
255
+
256
+ double proportion_of_all_lines_so_far;
257
+ if(direction == FACETS_ANCHOR_DIRECTION_FORWARD)
258
+ proportion_of_all_lines_so_far = (double)scanned_lines / (double)(current_msg_seqnum - jf->first_seqnum);
259
+ else
260
+ proportion_of_all_lines_so_far = (double)scanned_lines / (double)(jf->last_seqnum - current_msg_seqnum);
261
+
262
+ if(proportion_of_all_lines_so_far > 1.0)
263
+ proportion_of_all_lines_so_far = 1.0;
264
+
265
+ expected_matching_logs_by_seqnum = (size_t)(proportion_of_all_lines_so_far * (double)jf->messages_in_file);
266
+
267
+ proportion_by_seqnum = (double)scanned_lines / (double)expected_matching_logs_by_seqnum;
268
+
269
+ if (proportion_by_seqnum == 0 || proportion_by_seqnum > 1.0 || !isfinite(proportion_by_seqnum))
270
+ proportion_by_seqnum = 1.0;
271
+
272
+ remaining_logs_by_seqnum = expected_matching_logs_by_seqnum - scanned_lines;
273
+ if(!remaining_logs_by_seqnum) remaining_logs_by_seqnum = 1;
274
+ }
275
+#endif
276
+
277
+ if(remaining_logs_by_seqnum)
278
+ return remaining_logs_by_seqnum;
279
+
280
+ return sampling_running_file_query_estimate_remaining_lines_by_time(lqs, jf, direction, msg_ut);
281
+}
282
+
283
+static inline void sampling_decide_file_sampling_every(sd_journal *j,
284
+ LOGS_QUERY_STATUS *lqs, struct journal_file *jf, FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut) {
285
+ size_t files_matched = lqs->c.files_matched;
286
+ if(!files_matched) files_matched = 1;
287
+
288
+ size_t remaining_lines = sampling_running_file_query_estimate_remaining_lines(j, lqs, jf, direction, msg_ut);
289
+ size_t wanted_samples = (lqs->rq.sampling / 2) / files_matched;
290
+ if(!wanted_samples) wanted_samples = 1;
291
+
292
+ lqs->c.samples_per_file.every = remaining_lines / wanted_samples;
293
+
294
+ if(lqs->c.samples_per_file.every < 1)
295
+ lqs->c.samples_per_file.every = 1;
296
+}
297
+
298
+typedef enum {
299
+ SAMPLING_STOP_AND_ESTIMATE = -1,
300
+ SAMPLING_FULL = 0,
301
+ SAMPLING_SKIP_FIELDS = 1,
302
+} sampling_t;
303
+
304
+static inline sampling_t is_row_in_sample(
305
+ sd_journal *j, LOGS_QUERY_STATUS *lqs, struct journal_file *jf,
306
+ usec_t msg_ut, FACETS_ANCHOR_DIRECTION direction, bool candidate_to_keep) {
307
+ if(!lqs->rq.sampling || candidate_to_keep)
308
+ return SAMPLING_FULL;
309
+
310
+ if(unlikely(msg_ut < lqs->c.samples_per_time_slot.start_ut))
311
+ msg_ut = lqs->c.samples_per_time_slot.start_ut;
312
+ if(unlikely(msg_ut > lqs->c.samples_per_time_slot.end_ut))
313
+ msg_ut = lqs->c.samples_per_time_slot.end_ut;
314
+
315
+ size_t slot = (msg_ut - lqs->c.samples_per_time_slot.start_ut) / lqs->c.samples_per_time_slot.step_ut;
316
+ if(slot >= lqs->c.samples.slots)
317
+ slot = lqs->c.samples.slots - 1;
318
+
319
+ bool should_sample = false;
320
+
321
+ if(lqs->c.samples.sampled < lqs->c.samples.enable_after_samples ||
322
+ lqs->c.samples_per_file.sampled < lqs->c.samples_per_file.enable_after_samples ||
323
+ lqs->c.samples_per_time_slot.sampled[slot] < lqs->c.samples_per_time_slot.enable_after_samples)
324
+ should_sample = true;
325
+
326
+ else if(lqs->c.samples_per_file.recalibrate >= SYSTEMD_JOURNAL_SAMPLING_RECALIBRATE || !lqs->c.samples_per_file.every) {
327
+ // this is the first to be unsampled for this file
328
+ sampling_decide_file_sampling_every(j, lqs, jf, direction, msg_ut);
329
+ lqs->c.samples_per_file.recalibrate = 0;
330
+ should_sample = true;
331
+ }
332
+ else {
333
+ // we sample 1 every fqs->samples_per_file.every
334
+ if(lqs->c.samples_per_file.skipped >= lqs->c.samples_per_file.every) {
335
+ lqs->c.samples_per_file.skipped = 0;
336
+ should_sample = true;
337
+ }
338
+ else
339
+ lqs->c.samples_per_file.skipped++;
340
+ }
341
+
342
+ if(should_sample) {
343
+ lqs->c.samples.sampled++;
344
+ lqs->c.samples_per_file.sampled++;
345
+ lqs->c.samples_per_time_slot.sampled[slot]++;
346
+
347
+ return SAMPLING_FULL;
348
+ }
349
+
350
+ lqs->c.samples_per_file.recalibrate++;
351
+
352
+ lqs->c.samples.unsampled++;
353
+ lqs->c.samples_per_file.unsampled++;
354
+ lqs->c.samples_per_time_slot.unsampled[slot]++;
355
+
356
+ if(lqs->c.samples_per_file.unsampled > lqs->c.samples_per_file.sampled) {
357
+ double progress_by_time = sampling_running_file_query_progress_by_time(lqs, jf, direction, msg_ut);
358
+
359
+ if(progress_by_time > SYSTEMD_JOURNAL_ENABLE_ESTIMATIONS_FILE_PERCENTAGE)
360
+ return SAMPLING_STOP_AND_ESTIMATE;
361
+ }
362
+
363
+ return SAMPLING_SKIP_FIELDS;
364
+}
365
+
366
+static inline void sampling_update_running_query_file_estimates(
367
+ FACETS *facets, sd_journal *j,
368
+ LOGS_QUERY_STATUS *lqs, struct journal_file *jf, usec_t msg_ut, FACETS_ANCHOR_DIRECTION direction) {
369
+ usec_t total_time_ut, remaining_start_ut, remaining_end_ut;
370
+ sampling_running_file_query_remaining_time(
371
+ lqs, jf, direction, msg_ut, &total_time_ut, &remaining_start_ut, &remaining_end_ut);
372
+ size_t remaining_lines = sampling_running_file_query_estimate_remaining_lines(j, lqs, jf, direction, msg_ut);
373
+ facets_update_estimations(facets, remaining_start_ut, remaining_end_ut, remaining_lines);
374
+ lqs->c.samples.estimated += remaining_lines;
375
+ lqs->c.samples_per_file.estimated += remaining_lines;
376
+}
377
+
378
+#endif //NETDATA_SYSTEMD_JOURNAL_SAMPLING_H
src/collectors/systemd-journal.plugin/systemd-journal.c
+263
-1335
@@ -5,53 +5,98 @@
5
* GPL v3+
6
*/
7
8
-#include "systemd-internals.h"
9
-
8
/*
9
* TODO
12
- *
10
* _UDEV_DEVLINK is frequently set more than once per field - support multi-value faces
11
*
12
*/
13
17
-#define FACET_MAX_VALUE_LENGTH 8192
14
+#include "systemd-internals.h"
15
16
#define SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
17
#define SYSTEMD_JOURNAL_FUNCTION_NAME "systemd-journal"
21
-#define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 60
22
-#define SYSTEMD_JOURNAL_MAX_PARAMS 1000
23
-#define SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION (1 * 3600)
24
-#define SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY 200
25
-#define SYSTEMD_JOURNAL_DEFAULT_ITEMS_SAMPLING 1000000
26
-#define SYSTEMD_JOURNAL_SAMPLING_SLOTS 1000
27
-#define SYSTEMD_JOURNAL_SAMPLING_RECALIBRATE 10000
18
+#define SYSTEMD_JOURNAL_SAMPLING_SLOTS 1000
19
+#define SYSTEMD_JOURNAL_SAMPLING_RECALIBRATE 10000
20
29
-#define SYSTEMD_JOURNAL_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
21
+#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
22
+#define LQS_DEFAULT_SLICE_MODE 1
23
+#else
24
+#define LQS_DEFAULT_SLICE_MODE 0
25
+#endif
26
31
-#define JOURNAL_PARAMETER_HELP "help"
32
-#define JOURNAL_PARAMETER_AFTER "after"
33
-#define JOURNAL_PARAMETER_BEFORE "before"
34
-#define JOURNAL_PARAMETER_ANCHOR "anchor"
35
-#define JOURNAL_PARAMETER_LAST "last"
36
-#define JOURNAL_PARAMETER_QUERY "query"
37
-#define JOURNAL_PARAMETER_FACETS "facets"
38
-#define JOURNAL_PARAMETER_HISTOGRAM "histogram"
39
-#define JOURNAL_PARAMETER_DIRECTION "direction"
40
-#define JOURNAL_PARAMETER_IF_MODIFIED_SINCE "if_modified_since"
41
-#define JOURNAL_PARAMETER_DATA_ONLY "data_only"
42
-#define JOURNAL_PARAMETER_SOURCE "source"
43
-#define JOURNAL_PARAMETER_INFO "info"
44
-#define JOURNAL_PARAMETER_SLICE "slice"
45
-#define JOURNAL_PARAMETER_DELTA "delta"
46
-#define JOURNAL_PARAMETER_TAIL "tail"
47
-#define JOURNAL_PARAMETER_SAMPLING "sampling"
27
+// functions needed by LQS
28
+static SD_JOURNAL_FILE_SOURCE_TYPE get_internal_source_type(const char *value);
29
+
30
+// structures needed by LQS
31
+struct lqs_extension {
32
+ struct {
33
+ usec_t start_ut;
34
+ usec_t stop_ut;
35
+ usec_t first_msg_ut;
36
37
+ sd_id128_t first_msg_writer;
38
+ uint64_t first_msg_seqnum;
39
+ } query_file;
40
+
41
+ struct {
42
+ uint32_t enable_after_samples;
43
+ uint32_t slots;
44
+ uint32_t sampled;
45
+ uint32_t unsampled;
46
+ uint32_t estimated;
47
+ } samples;
48
+
49
+ struct {
50
+ uint32_t enable_after_samples;
51
+ uint32_t every;
52
+ uint32_t skipped;
53
+ uint32_t recalibrate;
54
+ uint32_t sampled;
55
+ uint32_t unsampled;
56
+ uint32_t estimated;
57
+ } samples_per_file;
58
+
59
+ struct {
60
+ usec_t start_ut;
61
+ usec_t end_ut;
62
+ usec_t step_ut;
63
+ uint32_t enable_after_samples;
64
+ uint32_t sampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
65
+ uint32_t unsampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
66
+ } samples_per_time_slot;
67
+
68
+ // per file progress info
69
+ // size_t cached_count;
70
+
71
+ // progress statistics
72
+ usec_t matches_setup_ut;
73
+ size_t rows_useful;
74
+ size_t rows_read;
75
+ size_t bytes_read;
76
+ size_t files_matched;
77
+ size_t file_working;
78
+};
79
+
80
+// prepare LQS
81
+#define LQS_FUNCTION_NAME SYSTEMD_JOURNAL_FUNCTION_NAME
82
+#define LQS_FUNCTION_DESCRIPTION SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION
83
+#define LQS_DEFAULT_ITEMS_PER_QUERY 200
84
+#define LQS_DEFAULT_ITEMS_SAMPLING 1000000
85
+#define LQS_SOURCE_TYPE SD_JOURNAL_FILE_SOURCE_TYPE
86
+#define LQS_SOURCE_TYPE_ALL SDJF_ALL
87
+#define LQS_SOURCE_TYPE_NONE SDJF_NONE
88
+#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) get_internal_source_type(value)
89
+#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) available_journal_file_sources_to_json_array(wb)
90
+#include "libnetdata/facets/logs_query_status.h"
91
+
92
+#include "systemd-journal-sampling.h"
93
+
94
+#define FACET_MAX_VALUE_LENGTH 8192
95
+#define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 60
96
+#define SYSTEMD_JOURNAL_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
97
#define JOURNAL_KEY_ND_JOURNAL_FILE "ND_JOURNAL_FILE"
98
#define JOURNAL_KEY_ND_JOURNAL_PROCESS "ND_JOURNAL_PROCESS"
51
-
52
-#define JOURNAL_DEFAULT_SLICE_MODE true
99
#define JOURNAL_DEFAULT_DIRECTION FACETS_ANCHOR_DIRECTION_BACKWARD
54
-
100
#define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL
101
102
#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS \
@@ -182,100 +227,27 @@
227
228
// ----------------------------------------------------------------------------
229
185
-typedef struct function_query_status {
186
- bool *cancelled; // a pointer to the cancelling boolean
187
- usec_t *stop_monotonic_ut;
188
-
189
- // request
190
- const char *transaction;
191
-
192
- SD_JOURNAL_FILE_SOURCE_TYPE source_type;
193
- SIMPLE_PATTERN *sources;
194
- usec_t after_ut;
195
- usec_t before_ut;
196
-
197
- struct {
198
- usec_t start_ut;
199
- usec_t stop_ut;
200
- } anchor;
201
-
202
- FACETS_ANCHOR_DIRECTION direction;
203
- size_t entries;
204
- usec_t if_modified_since;
205
- bool delta;
206
- bool tail;
207
- bool data_only;
208
- bool slice;
209
- size_t sampling;
210
- size_t filters;
211
- usec_t last_modified;
212
- const char *query;
213
- const char *histogram;
214
-
215
- struct {
216
- usec_t start_ut; // the starting time of the query - we start from this
217
- usec_t stop_ut; // the ending time of the query - we stop at this
218
- usec_t first_msg_ut;
219
-
220
- sd_id128_t first_msg_writer;
221
- uint64_t first_msg_seqnum;
222
- } query_file;
223
-
224
- struct {
225
- uint32_t enable_after_samples;
226
- uint32_t slots;
227
- uint32_t sampled;
228
- uint32_t unsampled;
229
- uint32_t estimated;
230
- } samples;
231
-
232
- struct {
233
- uint32_t enable_after_samples;
234
- uint32_t every;
235
- uint32_t skipped;
236
- uint32_t recalibrate;
237
- uint32_t sampled;
238
- uint32_t unsampled;
239
- uint32_t estimated;
240
- } samples_per_file;
241
-
242
- struct {
243
- usec_t start_ut;
244
- usec_t end_ut;
245
- usec_t step_ut;
246
- uint32_t enable_after_samples;
247
- uint32_t sampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
248
- uint32_t unsampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
249
- } samples_per_time_slot;
250
-
251
- // per file progress info
252
- // size_t cached_count;
230
+static SD_JOURNAL_FILE_SOURCE_TYPE get_internal_source_type(const char *value) {
231
+ if(strcmp(value, SDJF_SOURCE_ALL_NAME) == 0)
232
+ return SDJF_ALL;
233
+ else if(strcmp(value, SDJF_SOURCE_LOCAL_NAME) == 0)
234
+ return SDJF_LOCAL_ALL;
235
+ else if(strcmp(value, SDJF_SOURCE_REMOTES_NAME) == 0)
236
+ return SDJF_REMOTE_ALL;
237
+ else if(strcmp(value, SDJF_SOURCE_NAMESPACES_NAME) == 0)
238
+ return SDJF_LOCAL_NAMESPACE;
239
+ else if(strcmp(value, SDJF_SOURCE_LOCAL_SYSTEM_NAME) == 0)
240
+ return SDJF_LOCAL_SYSTEM;
241
+ else if(strcmp(value, SDJF_SOURCE_LOCAL_USERS_NAME) == 0)
242
+ return SDJF_LOCAL_USER;
243
+ else if(strcmp(value, SDJF_SOURCE_LOCAL_OTHER_NAME) == 0)
244
+ return SDJF_LOCAL_OTHER;
245
254
- // progress statistics
255
- usec_t matches_setup_ut;
256
- size_t rows_useful;
257
- size_t rows_read;
258
- size_t bytes_read;
259
- size_t files_matched;
260
- size_t file_working;
261
-} FUNCTION_QUERY_STATUS;
262
-
263
-static void log_fqs(FUNCTION_QUERY_STATUS *fqs, const char *msg) {
264
- netdata_log_error("ERROR: %s, on query "
265
- "timeframe [%"PRIu64" - %"PRIu64"], "
266
- "anchor [%"PRIu64" - %"PRIu64"], "
267
- "if_modified_since %"PRIu64", "
268
- "data_only:%s, delta:%s, tail:%s, direction:%s"
269
- , msg
270
- , fqs->after_ut, fqs->before_ut
271
- , fqs->anchor.start_ut, fqs->anchor.stop_ut
272
- , fqs->if_modified_since
273
- , fqs->data_only ? "true" : "false"
274
- , fqs->delta ? "true" : "false"
275
- , fqs->tail ? "tail" : "false"
276
- , fqs->direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
246
+ return SDJF_NONE;
247
}
248
249
+// ----------------------------------------------------------------------------
250
+
251
static inline bool netdata_systemd_journal_seek_to(sd_journal *j, usec_t timestamp) {
252
if(sd_journal_seek_realtime_usec(j, timestamp) < 0) {
253
netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to %" PRIu64, timestamp);
@@ -290,367 +262,6 @@ static inline bool netdata_systemd_journal_seek_to(sd_journal *j, usec_t timesta
262
263
#define JD_SOURCE_REALTIME_TIMESTAMP "_SOURCE_REALTIME_TIMESTAMP"
264
293
-// ----------------------------------------------------------------------------
294
-// sampling support
295
-
296
-static void sampling_query_init(FUNCTION_QUERY_STATUS *fqs, FACETS *facets) {
297
- if(!fqs->sampling)
298
- return;
299
-
300
- if(!fqs->slice) {
301
- // the user is doing a full data query
302
- // disable sampling
303
- fqs->sampling = 0;
304
- return;
305
- }
306
-
307
- if(fqs->data_only) {
308
- // the user is doing a data query
309
- // disable sampling
310
- fqs->sampling = 0;
311
- return;
312
- }
313
-
314
- if(!fqs->files_matched) {
315
- // no files have been matched
316
- // disable sampling
317
- fqs->sampling = 0;
318
- return;
319
- }
320
-
321
- fqs->samples.slots = facets_histogram_slots(facets);
322
- if(fqs->samples.slots < 2) fqs->samples.slots = 2;
323
- if(fqs->samples.slots > SYSTEMD_JOURNAL_SAMPLING_SLOTS)
324
- fqs->samples.slots = SYSTEMD_JOURNAL_SAMPLING_SLOTS;
325
-
326
- if(!fqs->after_ut || !fqs->before_ut || fqs->after_ut >= fqs->before_ut) {
327
- // we don't have enough information for sampling
328
- fqs->sampling = 0;
329
- return;
330
- }
331
-
332
- usec_t delta = fqs->before_ut - fqs->after_ut;
333
- usec_t step = delta / facets_histogram_slots(facets) - 1;
334
- if(step < 1) step = 1;
335
-
336
- fqs->samples_per_time_slot.start_ut = fqs->after_ut;
337
- fqs->samples_per_time_slot.end_ut = fqs->before_ut;
338
- fqs->samples_per_time_slot.step_ut = step;
339
-
340
- // the minimum number of rows to enable sampling
341
- fqs->samples.enable_after_samples = fqs->sampling / 2;
342
-
343
- size_t files_matched = fqs->files_matched;
344
- if(!files_matched)
345
- files_matched = 1;
346
-
347
- // the minimum number of rows per file to enable sampling
348
- fqs->samples_per_file.enable_after_samples = (fqs->sampling / 4) / files_matched;
349
- if(fqs->samples_per_file.enable_after_samples < fqs->entries)
350
- fqs->samples_per_file.enable_after_samples = fqs->entries;
351
-
352
- // the minimum number of rows per time slot to enable sampling
353
- fqs->samples_per_time_slot.enable_after_samples = (fqs->sampling / 4) / fqs->samples.slots;
354
- if(fqs->samples_per_time_slot.enable_after_samples < fqs->entries)
355
- fqs->samples_per_time_slot.enable_after_samples = fqs->entries;
356
-}
357
-
358
-static void sampling_file_init(FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf __maybe_unused) {
359
- fqs->samples_per_file.sampled = 0;
360
- fqs->samples_per_file.unsampled = 0;
361
- fqs->samples_per_file.estimated = 0;
362
- fqs->samples_per_file.every = 0;
363
- fqs->samples_per_file.skipped = 0;
364
- fqs->samples_per_file.recalibrate = 0;
365
-}
366
-
367
-static size_t sampling_file_lines_scanned_so_far(FUNCTION_QUERY_STATUS *fqs) {
368
- size_t sampled = fqs->samples_per_file.sampled + fqs->samples_per_file.unsampled;
369
- if(!sampled) sampled = 1;
370
- return sampled;
371
-}
372
-
373
-static void sampling_running_file_query_overlapping_timeframe_ut(
374
- FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf, FACETS_ANCHOR_DIRECTION direction,
375
- usec_t msg_ut, usec_t *after_ut, usec_t *before_ut) {
376
-
377
- // find the overlap of the query and file timeframes
378
- // taking into account the first message we encountered
379
-
380
- usec_t oldest_ut, newest_ut;
381
- if(direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
382
- // the first message we know (oldest)
383
- oldest_ut = fqs->query_file.first_msg_ut ? fqs->query_file.first_msg_ut : jf->msg_first_ut;
384
- if(!oldest_ut) oldest_ut = fqs->query_file.start_ut;
385
-
386
- if(jf->msg_last_ut)
387
- newest_ut = MIN(fqs->query_file.stop_ut, jf->msg_last_ut);
388
- else if(jf->file_last_modified_ut)
389
- newest_ut = MIN(fqs->query_file.stop_ut, jf->file_last_modified_ut);
390
- else
391
- newest_ut = fqs->query_file.stop_ut;
392
-
393
- if(msg_ut < oldest_ut)
394
- oldest_ut = msg_ut - 1;
395
- }
396
- else /* BACKWARD */ {
397
- // the latest message we know (newest)
398
- newest_ut = fqs->query_file.first_msg_ut ? fqs->query_file.first_msg_ut : jf->msg_last_ut;
399
- if(!newest_ut) newest_ut = fqs->query_file.start_ut;
400
-
401
- if(jf->msg_first_ut)
402
- oldest_ut = MAX(fqs->query_file.stop_ut, jf->msg_first_ut);
403
- else
404
- oldest_ut = fqs->query_file.stop_ut;
405
-
406
- if(newest_ut < msg_ut)
407
- newest_ut = msg_ut + 1;
408
- }
409
-
410
- *after_ut = oldest_ut;
411
- *before_ut = newest_ut;
412
-}
413
-
414
-static double sampling_running_file_query_progress_by_time(FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf,
415
- FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut) {
416
-
417
- usec_t after_ut, before_ut, elapsed_ut;
418
- sampling_running_file_query_overlapping_timeframe_ut(fqs, jf, direction, msg_ut, &after_ut, &before_ut);
419
-
420
- if(direction == FACETS_ANCHOR_DIRECTION_FORWARD)
421
- elapsed_ut = msg_ut - after_ut;
422
- else
423
- elapsed_ut = before_ut - msg_ut;
424
-
425
- usec_t total_ut = before_ut - after_ut;
426
- double progress = (double)elapsed_ut / (double)total_ut;
427
-
428
- return progress;
429
-}
430
-
431
-static usec_t sampling_running_file_query_remaining_time(FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf,
432
- FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut,
433
- usec_t *total_time_ut, usec_t *remaining_start_ut,
434
- usec_t *remaining_end_ut) {
435
- usec_t after_ut, before_ut;
436
- sampling_running_file_query_overlapping_timeframe_ut(fqs, jf, direction, msg_ut, &after_ut, &before_ut);
437
-
438
- // since we have a timestamp in msg_ut
439
- // this timestamp can extend the overlap
440
- if(msg_ut <= after_ut)
441
- after_ut = msg_ut - 1;
442
-
443
- if(msg_ut >= before_ut)
444
- before_ut = msg_ut + 1;
445
-
446
- // return the remaining duration
447
- usec_t remaining_from_ut, remaining_to_ut;
448
- if(direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
449
- remaining_from_ut = msg_ut;
450
- remaining_to_ut = before_ut;
451
- }
452
- else {
453
- remaining_from_ut = after_ut;
454
- remaining_to_ut = msg_ut;
455
- }
456
-
457
- usec_t remaining_ut = remaining_to_ut - remaining_from_ut;
458
-
459
- if(total_time_ut)
460
- *total_time_ut = (before_ut > after_ut) ? before_ut - after_ut : 1;
461
-
462
- if(remaining_start_ut)
463
- *remaining_start_ut = remaining_from_ut;
464
-
465
- if(remaining_end_ut)
466
- *remaining_end_ut = remaining_to_ut;
467
-
468
- return remaining_ut;
469
-}
470
-
471
-static size_t sampling_running_file_query_estimate_remaining_lines_by_time(FUNCTION_QUERY_STATUS *fqs,
472
- struct journal_file *jf,
473
- FACETS_ANCHOR_DIRECTION direction,
474
- usec_t msg_ut) {
475
- size_t scanned_lines = sampling_file_lines_scanned_so_far(fqs);
476
-
477
- // Calculate the proportion of time covered
478
- usec_t total_time_ut, remaining_start_ut, remaining_end_ut;
479
- usec_t remaining_time_ut = sampling_running_file_query_remaining_time(fqs, jf, direction, msg_ut, &total_time_ut,
480
- &remaining_start_ut, &remaining_end_ut);
481
- if (total_time_ut == 0) total_time_ut = 1;
482
-
483
- double proportion_by_time = (double) (total_time_ut - remaining_time_ut) / (double) total_time_ut;
484
-
485
- if (proportion_by_time == 0 || proportion_by_time > 1.0 || !isfinite(proportion_by_time))
486
- proportion_by_time = 1.0;
487
-
488
- // Estimate the total number of lines in the file
489
- size_t expected_matching_logs_by_time = (size_t)((double)scanned_lines / proportion_by_time);
490
-
491
- if(jf->messages_in_file && expected_matching_logs_by_time > jf->messages_in_file)
492
- expected_matching_logs_by_time = jf->messages_in_file;
493
-
494
- // Calculate the estimated number of remaining lines
495
- size_t remaining_logs_by_time = expected_matching_logs_by_time - scanned_lines;
496
- if (remaining_logs_by_time < 1) remaining_logs_by_time = 1;
497
-
498
-// nd_log(NDLS_COLLECTORS, NDLP_INFO,
499
-// "JOURNAL ESTIMATION: '%s' "
500
-// "scanned_lines=%zu [sampled=%zu, unsampled=%zu, estimated=%zu], "
501
-// "file [%"PRIu64" - %"PRIu64", duration %"PRId64", known lines in file %zu], "
502
-// "query [%"PRIu64" - %"PRIu64", duration %"PRId64"], "
503
-// "first message read from the file at %"PRIu64", current message at %"PRIu64", "
504
-// "proportion of time %.2f %%, "
505
-// "expected total lines in file %zu, "
506
-// "remaining lines %zu, "
507
-// "remaining time %"PRIu64" [%"PRIu64" - %"PRIu64", duration %"PRId64"]"
508
-// , jf->filename
509
-// , scanned_lines, fqs->samples_per_file.sampled, fqs->samples_per_file.unsampled, fqs->samples_per_file.estimated
510
-// , jf->msg_first_ut, jf->msg_last_ut, jf->msg_last_ut - jf->msg_first_ut, jf->messages_in_file
511
-// , fqs->query_file.start_ut, fqs->query_file.stop_ut, fqs->query_file.stop_ut - fqs->query_file.start_ut
512
-// , fqs->query_file.first_msg_ut, msg_ut
513
-// , proportion_by_time * 100.0
514
-// , expected_matching_logs_by_time
515
-// , remaining_logs_by_time
516
-// , remaining_time_ut, remaining_start_ut, remaining_end_ut, remaining_end_ut - remaining_start_ut
517
-// );
518
-
519
- return remaining_logs_by_time;
520
-}
521
-
522
-static size_t sampling_running_file_query_estimate_remaining_lines(sd_journal *j __maybe_unused, FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf, FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut) {
523
- size_t remaining_logs_by_seqnum = 0;
524
-
525
-#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
526
- size_t expected_matching_logs_by_seqnum = 0;
527
- double proportion_by_seqnum = 0.0;
528
- uint64_t current_msg_seqnum;
529
- sd_id128_t current_msg_writer;
530
- if(!fqs->query_file.first_msg_seqnum || sd_journal_get_seqnum(j, ¤t_msg_seqnum, ¤t_msg_writer) < 0) {
531
- fqs->query_file.first_msg_seqnum = 0;
532
- fqs->query_file.first_msg_writer = SD_ID128_NULL;
533
- }
534
- else if(jf->messages_in_file) {
535
- size_t scanned_lines = sampling_file_lines_scanned_so_far(fqs);
536
-
537
- double proportion_of_all_lines_so_far;
538
- if(direction == FACETS_ANCHOR_DIRECTION_FORWARD)
539
- proportion_of_all_lines_so_far = (double)scanned_lines / (double)(current_msg_seqnum - jf->first_seqnum);
540
- else
541
- proportion_of_all_lines_so_far = (double)scanned_lines / (double)(jf->last_seqnum - current_msg_seqnum);
542
-
543
- if(proportion_of_all_lines_so_far > 1.0)
544
- proportion_of_all_lines_so_far = 1.0;
545
-
546
- expected_matching_logs_by_seqnum = (size_t)(proportion_of_all_lines_so_far * (double)jf->messages_in_file);
547
-
548
- proportion_by_seqnum = (double)scanned_lines / (double)expected_matching_logs_by_seqnum;
549
-
550
- if (proportion_by_seqnum == 0 || proportion_by_seqnum > 1.0 || !isfinite(proportion_by_seqnum))
551
- proportion_by_seqnum = 1.0;
552
-
553
- remaining_logs_by_seqnum = expected_matching_logs_by_seqnum - scanned_lines;
554
- if(!remaining_logs_by_seqnum) remaining_logs_by_seqnum = 1;
555
- }
556
-#endif
557
-
558
- if(remaining_logs_by_seqnum)
559
- return remaining_logs_by_seqnum;
560
-
561
- return sampling_running_file_query_estimate_remaining_lines_by_time(fqs, jf, direction, msg_ut);
562
-}
563
-
564
-static void sampling_decide_file_sampling_every(sd_journal *j, FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf, FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut) {
565
- size_t files_matched = fqs->files_matched;
566
- if(!files_matched) files_matched = 1;
567
-
568
- size_t remaining_lines = sampling_running_file_query_estimate_remaining_lines(j, fqs, jf, direction, msg_ut);
569
- size_t wanted_samples = (fqs->sampling / 2) / files_matched;
570
- if(!wanted_samples) wanted_samples = 1;
571
-
572
- fqs->samples_per_file.every = remaining_lines / wanted_samples;
573
-
574
- if(fqs->samples_per_file.every < 1)
575
- fqs->samples_per_file.every = 1;
576
-}
577
-
578
-typedef enum {
579
- SAMPLING_STOP_AND_ESTIMATE = -1,
580
- SAMPLING_FULL = 0,
581
- SAMPLING_SKIP_FIELDS = 1,
582
-} sampling_t;
583
-
584
-static inline sampling_t is_row_in_sample(sd_journal *j, FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf, usec_t msg_ut, FACETS_ANCHOR_DIRECTION direction, bool candidate_to_keep) {
585
- if(!fqs->sampling || candidate_to_keep)
586
- return SAMPLING_FULL;
587
-
588
- if(unlikely(msg_ut < fqs->samples_per_time_slot.start_ut))
589
- msg_ut = fqs->samples_per_time_slot.start_ut;
590
- if(unlikely(msg_ut > fqs->samples_per_time_slot.end_ut))
591
- msg_ut = fqs->samples_per_time_slot.end_ut;
592
-
593
- size_t slot = (msg_ut - fqs->samples_per_time_slot.start_ut) / fqs->samples_per_time_slot.step_ut;
594
- if(slot >= fqs->samples.slots)
595
- slot = fqs->samples.slots - 1;
596
-
597
- bool should_sample = false;
598
-
599
- if(fqs->samples.sampled < fqs->samples.enable_after_samples ||
600
- fqs->samples_per_file.sampled < fqs->samples_per_file.enable_after_samples ||
601
- fqs->samples_per_time_slot.sampled[slot] < fqs->samples_per_time_slot.enable_after_samples)
602
- should_sample = true;
603
-
604
- else if(fqs->samples_per_file.recalibrate >= SYSTEMD_JOURNAL_SAMPLING_RECALIBRATE || !fqs->samples_per_file.every) {
605
- // this is the first to be unsampled for this file
606
- sampling_decide_file_sampling_every(j, fqs, jf, direction, msg_ut);
607
- fqs->samples_per_file.recalibrate = 0;
608
- should_sample = true;
609
- }
610
- else {
611
- // we sample 1 every fqs->samples_per_file.every
612
- if(fqs->samples_per_file.skipped >= fqs->samples_per_file.every) {
613
- fqs->samples_per_file.skipped = 0;
614
- should_sample = true;
615
- }
616
- else
617
- fqs->samples_per_file.skipped++;
618
- }
619
-
620
- if(should_sample) {
621
- fqs->samples.sampled++;
622
- fqs->samples_per_file.sampled++;
623
- fqs->samples_per_time_slot.sampled[slot]++;
624
-
625
- return SAMPLING_FULL;
626
- }
627
-
628
- fqs->samples_per_file.recalibrate++;
629
-
630
- fqs->samples.unsampled++;
631
- fqs->samples_per_file.unsampled++;
632
- fqs->samples_per_time_slot.unsampled[slot]++;
633
-
634
- if(fqs->samples_per_file.unsampled > fqs->samples_per_file.sampled) {
635
- double progress_by_time = sampling_running_file_query_progress_by_time(fqs, jf, direction, msg_ut);
636
-
637
- if(progress_by_time > SYSTEMD_JOURNAL_ENABLE_ESTIMATIONS_FILE_PERCENTAGE)
638
- return SAMPLING_STOP_AND_ESTIMATE;
639
- }
640
-
641
- return SAMPLING_SKIP_FIELDS;
642
-}
643
-
644
-static void sampling_update_running_query_file_estimates(FACETS *facets, sd_journal *j, FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf, usec_t msg_ut, FACETS_ANCHOR_DIRECTION direction) {
645
- usec_t total_time_ut, remaining_start_ut, remaining_end_ut;
646
- sampling_running_file_query_remaining_time(fqs, jf, direction, msg_ut, &total_time_ut, &remaining_start_ut,
647
- &remaining_end_ut);
648
- size_t remaining_lines = sampling_running_file_query_estimate_remaining_lines(j, fqs, jf, direction, msg_ut);
649
- facets_update_estimations(facets, remaining_start_ut, remaining_end_ut, remaining_lines);
650
- fqs->samples.estimated += remaining_lines;
651
- fqs->samples_per_file.estimated += remaining_lines;
652
-}
653
-
265
// ----------------------------------------------------------------------------
266
267
static inline size_t netdata_systemd_journal_process_row(sd_journal *j, FACETS *facets, struct journal_file *jf, usec_t *msg_ut) {
@@ -721,16 +332,17 @@ static inline ND_SD_JOURNAL_STATUS check_stop(const bool *cancelled, const usec_
332
333
ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
334
sd_journal *j, BUFFER *wb __maybe_unused, FACETS *facets,
724
- struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) {
335
+ struct journal_file *jf,
336
+ LOGS_QUERY_STATUS *fqs) {
337
338
usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
339
728
- usec_t start_ut = ((fqs->data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->before_ut) + anchor_delta;
729
- usec_t stop_ut = (fqs->data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->after_ut;
730
- bool stop_when_full = (fqs->data_only && !fqs->anchor.stop_ut);
340
+ usec_t start_ut = ((fqs->rq.data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->rq.before_ut) + anchor_delta;
341
+ usec_t stop_ut = (fqs->rq.data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->rq.after_ut;
342
+ bool stop_when_full = (fqs->rq.data_only && !fqs->anchor.stop_ut);
343
732
- fqs->query_file.start_ut = start_ut;
733
- fqs->query_file.stop_ut = stop_ut;
344
+ fqs->c.query_file.start_ut = start_ut;
345
+ fqs->c.query_file.stop_ut = stop_ut;
346
347
if(!netdata_systemd_journal_seek_to(j, start_ut))
348
return ND_SD_JOURNAL_FAILED_TO_SEEK;
@@ -765,12 +377,12 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
377
378
if(unlikely(!first_msg_ut)) {
379
first_msg_ut = msg_ut;
768
- fqs->query_file.first_msg_ut = msg_ut;
380
+ fqs->c.query_file.first_msg_ut = msg_ut;
381
382
#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
771
- if(sd_journal_get_seqnum(j, &fqs->query_file.first_msg_seqnum, &fqs->query_file.first_msg_writer) < 0) {
772
- fqs->query_file.first_msg_seqnum = 0;
773
- fqs->query_file.first_msg_writer = SD_ID128_NULL;
383
+ if(sd_journal_get_seqnum(j, &fqs->c.query_file.first_msg_seqnum, &fqs->c.query_file.first_msg_writer) < 0) {
384
+ fqs->c.query_file.first_msg_seqnum = 0;
385
+ fqs->c.query_file.first_msg_writer = SD_ID128_NULL;
386
}
387
#endif
388
}
@@ -794,7 +406,7 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
406
row_counter++;
407
if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
408
stop_when_full &&
797
- facets_rows(facets) >= fqs->entries)) {
409
+ facets_rows(facets) >= fqs->rq.entries)) {
410
// stop the data only query
411
usec_t oldest = facets_row_oldest_ut(facets);
412
if(oldest && msg_ut < (oldest - anchor_delta))
@@ -802,10 +414,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
414
}
415
416
if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
805
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter);
417
+ FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
418
last_row_counter = row_counter;
419
808
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes);
420
+ FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
421
last_bytes = bytes;
422
423
status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut);
@@ -819,10 +431,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
431
}
432
}
433
822
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter);
823
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes);
434
+ FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
435
+ FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
436
825
- fqs->rows_useful += rows_useful;
437
+ fqs->c.rows_useful += rows_useful;
438
439
if(errors_no_timestamp)
440
netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp);
@@ -835,16 +447,17 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
447
448
ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
449
sd_journal *j, BUFFER *wb __maybe_unused, FACETS *facets,
838
- struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) {
450
+ struct journal_file *jf,
451
+ LOGS_QUERY_STATUS *fqs) {
452
453
usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
454
842
- usec_t start_ut = (fqs->data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->after_ut;
843
- usec_t stop_ut = ((fqs->data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->before_ut) + anchor_delta;
844
- bool stop_when_full = (fqs->data_only && !fqs->anchor.stop_ut);
455
+ usec_t start_ut = (fqs->rq.data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->rq.after_ut;
456
+ usec_t stop_ut = ((fqs->rq.data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->rq.before_ut) + anchor_delta;
457
+ bool stop_when_full = (fqs->rq.data_only && !fqs->anchor.stop_ut);
458
846
- fqs->query_file.start_ut = start_ut;
847
- fqs->query_file.stop_ut = stop_ut;
459
+ fqs->c.query_file.start_ut = start_ut;
460
+ fqs->c.query_file.stop_ut = stop_ut;
461
462
if(!netdata_systemd_journal_seek_to(j, start_ut))
463
return ND_SD_JOURNAL_FAILED_TO_SEEK;
@@ -879,7 +492,7 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
492
493
if(unlikely(!first_msg_ut)) {
494
first_msg_ut = msg_ut;
882
- fqs->query_file.first_msg_ut = msg_ut;
495
+ fqs->c.query_file.first_msg_ut = msg_ut;
496
}
497
498
sampling_t sample = is_row_in_sample(j, fqs, jf, msg_ut,
@@ -901,7 +514,7 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
514
row_counter++;
515
if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
516
stop_when_full &&
904
- facets_rows(facets) >= fqs->entries)) {
517
+ facets_rows(facets) >= fqs->rq.entries)) {
518
// stop the data only query
519
usec_t newest = facets_row_newest_ut(facets);
520
if(newest && msg_ut > (newest + anchor_delta))
@@ -909,10 +522,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
522
}
523
524
if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
912
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter);
525
+ FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
526
last_row_counter = row_counter;
527
915
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes);
528
+ FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
529
last_bytes = bytes;
530
531
status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut);
@@ -926,10 +539,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
539
}
540
}
541
929
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter);
930
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes);
542
+ FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
543
+ FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
544
932
- fqs->rows_useful += rows_useful;
545
+ fqs->c.rows_useful += rows_useful;
546
547
if(errors_no_timestamp)
548
netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp);
@@ -963,7 +576,7 @@ bool netdata_systemd_journal_check_if_modified_since(sd_journal *j, usec_t seek_
576
}
577
578
#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
966
-static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets, FUNCTION_QUERY_STATUS *fqs) {
579
+static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets, LOGS_QUERY_STATUS *lqs) {
580
const char *field = NULL;
581
const void *data = NULL;
582
size_t data_length;
@@ -974,7 +587,7 @@ static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets,
587
SD_JOURNAL_FOREACH_FIELD(j, field) { // for each key
588
bool interesting;
589
977
- if(fqs->data_only)
590
+ if(lqs->rq.data_only)
591
interesting = facets_key_name_is_filter(facets, field);
592
else
593
interesting = facets_key_name_is_facet(facets, field);
@@ -1023,7 +636,7 @@ static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets,
636
}
637
638
if(failures) {
1026
- log_fqs(fqs, "failed to setup journal filter, will run the full query.");
639
+ lqs_log_error(lqs, "failed to setup journal filter, will run the full query.");
640
sd_journal_flush_matches(j);
641
return true;
642
}
@@ -1034,7 +647,8 @@ static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets,
647
648
static ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_one_file(
649
const char *filename, BUFFER *wb, FACETS *facets,
1037
- struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) {
650
+ struct journal_file *jf,
651
+ LOGS_QUERY_STATUS *fqs) {
652
653
sd_journal *j = NULL;
654
errno_clear();
@@ -1056,18 +670,18 @@ static ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_one_file(
670
bool matches_filters = true;
671
672
#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
1059
- if(fqs->slice) {
673
+ if(fqs->rq.slice) {
674
usec_t started = now_monotonic_usec();
675
1062
- matches_filters = netdata_systemd_filtering_by_journal(j, facets, fqs) || !fqs->filters;
676
+ matches_filters = netdata_systemd_filtering_by_journal(j, facets, fqs) || !fqs->rq.filters;
677
usec_t ended = now_monotonic_usec();
678
1065
- fqs->matches_setup_ut += (ended - started);
679
+ fqs->c.matches_setup_ut += (ended - started);
680
}
681
#endif // HAVE_SD_JOURNAL_RESTART_FIELDS
682
683
if(matches_filters) {
1070
- if(fqs->direction == FACETS_ANCHOR_DIRECTION_FORWARD)
684
+ if(fqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD)
685
status = netdata_systemd_journal_query_forward(j, wb, facets, jf, fqs);
686
else
687
status = netdata_systemd_journal_query_backward(j, wb, facets, jf, fqs);
@@ -1081,10 +695,10 @@ static ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_one_file(
695
return status;
696
}
697
1084
-static bool jf_is_mine(struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) {
698
+static bool jf_is_mine(struct journal_file *jf, LOGS_QUERY_STATUS *fqs) {
699
1086
- if((fqs->source_type == SDJF_NONE && !fqs->sources) || (jf->source_type & fqs->source_type) ||
1087
- (fqs->sources && simple_pattern_matches(fqs->sources, string2str(jf->source)))) {
700
+ if((fqs->rq.source_type == SDJF_NONE && !fqs->rq.sources) || (jf->source_type & fqs->rq.source_type) ||
701
+ (fqs->rq.sources && simple_pattern_matches(fqs->rq.sources, string2str(jf->source)))) {
702
703
if(!jf->msg_last_ut)
704
// the file is not scanned yet, or the timestamps have not been updated,
@@ -1095,22 +709,24 @@ static bool jf_is_mine(struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) {
709
usec_t first_ut = jf->msg_first_ut - anchor_delta;
710
usec_t last_ut = jf->msg_last_ut + anchor_delta;
711
1098
- if(last_ut >= fqs->after_ut && first_ut <= fqs->before_ut)
712
+ if(last_ut >= fqs->rq.after_ut && first_ut <= fqs->rq.before_ut)
713
return true;
714
}
715
716
return false;
717
}
718
1105
-static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QUERY_STATUS *fqs) {
719
+static int netdata_systemd_journal_query(BUFFER *wb, LOGS_QUERY_STATUS *lqs) {
720
+ FACETS *facets = lqs->facets;
721
+
722
ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_NO_FILE_MATCHED;
723
struct journal_file *jf;
724
1109
- fqs->files_matched = 0;
1110
- fqs->file_working = 0;
1111
- fqs->rows_useful = 0;
1112
- fqs->rows_read = 0;
1113
- fqs->bytes_read = 0;
725
+ lqs->c.files_matched = 0;
726
+ lqs->c.file_working = 0;
727
+ lqs->c.rows_useful = 0;
728
+ lqs->c.rows_read = 0;
729
+ lqs->c.bytes_read = 0;
730
731
size_t files_used = 0;
732
size_t files_max = dictionary_entries(journal_files_registry);
@@ -1119,26 +735,24 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
735
// count the files
736
bool files_are_newer = false;
737
dfe_start_read(journal_files_registry, jf) {
1122
- if(!jf_is_mine(jf, fqs))
738
+ if(!jf_is_mine(jf, lqs))
739
continue;
740
741
file_items[files_used++] = dictionary_acquired_item_dup(journal_files_registry, jf_dfe.item);
742
1127
- if(jf->msg_last_ut > fqs->if_modified_since)
743
+ if(jf->msg_last_ut > lqs->rq.if_modified_since)
744
files_are_newer = true;
745
}
746
dfe_done(jf);
747
1132
- fqs->files_matched = files_used;
748
+ lqs->c.files_matched = files_used;
749
1134
- if(fqs->if_modified_since && !files_are_newer) {
1135
- buffer_flush(wb);
1136
- return HTTP_RESP_NOT_MODIFIED;
1137
- }
750
+ if(lqs->rq.if_modified_since && !files_are_newer)
751
+ return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
752
753
// sort the files, so that they are optimal for facets
754
if(files_used >= 2) {
1141
- if (fqs->direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
755
+ if (lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
756
qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
757
journal_file_dict_items_backward_compar);
758
else
@@ -1153,38 +767,38 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
767
usec_t duration_ut = 0, max_duration_ut = 0;
768
usec_t progress_duration_ut = 0;
769
1156
- sampling_query_init(fqs, facets);
770
+ sampling_query_init(lqs, facets);
771
772
buffer_json_member_add_array(wb, "_journal_files");
773
for(size_t f = 0; f < files_used ;f++) {
774
const char *filename = dictionary_acquired_item_name(file_items[f]);
775
jf = dictionary_acquired_item_value(file_items[f]);
776
1163
- if(!jf_is_mine(jf, fqs))
777
+ if(!jf_is_mine(jf, lqs))
778
continue;
779
780
started_ut = ended_ut;
781
782
// do not even try to do the query if we expect it to pass the timeout
1169
- if(ended_ut + max_duration_ut * 3 >= *fqs->stop_monotonic_ut) {
783
+ if(ended_ut + max_duration_ut * 3 >= *lqs->stop_monotonic_ut) {
784
partial = true;
785
status = ND_SD_JOURNAL_TIMED_OUT;
786
break;
787
}
788
1175
- fqs->file_working++;
789
+ lqs->c.file_working++;
790
// fqs->cached_count = 0;
791
792
size_t fs_calls = fstat_thread_calls;
793
size_t fs_cached = fstat_thread_cached_responses;
1180
- size_t rows_useful = fqs->rows_useful;
1181
- size_t rows_read = fqs->rows_read;
1182
- size_t bytes_read = fqs->bytes_read;
1183
- size_t matches_setup_ut = fqs->matches_setup_ut;
794
+ size_t rows_useful = lqs->c.rows_useful;
795
+ size_t rows_read = lqs->c.rows_read;
796
+ size_t bytes_read = lqs->c.bytes_read;
797
+ size_t matches_setup_ut = lqs->c.matches_setup_ut;
798
1185
- sampling_file_init(fqs, jf);
799
+ sampling_file_init(lqs, jf);
800
1187
- ND_SD_JOURNAL_STATUS tmp_status = netdata_systemd_journal_query_one_file(filename, wb, facets, jf, fqs);
801
+ ND_SD_JOURNAL_STATUS tmp_status = netdata_systemd_journal_query_one_file(filename, wb, facets, jf, lqs);
802
803
// nd_log(NDLS_COLLECTORS, NDLP_INFO,
804
// "JOURNAL ESTIMATION FINAL: '%s' "
@@ -1198,10 +812,10 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
812
// , fqs->query_file.start_ut, fqs->query_file.stop_ut, fqs->query_file.stop_ut - fqs->query_file.start_ut
813
// );
814
1201
- rows_useful = fqs->rows_useful - rows_useful;
1202
- rows_read = fqs->rows_read - rows_read;
1203
- bytes_read = fqs->bytes_read - bytes_read;
1204
- matches_setup_ut = fqs->matches_setup_ut - matches_setup_ut;
815
+ rows_useful = lqs->c.rows_useful - rows_useful;
816
+ rows_read = lqs->c.rows_read - rows_read;
817
+ bytes_read = lqs->c.bytes_read - bytes_read;
818
+ matches_setup_ut = lqs->c.matches_setup_ut - matches_setup_ut;
819
fs_calls = fstat_thread_calls - fs_calls;
820
fs_cached = fstat_thread_cached_responses - fs_cached;
821
@@ -1215,7 +829,7 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
829
if(progress_duration_ut >= SYSTEMD_JOURNAL_PROGRESS_EVERY_UT) {
830
progress_duration_ut = 0;
831
netdata_mutex_lock(&stdout_mutex);
1218
- pluginsd_function_progress_to_stdout(fqs->transaction, f + 1, files_used);
832
+ pluginsd_function_progress_to_stdout(lqs->rq.transaction, f + 1, files_used);
833
netdata_mutex_unlock(&stdout_mutex);
834
}
835
@@ -1241,12 +855,12 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
855
buffer_json_member_add_uint64(wb, "fstat_query_calls", fs_calls);
856
buffer_json_member_add_uint64(wb, "fstat_query_cached_responses", fs_cached);
857
1244
- if(fqs->sampling) {
858
+ if(lqs->rq.sampling) {
859
buffer_json_member_add_object(wb, "_sampling");
860
{
1247
- buffer_json_member_add_uint64(wb, "sampled", fqs->samples_per_file.sampled);
1248
- buffer_json_member_add_uint64(wb, "unsampled", fqs->samples_per_file.unsampled);
1249
- buffer_json_member_add_uint64(wb, "estimated", fqs->samples_per_file.estimated);
861
+ buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples_per_file.sampled);
862
+ buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples_per_file.unsampled);
863
+ buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples_per_file.estimated);
864
}
865
buffer_json_object_close(wb); // _sampling
866
}
@@ -1290,10 +904,8 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
904
905
switch (status) {
906
case ND_SD_JOURNAL_OK:
1293
- if(fqs->if_modified_since && !fqs->rows_useful) {
1294
- buffer_flush(wb);
1295
- return HTTP_RESP_NOT_MODIFIED;
1296
- }
907
+ if(lqs->rq.if_modified_since && !lqs->c.rows_useful)
908
+ return rrd_call_function_error(wb, "no useful logs, not modified", HTTP_RESP_NOT_MODIFIED);
909
break;
910
911
case ND_SD_JOURNAL_TIMED_OUT:
@@ -1301,18 +913,19 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
913
break;
914
915
case ND_SD_JOURNAL_CANCELLED:
1304
- buffer_flush(wb);
1305
- return HTTP_RESP_CLIENT_CLOSED_REQUEST;
916
+ return rrd_call_function_error(wb, "client closed connection", HTTP_RESP_CLIENT_CLOSED_REQUEST);
917
918
case ND_SD_JOURNAL_NOT_MODIFIED:
1308
- buffer_flush(wb);
1309
- return HTTP_RESP_NOT_MODIFIED;
919
+ return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
920
1311
- default:
921
case ND_SD_JOURNAL_FAILED_TO_OPEN:
922
+ return rrd_call_function_error(wb, "failed to open journal", HTTP_RESP_INTERNAL_SERVER_ERROR);
923
+
924
case ND_SD_JOURNAL_FAILED_TO_SEEK:
1314
- buffer_flush(wb);
1315
- return HTTP_RESP_INTERNAL_SERVER_ERROR;
925
+ return rrd_call_function_error(wb, "failed to seek in journal", HTTP_RESP_INTERNAL_SERVER_ERROR);
926
+
927
+ default:
928
+ return rrd_call_function_error(wb, "unknown status", HTTP_RESP_INTERNAL_SERVER_ERROR);
929
}
930
931
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
@@ -1320,7 +933,7 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
933
buffer_json_member_add_string(wb, "type", "table");
934
935
// build a message for the query
1323
- if(!fqs->data_only) {
936
+ if(!lqs->rq.data_only) {
937
CLEAN_BUFFER *msg = buffer_create(0, NULL);
938
CLEAN_BUFFER *msg_description = buffer_create(0, NULL);
939
ND_LOG_FIELD_PRIORITY msg_priority = NDLP_INFO;
@@ -1339,17 +952,17 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
952
msg_priority = NDLP_WARNING;
953
}
954
1342
- if(fqs->samples.estimated || fqs->samples.unsampled) {
1343
- double percent = (double) (fqs->samples.sampled * 100.0 /
1344
- (fqs->samples.estimated + fqs->samples.unsampled + fqs->samples.sampled));
955
+ if(lqs->c.samples.estimated || lqs->c.samples.unsampled) {
956
+ double percent = (double) (lqs->c.samples.sampled * 100.0 /
957
+ (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
958
buffer_sprintf(msg, "%.2f%% real data", percent);
959
buffer_sprintf(msg_description, "ACTUAL DATA: The filters counters reflect %0.2f%% of the data. ", percent);
960
msg_priority = MIN(msg_priority, NDLP_NOTICE);
961
}
962
1350
- if(fqs->samples.unsampled) {
1351
- double percent = (double) (fqs->samples.unsampled * 100.0 /
1352
- (fqs->samples.estimated + fqs->samples.unsampled + fqs->samples.sampled));
963
+ if(lqs->c.samples.unsampled) {
964
+ double percent = (double) (lqs->c.samples.unsampled * 100.0 /
965
+ (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
966
buffer_sprintf(msg, ", %.2f%% unsampled", percent);
967
buffer_sprintf(msg_description
968
, "UNSAMPLED DATA: %0.2f%% of the events exist and have been counted, but their values have not been evaluated, so they are not included in the filters counters. "
@@ -1357,9 +970,9 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
970
msg_priority = MIN(msg_priority, NDLP_NOTICE);
971
}
972
1360
- if(fqs->samples.estimated) {
1361
- double percent = (double) (fqs->samples.estimated * 100.0 /
1362
- (fqs->samples.estimated + fqs->samples.unsampled + fqs->samples.sampled));
973
+ if(lqs->c.samples.estimated) {
974
+ double percent = (double) (lqs->c.samples.estimated * 100.0 /
975
+ (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
976
buffer_sprintf(msg, ", %.2f%% estimated", percent);
977
buffer_sprintf(msg_description
978
, "ESTIMATED DATA: The query selected a large amount of data, so to avoid delaying too much, the presented data are estimated by %0.2f%%. "
@@ -1377,18 +990,19 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
990
buffer_json_object_close(wb); // message
991
}
992
1380
- if(!fqs->data_only) {
993
+ if(!lqs->rq.data_only) {
994
buffer_json_member_add_time_t(wb, "update_every", 1);
995
buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
996
}
997
1385
- if(!fqs->data_only || fqs->tail)
1386
- buffer_json_member_add_uint64(wb, "last_modified", fqs->last_modified);
998
+ if(!lqs->rq.data_only || lqs->rq.tail)
999
+ buffer_json_member_add_uint64(wb, "last_modified", lqs->last_modified);
1000
1001
facets_sort_and_reorder_keys(facets);
1002
facets_report(facets, wb, used_hashes_registry);
1003
1391
- buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + (fqs->data_only ? 3600 : 0));
1004
+ wb->expires = now_realtime_sec() + (lqs->rq.data_only ? 3600 : 0);
1005
+ buffer_json_member_add_time_t(wb, "expires", wb->expires);
1006
1007
buffer_json_member_add_object(wb, "_fstat_caching");
1008
{
@@ -1397,581 +1011,24 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1011
}
1012
buffer_json_object_close(wb); // _fstat_caching
1013
1400
- if(fqs->sampling) {
1014
+ if(lqs->rq.sampling) {
1015
buffer_json_member_add_object(wb, "_sampling");
1016
{
1403
- buffer_json_member_add_uint64(wb, "sampled", fqs->samples.sampled);
1404
- buffer_json_member_add_uint64(wb, "unsampled", fqs->samples.unsampled);
1405
- buffer_json_member_add_uint64(wb, "estimated", fqs->samples.estimated);
1017
+ buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples.sampled);
1018
+ buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples.unsampled);
1019
+ buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples.estimated);
1020
}
1021
buffer_json_object_close(wb); // _sampling
1022
}
1023
1410
- buffer_json_finalize(wb);
1411
-
1412
- return HTTP_RESP_OK;
1024
+ wb->content_type = CT_APPLICATION_JSON;
1025
+ wb->response_code = HTTP_RESP_OK;
1026
+ return wb->response_code;
1027
}
1028
1415
-static void netdata_systemd_journal_function_help(const char *transaction) {
1416
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
1417
- buffer_sprintf(wb,
1418
- "%s / %s\n"
1419
- "\n"
1420
- "%s\n"
1421
- "\n"
1422
- "The following parameters are supported:\n"
1423
- "\n"
1424
- " "JOURNAL_PARAMETER_HELP"\n"
1425
- " Shows this help message.\n"
1426
- "\n"
1427
- " "JOURNAL_PARAMETER_INFO"\n"
1428
- " Request initial configuration information about the plugin.\n"
1429
- " The key entity returned is the required_params array, which includes\n"
1430
- " all the available systemd journal sources.\n"
1431
- " When `"JOURNAL_PARAMETER_INFO"` is requested, all other parameters are ignored.\n"
1432
- "\n"
1433
- " "JOURNAL_PARAMETER_DATA_ONLY":true or "JOURNAL_PARAMETER_DATA_ONLY":false\n"
1434
- " Quickly respond with data requested, without generating a\n"
1435
- " `histogram`, `facets` counters and `items`.\n"
1436
- "\n"
1437
- " "JOURNAL_PARAMETER_DELTA":true or "JOURNAL_PARAMETER_DELTA":false\n"
1438
- " When doing data only queries, include deltas for histogram, facets and items.\n"
1439
- "\n"
1440
- " "JOURNAL_PARAMETER_TAIL":true or "JOURNAL_PARAMETER_TAIL":false\n"
1441
- " When doing data only queries, respond with the newest messages,\n"
1442
- " and up to the anchor, but calculate deltas (if requested) for\n"
1443
- " the duration [anchor - before].\n"
1444
- "\n"
1445
- " "JOURNAL_PARAMETER_SLICE":true or "JOURNAL_PARAMETER_SLICE":false\n"
1446
- " When it is turned on, the plugin is executing filtering via libsystemd,\n"
1447
- " utilizing all the available indexes of the journal files.\n"
1448
- " When it is off, only the time constraint is handled by libsystemd and\n"
1449
- " all filtering is done by the plugin.\n"
1450
- " The default is: %s\n"
1451
- "\n"
1452
- " "JOURNAL_PARAMETER_SOURCE":SOURCE\n"
1453
- " Query only the specified journal sources.\n"
1454
- " Do an `"JOURNAL_PARAMETER_INFO"` query to find the sources.\n"
1455
- "\n"
1456
- " "JOURNAL_PARAMETER_BEFORE":TIMESTAMP_IN_SECONDS\n"
1457
- " Absolute or relative (to now) timestamp in seconds, to start the query.\n"
1458
- " The query is always executed from the most recent to the oldest log entry.\n"
1459
- " If not given the default is: now.\n"
1460
- "\n"
1461
- " "JOURNAL_PARAMETER_AFTER":TIMESTAMP_IN_SECONDS\n"
1462
- " Absolute or relative (to `before`) timestamp in seconds, to end the query.\n"
1463
- " If not given, the default is %d.\n"
1464
- "\n"
1465
- " "JOURNAL_PARAMETER_LAST":ITEMS\n"
1466
- " The number of items to return.\n"
1467
- " The default is %d.\n"
1468
- "\n"
1469
- " "JOURNAL_PARAMETER_SAMPLING":ITEMS\n"
1470
- " The number of log entries to sample to estimate facets counters and histogram.\n"
1471
- " The default is %d.\n"
1472
- "\n"
1473
- " "JOURNAL_PARAMETER_ANCHOR":TIMESTAMP_IN_MICROSECONDS\n"
1474
- " Return items relative to this timestamp.\n"
1475
- " The exact items to be returned depend on the query `"JOURNAL_PARAMETER_DIRECTION"`.\n"
1476
- "\n"
1477
- " "JOURNAL_PARAMETER_DIRECTION":forward or "JOURNAL_PARAMETER_DIRECTION":backward\n"
1478
- " When set to `backward` (default) the items returned are the newest before the\n"
1479
- " `"JOURNAL_PARAMETER_ANCHOR"`, (or `"JOURNAL_PARAMETER_BEFORE"` if `"JOURNAL_PARAMETER_ANCHOR"` is not set)\n"
1480
- " When set to `forward` the items returned are the oldest after the\n"
1481
- " `"JOURNAL_PARAMETER_ANCHOR"`, (or `"JOURNAL_PARAMETER_AFTER"` if `"JOURNAL_PARAMETER_ANCHOR"` is not set)\n"
1482
- " The default is: %s\n"
1483
- "\n"
1484
- " "JOURNAL_PARAMETER_QUERY":SIMPLE_PATTERN\n"
1485
- " Do a full text search to find the log entries matching the pattern given.\n"
1486
- " The plugin is searching for matches on all fields of the database.\n"
1487
- "\n"
1488
- " "JOURNAL_PARAMETER_IF_MODIFIED_SINCE":TIMESTAMP_IN_MICROSECONDS\n"
1489
- " Each successful response, includes a `last_modified` field.\n"
1490
- " By providing the timestamp to the `"JOURNAL_PARAMETER_IF_MODIFIED_SINCE"` parameter,\n"
1491
- " the plugin will return 200 with a successful response, or 304 if the source has not\n"
1492
- " been modified since that timestamp.\n"
1493
- "\n"
1494
- " "JOURNAL_PARAMETER_HISTOGRAM":facet_id\n"
1495
- " Use the given `facet_id` for the histogram.\n"
1496
- " This parameter is ignored in `"JOURNAL_PARAMETER_DATA_ONLY"` mode.\n"
1497
- "\n"
1498
- " "JOURNAL_PARAMETER_FACETS":facet_id1,facet_id2,facet_id3,...\n"
1499
- " Add the given facets to the list of fields for which analysis is required.\n"
1500
- " The plugin will offer both a histogram and facet value counters for its values.\n"
1501
- " This parameter is ignored in `"JOURNAL_PARAMETER_DATA_ONLY"` mode.\n"
1502
- "\n"
1503
- " facet_id:value_id1,value_id2,value_id3,...\n"
1504
- " Apply filters to the query, based on the facet IDs returned.\n"
1505
- " Each `facet_id` can be given once, but multiple `facet_ids` can be given.\n"
1506
- "\n"
1507
- , program_name
1508
- , SYSTEMD_JOURNAL_FUNCTION_NAME
1509
- , SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION
1510
- , JOURNAL_DEFAULT_SLICE_MODE ? "true" : "false" // slice
1511
- , -SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION
1512
- , SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY
1513
- , SYSTEMD_JOURNAL_DEFAULT_ITEMS_SAMPLING
1514
- , JOURNAL_DEFAULT_DIRECTION == FACETS_ANCHOR_DIRECTION_BACKWARD ? "backward" : "forward"
1515
- );
1516
-
1517
- netdata_mutex_lock(&stdout_mutex);
1518
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);
1519
- netdata_mutex_unlock(&stdout_mutex);
1520
-}
1521
-
1522
-typedef struct {
1523
- FACET_KEY_OPTIONS default_facet;
1524
- bool info;
1525
- bool data_only;
1526
- bool slice;
1527
- bool delta;
1528
- bool tail;
1529
- time_t after_s;
1530
- time_t before_s;
1531
- usec_t anchor;
1532
- usec_t if_modified_since;
1533
- size_t last;
1534
- FACETS_ANCHOR_DIRECTION direction;
1535
- const char *query;
1536
- const char *chart;
1537
- SIMPLE_PATTERN *sources;
1538
- SD_JOURNAL_FILE_SOURCE_TYPE source_type;
1539
- size_t filters;
1540
- size_t sampling;
1541
-} JOURNAL_QUERY;
1542
-
1543
-static SD_JOURNAL_FILE_SOURCE_TYPE get_internal_source_type(const char *value) {
1544
- if(strcmp(value, SDJF_SOURCE_ALL_NAME) == 0)
1545
- return SDJF_ALL;
1546
- else if(strcmp(value, SDJF_SOURCE_LOCAL_NAME) == 0)
1547
- return SDJF_LOCAL_ALL;
1548
- else if(strcmp(value, SDJF_SOURCE_REMOTES_NAME) == 0)
1549
- return SDJF_REMOTE_ALL;
1550
- else if(strcmp(value, SDJF_SOURCE_NAMESPACES_NAME) == 0)
1551
- return SDJF_LOCAL_NAMESPACE;
1552
- else if(strcmp(value, SDJF_SOURCE_LOCAL_SYSTEM_NAME) == 0)
1553
- return SDJF_LOCAL_SYSTEM;
1554
- else if(strcmp(value, SDJF_SOURCE_LOCAL_USERS_NAME) == 0)
1555
- return SDJF_LOCAL_USER;
1556
- else if(strcmp(value, SDJF_SOURCE_LOCAL_OTHER_NAME) == 0)
1557
- return SDJF_LOCAL_OTHER;
1558
-
1559
- return SDJF_NONE;
1560
-}
1561
-
1562
-static FACETS_ANCHOR_DIRECTION get_direction(const char *value) {
1563
- return strcasecmp(value, "forward") == 0 ? FACETS_ANCHOR_DIRECTION_FORWARD : FACETS_ANCHOR_DIRECTION_BACKWARD;
1564
-}
1565
-
1566
-struct post_query_data {
1567
- const char *transaction;
1568
- FACETS *facets;
1569
- JOURNAL_QUERY *q;
1570
- BUFFER *wb;
1571
-};
1572
-
1573
-static bool parse_json_payload(json_object *jobj, const char *path, void *data, BUFFER *error) {
1574
- struct post_query_data *qd = data;
1575
- JOURNAL_QUERY *q = qd->q;
1576
- BUFFER *wb = qd->wb;
1577
- FACETS *facets = qd->facets;
1578
- // const char *transaction = qd->transaction;
1579
-
1580
- buffer_flush(error);
1581
-
1582
- JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_INFO, q->info, error, false);
1583
- JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_DELTA, q->delta, error, false);
1584
- JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_TAIL, q->tail, error, false);
1585
- JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_SLICE, q->slice, error, false);
1586
- JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_DATA_ONLY, q->data_only, error, false);
1587
- JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_SAMPLING, q->sampling, error, false);
1588
- JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_AFTER, q->after_s, error, false);
1589
- JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_BEFORE, q->before_s, error, false);
1590
- JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_IF_MODIFIED_SINCE, q->if_modified_since, error, false);
1591
- JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_ANCHOR, q->anchor, error, false);
1592
- JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_LAST, q->last, error, false);
1593
- JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_DIRECTION, get_direction, q->direction, error, false);
1594
- JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_QUERY, q->query, error, false);
1595
- JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, JOURNAL_PARAMETER_HISTOGRAM, q->chart, error, false);
1596
-
1597
- json_object *sources;
1598
- if (json_object_object_get_ex(jobj, JOURNAL_PARAMETER_SOURCE, &sources)) {
1599
- if (json_object_get_type(sources) != json_type_array) {
1600
- buffer_sprintf(error, "member '%s' is not an array", JOURNAL_PARAMETER_SOURCE);
1601
- return false;
1602
- }
1603
-
1604
- buffer_json_member_add_array(wb, JOURNAL_PARAMETER_SOURCE);
1605
-
1606
- CLEAN_BUFFER *sources_list = buffer_create(0, NULL);
1607
-
1608
- q->source_type = SDJF_NONE;
1609
-
1610
- size_t sources_len = json_object_array_length(sources);
1611
- for (size_t i = 0; i < sources_len; i++) {
1612
- json_object *src = json_object_array_get_idx(sources, i);
1613
-
1614
- if (json_object_get_type(src) != json_type_string) {
1615
- buffer_sprintf(error, "sources array item %zu is not a string", i);
1616
- return false;
1617
- }
1618
-
1619
- const char *value = json_object_get_string(src);
1620
- buffer_json_add_array_item_string(wb, value);
1621
-
1622
- SD_JOURNAL_FILE_SOURCE_TYPE t = get_internal_source_type(value);
1623
- if(t != SDJF_NONE) {
1624
- q->source_type |= t;
1625
- value = NULL;
1626
- }
1627
- else {
1628
- // else, match the source, whatever it is
1629
- if(buffer_strlen(sources_list))
1630
- buffer_putc(sources_list, '|');
1631
-
1632
- buffer_strcat(sources_list, value);
1633
- }
1634
- }
1635
-
1636
- if(buffer_strlen(sources_list)) {
1637
- simple_pattern_free(q->sources);
1638
- q->sources = simple_pattern_create(buffer_tostring(sources_list), "|", SIMPLE_PATTERN_EXACT, false);
1639
- }
1640
-
1641
- buffer_json_array_close(wb); // source
1642
- }
1643
-
1644
- json_object *fcts;
1645
- if (json_object_object_get_ex(jobj, JOURNAL_PARAMETER_FACETS, &fcts)) {
1646
- if (json_object_get_type(sources) != json_type_array) {
1647
- buffer_sprintf(error, "member '%s' is not an array", JOURNAL_PARAMETER_FACETS);
1648
- return false;
1649
- }
1650
-
1651
- q->default_facet = FACET_KEY_OPTION_NONE;
1652
- facets_reset_and_disable_all_facets(facets);
1653
-
1654
- buffer_json_member_add_array(wb, JOURNAL_PARAMETER_FACETS);
1655
-
1656
- size_t facets_len = json_object_array_length(fcts);
1657
- for (size_t i = 0; i < facets_len; i++) {
1658
- json_object *fct = json_object_array_get_idx(fcts, i);
1659
-
1660
- if (json_object_get_type(fct) != json_type_string) {
1661
- buffer_sprintf(error, "facets array item %zu is not a string", i);
1662
- return false;
1663
- }
1664
-
1665
- const char *value = json_object_get_string(fct);
1666
- facets_register_facet(facets, value, FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS|FACET_KEY_OPTION_REORDER);
1667
- buffer_json_add_array_item_string(wb, value);
1668
- }
1669
-
1670
- buffer_json_array_close(wb); // facets
1671
- }
1672
-
1673
- json_object *selections;
1674
- if (json_object_object_get_ex(jobj, "selections", &selections)) {
1675
- if (json_object_get_type(selections) != json_type_object) {
1676
- buffer_sprintf(error, "member 'selections' is not an object");
1677
- return false;
1678
- }
1679
-
1680
- buffer_json_member_add_object(wb, "selections");
1681
-
1682
- json_object_object_foreach(selections, key, val) {
1683
- if (json_object_get_type(val) != json_type_array) {
1684
- buffer_sprintf(error, "selection '%s' is not an array", key);
1685
- return false;
1686
- }
1687
-
1688
- buffer_json_member_add_array(wb, key);
1689
-
1690
- size_t values_len = json_object_array_length(val);
1691
- for (size_t i = 0; i < values_len; i++) {
1692
- json_object *value_obj = json_object_array_get_idx(val, i);
1693
-
1694
- if (json_object_get_type(value_obj) != json_type_string) {
1695
- buffer_sprintf(error, "selection '%s' array item %zu is not a string", key, i);
1696
- return false;
1697
- }
1698
-
1699
- const char *value = json_object_get_string(value_obj);
1700
-
1701
- // Call facets_register_facet_id_filter for each value
1702
- facets_register_facet_filter(
1703
- facets, key, value, FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);
1704
-
1705
- buffer_json_add_array_item_string(wb, value);
1706
- q->filters++;
1707
- }
1708
-
1709
- buffer_json_array_close(wb); // key
1710
- }
1711
-
1712
- buffer_json_object_close(wb); // selections
1713
- }
1714
-
1715
- return true;
1716
-}
1717
-
1718
-static bool parse_post_params(FACETS *facets, JOURNAL_QUERY *q, BUFFER *wb, BUFFER *payload, const char *transaction) {
1719
- struct post_query_data qd = {
1720
- .transaction = transaction,
1721
- .facets = facets,
1722
- .q = q,
1723
- .wb = wb,
1724
- };
1725
-
1726
- int code;
1727
- CLEAN_JSON_OBJECT *jobj = json_parse_function_payload_or_error(wb, payload, &code, parse_json_payload, &qd);
1728
- if(!jobj || code != HTTP_RESP_OK) {
1729
- netdata_mutex_lock(&stdout_mutex);
1730
- pluginsd_function_result_to_stdout(transaction, code, "application/json", now_realtime_sec() + 1, wb);
1731
- netdata_mutex_unlock(&stdout_mutex);
1732
- return false;
1733
- }
1734
-
1735
- return true;
1736
-}
1737
-
1738
-static bool parse_get_params(FACETS *facets, JOURNAL_QUERY *q, BUFFER *wb, char *function, const char *transaction) {
1739
- buffer_json_member_add_object(wb, "_request");
1740
-
1741
- char *words[SYSTEMD_JOURNAL_MAX_PARAMS] = { NULL };
1742
- size_t num_words = quoted_strings_splitter_pluginsd(function, words, SYSTEMD_JOURNAL_MAX_PARAMS);
1743
- for(int i = 1; i < SYSTEMD_JOURNAL_MAX_PARAMS ;i++) {
1744
- char *keyword = get_word(words, num_words, i);
1745
- if(!keyword) break;
1746
-
1747
- if(strcmp(keyword, JOURNAL_PARAMETER_HELP) == 0) {
1748
- netdata_systemd_journal_function_help(transaction);
1749
- return false;
1750
- }
1751
- else if(strcmp(keyword, JOURNAL_PARAMETER_INFO) == 0) {
1752
- q->info = true;
1753
- }
1754
- else if(strncmp(keyword, JOURNAL_PARAMETER_DELTA ":", sizeof(JOURNAL_PARAMETER_DELTA ":") - 1) == 0) {
1755
- char *v = &keyword[sizeof(JOURNAL_PARAMETER_DELTA ":") - 1];
1756
-
1757
- if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
1758
- q->delta = false;
1759
- else
1760
- q->delta = true;
1761
- }
1762
- else if(strncmp(keyword, JOURNAL_PARAMETER_TAIL ":", sizeof(JOURNAL_PARAMETER_TAIL ":") - 1) == 0) {
1763
- char *v = &keyword[sizeof(JOURNAL_PARAMETER_TAIL ":") - 1];
1764
-
1765
- if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
1766
- q->tail = false;
1767
- else
1768
- q->tail = true;
1769
- }
1770
- else if(strncmp(keyword, JOURNAL_PARAMETER_SAMPLING ":", sizeof(JOURNAL_PARAMETER_SAMPLING ":") - 1) == 0) {
1771
- q->sampling = str2ul(&keyword[sizeof(JOURNAL_PARAMETER_SAMPLING ":") - 1]);
1772
- }
1773
- else if(strncmp(keyword, JOURNAL_PARAMETER_DATA_ONLY ":", sizeof(JOURNAL_PARAMETER_DATA_ONLY ":") - 1) == 0) {
1774
- char *v = &keyword[sizeof(JOURNAL_PARAMETER_DATA_ONLY ":") - 1];
1775
-
1776
- if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
1777
- q->data_only = false;
1778
- else
1779
- q->data_only = true;
1780
- }
1781
- else if(strncmp(keyword, JOURNAL_PARAMETER_SLICE ":", sizeof(JOURNAL_PARAMETER_SLICE ":") - 1) == 0) {
1782
- char *v = &keyword[sizeof(JOURNAL_PARAMETER_SLICE ":") - 1];
1783
-
1784
- if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
1785
- q->slice = false;
1786
- else
1787
- q->slice = true;
1788
- }
1789
- else if(strncmp(keyword, JOURNAL_PARAMETER_SOURCE ":", sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1) == 0) {
1790
- const char *value = &keyword[sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1];
1791
-
1792
- buffer_json_member_add_array(wb, JOURNAL_PARAMETER_SOURCE);
1793
-
1794
- CLEAN_BUFFER *sources_list = buffer_create(0, NULL);
1795
-
1796
- q->source_type = SDJF_NONE;
1797
- while(value) {
1798
- char *sep = strchr(value, ',');
1799
- if(sep)
1800
- *sep++ = '\0';
1801
-
1802
- buffer_json_add_array_item_string(wb, value);
1803
-
1804
- SD_JOURNAL_FILE_SOURCE_TYPE t = get_internal_source_type(value);
1805
- if(t != SDJF_NONE) {
1806
- q->source_type |= t;
1807
- value = NULL;
1808
- }
1809
- else {
1810
- // else, match the source, whatever it is
1811
- if(buffer_strlen(sources_list))
1812
- buffer_putc(sources_list, '|');
1813
-
1814
- buffer_strcat(sources_list, value);
1815
- }
1816
-
1817
- value = sep;
1818
- }
1819
-
1820
- if(buffer_strlen(sources_list)) {
1821
- simple_pattern_free(q->sources);
1822
- q->sources = simple_pattern_create(buffer_tostring(sources_list), "|", SIMPLE_PATTERN_EXACT, false);
1823
- }
1824
-
1825
- buffer_json_array_close(wb); // source
1826
- }
1827
- else if(strncmp(keyword, JOURNAL_PARAMETER_AFTER ":", sizeof(JOURNAL_PARAMETER_AFTER ":") - 1) == 0) {
1828
- q->after_s = str2l(&keyword[sizeof(JOURNAL_PARAMETER_AFTER ":") - 1]);
1829
- }
1830
- else if(strncmp(keyword, JOURNAL_PARAMETER_BEFORE ":", sizeof(JOURNAL_PARAMETER_BEFORE ":") - 1) == 0) {
1831
- q->before_s = str2l(&keyword[sizeof(JOURNAL_PARAMETER_BEFORE ":") - 1]);
1832
- }
1833
- else if(strncmp(keyword, JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":", sizeof(JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":") - 1) == 0) {
1834
- q->if_modified_since = str2ull(&keyword[sizeof(JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":") - 1], NULL);
1835
- }
1836
- else if(strncmp(keyword, JOURNAL_PARAMETER_ANCHOR ":", sizeof(JOURNAL_PARAMETER_ANCHOR ":") - 1) == 0) {
1837
- q->anchor = str2ull(&keyword[sizeof(JOURNAL_PARAMETER_ANCHOR ":") - 1], NULL);
1838
- }
1839
- else if(strncmp(keyword, JOURNAL_PARAMETER_DIRECTION ":", sizeof(JOURNAL_PARAMETER_DIRECTION ":") - 1) == 0) {
1840
- q->direction = get_direction(&keyword[sizeof(JOURNAL_PARAMETER_DIRECTION ":") - 1]);
1841
- }
1842
- else if(strncmp(keyword, JOURNAL_PARAMETER_LAST ":", sizeof(JOURNAL_PARAMETER_LAST ":") - 1) == 0) {
1843
- q->last = str2ul(&keyword[sizeof(JOURNAL_PARAMETER_LAST ":") - 1]);
1844
- }
1845
- else if(strncmp(keyword, JOURNAL_PARAMETER_QUERY ":", sizeof(JOURNAL_PARAMETER_QUERY ":") - 1) == 0) {
1846
- freez((void *)q->query);
1847
- q->query= strdupz(&keyword[sizeof(JOURNAL_PARAMETER_QUERY ":") - 1]);
1848
- }
1849
- else if(strncmp(keyword, JOURNAL_PARAMETER_HISTOGRAM ":", sizeof(JOURNAL_PARAMETER_HISTOGRAM ":") - 1) == 0) {
1850
- freez((void *)q->chart);
1851
- q->chart = strdupz(&keyword[sizeof(JOURNAL_PARAMETER_HISTOGRAM ":") - 1]);
1852
- }
1853
- else if(strncmp(keyword, JOURNAL_PARAMETER_FACETS ":", sizeof(JOURNAL_PARAMETER_FACETS ":") - 1) == 0) {
1854
- q->default_facet = FACET_KEY_OPTION_NONE;
1855
- facets_reset_and_disable_all_facets(facets);
1856
-
1857
- char *value = &keyword[sizeof(JOURNAL_PARAMETER_FACETS ":") - 1];
1858
- if(*value) {
1859
- buffer_json_member_add_array(wb, JOURNAL_PARAMETER_FACETS);
1860
-
1861
- while(value) {
1862
- char *sep = strchr(value, ',');
1863
- if(sep)
1864
- *sep++ = '\0';
1865
-
1866
- facets_register_facet_id(facets, value, FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS|FACET_KEY_OPTION_REORDER);
1867
- buffer_json_add_array_item_string(wb, value);
1868
-
1869
- value = sep;
1870
- }
1871
-
1872
- buffer_json_array_close(wb); // JOURNAL_PARAMETER_FACETS
1873
- }
1874
- }
1875
- else {
1876
- char *value = strchr(keyword, ':');
1877
- if(value) {
1878
- *value++ = '\0';
1879
-
1880
- buffer_json_member_add_array(wb, keyword);
1881
-
1882
- while(value) {
1883
- char *sep = strchr(value, ',');
1884
- if(sep)
1885
- *sep++ = '\0';
1886
-
1887
- facets_register_facet_filter_id(
1888
- facets,
1889
- keyword,
1890
- value,
1891
- FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);
1892
-
1893
- buffer_json_add_array_item_string(wb, value);
1894
- q->filters++;
1895
-
1896
- value = sep;
1897
- }
1898
-
1899
- buffer_json_array_close(wb); // keyword
1900
- }
1901
- }
1902
- }
1903
-
1904
- return true;
1905
-}
1906
-
1907
-void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
1908
- BUFFER *payload, HTTP_ACCESS access __maybe_unused,
1909
- const char *source __maybe_unused, void *data __maybe_unused) {
1910
- fstat_thread_calls = 0;
1911
- fstat_thread_cached_responses = 0;
1912
-
1913
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
1914
- buffer_flush(wb);
1915
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
1916
-
1917
- FUNCTION_QUERY_STATUS tmp_fqs = {
1918
- .cancelled = cancelled,
1919
- .stop_monotonic_ut = stop_monotonic_ut,
1920
- };
1921
- FUNCTION_QUERY_STATUS *fqs = NULL;
1922
-
1923
- FACETS *facets = facets_create(50, FACETS_OPTION_ALL_KEYS_FTS,
1924
- SYSTEMD_ALWAYS_VISIBLE_KEYS,
1925
- SYSTEMD_KEYS_INCLUDED_IN_FACETS,
1926
- SYSTEMD_KEYS_EXCLUDED_FROM_FACETS);
1927
-
1928
- facets_accepted_param(facets, JOURNAL_PARAMETER_INFO);
1929
- facets_accepted_param(facets, JOURNAL_PARAMETER_SOURCE);
1930
- facets_accepted_param(facets, JOURNAL_PARAMETER_AFTER);
1931
- facets_accepted_param(facets, JOURNAL_PARAMETER_BEFORE);
1932
- facets_accepted_param(facets, JOURNAL_PARAMETER_ANCHOR);
1933
- facets_accepted_param(facets, JOURNAL_PARAMETER_DIRECTION);
1934
- facets_accepted_param(facets, JOURNAL_PARAMETER_LAST);
1935
- facets_accepted_param(facets, JOURNAL_PARAMETER_QUERY);
1936
- facets_accepted_param(facets, JOURNAL_PARAMETER_FACETS);
1937
- facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM);
1938
- facets_accepted_param(facets, JOURNAL_PARAMETER_IF_MODIFIED_SINCE);
1939
- facets_accepted_param(facets, JOURNAL_PARAMETER_DATA_ONLY);
1940
- facets_accepted_param(facets, JOURNAL_PARAMETER_DELTA);
1941
- facets_accepted_param(facets, JOURNAL_PARAMETER_TAIL);
1942
- facets_accepted_param(facets, JOURNAL_PARAMETER_SAMPLING);
1943
-
1944
-#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
1945
- facets_accepted_param(facets, JOURNAL_PARAMETER_SLICE);
1946
-#endif // HAVE_SD_JOURNAL_RESTART_FIELDS
1947
-
1948
- // ------------------------------------------------------------------------
1949
- // parse the parameters
1950
-
1951
- JOURNAL_QUERY q = {
1952
- .default_facet = FACET_KEY_OPTION_FACET,
1953
- .info = false,
1954
- .data_only = false,
1955
- .slice = JOURNAL_DEFAULT_SLICE_MODE,
1956
- .delta = false,
1957
- .tail = false,
1958
- .after_s = 0,
1959
- .before_s = 0,
1960
- .anchor = 0,
1961
- .if_modified_since = 0,
1962
- .last = 0,
1963
- .direction = JOURNAL_DEFAULT_DIRECTION,
1964
- .query = NULL,
1965
- .chart = NULL,
1966
- .sources = NULL,
1967
- .source_type = SDJF_ALL,
1968
- .filters = 0,
1969
- .sampling = SYSTEMD_JOURNAL_DEFAULT_ITEMS_SAMPLING,
1970
- };
1971
-
1972
- if( (payload && !parse_post_params(facets, &q, wb, payload, transaction)) ||
1973
- (!payload && !parse_get_params(facets, &q, wb, function, transaction)) )
1974
- goto cleanup;
1029
+static void systemd_journal_register_transformations(LOGS_QUERY_STATUS *lqs) {
1030
+ FACETS *facets = lqs->facets;
1031
+ LOGS_QUERY_REQUEST *rq = &lqs->rq;
1032
1033
// ----------------------------------------------------------------------------------------------------------------
1034
// register the fields in the order you want them on the dashboard
@@ -1979,8 +1036,7 @@ void function_systemd_journal(const char *transaction, char *function, usec_t *s
1036
facets_register_row_severity(facets, syslog_priority_to_facet_severity, NULL);
1037
1038
facets_register_key_name(
1982
- facets, "_HOSTNAME",
1983
- q.default_facet | FACET_KEY_OPTION_VISIBLE);
1039
+ facets, "_HOSTNAME", rq->default_facet | FACET_KEY_OPTION_VISIBLE);
1040
1041
facets_register_dynamic_key_name(
1042
facets, JOURNAL_KEY_ND_JOURNAL_PROCESS,
@@ -2000,19 +1056,19 @@ void function_systemd_journal(const char *transaction, char *function, usec_t *s
1056
1057
facets_register_key_name_transformation(
1058
facets, "PRIORITY",
2003
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW |
1059
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW |
1060
FACET_KEY_OPTION_EXPANDED_FILTER,
1061
netdata_systemd_journal_transform_priority, NULL);
1062
1063
facets_register_key_name_transformation(
1064
facets, "SYSLOG_FACILITY",
2009
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW |
1065
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW |
1066
FACET_KEY_OPTION_EXPANDED_FILTER,
1067
netdata_systemd_journal_transform_syslog_facility, NULL);
1068
1069
facets_register_key_name_transformation(
1070
facets, "ERRNO",
2015
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1071
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1072
netdata_systemd_journal_transform_errno, NULL);
1073
1074
facets_register_key_name(
@@ -2020,56 +1076,53 @@ void function_systemd_journal(const char *transaction, char *function, usec_t *s
1076
FACET_KEY_OPTION_NEVER_FACET);
1077
1078
facets_register_key_name(
2023
- facets, "SYSLOG_IDENTIFIER",
2024
- q.default_facet);
1079
+ facets, "SYSLOG_IDENTIFIER", rq->default_facet);
1080
1081
facets_register_key_name(
2027
- facets, "UNIT",
2028
- q.default_facet);
1082
+ facets, "UNIT", rq->default_facet);
1083
1084
facets_register_key_name(
2031
- facets, "USER_UNIT",
2032
- q.default_facet);
1085
+ facets, "USER_UNIT", rq->default_facet);
1086
1087
facets_register_key_name_transformation(
1088
facets, "MESSAGE_ID",
2036
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW |
1089
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW |
1090
FACET_KEY_OPTION_EXPANDED_FILTER,
1091
netdata_systemd_journal_transform_message_id, NULL);
1092
1093
facets_register_key_name_transformation(
1094
facets, "_BOOT_ID",
2042
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1095
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1096
netdata_systemd_journal_transform_boot_id, NULL);
1097
1098
facets_register_key_name_transformation(
1099
facets, "_SYSTEMD_OWNER_UID",
2047
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1100
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1101
netdata_systemd_journal_transform_uid, NULL);
1102
1103
facets_register_key_name_transformation(
1104
facets, "_UID",
2052
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1105
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1106
netdata_systemd_journal_transform_uid, NULL);
1107
1108
facets_register_key_name_transformation(
1109
facets, "OBJECT_SYSTEMD_OWNER_UID",
2057
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1110
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1111
netdata_systemd_journal_transform_uid, NULL);
1112
1113
facets_register_key_name_transformation(
1114
facets, "OBJECT_UID",
2062
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1115
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1116
netdata_systemd_journal_transform_uid, NULL);
1117
1118
facets_register_key_name_transformation(
1119
facets, "_GID",
2067
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1120
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1121
netdata_systemd_journal_transform_gid, NULL);
1122
1123
facets_register_key_name_transformation(
1124
facets, "OBJECT_GID",
2072
- q.default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1125
+ rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1126
netdata_systemd_journal_transform_gid, NULL);
1127
1128
facets_register_key_name_transformation(
@@ -2091,189 +1144,64 @@ void function_systemd_journal(const char *transaction, char *function, usec_t *s
1144
facets, "_SOURCE_REALTIME_TIMESTAMP",
1145
FACET_KEY_OPTION_TRANSFORM_VIEW,
1146
netdata_systemd_journal_transform_timestamp_usec, NULL);
1147
+}
1148
2095
- // ------------------------------------------------------------------------
2096
- // put this request into the progress db
2097
-
2098
- fqs = &tmp_fqs;
2099
-
2100
- // ------------------------------------------------------------------------
2101
- // validate parameters
2102
-
2103
- time_t now_s = now_realtime_sec();
2104
- time_t expires = now_s + 1;
2105
-
2106
- if(!q.after_s && !q.before_s) {
2107
- q.before_s = now_s;
2108
- q.after_s = q.before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
2109
- }
2110
- else
2111
- rrdr_relative_window_to_absolute(&q.after_s, &q.before_s, now_s);
2112
-
2113
- if(q.after_s > q.before_s) {
2114
- time_t tmp = q.after_s;
2115
- q.after_s = q.before_s;
2116
- q.before_s = tmp;
2117
- }
2118
-
2119
- if(q.after_s == q.before_s)
2120
- q.after_s = q.before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
2121
-
2122
- if(!q.last)
2123
- q.last = SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY;
2124
-
2125
- // ------------------------------------------------------------------------
2126
- // set query time-frame, anchors and direction
2127
-
2128
- fqs->transaction = transaction;
2129
- fqs->after_ut = q.after_s * USEC_PER_SEC;
2130
- fqs->before_ut = (q.before_s * USEC_PER_SEC) + USEC_PER_SEC - 1;
2131
- fqs->if_modified_since = q.if_modified_since;
2132
- fqs->data_only = q.data_only;
2133
- fqs->delta = (fqs->data_only) ? q.delta : false;
2134
- fqs->tail = (fqs->data_only && fqs->if_modified_since) ? q.tail : false;
2135
- fqs->sources = q.sources;
2136
- fqs->source_type = q.source_type;
2137
- fqs->entries = q.last;
2138
- fqs->last_modified = 0;
2139
- fqs->filters = q.filters;
2140
- fqs->query = (q.query && *q.query) ? q.query : NULL;
2141
- fqs->histogram = (q.chart && *q.chart) ? q.chart : NULL;
2142
- fqs->direction = q.direction;
2143
- fqs->anchor.start_ut = q.anchor;
2144
- fqs->anchor.stop_ut = 0;
2145
- fqs->sampling = q.sampling;
2146
-
2147
- if(fqs->anchor.start_ut && fqs->tail) {
2148
- // a tail request
2149
- // we need the top X entries from BEFORE
2150
- // but, we need to calculate the facets and the
2151
- // histogram up to the anchor
2152
- fqs->direction = q.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
2153
- fqs->anchor.start_ut = 0;
2154
- fqs->anchor.stop_ut = q.anchor;
2155
- }
2156
-
2157
- if(q.anchor && q.anchor < fqs->after_ut) {
2158
- log_fqs(fqs, "received anchor is too small for query timeframe, ignoring anchor");
2159
- q.anchor = 0;
2160
- fqs->anchor.start_ut = 0;
2161
- fqs->anchor.stop_ut = 0;
2162
- fqs->direction = q.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
2163
- }
2164
- else if(q.anchor > fqs->before_ut) {
2165
- log_fqs(fqs, "received anchor is too big for query timeframe, ignoring anchor");
2166
- q.anchor = 0;
2167
- fqs->anchor.start_ut = 0;
2168
- fqs->anchor.stop_ut = 0;
2169
- fqs->direction = q.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
2170
- }
2171
-
2172
- facets_set_anchor(facets, fqs->anchor.start_ut, fqs->anchor.stop_ut, fqs->direction);
2173
-
2174
- facets_set_additional_options(facets,
2175
- ((fqs->data_only) ? FACETS_OPTION_DATA_ONLY : 0) |
2176
- ((fqs->delta) ? FACETS_OPTION_SHOW_DELTAS : 0));
2177
-
2178
- // ------------------------------------------------------------------------
2179
- // set the rest of the query parameters
2180
-
2181
-
2182
- facets_set_items(facets, fqs->entries);
2183
- facets_set_query(facets, fqs->query);
1149
+void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
1150
+ BUFFER *payload, HTTP_ACCESS access __maybe_unused,
1151
+ const char *source __maybe_unused, void *data __maybe_unused) {
1152
+ fstat_thread_calls = 0;
1153
+ fstat_thread_cached_responses = 0;
1154
1155
#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
2186
- fqs->slice = q.slice;
2187
- if(q.slice)
2188
- facets_enable_slice_mode(facets);
1156
+ bool have_slice = true;
1157
#else
2190
- fqs->slice = false;
2191
-#endif
2192
-
2193
- if(fqs->histogram)
2194
- facets_set_timeframe_and_histogram_by_id(facets, fqs->histogram, fqs->after_ut, fqs->before_ut);
2195
- else
2196
- facets_set_timeframe_and_histogram_by_name(facets, "PRIORITY", fqs->after_ut, fqs->before_ut);
1158
+ bool have_slice = false;
1159
+#endif // HAVE_SD_JOURNAL_RESTART_FIELDS
1160
1161
+ LOGS_QUERY_STATUS tmp_fqs = {
1162
+ .facets = lqs_facets_create(
1163
+ LQS_DEFAULT_ITEMS_PER_QUERY,
1164
+ FACETS_OPTION_ALL_KEYS_FTS,
1165
+ SYSTEMD_ALWAYS_VISIBLE_KEYS,
1166
+ SYSTEMD_KEYS_INCLUDED_IN_FACETS,
1167
+ SYSTEMD_KEYS_EXCLUDED_FROM_FACETS,
1168
+ have_slice),
1169
2199
- // ------------------------------------------------------------------------
2200
- // complete the request object
2201
-
2202
- buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_INFO, false);
2203
- buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_SLICE, fqs->slice);
2204
- buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DATA_ONLY, fqs->data_only);
2205
- buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DELTA, fqs->delta);
2206
- buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_TAIL, fqs->tail);
2207
- buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_SAMPLING, fqs->sampling);
2208
- buffer_json_member_add_uint64(wb, "source_type", fqs->source_type);
2209
- buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_AFTER, fqs->after_ut / USEC_PER_SEC);
2210
- buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_BEFORE, fqs->before_ut / USEC_PER_SEC);
2211
- buffer_json_member_add_uint64(wb, "if_modified_since", fqs->if_modified_since);
2212
- buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_ANCHOR, q.anchor);
2213
- buffer_json_member_add_string(wb, JOURNAL_PARAMETER_DIRECTION, fqs->direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
2214
- buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_LAST, fqs->entries);
2215
- buffer_json_member_add_string(wb, JOURNAL_PARAMETER_QUERY, fqs->query);
2216
- buffer_json_member_add_string(wb, JOURNAL_PARAMETER_HISTOGRAM, fqs->histogram);
2217
- buffer_json_object_close(wb); // request
2218
-
2219
- buffer_json_journal_versions(wb);
1170
+ .rq = LOGS_QUERY_REQUEST_DEFAULTS(transaction, LQS_DEFAULT_SLICE_MODE, JOURNAL_DEFAULT_DIRECTION),
1171
2221
- // ------------------------------------------------------------------------
2222
- // run the request
1172
+ .cancelled = cancelled,
1173
+ .stop_monotonic_ut = stop_monotonic_ut,
1174
+ };
1175
+ LOGS_QUERY_STATUS *lqs = &tmp_fqs;
1176
2224
- int response;
1177
+ CLEAN_BUFFER *wb = lqs_create_output_buffer();
1178
2226
- if(q.info) {
2227
- buffer_json_member_add_uint64(wb, "v", 3);
2228
- facets_accepted_parameters_to_json_array(facets, wb, false);
2229
- buffer_json_member_add_array(wb, "required_params");
2230
- {
2231
- buffer_json_add_array_item_object(wb);
2232
- {
2233
- buffer_json_member_add_string(wb, "id", "source");
2234
- buffer_json_member_add_string(wb, "name", "source");
2235
- buffer_json_member_add_string(wb, "help", "Select the SystemD Journal source to query");
2236
- buffer_json_member_add_string(wb, "type", "multiselect");
2237
- buffer_json_member_add_array(wb, "options");
2238
- {
2239
- available_journal_file_sources_to_json_array(wb);
2240
- }
2241
- buffer_json_array_close(wb); // options array
2242
- }
2243
- buffer_json_object_close(wb); // required params object
2244
- }
2245
- buffer_json_array_close(wb); // required_params array
1179
+ // ------------------------------------------------------------------------
1180
+ // parse the parameters
1181
2247
- facets_table_config(wb);
1182
+ if(lqs_request_parse_and_validate(lqs, wb, function, payload, have_slice, "PRIORITY")) {
1183
+ systemd_journal_register_transformations(lqs);
1184
2249
- buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
2250
- buffer_json_member_add_string(wb, "type", "table");
2251
- buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
2252
- buffer_json_finalize(wb);
2253
- response = HTTP_RESP_OK;
2254
- goto output;
2255
- }
1185
+ // ------------------------------------------------------------------------
1186
+ // add versions to the response
1187
2257
- response = netdata_systemd_journal_query(wb, facets, fqs);
1188
+ buffer_json_journal_versions(wb);
1189
2259
- // ------------------------------------------------------------------------
2260
- // handle error response
1190
+ // ------------------------------------------------------------------------
1191
+ // run the request
1192
2262
- if(response != HTTP_RESP_OK) {
2263
- netdata_mutex_lock(&stdout_mutex);
2264
- pluginsd_function_json_error_to_stdout(transaction, response, "failed");
2265
- netdata_mutex_unlock(&stdout_mutex);
2266
- goto cleanup;
1193
+ if (lqs->rq.info)
1194
+ lqs_info_response(wb, lqs->facets);
1195
+ else {
1196
+ netdata_systemd_journal_query(wb, lqs);
1197
+ if (wb->response_code == HTTP_RESP_OK)
1198
+ buffer_json_finalize(wb);
1199
+ }
1200
}
1201
2269
-output:
1202
netdata_mutex_lock(&stdout_mutex);
2271
- pluginsd_function_result_to_stdout(transaction, response, "application/json", expires, wb);
1203
+ pluginsd_function_result_to_stdout(transaction, wb);
1204
netdata_mutex_unlock(&stdout_mutex);
1205
2274
-cleanup:
2275
- freez((void *)q.query);
2276
- freez((void *)q.chart);
2277
- simple_pattern_free(q.sources);
2278
- facets_destroy(facets);
1206
+ lqs_cleanup(lqs);
1207
}
src/collectors/systemd-journal.plugin/systemd-main.c
+2
-2
@@ -114,10 +114,10 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
114
115
// ------------------------------------------------------------------------
116
117
- usec_t step_ut = 100 * USEC_PER_MS;
117
+ const usec_t step_ut = 100 * USEC_PER_MS;
118
usec_t send_newline_ut = 0;
119
usec_t since_last_scan_ut = SYSTEMD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
120
- bool tty = isatty(fileno(stdout)) == 1;
120
+ const bool tty = isatty(fileno(stdout)) == 1;
121
122
heartbeat_t hb;
123
heartbeat_init(&hb);
src/collectors/systemd-journal.plugin/systemd-units.c
+12
-3
@@ -1153,7 +1153,10 @@ static void netdata_systemd_units_function_help(const char *transaction) {
1153
);
1154
1155
netdata_mutex_lock(&stdout_mutex);
1156
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);
1156
+ wb->response_code = HTTP_RESP_OK;
1157
+ wb->content_type = CT_TEXT_PLAIN;
1158
+ wb->expires = now_realtime_sec() + 3600;
1159
+ pluginsd_function_result_to_stdout(transaction, wb);
1160
netdata_mutex_unlock(&stdout_mutex);
1161
1162
buffer_free(wb);
@@ -1169,7 +1172,10 @@ static void netdata_systemd_units_function_info(const char *transaction) {
1172
1173
buffer_json_finalize(wb);
1174
netdata_mutex_lock(&stdout_mutex);
1172
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);
1175
+ wb->response_code = HTTP_RESP_OK;
1176
+ wb->content_type = CT_TEXT_PLAIN;
1177
+ wb->expires = now_realtime_sec() + 3600;
1178
+ pluginsd_function_result_to_stdout(transaction, wb);
1179
netdata_mutex_unlock(&stdout_mutex);
1180
1181
buffer_free(wb);
@@ -1958,7 +1964,10 @@ void function_systemd_units(const char *transaction, char *function,
1964
buffer_json_finalize(wb);
1965
1966
netdata_mutex_lock(&stdout_mutex);
1961
- pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", now_realtime_sec() + 1, wb);
1967
+ wb->response_code = HTTP_RESP_OK;
1968
+ wb->content_type = CT_APPLICATION_JSON;
1969
+ wb->expires = now_realtime_sec() + 1;
1970
+ pluginsd_function_result_to_stdout(transaction, wb);
1971
netdata_mutex_unlock(&stdout_mutex);
1972
1973
buffer_free(wb);
src/collectors/windows-events.plugin/windows-events-query.c
new
+674
@@ -0,0 +1,674 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "windows-events-query.h"
4
+
5
+static uint64_t wevt_log_file_size(const wchar_t *channel);
6
+
7
+#define FIELD_CHANNEL (0)
8
+#define FIELD_PROVIDER_NAME (1)
9
+#define FIELD_EVENT_SOURCE_NAME (2)
10
+#define FIELD_PROVIDER_GUID (3)
11
+#define FIELD_RECORD_NUMBER (4)
12
+#define FIELD_EVENT_ID (5)
13
+#define FIELD_LEVEL (6)
14
+#define FIELD_KEYWORDS (7)
15
+#define FIELD_TIME_CREATED (8)
16
+#define FIELD_COMPUTER_NAME (9)
17
+#define FIELD_USER_ID (10)
18
+#define FIELD_CORRELATION_ACTIVITY_ID (11)
19
+#define FIELD_OPCODE (12)
20
+#define FIELD_VERSION (13)
21
+#define FIELD_TASK (14)
22
+#define FIELD_PROCESS_ID (15)
23
+#define FIELD_THREAD_ID (16)
24
+#define FIELD_EVENT_DATA (17)
25
+
26
+// These are the fields we extract from the logs
27
+static const wchar_t *RENDER_ITEMS[] = {
28
+ L"/Event/System/Channel",
29
+ L"/Event/System/Provider/@Name",
30
+ L"/Event/System/Provider/@EventSourceName",
31
+ L"/Event/System/Provider/@Guid",
32
+ L"/Event/System/EventRecordID",
33
+ L"/Event/System/EventID",
34
+ L"/Event/System/Level",
35
+ L"/Event/System/Keywords",
36
+ L"/Event/System/TimeCreated/@SystemTime",
37
+ L"/Event/System/Computer",
38
+ L"/Event/System/Security/@UserID",
39
+ L"/Event/System/Correlation/@ActivityID",
40
+ L"/Event/System/Opcode",
41
+ L"/Event/System/Version",
42
+ L"/Event/System/Task",
43
+ L"/Event/System/Execution/@ProcessID",
44
+ L"/Event/System/Execution/@ThreadID",
45
+ L"/Event/EventData/Data"
46
+};
47
+
48
+static bool wevt_GUID_to_ND_UUID(ND_UUID *nd_uuid, const GUID *guid) {
49
+ if(guid && sizeof(GUID) == sizeof(ND_UUID)) {
50
+ memcpy(nd_uuid->uuid, guid, sizeof(ND_UUID));
51
+ return true;
52
+ }
53
+ else {
54
+ *nd_uuid = UUID_ZERO;
55
+ return false;
56
+ }
57
+}
58
+
59
+static uint64_t wevt_get_unsigned_by_type(WEVT_LOG *log, size_t field) {
60
+ switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
61
+ case EvtVarTypeHexInt64: return log->ops.content.data[field].UInt64Val;
62
+ case EvtVarTypeHexInt32: return log->ops.content.data[field].UInt32Val;
63
+ case EvtVarTypeUInt64: return log->ops.content.data[field].UInt64Val;
64
+ case EvtVarTypeUInt32: return log->ops.content.data[field].UInt32Val;
65
+ case EvtVarTypeUInt16: return log->ops.content.data[field].UInt16Val;
66
+ case EvtVarTypeInt64: return ABS(log->ops.content.data[field].Int64Val);
67
+ case EvtVarTypeInt32: return ABS(log->ops.content.data[field].Int32Val);
68
+ case EvtVarTypeByte: return log->ops.content.data[field].ByteVal;
69
+ case EvtVarTypeInt16: return ABS(log->ops.content.data[field].Int16Val);
70
+ case EvtVarTypeSByte: return ABS(log->ops.content.data[field].SByteVal);
71
+ case EvtVarTypeSingle: return ABS(log->ops.content.data[field].SingleVal);
72
+ case EvtVarTypeDouble: return ABS(log->ops.content.data[field].DoubleVal);
73
+ case EvtVarTypeBoolean: return log->ops.content.data[field].BooleanVal ? 1 : 0;
74
+ case EvtVarTypeSizeT: return log->ops.content.data[field].SizeTVal;
75
+ default: return 0;
76
+ }
77
+}
78
+
79
+static uint64_t wevt_get_filetime_to_ns_by_type(WEVT_LOG *log, size_t field) {
80
+ switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
81
+ case EvtVarTypeFileTime:
82
+ case EvtVarTypeUInt64:
83
+ return os_windows_ulonglong_to_unix_epoch_ns(log->ops.content.data[field].FileTimeVal);
84
+
85
+ default: return 0;
86
+ }
87
+}
88
+
89
+static bool wevt_get_uuid_by_type(WEVT_LOG *log, size_t field, ND_UUID *dst) {
90
+ switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
91
+ case EvtVarTypeGuid:
92
+ return wevt_GUID_to_ND_UUID(dst, log->ops.content.data[field].GuidVal);
93
+
94
+ default:
95
+ return wevt_GUID_to_ND_UUID(dst, NULL);
96
+ }
97
+}
98
+
99
+static void wevt_empty_utf8(TXT_UTF8 *dst) {
100
+ txt_utf8_resize(dst, 1);
101
+ dst->data[0] = '\0';
102
+ dst->used = 1;
103
+}
104
+
105
+static bool wevt_get_message_utf8(WEVT_LOG *log, EVT_HANDLE event_handle, TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS what) {
106
+ DWORD size = 0;
107
+
108
+ if(!log->ops.unicode.data) {
109
+ EvtFormatMessage(NULL, event_handle, 0, 0, NULL, what, 0, NULL, &size);
110
+ if(!size) {
111
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() to get message size failed.");
112
+ goto cleanup;
113
+ }
114
+ txt_unicode_resize(&log->ops.unicode, size);
115
+ }
116
+
117
+ // First, try to get the message using the existing buffer
118
+ if (!EvtFormatMessage(NULL, event_handle, 0, 0, NULL, what, log->ops.unicode.size, log->ops.unicode.data, &size) || !log->ops.unicode.data) {
119
+ if (log->ops.unicode.data && GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
120
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() failed.");
121
+ goto cleanup;
122
+ }
123
+
124
+ // Try again with the resized buffer
125
+ txt_unicode_resize(&log->ops.unicode, size);
126
+ if (!EvtFormatMessage(NULL, event_handle, 0, 0, NULL, what, log->ops.unicode.size, log->ops.unicode.data, &size)) {
127
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() failed after resizing buffer.");
128
+ goto cleanup;
129
+ }
130
+ }
131
+
132
+ // EvtFormatMessage pads it with zeros at the end
133
+ while(size >= 2 && log->ops.unicode.data[size - 2] == 0)
134
+ size--;
135
+
136
+ log->ops.unicode.used = size;
137
+
138
+ internal_fatal(wcslen(log->ops.unicode.data) + 1 != (size_t)log->ops.unicode.used,
139
+ "Wrong unicode string length!");
140
+
141
+ return wevt_str_unicode_to_utf8(dst, &log->ops.unicode);
142
+
143
+cleanup:
144
+ wevt_empty_utf8(dst);
145
+ return false;
146
+}
147
+
148
+static bool wevt_get_utf8_by_type(WEVT_LOG *log, size_t field, TXT_UTF8 *dst) {
149
+ switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
150
+ case EvtVarTypeString:
151
+ return wevt_str_wchar_to_utf8(dst, log->ops.content.data[field].StringVal, -1);
152
+
153
+ default:
154
+ wevt_empty_utf8(dst);
155
+ return false;
156
+ }
157
+}
158
+
159
+static bool wevt_get_sid_by_type(WEVT_LOG *log, size_t field, TXT_UTF8 *dst) {
160
+ switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
161
+ case EvtVarTypeSid:
162
+ return wevt_convert_user_id_to_name(log->ops.content.data[field].SidVal, dst);
163
+
164
+ default:
165
+ wevt_empty_utf8(dst);
166
+ return false;
167
+ }
168
+}
169
+
170
+void wevt_field_to_buffer_append(BUFFER *wb, EVT_VARIANT *d, const char *prefix, const char *suffix) {
171
+ buffer_strcat(wb, prefix);
172
+
173
+ switch(d->Type & EVT_VARIANT_TYPE_MASK) {
174
+ case EvtVarTypeNull:
175
+ buffer_strcat(wb, "(null)");
176
+ break;
177
+
178
+ case EvtVarTypeString:
179
+ buffer_sprintf(wb, "%ls", d->StringVal);
180
+ break;
181
+
182
+ case EvtVarTypeSByte:
183
+ buffer_sprintf(wb, "%d", (int) d->SByteVal);
184
+ break;
185
+
186
+ case EvtVarTypeByte:
187
+ buffer_sprintf(wb, "%u", (unsigned) d->ByteVal);
188
+ break;
189
+
190
+ case EvtVarTypeInt16:
191
+ buffer_sprintf(wb, "%d", (int) d->Int16Val);
192
+ break;
193
+
194
+ case EvtVarTypeUInt16:
195
+ buffer_sprintf(wb, "%u", (unsigned) d->UInt16Val);
196
+ break;
197
+
198
+ case EvtVarTypeInt32:
199
+ buffer_sprintf(wb, "%d", (int) d->Int32Val);
200
+ break;
201
+
202
+ case EvtVarTypeUInt32:
203
+ case EvtVarTypeHexInt32:
204
+ buffer_sprintf(wb, "%u", (unsigned) d->UInt32Val);
205
+ break;
206
+
207
+ case EvtVarTypeInt64:
208
+ buffer_sprintf(wb, "%" PRIi64, (uint64_t)d->Int64Val);
209
+ break;
210
+
211
+ case EvtVarTypeUInt64:
212
+ case EvtVarTypeHexInt64:
213
+ buffer_sprintf(wb, "%" PRIu64, (uint64_t)d->UInt64Val);
214
+ break;
215
+
216
+ case EvtVarTypeSizeT:
217
+ buffer_sprintf(wb, "%zu", d->SizeTVal);
218
+ break;
219
+
220
+ case EvtVarTypeSysTime:
221
+ buffer_sprintf(wb, "%04d-%02d-%02dT%02d:%02d:%02d.%03d (SystemTime)",
222
+ d->SysTimeVal->wYear, d->SysTimeVal->wMonth, d->SysTimeVal->wDay,
223
+ d->SysTimeVal->wHour, d->SysTimeVal->wMinute, d->SysTimeVal->wSecond,
224
+ d->SysTimeVal->wMilliseconds);
225
+ break;
226
+
227
+ case EvtVarTypeGuid: {
228
+ char uuid_str[UUID_COMPACT_STR_LEN];
229
+ ND_UUID uuid;
230
+ if (wevt_GUID_to_ND_UUID(&uuid, d->GuidVal)) {
231
+ uuid_unparse_lower_compact(uuid.uuid, uuid_str);
232
+ buffer_strcat(wb, uuid_str);
233
+ buffer_strcat(wb, " (GUID)");
234
+ }
235
+ break;
236
+ }
237
+
238
+ case EvtVarTypeSingle:
239
+ buffer_print_netdata_double(wb, d->SingleVal);
240
+ break;
241
+
242
+ case EvtVarTypeDouble:
243
+ buffer_print_netdata_double(wb, d->DoubleVal);
244
+ break;
245
+
246
+ case EvtVarTypeBoolean:
247
+ buffer_strcat(wb, d->BooleanVal ? "true" : "false");
248
+ break;
249
+
250
+ case EvtVarTypeFileTime: {
251
+ nsec_t ns = os_windows_ulonglong_to_unix_epoch_ns(d->FileTimeVal);
252
+ char buf[RFC3339_MAX_LENGTH];
253
+ rfc3339_datetime_ut(buf, sizeof(buf), ns, 2, true);
254
+ buffer_strcat(wb, buf);
255
+ buffer_strcat(wb, " (FileTime)");
256
+ break;
257
+ }
258
+
259
+ case EvtVarTypeBinary:
260
+ buffer_strcat(wb, "(binary data)");
261
+ break;
262
+
263
+ case EvtVarTypeSid:
264
+ buffer_strcat(wb, "(user id data)");
265
+ break;
266
+
267
+ case EvtVarTypeAnsiString:
268
+ case EvtVarTypeEvtHandle:
269
+ case EvtVarTypeEvtXml:
270
+ default:
271
+ buffer_sprintf(wb, "(unsupported data type: %u)", d->Type);
272
+ break;
273
+ }
274
+
275
+ buffer_strcat(wb, suffix);
276
+}
277
+
278
+static bool wevt_format_summary_array_traversal(WEVT_LOG *log, size_t field, WEVT_EVENT *ev) {
279
+ EVT_VARIANT *d = &log->ops.content.data[field];
280
+
281
+ if (!log->ops.message)
282
+ log->ops.message = buffer_create(0, NULL);
283
+
284
+ BUFFER *wb = log->ops.message;
285
+ buffer_flush(wb);
286
+
287
+ // Check if there is an event description
288
+ if (log->ops.event.data && *log->ops.event.data) {
289
+ buffer_strcat(wb, log->ops.event.data);
290
+ buffer_strcat(wb, "\nRelated data:\n");
291
+ } else {
292
+ buffer_sprintf(wb, "Event %" PRIu64 ", with the following related data:\n", (uint64_t)ev->event_id);
293
+ }
294
+
295
+ // Check if the field contains an array or a single value
296
+ bool is_array = (d->Type & EVT_VARIANT_TYPE_ARRAY) != 0;
297
+ EVT_VARIANT_TYPE base_type = (EVT_VARIANT_TYPE)(d->Type & EVT_VARIANT_TYPE_MASK);
298
+
299
+ if (is_array) {
300
+ DWORD count = d->Count; // Number of elements in the array
301
+
302
+ for (DWORD i = 0; i < count; i++) {
303
+ EVT_VARIANT array_element = {
304
+ .Type = base_type,
305
+ .Count = 0,
306
+ };
307
+
308
+ // Point the array element to the correct data
309
+ switch (base_type) {
310
+ case EvtVarTypeBoolean:
311
+ array_element.BooleanVal = d->BooleanArr[i];
312
+ break;
313
+ case EvtVarTypeSByte:
314
+ array_element.SByteVal = d->SByteArr[i];
315
+ break;
316
+ case EvtVarTypeByte:
317
+ array_element.ByteVal = d->ByteArr[i];
318
+ break;
319
+ case EvtVarTypeInt16:
320
+ array_element.Int16Val = d->Int16Arr[i];
321
+ break;
322
+ case EvtVarTypeUInt16:
323
+ array_element.UInt16Val = d->UInt16Arr[i];
324
+ break;
325
+ case EvtVarTypeInt32:
326
+ array_element.Int32Val = d->Int32Arr[i];
327
+ break;
328
+ case EvtVarTypeUInt32:
329
+ array_element.UInt32Val = d->UInt32Arr[i];
330
+ break;
331
+ case EvtVarTypeInt64:
332
+ array_element.Int64Val = d->Int64Arr[i];
333
+ break;
334
+ case EvtVarTypeUInt64:
335
+ array_element.UInt64Val = d->UInt64Arr[i];
336
+ break;
337
+ case EvtVarTypeSingle:
338
+ array_element.SingleVal = d->SingleArr[i];
339
+ break;
340
+ case EvtVarTypeDouble:
341
+ array_element.DoubleVal = d->DoubleArr[i];
342
+ break;
343
+ case EvtVarTypeFileTime:
344
+ array_element.FileTimeVal = ((ULONGLONG)d->FileTimeArr[i].dwLowDateTime |
345
+ ((ULONGLONG)d->FileTimeArr[i].dwHighDateTime << 32));
346
+ break;
347
+ case EvtVarTypeSysTime:
348
+ array_element.SysTimeVal = &d->SysTimeArr[i];
349
+ break;
350
+ case EvtVarTypeGuid:
351
+ array_element.GuidVal = &d->GuidArr[i];
352
+ break;
353
+ case EvtVarTypeString:
354
+ array_element.StringVal = d->StringArr[i];
355
+ break;
356
+ case EvtVarTypeAnsiString:
357
+ array_element.AnsiStringVal = d->AnsiStringArr[i];
358
+ break;
359
+ case EvtVarTypeSid:
360
+ array_element.SidVal = d->SidArr[i];
361
+ break;
362
+ case EvtVarTypeSizeT:
363
+ array_element.SizeTVal = d->SizeTArr[i];
364
+ break;
365
+ case EvtVarTypeEvtXml:
366
+ array_element.XmlVal = d->XmlValArr[i];
367
+ break;
368
+ default:
369
+ buffer_sprintf(wb, " - Unsupported array type: %u\n", base_type);
370
+ continue;
371
+ }
372
+
373
+ // Call the field appending function for each array element
374
+ wevt_field_to_buffer_append(wb, &array_element, " - ", "\n");
375
+ }
376
+ } else {
377
+ // Handle single values, pass the EVT_VARIANT directly
378
+ wevt_field_to_buffer_append(wb, d, " - ", "\n");
379
+ }
380
+
381
+ return true;
382
+}
383
+
384
+static bool wevt_format_summary(WEVT_LOG *log, WEVT_EVENT *ev) {
385
+ if (!log->ops.message)
386
+ log->ops.message = buffer_create(0, NULL);
387
+
388
+ BUFFER *wb = log->ops.message;
389
+ buffer_flush(wb);
390
+
391
+ // Check if there is an event description
392
+ if (log->ops.event.data && *log->ops.event.data)
393
+ buffer_strcat(wb, log->ops.event.data);
394
+ else
395
+ buffer_sprintf(wb, "Event %" PRIu64, (uint64_t) ev->event_id);
396
+
397
+ const char *xml = log->ops.xml.data;
398
+ const char *event_data_start = strstr(xml, "<EventData>");
399
+ if(event_data_start)
400
+ event_data_start = &event_data_start[11];
401
+
402
+ const char *event_data_end = event_data_start ? strstr(xml, "</EventData>") : NULL;
403
+
404
+ if(event_data_start && event_data_end) {
405
+ buffer_strcat(wb, "\n\nRelated data:\n\n");
406
+ // copy the event data block
407
+ buffer_fast_strcat(wb, event_data_start, event_data_end - event_data_start);
408
+ }
409
+
410
+ return true;
411
+}
412
+
413
+bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
414
+ DWORD size_required_next = 0, size = 0, bookmarkedCount = 0;
415
+ EVT_HANDLE tmp_event_bookmark = NULL;
416
+ bool ret = false;
417
+
418
+ fatal_assert(log && log->event_query && log->render_context);
419
+
420
+ if (!EvtNext(log->event_query, 1, &tmp_event_bookmark, INFINITE, 0, &size_required_next))
421
+ goto cleanup; // no data available, return failure
422
+
423
+ // obtain the information from selected events
424
+ if (!EvtRender(log->render_context, tmp_event_bookmark, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &size, &bookmarkedCount)) {
425
+ // information exceeds the allocated space
426
+ if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
427
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed");
428
+ goto cleanup;
429
+ }
430
+
431
+ freez(log->ops.content.data);
432
+ log->ops.content.size = size;
433
+ log->ops.content.data = (EVT_VARIANT *)mallocz(log->ops.content.size);
434
+
435
+ if (!EvtRender(log->render_context, tmp_event_bookmark, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &size, &bookmarkedCount)) {
436
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, after size increase");
437
+ goto cleanup;
438
+ }
439
+ }
440
+ log->ops.content.len = size;
441
+
442
+ ev->id = wevt_get_unsigned_by_type(log, FIELD_RECORD_NUMBER);
443
+ ev->event_id = wevt_get_unsigned_by_type(log, FIELD_EVENT_ID);
444
+ ev->level = wevt_get_unsigned_by_type(log, FIELD_LEVEL);
445
+ ev->keyword = wevt_get_unsigned_by_type(log, FIELD_KEYWORDS);
446
+ ev->created_ns = wevt_get_filetime_to_ns_by_type(log, FIELD_TIME_CREATED);
447
+ ev->opcode = wevt_get_unsigned_by_type(log, FIELD_OPCODE);
448
+ ev->version = wevt_get_unsigned_by_type(log, FIELD_VERSION);
449
+ ev->task = wevt_get_unsigned_by_type(log, FIELD_TASK);
450
+ ev->process_id = wevt_get_unsigned_by_type(log, FIELD_PROCESS_ID);
451
+ ev->thread_id = wevt_get_unsigned_by_type(log, FIELD_THREAD_ID);
452
+
453
+ if(full) {
454
+ wevt_get_utf8_by_type(log, FIELD_CHANNEL, &log->ops.channel);
455
+ wevt_get_utf8_by_type(log, FIELD_PROVIDER_NAME, &log->ops.provider);
456
+ wevt_get_utf8_by_type(log, FIELD_EVENT_SOURCE_NAME, &log->ops.source);
457
+ wevt_get_uuid_by_type(log, FIELD_PROVIDER_GUID, &ev->provider);
458
+ wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.level, EvtFormatMessageLevel);
459
+
460
+ if(ev->event_id)
461
+ wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.event, EvtFormatMessageEvent);
462
+ else
463
+ wevt_empty_utf8(&log->ops.event);
464
+
465
+ if(ev->keyword)
466
+ wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.keyword, EvtFormatMessageKeyword);
467
+ else
468
+ wevt_empty_utf8(&log->ops.keyword);
469
+
470
+ if(ev->opcode)
471
+ wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.opcode, EvtFormatMessageOpcode);
472
+ else
473
+ wevt_empty_utf8(&log->ops.keyword);
474
+
475
+ // ComputerName
476
+ wevt_get_utf8_by_type(log, FIELD_COMPUTER_NAME, &log->ops.computer);
477
+
478
+ // User
479
+ wevt_get_sid_by_type(log, FIELD_USER_ID, &log->ops.user);
480
+
481
+ // CorrelationActivityID
482
+ wevt_get_uuid_by_type(log, FIELD_CORRELATION_ACTIVITY_ID, &ev->correlation_activity_id);
483
+
484
+ // Full XML of the entire message
485
+ wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.xml, EvtFormatMessageXml);
486
+
487
+ // Format a text message for the users to see
488
+ // wevt_format_summary(log, ev);
489
+ }
490
+
491
+ ret = true;
492
+
493
+cleanup:
494
+ if (tmp_event_bookmark)
495
+ EvtClose(tmp_event_bookmark);
496
+
497
+ return ret;
498
+}
499
+
500
+void wevt_query_done(WEVT_LOG *log) {
501
+ if (!log->event_query) return;
502
+
503
+ EvtClose(log->event_query);
504
+ log->event_query = NULL;
505
+}
506
+
507
+void wevt_closelog6(WEVT_LOG *log) {
508
+ wevt_query_done(log);
509
+
510
+ if (log->render_context)
511
+ EvtClose(log->render_context);
512
+
513
+ freez(log->ops.content.data);
514
+ txt_unicode_cleanup(&log->ops.unicode);
515
+ txt_utf8_cleanup(&log->ops.channel);
516
+ txt_utf8_cleanup(&log->ops.provider);
517
+ txt_utf8_cleanup(&log->ops.source);
518
+ txt_utf8_cleanup(&log->ops.computer);
519
+ txt_utf8_cleanup(&log->ops.event);
520
+ txt_utf8_cleanup(&log->ops.user);
521
+ txt_utf8_cleanup(&log->ops.opcode);
522
+ txt_utf8_cleanup(&log->ops.level);
523
+ txt_utf8_cleanup(&log->ops.keyword);
524
+ txt_utf8_cleanup(&log->ops.xml);
525
+ buffer_free(log->ops.message);
526
+ freez(log);
527
+}
528
+
529
+bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention) {
530
+ bool ret = false;
531
+
532
+ // get the number of the oldest record in the log
533
+ // "EvtGetLogInfo()" does not work properly with "EvtLogOldestRecordNumber"
534
+ // we have to get it from the first EventRecordID
535
+
536
+ // query the eventlog
537
+ log->event_query = EvtQuery(NULL, channel, NULL, EvtQueryChannelPath);
538
+ if (!log->event_query) {
539
+ if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
540
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, channel '%s' not found, cannot open log", channel2utf8(channel));
541
+ else
542
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() on channel '%s' failed, cannot open log", channel2utf8(channel));
543
+
544
+ goto cleanup;
545
+ }
546
+
547
+ if (!wevt_get_next_event(log, &retention->first_event, false))
548
+ goto cleanup;
549
+
550
+ if (!retention->first_event.id) {
551
+ // no data in the event log
552
+ retention->first_event = retention->last_event = WEVT_EVENT_EMPTY;
553
+ ret = true;
554
+ goto cleanup;
555
+ }
556
+ EvtClose(log->event_query);
557
+
558
+ log->event_query = EvtQuery(NULL, channel, NULL, EvtQueryChannelPath | EvtQueryReverseDirection);
559
+ if (!log->event_query) {
560
+ if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
561
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, channel '%s' not found", channel2utf8(channel));
562
+ else
563
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() on channel '%s' failed", channel2utf8(channel));
564
+
565
+ goto cleanup;
566
+ }
567
+
568
+ if (!wevt_get_next_event(log, &retention->last_event, false) || retention->last_event.id == 0) {
569
+ // no data in eventlog
570
+ retention->last_event = retention->first_event;
571
+ }
572
+ retention->last_event.id += 1; // we should read the last record
573
+ ret = true;
574
+
575
+cleanup:
576
+ wevt_query_done(log);
577
+
578
+ if(ret) {
579
+ retention->entries = retention->last_event.id - retention->first_event.id;
580
+
581
+ if(retention->last_event.created_ns >= retention->first_event.created_ns)
582
+ retention->duration_ns = retention->last_event.created_ns - retention->first_event.created_ns;
583
+ else
584
+ retention->duration_ns = retention->first_event.created_ns - retention->last_event.created_ns;
585
+
586
+ retention->size_bytes = wevt_log_file_size(channel);
587
+ }
588
+ else
589
+ memset(retention, 0, sizeof(*retention));
590
+
591
+ return ret;
592
+}
593
+
594
+WEVT_LOG *wevt_openlog6(void) {
595
+ size_t RENDER_ITEMS_count = (sizeof(RENDER_ITEMS) / sizeof(const wchar_t *));
596
+
597
+ WEVT_LOG *log = callocz(1, sizeof(*log));
598
+
599
+ // create the system render
600
+ log->render_context = EvtCreateRenderContext(RENDER_ITEMS_count, RENDER_ITEMS, EvtRenderContextValues);
601
+ if (!log->render_context) {
602
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtCreateRenderContext failed.");
603
+ goto cleanup;
604
+ }
605
+
606
+cleanup:
607
+ return log;
608
+}
609
+
610
+static uint64_t wevt_log_file_size(const wchar_t *channel) {
611
+ EVT_HANDLE hLog = NULL;
612
+ EVT_VARIANT evtVariant;
613
+ DWORD bufferUsed = 0;
614
+ uint64_t file_size = 0;
615
+
616
+ // Open the event log channel
617
+ hLog = EvtOpenLog(NULL, channel, EvtOpenChannelPath);
618
+ if (!hLog) {
619
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtOpenLog() on channel '%s' failed", channel2utf8(channel));
620
+ goto cleanup;
621
+ }
622
+
623
+ // Get the file size of the log
624
+ if (!EvtGetLogInfo(hLog, EvtLogFileSize, sizeof(evtVariant), &evtVariant, &bufferUsed)) {
625
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetLogInfo() on channel '%s' failed", channel2utf8(channel));
626
+ goto cleanup;
627
+ }
628
+
629
+ // Extract the file size from the EVT_VARIANT structure
630
+ file_size = evtVariant.UInt64Val;
631
+
632
+cleanup:
633
+ if (hLog)
634
+ EvtClose(hLog);
635
+
636
+ return file_size;
637
+}
638
+
639
+EVT_HANDLE wevt_query(LPCWSTR channel, usec_t seek_to, bool backward) {
640
+ // Convert microseconds to nanoseconds first (correct handling of precision)
641
+ if(backward) seek_to += USEC_PER_MS; // for backward mode, add a millisecond to the seek time.
642
+
643
+ // Convert the microseconds since Unix epoch to FILETIME (used in Windows APIs)
644
+ FILETIME fileTime = os_unix_epoch_ut_to_filetime(seek_to);
645
+
646
+ // Convert FILETIME to SYSTEMTIME for use in XPath
647
+ SYSTEMTIME systemTime;
648
+ if (!FileTimeToSystemTime(&fileTime, &systemTime)) {
649
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "FileTimeToSystemTime() failed");
650
+ return NULL;
651
+ }
652
+
653
+ // Format SYSTEMTIME into ISO 8601 format (YYYY-MM-DDTHH:MM:SS.sssZ)
654
+ static __thread WCHAR query[4096];
655
+ swprintf(query, 4096,
656
+ L"Event/System[TimeCreated[@SystemTime%ls\"%04d-%02d-%02dT%02d:%02d:%02d.%03dZ\"]]",
657
+ backward ? L"<=" : L">=", // Use <= if backward, >= if forward
658
+ systemTime.wYear, systemTime.wMonth, systemTime.wDay,
659
+ systemTime.wHour, systemTime.wMinute, systemTime.wSecond, systemTime.wMilliseconds);
660
+
661
+ // Execute the query
662
+ EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (backward ? EvtQueryReverseDirection : EvtQueryForwardDirection));
663
+ if (!hQuery) {
664
+ wchar_t wbuf[1024];
665
+ DWORD wbuf_used;
666
+ EvtGetExtendedStatus(sizeof(wbuf), wbuf, &wbuf_used);
667
+
668
+ char buf[1024];
669
+ rfc3339_datetime_ut(buf, sizeof(buf), seek_to, 3, true);
670
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, seek to '%s', query: %s | extended info: %ls", buf, query2utf8(query), wbuf);
671
+ }
672
+
673
+ return hQuery;
674
+}
src/collectors/windows-events.plugin/windows-events-query.h
new
+81
@@ -0,0 +1,81 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WINDOWS_EVENTS_QUERY_H
4
+#define NETDATA_WINDOWS_EVENTS_QUERY_H
5
+
6
+#include "windows-events.h"
7
+
8
+typedef struct wevt_event {
9
+ uint64_t id; // EventRecordId (unique and sequential per channel)
10
+ uint16_t event_id; // This is the template that defines the message to be shown
11
+ uint16_t opcode;
12
+ uint8_t level; // The severity of event
13
+ uint8_t version;
14
+ uint16_t task;
15
+ uint32_t process_id;
16
+ uint32_t thread_id;
17
+ uint64_t keyword; // Categorization of the event
18
+ ND_UUID provider;
19
+ ND_UUID correlation_activity_id;
20
+ nsec_t created_ns;
21
+} WEVT_EVENT;
22
+
23
+#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
24
+
25
+typedef struct {
26
+ WEVT_EVENT first_event;
27
+ WEVT_EVENT last_event;
28
+
29
+ uint64_t entries;
30
+ nsec_t duration_ns;
31
+ uint64_t size_bytes;
32
+} EVT_RETENTION;
33
+
34
+typedef struct wevt_log {
35
+ EVT_HANDLE event_query;
36
+ EVT_HANDLE render_context;
37
+
38
+ struct {
39
+ // temp buffer used for rendering event log messages
40
+ // never use directly
41
+ struct {
42
+ EVT_VARIANT *data;
43
+ size_t size;
44
+ size_t len;
45
+ } content;
46
+
47
+ // temp buffer used for fetching and converting UNICODE and UTF-8
48
+ // every string operation overwrites it, multiple times per event log entry
49
+ // it can be used within any function, for its own purposes,
50
+ // but never share between functions
51
+ TXT_UNICODE unicode;
52
+
53
+ // string attributes of the current event log entry
54
+ // valid until another event if fetched
55
+ TXT_UTF8 channel;
56
+ TXT_UTF8 provider;
57
+ TXT_UTF8 source;
58
+ TXT_UTF8 computer;
59
+ TXT_UTF8 event;
60
+ TXT_UTF8 user;
61
+ TXT_UTF8 opcode;
62
+ TXT_UTF8 level;
63
+ TXT_UTF8 keyword;
64
+ TXT_UTF8 xml;
65
+
66
+ BUFFER *message;
67
+ } ops;
68
+
69
+} WEVT_LOG;
70
+
71
+WEVT_LOG *wevt_openlog6(void);
72
+void wevt_closelog6(WEVT_LOG *log);
73
+
74
+bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention);
75
+
76
+EVT_HANDLE wevt_query(LPCWSTR channel, usec_t seek_to, bool backward);
77
+void wevt_query_done(WEVT_LOG *log);
78
+
79
+bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
80
+
81
+#endif //NETDATA_WINDOWS_EVENTS_QUERY_H
src/collectors/windows-events.plugin/windows-events-sid.c
new
+120
@@ -0,0 +1,120 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "windows-events-sid.h"
4
+#include <sddl.h>
5
+
6
+typedef struct {
7
+ size_t len;
8
+ uint8_t sid[];
9
+} SID_KEY;
10
+
11
+typedef struct {
12
+ const char *user;
13
+ size_t user_len;
14
+
15
+ // this needs to be last, because of its variable size
16
+ SID_KEY key;
17
+} SID_VALUE;
18
+
19
+#define SIMPLE_HASHTABLE_NAME _SID
20
+#define SIMPLE_HASHTABLE_VALUE_TYPE SID_VALUE
21
+#define SIMPLE_HASHTABLE_KEY_TYPE SID_KEY
22
+#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION sid_value_to_key
23
+#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION sid_cache_compar
24
+#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
25
+#include "libnetdata/simple_hashtable.h"
26
+
27
+static struct {
28
+ SPINLOCK spinlock;
29
+ bool initialized;
30
+ struct simple_hashtable_SID hashtable;
31
+} sid_globals = {
32
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
33
+ .hashtable = { 0 },
34
+};
35
+
36
+static inline SID_KEY *sid_value_to_key(SID_VALUE *s) {
37
+ return &s->key;
38
+}
39
+
40
+static inline bool sid_cache_compar(SID_KEY *a, SID_KEY *b) {
41
+ return a->len == b->len && memcmp(&a->sid, &b->sid, a->len) == 0;
42
+}
43
+
44
+static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
45
+ if(found && found->user) {
46
+ txt_utf8_resize(dst, found->user_len + 1);
47
+ memcpy(dst->data, found->user, found->user_len + 1);
48
+ dst->used = found->user_len + 1;
49
+ return true;
50
+ }
51
+
52
+ txt_utf8_resize(dst, 1);
53
+ dst->data[0] = '\0';
54
+ dst->used = 1;
55
+ return false;
56
+}
57
+
58
+static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
59
+ static __thread wchar_t account_unicode[256];
60
+ static __thread wchar_t domain_unicode[256];
61
+ DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
62
+ DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
63
+ SID_NAME_USE sid_type;
64
+
65
+ txt_utf8_resize(dst, 1024);
66
+
67
+ if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
68
+ const char *user = account2utf8(account_unicode);
69
+ const char *domain = domain2utf8(domain_unicode);
70
+ dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
71
+ }
72
+ else {
73
+ wchar_t *sid_string = NULL;
74
+ if (ConvertSidToStringSidW(sid, &sid_string)) {
75
+ const char *user = account2utf8(sid_string);
76
+ dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
77
+ }
78
+ else
79
+ dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
80
+ }
81
+}
82
+
83
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
84
+ if(!sid || !IsValidSid(sid))
85
+ return update_user(NULL, dst);
86
+
87
+ size_t size = GetLengthSid(sid);
88
+
89
+ size_t tmp_size = sizeof(SID_VALUE) + size;
90
+ size_t tmp_key_size = sizeof(SID_KEY) + size;
91
+ uint8_t buf[tmp_size];
92
+ SID_VALUE *tmp = (SID_VALUE *)&buf;
93
+ memcpy(&tmp->key.sid, sid, size);
94
+ tmp->key.len = size;
95
+
96
+ spinlock_lock(&sid_globals.spinlock);
97
+ if(!sid_globals.initialized) {
98
+ simple_hashtable_init_SID(&sid_globals.hashtable, 100);
99
+ sid_globals.initialized = true;
100
+ }
101
+ SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
102
+ spinlock_unlock(&sid_globals.spinlock);
103
+ if(found) return update_user(found, dst);
104
+
105
+ // allocate the SID_VALUE
106
+ found = mallocz(tmp_size);
107
+ memcpy(found, buf, tmp_size);
108
+
109
+ // lookup the user
110
+ lookup_user(sid, dst);
111
+ found->user = strdupz(dst->data);
112
+ found->user_len = dst->used - 1;
113
+
114
+ // add it to the cache
115
+ spinlock_lock(&sid_globals.spinlock);
116
+ simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
117
+ spinlock_unlock(&sid_globals.spinlock);
118
+
119
+ return update_user(found, dst);
120
+}
src/collectors/windows-events.plugin/windows-events-sid.h
new
+11
@@ -0,0 +1,11 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WINDOWS_EVENTS_SID_H
4
+#define NETDATA_WINDOWS_EVENTS_SID_H
5
+
6
+#include "windows-events.h"
7
+
8
+struct wevt_log;
9
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
10
+
11
+#endif //NETDATA_WINDOWS_EVENTS_SID_H
src/collectors/windows-events.plugin/windows-events-sources.c
new
+267
@@ -0,0 +1,267 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "windows-events-sources.h"
4
+
5
+DICTIONARY *wevt_sources = NULL;
6
+DICTIONARY *used_hashes_registry = NULL;
7
+static usec_t wevt_session = 0;
8
+
9
+WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
10
+ if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
11
+ return WEVTS_ALL;
12
+
13
+ return WEVTS_NONE;
14
+}
15
+
16
+void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
17
+ LOGS_QUERY_SOURCE *src = value;
18
+ freez((void *)src->fullname);
19
+ string_freez(src->source);
20
+
21
+ src->fullname = NULL;
22
+ src->source = NULL;
23
+}
24
+
25
+static bool wevt_sources_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
26
+ LOGS_QUERY_SOURCE *src_old = old_value;
27
+ LOGS_QUERY_SOURCE *src_new = new_value;
28
+
29
+ bool ret = false;
30
+ if(src_new->last_scan_monotonic_ut > src_old->last_scan_monotonic_ut) {
31
+ src_old->last_scan_monotonic_ut = src_new->last_scan_monotonic_ut;
32
+
33
+ if (src_old->source != src_new->source) {
34
+ string_freez(src_old->source);
35
+ src_old->source = src_new->source;
36
+ src_new->source = NULL;
37
+ }
38
+ src_old->source_type = src_new->source_type;
39
+
40
+ src_old->msg_first_ut = src_new->msg_first_ut;
41
+ src_old->msg_last_ut = src_new->msg_last_ut;
42
+ src_old->msg_first_id = src_new->msg_first_id;
43
+ src_old->msg_last_id = src_new->msg_last_id;
44
+ src_old->entries = src_new->entries;
45
+ src_old->size = src_new->size;
46
+
47
+ ret = true;
48
+ }
49
+
50
+ freez((void *)src_new->fullname);
51
+ string_freez(src_new->source);
52
+ src_new->fullname = NULL;
53
+ src_new->source = NULL;
54
+
55
+ return ret;
56
+}
57
+
58
+void wevt_sources_init(void) {
59
+ wevt_session = now_realtime_usec();
60
+
61
+ used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
62
+
63
+ wevt_sources = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
64
+ NULL, sizeof(LOGS_QUERY_SOURCE));
65
+
66
+ dictionary_register_delete_callback(wevt_sources, wevt_sources_del_cb, NULL);
67
+ dictionary_register_conflict_callback(wevt_sources, wevt_sources_conflict_cb, NULL);
68
+}
69
+
70
+void buffer_json_wevt_versions(BUFFER *wb __maybe_unused) {
71
+ buffer_json_member_add_object(wb, "versions");
72
+ {
73
+ buffer_json_member_add_uint64(wb, "sources",
74
+ wevt_session + dictionary_version(wevt_sources));
75
+ }
76
+ buffer_json_object_close(wb);
77
+}
78
+
79
+// --------------------------------------------------------------------------------------------------------------------
80
+
81
+int wevt_sources_dict_items_backward_compar(const void *a, const void *b) {
82
+ const DICTIONARY_ITEM **da = (const DICTIONARY_ITEM **)a, **db = (const DICTIONARY_ITEM **)b;
83
+ LOGS_QUERY_SOURCE *sa = dictionary_acquired_item_value(*da);
84
+ LOGS_QUERY_SOURCE *sb = dictionary_acquired_item_value(*db);
85
+
86
+ // compare the last message timestamps
87
+ if(sa->msg_last_ut < sb->msg_last_ut)
88
+ return 1;
89
+
90
+ if(sa->msg_last_ut > sb->msg_last_ut)
91
+ return -1;
92
+
93
+ // compare the first message timestamps
94
+ if(sa->msg_first_ut < sb->msg_first_ut)
95
+ return 1;
96
+
97
+ if(sa->msg_first_ut > sb->msg_first_ut)
98
+ return -1;
99
+
100
+ return 0;
101
+}
102
+
103
+int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
104
+ return -wevt_sources_dict_items_backward_compar(a, b);
105
+}
106
+
107
+// --------------------------------------------------------------------------------------------------------------------
108
+
109
+struct wevt_source {
110
+ usec_t first_ut;
111
+ usec_t last_ut;
112
+ size_t count;
113
+ uint64_t size;
114
+};
115
+
116
+static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) {
117
+ const struct wevt_source *s = entry;
118
+ BUFFER *wb = data;
119
+
120
+ const char *name = dictionary_acquired_item_name(item);
121
+
122
+ buffer_json_add_array_item_object(wb);
123
+ {
124
+ char size_for_humans[128];
125
+ size_snprintf(size_for_humans, sizeof(size_for_humans), s->size, "B", false);
126
+
127
+ char duration_for_humans[128];
128
+ duration_snprintf(duration_for_humans, sizeof(duration_for_humans),
129
+ (time_t)((s->last_ut - s->first_ut) / USEC_PER_SEC), "s", true);
130
+
131
+ char info[1024];
132
+ snprintfz(info, sizeof(info), "%zu channels, with a total size of %s, covering %s",
133
+ s->count, size_for_humans, duration_for_humans);
134
+
135
+ buffer_json_member_add_string(wb, "id", name);
136
+ buffer_json_member_add_string(wb, "name", name);
137
+ buffer_json_member_add_string(wb, "pill", size_for_humans);
138
+ buffer_json_member_add_string(wb, "info", info);
139
+ }
140
+ buffer_json_object_close(wb); // options object
141
+
142
+ return 1;
143
+}
144
+
145
+static bool wevt_source_merge_sizes(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value , void *data __maybe_unused) {
146
+ struct wevt_source *old_v = old_value;
147
+ const struct wevt_source *new_v = new_value;
148
+
149
+ old_v->count += new_v->count;
150
+ old_v->size += new_v->size;
151
+
152
+ if(new_v->first_ut && new_v->first_ut < old_v->first_ut)
153
+ old_v->first_ut = new_v->first_ut;
154
+
155
+ if(new_v->last_ut && new_v->last_ut > old_v->last_ut)
156
+ old_v->last_ut = new_v->last_ut;
157
+
158
+ return false;
159
+}
160
+
161
+void wevt_sources_to_json_array(BUFFER *wb) {
162
+ DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_NAME_LINK_DONT_CLONE|DICT_OPTION_DONT_OVERWRITE_VALUE);
163
+ dictionary_register_conflict_callback(dict, wevt_source_merge_sizes, NULL);
164
+
165
+ struct wevt_source t = { 0 };
166
+
167
+ LOGS_QUERY_SOURCE *src;
168
+ dfe_start_read(wevt_sources, src) {
169
+ t.first_ut = src->msg_first_ut;
170
+ t.last_ut = src->msg_last_ut;
171
+ t.count = 1;
172
+ t.size = src->size;
173
+
174
+ dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
175
+
176
+ if(src->source)
177
+ dictionary_set(dict, string2str(src->source), &t, sizeof(t));
178
+ }
179
+ dfe_done(jf);
180
+
181
+ dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
182
+}
183
+
184
+void wevt_sources_scan(void) {
185
+ static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
186
+ LPWSTR channel = NULL;
187
+ EVT_HANDLE hChannelEnum = NULL;
188
+
189
+ if(spinlock_trylock(&spinlock)) {
190
+ const usec_t now_monotonic_ut = now_monotonic_usec();
191
+
192
+ DWORD dwChannelBufferSize = 0;
193
+ DWORD dwChannelBufferUsed = 0;
194
+ DWORD status = ERROR_SUCCESS;
195
+
196
+ // Open a handle to enumerate the event channels
197
+ hChannelEnum = EvtOpenChannelEnum(NULL, 0);
198
+ if (!hChannelEnum) {
199
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "WINDOWS EVENTS: EvtOpenChannelEnum() failed with %" PRIu64 "\n",
200
+ (uint64_t)GetLastError());
201
+ goto cleanup;
202
+ }
203
+
204
+ WEVT_LOG *log = wevt_openlog6();
205
+ if(!log) goto cleanup;
206
+
207
+ while (true) {
208
+ if (!EvtNextChannelPath(hChannelEnum, dwChannelBufferSize, channel, &dwChannelBufferUsed)) {
209
+ status = GetLastError();
210
+ if (status == ERROR_NO_MORE_ITEMS)
211
+ break; // No more channels
212
+ else if (status == ERROR_INSUFFICIENT_BUFFER) {
213
+ dwChannelBufferSize = dwChannelBufferUsed;
214
+ freez(channel);
215
+ channel = mallocz(dwChannelBufferSize * sizeof(WCHAR));
216
+ continue;
217
+ } else {
218
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
219
+ "WINDOWS EVENTS: EvtNextChannelPath() failed\n");
220
+ break;
221
+ }
222
+ }
223
+
224
+ EVT_RETENTION retention;
225
+ if(!wevt_channel_retention(log, channel, &retention))
226
+ continue;
227
+
228
+ const char *name = channel2utf8(channel);
229
+ const char *fullname = strdupz(name);
230
+ char *slash = strchr(name, '/');
231
+ if(slash) *slash = '\0';
232
+
233
+ LOGS_QUERY_SOURCE src = {
234
+ .entries = retention.entries,
235
+ .fullname = fullname,
236
+ .fullname_len = strlen(fullname),
237
+ .last_scan_monotonic_ut = now_monotonic_usec(),
238
+ .msg_first_id = retention.first_event.id,
239
+ .msg_last_id = retention.last_event.id,
240
+ .msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC,
241
+ .msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
242
+ .size = retention.size_bytes,
243
+ .source_type = WEVTS_ALL,
244
+ .source = string_strdupz(name),
245
+ };
246
+
247
+ dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
248
+ }
249
+
250
+ wevt_closelog6(log);
251
+
252
+ LOGS_QUERY_SOURCE *src;
253
+ dfe_start_write(wevt_sources, src)
254
+ {
255
+ if(src->last_scan_monotonic_ut < now_monotonic_ut)
256
+ dictionary_del(wevt_sources, src->fullname);
257
+ }
258
+ dfe_done(src);
259
+ dictionary_garbage_collect(wevt_sources);
260
+
261
+ spinlock_unlock(&spinlock);
262
+ }
263
+
264
+cleanup:
265
+ freez(channel);
266
+ EvtClose(hChannelEnum);
267
+}
src/collectors/windows-events.plugin/windows-events-sources.h
new
+45
@@ -0,0 +1,45 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WINDOWS_EVENTS_SOURCES_H
4
+#define NETDATA_WINDOWS_EVENTS_SOURCES_H
5
+
6
+#include "windows-events.h"
7
+
8
+typedef enum {
9
+ WEVTS_NONE = 0,
10
+ WEVTS_ALL = (1 << 0),
11
+} WEVT_SOURCE_TYPE;
12
+
13
+typedef struct {
14
+ const char *fullname;
15
+ size_t fullname_len;
16
+
17
+ STRING *source;
18
+ WEVT_SOURCE_TYPE source_type;
19
+ usec_t msg_first_ut;
20
+ usec_t msg_last_ut;
21
+ size_t size;
22
+
23
+ usec_t last_scan_monotonic_ut;
24
+
25
+ uint64_t msg_first_id;
26
+ uint64_t msg_last_id;
27
+ uint64_t entries;
28
+} LOGS_QUERY_SOURCE;
29
+
30
+extern DICTIONARY *wevt_sources;
31
+extern DICTIONARY *used_hashes_registry;
32
+
33
+#define WEVT_SOURCE_ALL_NAME "All"
34
+
35
+void wevt_sources_init(void);
36
+void wevt_sources_scan(void);
37
+void buffer_json_wevt_versions(BUFFER *wb);
38
+
39
+void wevt_sources_to_json_array(BUFFER *wb);
40
+WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value);
41
+
42
+int wevt_sources_dict_items_backward_compar(const void *a, const void *b);
43
+int wevt_sources_dict_items_forward_compar(const void *a, const void *b);
44
+
45
+#endif //NETDATA_WINDOWS_EVENTS_SOURCES_H
src/collectors/windows-events.plugin/windows-events-unicode.c
new
+121
@@ -0,0 +1,121 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "windows-events-unicode.h"
4
+
5
+inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
6
+ if (src) {
7
+ // Convert from UTF-8 to wide char (UTF-16)
8
+ if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
9
+ wcsncpy(dst, L"[failed conv.]", dst_size - 1);
10
+ }
11
+ else
12
+ wcsncpy(dst, L"[null]", dst_size - 1);
13
+}
14
+
15
+inline void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src) {
16
+ if (src) {
17
+ if(WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, (int)dst_size, NULL, NULL) == 0)
18
+ strncpyz(dst, "[failed conv.]", dst_size - 1);
19
+ }
20
+ else
21
+ strncpyz(dst, "[null]", dst_size - 1);
22
+}
23
+
24
+wchar_t *channel2unicode(const char *utf8str) {
25
+ static __thread wchar_t buffer[1024];
26
+ utf82unicode(buffer, sizeof(buffer) / sizeof(buffer[0]), utf8str);
27
+ return buffer;
28
+}
29
+
30
+char *channel2utf8(const wchar_t *channel) {
31
+ static __thread char buffer[1024];
32
+ unicode2utf8(buffer, sizeof(buffer), channel);
33
+ return buffer;
34
+}
35
+
36
+char *account2utf8(const wchar_t *user) {
37
+ static __thread char buffer[1024];
38
+ unicode2utf8(buffer, sizeof(buffer), user);
39
+ return buffer;
40
+}
41
+
42
+char *domain2utf8(const wchar_t *domain) {
43
+ static __thread char buffer[1024];
44
+ unicode2utf8(buffer, sizeof(buffer), domain);
45
+ return buffer;
46
+}
47
+
48
+char *query2utf8(const wchar_t *query) {
49
+ static __thread char buffer[16384];
50
+ unicode2utf8(buffer, sizeof(buffer), query);
51
+ return buffer;
52
+}
53
+
54
+bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null) {
55
+ if(!src || !src_len_with_null)
56
+ goto cleanup;
57
+
58
+ // make sure the input is null terminated at the exact point we need it
59
+ // (otherwise, the output will not be null terminated either)
60
+ fatal_assert(src_len_with_null == -1 || (src_len_with_null >= 1 && src[src_len_with_null - 1] == 0));
61
+
62
+ // Try to convert using the existing buffer (if it exists, otherwise get the required buffer size)
63
+ int size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
64
+ if(size <= 0 || !utf8->data) {
65
+ // we have to set a buffer, or increase it
66
+
67
+ if(utf8->data) {
68
+ // we need to increase it the buffer size
69
+
70
+ if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
71
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed.");
72
+ goto cleanup;
73
+ }
74
+
75
+ // we have to find the required buffer size
76
+ size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, NULL, 0, NULL, NULL);
77
+ if(size <= 0)
78
+ goto cleanup;
79
+ }
80
+
81
+ // Retry conversion with the new buffer
82
+ txt_utf8_resize(utf8, size);
83
+ size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
84
+ if (size <= 0) {
85
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed after resizing.");
86
+ goto cleanup;
87
+ }
88
+ }
89
+
90
+ // Make sure it is not zero padded at the end
91
+ while(size >= 2 && utf8->data[size - 2] == 0)
92
+ size--;
93
+
94
+ utf8->used = (size_t)size;
95
+
96
+ internal_fatal(strlen(utf8->data) + 1 != utf8->used,
97
+ "Wrong UTF8 string length");
98
+
99
+ return true;
100
+
101
+cleanup:
102
+ txt_utf8_resize(utf8, 128);
103
+ if(src)
104
+ utf8->used = snprintfz(utf8->data, utf8->size, "[failed conv.]") + 1;
105
+ else {
106
+ utf8->data[0] = '\0';
107
+ utf8->used = 1;
108
+ }
109
+
110
+ return false;
111
+}
112
+
113
+bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode) {
114
+ fatal_assert(utf8 && ((utf8->data && utf8->size) || (!utf8->data && !utf8->size)));
115
+ fatal_assert(unicode && ((unicode->data && unicode->size) || (!unicode->data && !unicode->size)));
116
+
117
+ // pass the entire unicode size, including the null terminator
118
+ // so that the resulting utf8 message will be null terminated too.
119
+ return wevt_str_wchar_to_utf8(utf8, unicode->data, (int)unicode->used);
120
+}
121
+
src/collectors/windows-events.plugin/windows-events-unicode.h
new
+72
@@ -0,0 +1,72 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WINDOWS_EVENTS_UNICODE_H
4
+#define NETDATA_WINDOWS_EVENTS_UNICODE_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+#include <windows.h>
8
+#include <wchar.h>
9
+
10
+typedef struct {
11
+ char *data;
12
+ size_t size; // the allocated size of data buffer
13
+ size_t used; // the used size of the data buffer (including null terminators, if any)
14
+} TXT_UTF8;
15
+
16
+typedef struct {
17
+ wchar_t *data;
18
+ size_t size; // the allocated size of data buffer
19
+ size_t used; // the used size of the data buffer (including null terminators, if any)
20
+} TXT_UNICODE;
21
+
22
+static inline size_t compute_new_size(size_t old_size, size_t required_size) {
23
+ size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048;
24
+ size = (size / 2048) * 2048;
25
+
26
+ if(size < old_size * 2)
27
+ size = old_size * 2;
28
+
29
+ return size;
30
+}
31
+
32
+static inline void txt_utf8_cleanup(TXT_UTF8 *utf8) {
33
+ freez(utf8->data);
34
+}
35
+
36
+static inline void txt_utf8_resize(TXT_UTF8 *utf8, size_t required_size) {
37
+ if(required_size < utf8->size)
38
+ return;
39
+
40
+ txt_utf8_cleanup(utf8);
41
+ utf8->size = compute_new_size(utf8->size, required_size);
42
+ utf8->data = mallocz(utf8->size);
43
+}
44
+
45
+static inline void txt_unicode_cleanup(TXT_UNICODE *unicode) {
46
+ freez(unicode->data);
47
+}
48
+
49
+static inline void txt_unicode_resize(TXT_UNICODE *unicode, size_t required_size) {
50
+ if(required_size < unicode->size)
51
+ return;
52
+
53
+ txt_unicode_cleanup(unicode);
54
+ unicode->size = compute_new_size(unicode->size, required_size);
55
+ unicode->data = mallocz(unicode->size * sizeof(wchar_t));
56
+}
57
+
58
+bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode);
59
+bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null);
60
+
61
+void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src);
62
+void utf82unicode(wchar_t *dst, size_t dst_size, const char *src);
63
+
64
+char *account2utf8(const wchar_t *user);
65
+char *domain2utf8(const wchar_t *domain);
66
+
67
+char *channel2utf8(const wchar_t *channel);
68
+wchar_t *channel2unicode(const char *utf8str);
69
+
70
+char *query2utf8(const wchar_t *query);
71
+
72
+#endif //NETDATA_WINDOWS_EVENTS_UNICODE_H
src/collectors/windows-events.plugin/windows-events-xml.c
new
+211
@@ -0,0 +1,211 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "windows-events-xml.h"
4
+
5
+#include <string.h>
6
+#include <stdio.h>
7
+
8
+#define INDENT_STEP 2
9
+#define A_LOT_OF_SPACES " "
10
+
11
+// Helper: Add indentation
12
+static inline void buffer_add_xml_indent(BUFFER *buffer, const int level) {
13
+ size_t total_spaces = (size_t)level * INDENT_STEP;
14
+ const size_t step = sizeof(A_LOT_OF_SPACES) - 1;
15
+ while (total_spaces > 0) {
16
+ const size_t spaces_to_add = (total_spaces > step) ? step : total_spaces;
17
+ buffer_fast_strcat(buffer, A_LOT_OF_SPACES, spaces_to_add);
18
+ total_spaces -= spaces_to_add;
19
+ }
20
+}
21
+
22
+const char *append_the_rest(BUFFER *buffer, const char *xml, const char *end) {
23
+ if(xml >= end) return end;
24
+ buffer_fast_strcat(buffer, xml, end - xml);
25
+ return end;
26
+}
27
+
28
+static const char *parse_node(BUFFER *buffer, const char *xml, const char *end, int level);
29
+
30
+// Helper: Parse the value (between > and <) and return the next position to parse
31
+const char *parse_value_and_closing_tag(BUFFER *buffer, const char *xml, const char *end, int level) {
32
+ const char *start = xml;
33
+ bool has_subnodes = false;
34
+
35
+ while (xml < end) {
36
+ if(*xml == '<') {
37
+ if(xml + 1 < end && *(xml + 1) == '/') {
38
+ // a closing tag
39
+ xml += 2;
40
+
41
+ while(xml < end && *xml != '>')
42
+ xml++;
43
+
44
+ if(xml < end && *xml == '>')
45
+ xml++;
46
+
47
+ if(has_subnodes) {
48
+ buffer_putc(buffer, '\n');
49
+ buffer_add_xml_indent(buffer, level);
50
+ }
51
+
52
+ buffer_fast_strcat(buffer, start, xml - start);
53
+ return xml;
54
+ }
55
+ else {
56
+ // an opening tag
57
+ buffer_fast_strcat(buffer, start, xml - start);
58
+ xml = start = parse_node(buffer, xml, end, level + 1);
59
+ while(xml < end && isspace(*xml))
60
+ xml++;
61
+ has_subnodes = true;
62
+ }
63
+ }
64
+ else
65
+ xml++;
66
+ }
67
+
68
+ return append_the_rest(buffer, start, end);
69
+}
70
+
71
+// Parse a field value and return the next position to parse
72
+const char *parse_field_value(BUFFER *buffer, const char *xml, const char *end) {
73
+ const char quote = *xml;
74
+
75
+ if(quote != '"' && quote != '\'')
76
+ return append_the_rest(buffer, xml, end);
77
+
78
+ const char *start = xml++;
79
+
80
+ while (xml < end && *xml != quote) {
81
+ if (*xml == '\\') {
82
+ xml++; // Skip escape character
83
+
84
+ if(xml < end)
85
+ xml++;
86
+
87
+ continue;
88
+ }
89
+
90
+ xml++;
91
+ }
92
+
93
+ if(xml < end && *xml == quote) {
94
+ xml++; // Move past the closing quote
95
+ buffer_fast_strcat(buffer, start, xml - start);
96
+ return xml;
97
+ }
98
+
99
+ return append_the_rest(buffer, start, end);
100
+}
101
+
102
+// Parse a field name and return the next position to parse
103
+const char *parse_field(BUFFER *buffer, const char *xml, const char *end) {
104
+ while(isspace(*xml) && xml < end) xml++;
105
+
106
+ const char *start = xml;
107
+
108
+ while (*xml != '=' && xml < end)
109
+ xml++;
110
+
111
+ // Append the field name
112
+ buffer_fast_strcat(buffer, start, xml - start);
113
+
114
+ if(xml < end && *xml == '=') {
115
+ xml++;
116
+
117
+ buffer_putc(buffer, '=');
118
+
119
+ if(xml < end && (*xml == '"' || *xml == '\''))
120
+ xml = parse_field_value(buffer, xml, end);
121
+
122
+ return xml; // Return the next character to parse
123
+ }
124
+
125
+ return append_the_rest(buffer, start, end);
126
+}
127
+
128
+// Parse a node (handles fields and subnodes) and return the next position to parse
129
+static inline const char *parse_node(BUFFER *buffer, const char *xml, const char *end, int level) {
130
+ if(*xml != '<')
131
+ return append_the_rest(buffer, xml, end);
132
+
133
+ const char *start = xml++; // skip the <
134
+
135
+ buffer_putc(buffer, '\n');
136
+ buffer_add_xml_indent(buffer, level);
137
+
138
+ // skip spaces before the tag name
139
+ while(xml < end && isspace(*xml)) xml++;
140
+
141
+ // Parse the tag name
142
+ while (xml < end && *xml != '>' && *xml != '/') {
143
+ xml++;
144
+
145
+ if(xml < end && isspace(*xml)) {
146
+ xml++;
147
+
148
+ while(xml < end && isspace(*xml))
149
+ xml++;
150
+
151
+ if(xml < end && *xml == '/') {
152
+ // an opening tag that is self-closing
153
+ xml++;
154
+ if(xml < end && *xml == '>') {
155
+ xml++;
156
+ buffer_fast_strcat(buffer, start, xml - start);
157
+ return xml;
158
+ }
159
+ else
160
+ return append_the_rest(buffer, start, end);
161
+ }
162
+ else if(xml < end && *xml == '>') {
163
+ // the end of an opening tag
164
+ xml++;
165
+ buffer_fast_strcat(buffer, start, xml - start);
166
+ return parse_value_and_closing_tag(buffer, xml, end, level);
167
+ }
168
+ else {
169
+ buffer_fast_strcat(buffer, start, xml - start);
170
+ xml = start = parse_field(buffer, xml, end);
171
+ while(xml < end && isspace(*xml))
172
+ xml++;
173
+ }
174
+ }
175
+ }
176
+
177
+ bool self_closing_tag = false;
178
+ if(xml < end && *xml == '/') {
179
+ self_closing_tag = true;
180
+ xml++;
181
+ }
182
+
183
+ if(xml < end && *xml == '>') {
184
+ xml++;
185
+ buffer_fast_strcat(buffer, start, xml - start);
186
+
187
+ if(self_closing_tag)
188
+ return xml;
189
+
190
+ return parse_value_and_closing_tag(buffer, xml, end, level);
191
+ }
192
+
193
+ return append_the_rest(buffer, start, end);
194
+}
195
+
196
+// Main pretty-print XML function
197
+void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len) {
198
+ const char *end = xml + xml_len;
199
+
200
+ while(xml < end) {
201
+ while(xml < end && isspace(*xml))
202
+ xml++;
203
+
204
+ if(xml < end && *xml == '<')
205
+ xml = parse_node(buffer, xml, end, 1);
206
+ else {
207
+ append_the_rest(buffer, xml, end);
208
+ return;
209
+ }
210
+ }
211
+}
src/collectors/windows-events.plugin/windows-events-xml.h
new
+10
@@ -0,0 +1,10 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef WINDOWS_EVENTS_XML_H
4
+#define WINDOWS_EVENTS_XML_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+
8
+void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len);
9
+
10
+#endif //WINDOWS_EVENTS_XML_H
src/collectors/windows-events.plugin/windows-events.c
new
+1191
@@ -0,0 +1,1191 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "libnetdata/libnetdata.h"
4
+#include "libnetdata/required_dummies.h"
5
+
6
+#include "windows-events.h"
7
+
8
+#define WINDOWS_EVENTS_WORKER_THREADS 5
9
+#define WINDOWS_EVENTS_DEFAULT_TIMEOUT 600
10
+#define WINDOWS_EVENTS_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
11
+#define WINDOWS_EVENTS_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
12
+
13
+#define FUNCTION_PROGRESS_UPDATE_ROWS(rows_read, rows) __atomic_fetch_add(&(rows_read), rows, __ATOMIC_RELAXED)
14
+#define FUNCTION_PROGRESS_UPDATE_BYTES(bytes_read, bytes) __atomic_fetch_add(&(bytes_read), bytes, __ATOMIC_RELAXED)
15
+#define FUNCTION_PROGRESS_EVERY_ROWS (1ULL << 13)
16
+#define FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS (1ULL << 7)
17
+#define ANCHOR_DELTA_UT (10 * USEC_PER_SEC)
18
+
19
+netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER;
20
+static bool plugin_should_exit = false;
21
+
22
+#define WEVT_FUNCTION_DESCRIPTION "View, search and analyze the Microsoft Windows Events log."
23
+#define WEVT_FUNCTION_NAME "windows-events"
24
+
25
+// functions needed by LQS
26
+
27
+// structures needed by LQS
28
+struct lqs_extension {
29
+ struct {
30
+ usec_t start_ut;
31
+ usec_t stop_ut;
32
+ usec_t first_msg_ut;
33
+
34
+ uint64_t first_msg_seqnum;
35
+ } query_file;
36
+
37
+ // struct {
38
+ // uint32_t enable_after_samples;
39
+ // uint32_t slots;
40
+ // uint32_t sampled;
41
+ // uint32_t unsampled;
42
+ // uint32_t estimated;
43
+ // } samples;
44
+
45
+ // struct {
46
+ // uint32_t enable_after_samples;
47
+ // uint32_t every;
48
+ // uint32_t skipped;
49
+ // uint32_t recalibrate;
50
+ // uint32_t sampled;
51
+ // uint32_t unsampled;
52
+ // uint32_t estimated;
53
+ // } samples_per_file;
54
+
55
+ // struct {
56
+ // usec_t start_ut;
57
+ // usec_t end_ut;
58
+ // usec_t step_ut;
59
+ // uint32_t enable_after_samples;
60
+ // uint32_t sampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
61
+ // uint32_t unsampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
62
+ // } samples_per_time_slot;
63
+
64
+ // per file progress info
65
+ // size_t cached_count;
66
+
67
+ // progress statistics
68
+ usec_t matches_setup_ut;
69
+ size_t rows_useful;
70
+ size_t rows_read;
71
+ size_t bytes_read;
72
+ size_t files_matched;
73
+ size_t file_working;
74
+};
75
+
76
+// prepare LQS
77
+#define LQS_DEFAULT_SLICE_MODE 0
78
+#define LQS_FUNCTION_NAME WEVT_FUNCTION_NAME
79
+#define LQS_FUNCTION_DESCRIPTION WEVT_FUNCTION_DESCRIPTION
80
+#define LQS_DEFAULT_ITEMS_PER_QUERY 200
81
+#define LQS_DEFAULT_ITEMS_SAMPLING 1000000
82
+#define LQS_SOURCE_TYPE WEVT_SOURCE_TYPE
83
+#define LQS_SOURCE_TYPE_ALL WEVTS_ALL
84
+#define LQS_SOURCE_TYPE_NONE WEVTS_NONE
85
+#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) wevt_internal_source_type(value)
86
+#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) wevt_sources_to_json_array(wb)
87
+#include "libnetdata/facets/logs_query_status.h"
88
+
89
+#define WEVT_FIELD_COMPUTER "Computer"
90
+#define WEVT_FIELD_CHANNEL "Channel"
91
+#define WEVT_FIELD_PROVIDER "Provider"
92
+#define WEVT_FIELD_SOURCE "Source"
93
+#define WEVT_FIELD_EVENTRECORDID "EventRecordID"
94
+#define WEVT_FIELD_EVENTID "EventID"
95
+#define WEVT_FIELD_LEVELID "LevelID"
96
+#define WEVT_FIELD_LEVEL "Level"
97
+#define WEVT_FIELD_KEYWORDID "KeywordID"
98
+#define WEVT_FIELD_KEYWORD "Keyword"
99
+#define WEVT_FIELD_OPCODEID "OpcodeID"
100
+#define WEVT_FIELD_OPCODE "Opcode"
101
+#define WEVT_FIELD_USER "User"
102
+#define WEVT_FIELD_EVENT "Event"
103
+#define WEVT_FIELD_TASK "Task"
104
+#define WEVT_FIELD_PROCESSID "ProcessID"
105
+#define WEVT_FIELD_THREADID "ThreadID"
106
+#define WEVT_FIELD_XML "XML"
107
+#define WEVT_FIELD_MESSAGE "Message"
108
+
109
+#define WEVT_ALWAYS_VISIBLE_KEYS NULL
110
+
111
+#define WEVT_KEYS_EXCLUDED_FROM_FACETS \
112
+ "|" WEVT_FIELD_EVENT \
113
+ "|" WEVT_FIELD_MESSAGE \
114
+ "|" WEVT_FIELD_XML \
115
+ ""
116
+
117
+#define WEVT_KEYS_INCLUDED_IN_FACETS \
118
+ "|" WEVT_FIELD_PROVIDER \
119
+ "|" WEVT_FIELD_SOURCE \
120
+ "|" WEVT_FIELD_CHANNEL \
121
+ "|" WEVT_FIELD_LEVEL \
122
+ "|" WEVT_FIELD_KEYWORD \
123
+ "|" WEVT_FIELD_OPCODE \
124
+ "|" WEVT_FIELD_COMPUTER \
125
+ "|" WEVT_FIELD_USER \
126
+ ""
127
+
128
+static inline WEVT_QUERY_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) {
129
+ if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
130
+ internal_error(true, "Function has been cancelled");
131
+ return WEVT_CANCELLED;
132
+ }
133
+
134
+ if(now_monotonic_usec() > __atomic_load_n(stop_monotonic_ut, __ATOMIC_RELAXED)) {
135
+ internal_error(true, "Function timed out");
136
+ return WEVT_TIMED_OUT;
137
+ }
138
+
139
+ return WEVT_OK;
140
+}
141
+
142
+FACET_ROW_SEVERITY wevt_levelid_to_facet_severity(FACETS *facets __maybe_unused, FACET_ROW *row, void *data __maybe_unused) {
143
+ FACET_ROW_KEY_VALUE *levelid_rkv = dictionary_get(row->dict, WEVT_FIELD_LEVELID);
144
+ if(!levelid_rkv || levelid_rkv->empty)
145
+ return FACET_ROW_SEVERITY_NORMAL;
146
+
147
+ int windows_event_level = str2i(buffer_tostring(levelid_rkv->wb));
148
+
149
+ switch (windows_event_level) {
150
+ case 5: // Verbose
151
+ return FACET_ROW_SEVERITY_DEBUG;
152
+ case 4: // Information
153
+ return FACET_ROW_SEVERITY_NORMAL;
154
+ case 3: // Warning
155
+ return FACET_ROW_SEVERITY_WARNING;
156
+ case 2: // Error
157
+ return FACET_ROW_SEVERITY_CRITICAL;
158
+ case 1: // Critical
159
+ return FACET_ROW_SEVERITY_CRITICAL;
160
+ default: // Any unhandled or special levels
161
+ return FACET_ROW_SEVERITY_NORMAL;
162
+ }
163
+}
164
+
165
+void wevt_render_message(
166
+ FACETS *facets __maybe_unused,
167
+ BUFFER *json_array,
168
+ FACET_ROW_KEY_VALUE *rkv,
169
+ FACET_ROW *row,
170
+ void *data __maybe_unused) {
171
+
172
+ buffer_flush(rkv->wb);
173
+
174
+ const FACET_ROW_KEY_VALUE *event_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENT);
175
+ if(!event_rkv || !buffer_strlen(event_rkv->wb)) {
176
+ const FACET_ROW_KEY_VALUE *event_id_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENTID);
177
+ if(event_id_rkv && buffer_strlen(event_id_rkv->wb)) {
178
+ buffer_fast_strcat(rkv->wb, "EventID ", 8);
179
+ buffer_fast_strcat(rkv->wb, buffer_tostring(event_id_rkv->wb), buffer_strlen(event_id_rkv->wb));
180
+ }
181
+ else
182
+ buffer_strcat(rkv->wb, "Unknown EventID ");
183
+
184
+ const FACET_ROW_KEY_VALUE *provider_rkv = dictionary_get(row->dict, WEVT_FIELD_PROVIDER);
185
+ if(provider_rkv && buffer_strlen(provider_rkv->wb)) {
186
+ buffer_fast_strcat(rkv->wb, " of ", 4);
187
+ buffer_fast_strcat(rkv->wb, buffer_tostring(provider_rkv->wb), buffer_strlen(provider_rkv->wb));
188
+ buffer_putc(rkv->wb, '.');
189
+ }
190
+ else
191
+ buffer_strcat(rkv->wb, "of unknown Provider.");
192
+ }
193
+ else
194
+ buffer_fast_strcat(rkv->wb, buffer_tostring(event_rkv->wb), buffer_strlen(event_rkv->wb));
195
+
196
+ const FACET_ROW_KEY_VALUE *xml_rkv = dictionary_get(row->dict, WEVT_FIELD_XML);
197
+ if(xml_rkv && buffer_strlen(xml_rkv->wb)) {
198
+ const char *xml = buffer_tostring(xml_rkv->wb);
199
+
200
+ bool added_event_data = false, added_user_data = false;
201
+
202
+ const char *data_start = strstr(xml, "<EventData>");
203
+ if(data_start) {
204
+ data_start = &data_start[11];
205
+ const char *data_end = strstr(data_start, "</EventData>");
206
+
207
+ if(data_start && data_end) {
208
+ buffer_fast_strcat(rkv->wb, "\n\nRelated event data:\n", 22);
209
+ // copy the event data block
210
+ buffer_pretty_print_xml(rkv->wb, data_start, data_end - data_start);
211
+ added_event_data = true;
212
+ }
213
+ }
214
+
215
+ data_start = strstr(xml, "<UserData>");
216
+ if(data_start) {
217
+ data_start = &data_start[10];
218
+ const char *data_end = strstr(data_start, "</UserData>");
219
+
220
+ if(data_start && data_end) {
221
+ buffer_fast_strcat(rkv->wb, "\n\nRelated user data:\n", 21);
222
+ // copy the event data block
223
+ buffer_pretty_print_xml(rkv->wb, data_start, data_end - data_start);
224
+ added_user_data = true;
225
+ }
226
+ }
227
+
228
+ if(!added_event_data && !added_user_data)
229
+ buffer_strcat(rkv->wb, " Without any related data.");
230
+ }
231
+
232
+ buffer_json_add_array_item_string(json_array, buffer_tostring(rkv->wb));
233
+}
234
+
235
+static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
236
+ FACETS *facets = lqs->facets;
237
+ LOGS_QUERY_REQUEST *rq = &lqs->rq;
238
+
239
+ facets_register_row_severity(facets, wevt_levelid_to_facet_severity, NULL);
240
+
241
+ facets_register_key_name(
242
+ facets, WEVT_FIELD_COMPUTER,
243
+ rq->default_facet | FACET_KEY_OPTION_VISIBLE);
244
+
245
+ facets_register_key_name(
246
+ facets, WEVT_FIELD_CHANNEL,
247
+ rq->default_facet |
248
+ FACET_KEY_OPTION_FTS);
249
+
250
+ facets_register_key_name(
251
+ facets, WEVT_FIELD_PROVIDER,
252
+ rq->default_facet |
253
+ FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
254
+
255
+ facets_register_key_name(
256
+ facets, WEVT_FIELD_SOURCE,
257
+ rq->default_facet |
258
+ FACET_KEY_OPTION_FTS);
259
+
260
+ facets_register_key_name(
261
+ facets, WEVT_FIELD_EVENTID,
262
+ rq->default_facet |
263
+ FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
264
+
265
+ facets_register_key_name(
266
+ facets, WEVT_FIELD_LEVELID,
267
+ rq->default_facet);
268
+
269
+ facets_register_key_name(
270
+ facets, WEVT_FIELD_LEVEL,
271
+ rq->default_facet | FACET_KEY_OPTION_FTS);
272
+
273
+ facets_register_key_name(
274
+ facets, WEVT_FIELD_KEYWORDID,
275
+ rq->default_facet | FACET_KEY_OPTION_FTS);
276
+
277
+ facets_register_key_name(
278
+ facets, WEVT_FIELD_KEYWORD,
279
+ rq->default_facet | FACET_KEY_OPTION_FTS);
280
+
281
+ facets_register_key_name(
282
+ facets, WEVT_FIELD_OPCODEID,
283
+ rq->default_facet | FACET_KEY_OPTION_FTS);
284
+
285
+ facets_register_key_name(
286
+ facets, WEVT_FIELD_OPCODE,
287
+ rq->default_facet | FACET_KEY_OPTION_FTS);
288
+
289
+ facets_register_key_name(
290
+ facets, WEVT_FIELD_USER,
291
+ rq->default_facet | FACET_KEY_OPTION_FTS);
292
+
293
+ facets_register_key_name(
294
+ facets, WEVT_FIELD_TASK,
295
+ rq->default_facet | FACET_KEY_OPTION_FTS);
296
+
297
+ facets_register_key_name(
298
+ facets, WEVT_FIELD_PROCESSID,
299
+ rq->default_facet | FACET_KEY_OPTION_FTS);
300
+
301
+ facets_register_key_name(
302
+ facets, WEVT_FIELD_THREADID,
303
+ rq->default_facet | FACET_KEY_OPTION_FTS);
304
+
305
+ facets_register_key_name(
306
+ facets, WEVT_FIELD_EVENT,
307
+ FACET_KEY_OPTION_NEVER_FACET |
308
+ FACET_KEY_OPTION_FTS);
309
+
310
+ facets_register_key_name(
311
+ facets, WEVT_FIELD_TASK,
312
+ rq->default_facet | FACET_KEY_OPTION_FTS);
313
+
314
+ facets_register_key_name(
315
+ facets, WEVT_FIELD_PROCESSID,
316
+ rq->default_facet | FACET_KEY_OPTION_FTS);
317
+
318
+ facets_register_key_name(
319
+ facets, WEVT_FIELD_THREADID,
320
+ rq->default_facet | FACET_KEY_OPTION_FTS);
321
+
322
+ facets_register_key_name(
323
+ facets, WEVT_FIELD_XML,
324
+ FACET_KEY_OPTION_NEVER_FACET |
325
+ FACET_KEY_OPTION_FTS);
326
+
327
+ // facets_register_key_name(
328
+ // facets, WEVT_FIELD_MESSAGE,
329
+ // FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT |
330
+ // FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
331
+
332
+ facets_register_dynamic_key_name(
333
+ facets, WEVT_FIELD_MESSAGE,
334
+ FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE,
335
+ wevt_render_message, NULL);
336
+}
337
+
338
+static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUERY_SOURCE *src, usec_t *msg_ut __maybe_unused, WEVT_EVENT *e) {
339
+ size_t len, bytes = log->ops.content.len;
340
+
341
+ bytes += log->ops.provider.used * 2; // unicode is double
342
+ facets_add_key_value_length(facets,
343
+ WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
344
+ log->ops.provider.data, log->ops.provider.used - 1);
345
+
346
+ bytes += log->ops.source.used * 2;
347
+ facets_add_key_value_length(facets,
348
+ WEVT_FIELD_SOURCE, sizeof(WEVT_FIELD_SOURCE) - 1,
349
+ log->ops.source.data, log->ops.source.used - 1);
350
+
351
+ if(log->ops.channel.used > 1) {
352
+ bytes += log->ops.channel.used * 2;
353
+ facets_add_key_value_length(facets,
354
+ WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
355
+ log->ops.channel.data, log->ops.channel.used - 1);
356
+ }
357
+ else {
358
+ bytes += src->fullname_len * 2;
359
+ facets_add_key_value_length(facets,
360
+ WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
361
+ src->fullname, src->fullname_len);
362
+ }
363
+
364
+ static __thread char id_str[24];
365
+ len = print_uint64(id_str, e->id);
366
+ bytes += len;
367
+ facets_add_key_value_length(facets,
368
+ WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
369
+ id_str, len);
370
+
371
+ static __thread char level_id_str[24];
372
+ len = print_uint64(level_id_str, e->level);
373
+ bytes += len;
374
+ facets_add_key_value_length(facets,
375
+ WEVT_FIELD_LEVELID, sizeof(WEVT_FIELD_LEVELID) - 1,
376
+ level_id_str, len);
377
+
378
+ bytes += log->ops.level.used * 2;
379
+ facets_add_key_value_length(facets,
380
+ WEVT_FIELD_LEVEL, sizeof(WEVT_FIELD_LEVEL) - 1,
381
+ log->ops.level.data, log->ops.level.used - 1);
382
+
383
+ if(e->keyword) {
384
+ static __thread char keyword_id_str[24];
385
+ len = print_uint64_hex(keyword_id_str, e->keyword);
386
+ bytes += len;
387
+ facets_add_key_value_length(facets,
388
+ WEVT_FIELD_KEYWORDID, sizeof(WEVT_FIELD_KEYWORDID) - 1,
389
+ keyword_id_str, len);
390
+
391
+ bytes += log->ops.keyword.used * 2;
392
+ facets_add_key_value_length(facets,
393
+ WEVT_FIELD_KEYWORD, sizeof(WEVT_FIELD_KEYWORD) - 1,
394
+ log->ops.keyword.data, log->ops.keyword.used - 1);
395
+ }
396
+
397
+ bytes += log->ops.computer.used * 2;
398
+ facets_add_key_value_length(facets,
399
+ WEVT_FIELD_COMPUTER, sizeof(WEVT_FIELD_COMPUTER) - 1,
400
+ log->ops.computer.data, log->ops.computer.used - 1);
401
+
402
+ if(e->opcode) {
403
+ static __thread char opcode_id_str[24];
404
+ len = print_uint64(opcode_id_str, e->opcode);
405
+ bytes += len;
406
+ facets_add_key_value_length(facets,
407
+ WEVT_FIELD_OPCODEID, sizeof(WEVT_FIELD_OPCODEID) - 1,
408
+ opcode_id_str, len);
409
+
410
+ bytes += log->ops.opcode.used * 2;
411
+ facets_add_key_value_length(facets,
412
+ WEVT_FIELD_OPCODE, sizeof(WEVT_FIELD_OPCODE) - 1,
413
+ log->ops.opcode.data, log->ops.opcode.used - 1);
414
+ }
415
+
416
+ bytes += log->ops.user.used * 2;
417
+ facets_add_key_value_length(facets,
418
+ WEVT_FIELD_USER, sizeof(WEVT_FIELD_USER) - 1,
419
+ log->ops.user.data, log->ops.user.used - 1);
420
+
421
+ if(e->event_id) {
422
+ static __thread char event_id_str[24];
423
+ len = print_uint64(event_id_str, e->event_id);
424
+ bytes += len;
425
+ facets_add_key_value_length(facets,
426
+ WEVT_FIELD_EVENTID, sizeof(WEVT_FIELD_EVENTID) - 1,
427
+ event_id_str, len);
428
+
429
+ bytes += log->ops.event.used * 2;
430
+ facets_add_key_value_length(facets,
431
+ WEVT_FIELD_EVENT, sizeof(WEVT_FIELD_EVENT) - 1,
432
+ log->ops.event.data, log->ops.event.used - 1);
433
+ }
434
+
435
+ if(e->task) {
436
+ static __thread char task_str[24];
437
+ len = print_uint64(task_str, e->task);
438
+ bytes += len;
439
+ facets_add_key_value_length(facets,
440
+ WEVT_FIELD_TASK, sizeof(WEVT_FIELD_TASK) - 1,
441
+ task_str, len);
442
+ }
443
+
444
+ if(e->process_id) {
445
+ static __thread char process_id_str[24];
446
+ len = print_uint64(process_id_str, e->process_id);
447
+ bytes += len;
448
+ facets_add_key_value_length(facets,
449
+ WEVT_FIELD_PROCESSID, sizeof(WEVT_FIELD_PROCESSID) - 1,
450
+ process_id_str, len);
451
+ }
452
+
453
+ if(e->thread_id) {
454
+ static __thread char thread_id_str[24];
455
+ len = print_uint64(thread_id_str, e->thread_id);
456
+ bytes += len;
457
+ facets_add_key_value_length(facets,
458
+ WEVT_FIELD_THREADID, sizeof(WEVT_FIELD_THREADID) - 1,
459
+ thread_id_str, len);
460
+ }
461
+
462
+ bytes += log->ops.xml.used * 2;
463
+ facets_add_key_value_length(facets,
464
+ WEVT_FIELD_XML, sizeof(WEVT_FIELD_XML) - 1,
465
+ log->ops.xml.data, log->ops.xml.used - 1);
466
+
467
+ // facets_add_key_value_length(facets,
468
+ // WEVT_FIELD_MESSAGE, sizeof(WEVT_FIELD_MESSAGE) - 1,
469
+ // buffer_tostring(log->ops.message), buffer_strlen(log->ops.message));
470
+
471
+ return bytes;
472
+}
473
+
474
+static WEVT_QUERY_STATUS wevt_query_backward(
475
+ WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
476
+ LOGS_QUERY_SOURCE *src,
477
+ LOGS_QUERY_STATUS *lqs)
478
+{
479
+
480
+ usec_t anchor_delta = ANCHOR_DELTA_UT;
481
+
482
+ usec_t start_ut = ((lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.before_ut) + anchor_delta;
483
+ usec_t stop_ut = (lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.after_ut;
484
+ bool stop_when_full = (lqs->rq.data_only && !lqs->anchor.stop_ut);
485
+
486
+ lqs->c.query_file.start_ut = start_ut;
487
+ lqs->c.query_file.stop_ut = stop_ut;
488
+
489
+ log->event_query = wevt_query(channel2unicode(src->fullname), start_ut, true);
490
+ if(!log->event_query)
491
+ return WEVT_FAILED_TO_SEEK;
492
+
493
+ size_t errors_no_timestamp = 0;
494
+ usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
495
+ usec_t first_msg_ut = 0; // the first message we got from the db
496
+ size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
497
+ size_t bytes = 0, last_bytes = 0;
498
+
499
+ usec_t last_usec_from = 0;
500
+ usec_t last_usec_to = 0;
501
+
502
+ WEVT_QUERY_STATUS status = WEVT_OK;
503
+
504
+ facets_rows_begin(facets);
505
+ WEVT_EVENT e;
506
+ while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
507
+ usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
508
+
509
+ if(unlikely(!msg_ut)) {
510
+ errors_no_timestamp++;
511
+ continue;
512
+ }
513
+
514
+ if (unlikely(msg_ut > start_ut))
515
+ continue;
516
+
517
+ if (unlikely(msg_ut < stop_ut))
518
+ break;
519
+
520
+ if(unlikely(msg_ut > latest_msg_ut))
521
+ latest_msg_ut = msg_ut;
522
+
523
+ if(unlikely(!first_msg_ut)) {
524
+ first_msg_ut = msg_ut;
525
+ lqs->c.query_file.first_msg_ut = msg_ut;
526
+ }
527
+
528
+// sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
529
+// FACETS_ANCHOR_DIRECTION_BACKWARD,
530
+// facets_row_candidate_to_keep(facets, msg_ut));
531
+//
532
+// if(sample == SAMPLING_FULL) {
533
+ bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
534
+
535
+ // make sure each line gets a unique timestamp
536
+ if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
537
+ msg_ut = --last_usec_from;
538
+ else
539
+ last_usec_from = last_usec_to = msg_ut;
540
+
541
+ if(facets_row_finished(facets, msg_ut))
542
+ rows_useful++;
543
+
544
+ row_counter++;
545
+ if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
546
+ stop_when_full &&
547
+ facets_rows(facets) >= lqs->rq.entries)) {
548
+ // stop the data only query
549
+ usec_t oldest = facets_row_oldest_ut(facets);
550
+ if(oldest && msg_ut < (oldest - anchor_delta))
551
+ break;
552
+ }
553
+
554
+ if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
555
+ FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
556
+ last_row_counter = row_counter;
557
+
558
+ FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
559
+ last_bytes = bytes;
560
+
561
+ status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
562
+ }
563
+// }
564
+// else if(sample == SAMPLING_SKIP_FIELDS)
565
+// facets_row_finished_unsampled(facets, msg_ut);
566
+// else {
567
+// sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_BACKWARD);
568
+// break;
569
+// }
570
+ }
571
+
572
+ FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
573
+ FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
574
+
575
+ lqs->c.rows_useful += rows_useful;
576
+
577
+ if(errors_no_timestamp)
578
+ netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
579
+
580
+ if(latest_msg_ut > lqs->last_modified)
581
+ lqs->last_modified = latest_msg_ut;
582
+
583
+ wevt_query_done(log);
584
+
585
+ return status;
586
+}
587
+
588
+static WEVT_QUERY_STATUS wevt_query_forward(
589
+ WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
590
+ LOGS_QUERY_SOURCE *src,
591
+ LOGS_QUERY_STATUS *lqs)
592
+{
593
+ usec_t anchor_delta = ANCHOR_DELTA_UT;
594
+
595
+ usec_t start_ut = (lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.after_ut;
596
+ usec_t stop_ut = ((lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.before_ut) + anchor_delta;
597
+ bool stop_when_full = (lqs->rq.data_only && !lqs->anchor.stop_ut);
598
+
599
+ lqs->c.query_file.start_ut = start_ut;
600
+ lqs->c.query_file.stop_ut = stop_ut;
601
+
602
+ log->event_query = wevt_query(channel2unicode(src->fullname), start_ut, false);
603
+ if(!log->event_query)
604
+ return WEVT_FAILED_TO_SEEK;
605
+
606
+ size_t errors_no_timestamp = 0;
607
+ usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
608
+ usec_t first_msg_ut = 0; // the first message we got from the db
609
+ size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
610
+ size_t bytes = 0, last_bytes = 0;
611
+
612
+ usec_t last_usec_from = 0;
613
+ usec_t last_usec_to = 0;
614
+
615
+ WEVT_QUERY_STATUS status = WEVT_OK;
616
+
617
+ facets_rows_begin(facets);
618
+ WEVT_EVENT e;
619
+ while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
620
+ usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
621
+
622
+ if(unlikely(!msg_ut)) {
623
+ errors_no_timestamp++;
624
+ continue;
625
+ }
626
+
627
+ if (unlikely(msg_ut < start_ut))
628
+ continue;
629
+
630
+ if (unlikely(msg_ut > stop_ut))
631
+ break;
632
+
633
+ if(likely(msg_ut > latest_msg_ut))
634
+ latest_msg_ut = msg_ut;
635
+
636
+ if(unlikely(!first_msg_ut)) {
637
+ first_msg_ut = msg_ut;
638
+ lqs->c.query_file.first_msg_ut = msg_ut;
639
+ }
640
+
641
+// sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
642
+// FACETS_ANCHOR_DIRECTION_FORWARD,
643
+// facets_row_candidate_to_keep(facets, msg_ut));
644
+//
645
+// if(sample == SAMPLING_FULL) {
646
+ bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
647
+
648
+ // make sure each line gets a unique timestamp
649
+ if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
650
+ msg_ut = ++last_usec_to;
651
+ else
652
+ last_usec_from = last_usec_to = msg_ut;
653
+
654
+ if(facets_row_finished(facets, msg_ut))
655
+ rows_useful++;
656
+
657
+ row_counter++;
658
+ if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
659
+ stop_when_full &&
660
+ facets_rows(facets) >= lqs->rq.entries)) {
661
+ // stop the data only query
662
+ usec_t newest = facets_row_newest_ut(facets);
663
+ if(newest && msg_ut > (newest + anchor_delta))
664
+ break;
665
+ }
666
+
667
+ if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
668
+ FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
669
+ last_row_counter = row_counter;
670
+
671
+ FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
672
+ last_bytes = bytes;
673
+
674
+ status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
675
+ }
676
+// }
677
+// else if(sample == SAMPLING_SKIP_FIELDS)
678
+// facets_row_finished_unsampled(facets, msg_ut);
679
+// else {
680
+// sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_FORWARD);
681
+// break;
682
+// }
683
+ }
684
+
685
+ FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
686
+ FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
687
+
688
+ lqs->c.rows_useful += rows_useful;
689
+
690
+ if(errors_no_timestamp)
691
+ netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
692
+
693
+ if(latest_msg_ut > lqs->last_modified)
694
+ lqs->last_modified = latest_msg_ut;
695
+
696
+ wevt_query_done(log);
697
+
698
+ return status;
699
+}
700
+
701
+static WEVT_QUERY_STATUS wevt_query_one_channel(
702
+ WEVT_LOG *log,
703
+ BUFFER *wb, FACETS *facets,
704
+ LOGS_QUERY_SOURCE *src,
705
+ LOGS_QUERY_STATUS *lqs) {
706
+
707
+ errno_clear();
708
+
709
+ WEVT_QUERY_STATUS status;
710
+ bool matches_filters = true;
711
+
712
+ // if(lqs->rq.slice) {
713
+ // usec_t started = now_monotonic_usec();
714
+ //
715
+ // matches_filters = netdata_systemd_filtering_by_journal(j, facets, lqs) || !lqs->rq.filters;
716
+ // usec_t ended = now_monotonic_usec();
717
+ //
718
+ // lqs->c.matches_setup_ut += (ended - started);
719
+ // }
720
+
721
+ if(matches_filters) {
722
+ if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD)
723
+ status = wevt_query_forward(log, wb, facets, src, lqs);
724
+ else
725
+ status = wevt_query_backward(log, wb, facets, src, lqs);
726
+ }
727
+ else
728
+ status = WEVT_NO_CHANNEL_MATCHED;
729
+
730
+ return status;
731
+}
732
+
733
+static bool source_is_mine(LOGS_QUERY_SOURCE *src, LOGS_QUERY_STATUS *lqs) {
734
+ if((lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) || (src->source_type & lqs->rq.source_type) ||
735
+ (lqs->rq.sources && simple_pattern_matches(lqs->rq.sources, string2str(src->source)))) {
736
+
737
+ if(!src->msg_last_ut)
738
+ // the file is not scanned yet, or the timestamps have not been updated,
739
+ // so we don't know if it can contribute or not - let's add it.
740
+ return true;
741
+
742
+ usec_t anchor_delta = 0;
743
+ usec_t first_ut = src->msg_first_ut - anchor_delta;
744
+ usec_t last_ut = src->msg_last_ut + anchor_delta;
745
+
746
+ if(last_ut >= lqs->rq.after_ut && first_ut <= lqs->rq.before_ut)
747
+ return true;
748
+ }
749
+
750
+ return false;
751
+}
752
+
753
+static int wevt_master_query(BUFFER *wb __maybe_unused, LOGS_QUERY_STATUS *lqs __maybe_unused) {
754
+ // make sure the sources list is updated
755
+ wevt_sources_scan();
756
+
757
+ FACETS *facets = lqs->facets;
758
+
759
+ WEVT_QUERY_STATUS status = WEVT_NO_CHANNEL_MATCHED;
760
+
761
+ lqs->c.files_matched = 0;
762
+ lqs->c.file_working = 0;
763
+ lqs->c.rows_useful = 0;
764
+ lqs->c.rows_read = 0;
765
+ lqs->c.bytes_read = 0;
766
+
767
+ size_t files_used = 0;
768
+ size_t files_max = dictionary_entries(wevt_sources);
769
+ const DICTIONARY_ITEM *file_items[files_max];
770
+
771
+ // count the files
772
+ bool files_are_newer = false;
773
+ LOGS_QUERY_SOURCE *src;
774
+ dfe_start_read(wevt_sources, src) {
775
+ if(!source_is_mine(src, lqs))
776
+ continue;
777
+
778
+ file_items[files_used++] = dictionary_acquired_item_dup(wevt_sources, src_dfe.item);
779
+
780
+ if(src->msg_last_ut > lqs->rq.if_modified_since)
781
+ files_are_newer = true;
782
+ }
783
+ dfe_done(jf);
784
+
785
+ lqs->c.files_matched = files_used;
786
+
787
+ if(lqs->rq.if_modified_since && !files_are_newer)
788
+ return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
789
+
790
+ // sort the files, so that they are optimal for facets
791
+ if(files_used >= 2) {
792
+ if (lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
793
+ qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
794
+ wevt_sources_dict_items_backward_compar);
795
+ else
796
+ qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
797
+ wevt_sources_dict_items_forward_compar);
798
+ }
799
+
800
+ bool partial = false;
801
+ usec_t query_started_ut = now_monotonic_usec();
802
+ usec_t started_ut = query_started_ut;
803
+ usec_t ended_ut = started_ut;
804
+ usec_t duration_ut = 0, max_duration_ut = 0;
805
+ usec_t progress_duration_ut = 0;
806
+
807
+ WEVT_LOG *log = wevt_openlog6();
808
+ if(!log) {
809
+ netdata_log_error("WEVT: cannot windows event log");
810
+ return rrd_call_function_error(wb, "cannot open windows events log", HTTP_RESP_INTERNAL_SERVER_ERROR);
811
+ }
812
+
813
+ // sampling_query_init(lqs, facets);
814
+
815
+ buffer_json_member_add_array(wb, "_channels");
816
+ for(size_t f = 0; f < files_used ;f++) {
817
+ const char *filename = dictionary_acquired_item_name(file_items[f]);
818
+ src = dictionary_acquired_item_value(file_items[f]);
819
+
820
+ if(!source_is_mine(src, lqs))
821
+ continue;
822
+
823
+ started_ut = ended_ut;
824
+
825
+ // do not even try to do the query if we expect it to pass the timeout
826
+ if(ended_ut + max_duration_ut * 3 >= *lqs->stop_monotonic_ut) {
827
+ partial = true;
828
+ status = WEVT_TIMED_OUT;
829
+ break;
830
+ }
831
+
832
+ lqs->c.file_working++;
833
+
834
+ size_t rows_useful = lqs->c.rows_useful;
835
+ size_t rows_read = lqs->c.rows_read;
836
+ size_t bytes_read = lqs->c.bytes_read;
837
+ size_t matches_setup_ut = lqs->c.matches_setup_ut;
838
+
839
+ // sampling_file_init(lqs, src);
840
+
841
+ WEVT_QUERY_STATUS tmp_status = wevt_query_one_channel(log, wb, facets, src, lqs);
842
+
843
+ rows_useful = lqs->c.rows_useful - rows_useful;
844
+ rows_read = lqs->c.rows_read - rows_read;
845
+ bytes_read = lqs->c.bytes_read - bytes_read;
846
+ matches_setup_ut = lqs->c.matches_setup_ut - matches_setup_ut;
847
+
848
+ ended_ut = now_monotonic_usec();
849
+ duration_ut = ended_ut - started_ut;
850
+
851
+ if(duration_ut > max_duration_ut)
852
+ max_duration_ut = duration_ut;
853
+
854
+ progress_duration_ut += duration_ut;
855
+ if(progress_duration_ut >= WINDOWS_EVENTS_PROGRESS_EVERY_UT) {
856
+ progress_duration_ut = 0;
857
+ netdata_mutex_lock(&stdout_mutex);
858
+ pluginsd_function_progress_to_stdout(lqs->rq.transaction, f + 1, files_used);
859
+ netdata_mutex_unlock(&stdout_mutex);
860
+ }
861
+
862
+ buffer_json_add_array_item_object(wb); // channel source
863
+ {
864
+ // information about the file
865
+ buffer_json_member_add_string(wb, "_name", filename);
866
+ buffer_json_member_add_uint64(wb, "_source_type", src->source_type);
867
+ buffer_json_member_add_string(wb, "_source", string2str(src->source));
868
+ buffer_json_member_add_uint64(wb, "_msg_first_ut", src->msg_first_ut);
869
+ buffer_json_member_add_uint64(wb, "_msg_last_ut", src->msg_last_ut);
870
+
871
+ // information about the current use of the file
872
+ buffer_json_member_add_uint64(wb, "duration_ut", ended_ut - started_ut);
873
+ buffer_json_member_add_uint64(wb, "rows_read", rows_read);
874
+ buffer_json_member_add_uint64(wb, "rows_useful", rows_useful);
875
+ buffer_json_member_add_double(wb, "rows_per_second", (double) rows_read / (double) duration_ut * (double) USEC_PER_SEC);
876
+ buffer_json_member_add_uint64(wb, "bytes_read", bytes_read);
877
+ buffer_json_member_add_double(wb, "bytes_per_second", (double) bytes_read / (double) duration_ut * (double) USEC_PER_SEC);
878
+ buffer_json_member_add_uint64(wb, "duration_matches_ut", matches_setup_ut);
879
+
880
+ // if(lqs->rq.sampling) {
881
+ // buffer_json_member_add_object(wb, "_sampling");
882
+ // {
883
+ // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples_per_file.sampled);
884
+ // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples_per_file.unsampled);
885
+ // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples_per_file.estimated);
886
+ // }
887
+ // buffer_json_object_close(wb); // _sampling
888
+ // }
889
+ }
890
+ buffer_json_object_close(wb); // channel source
891
+
892
+ bool stop = false;
893
+ switch(tmp_status) {
894
+ case WEVT_OK:
895
+ case WEVT_NO_CHANNEL_MATCHED:
896
+ status = (status == WEVT_OK) ? WEVT_OK : tmp_status;
897
+ break;
898
+
899
+ case WEVT_FAILED_TO_OPEN:
900
+ case WEVT_FAILED_TO_SEEK:
901
+ partial = true;
902
+ if(status == WEVT_NO_CHANNEL_MATCHED)
903
+ status = tmp_status;
904
+ break;
905
+
906
+ case WEVT_CANCELLED:
907
+ case WEVT_TIMED_OUT:
908
+ partial = true;
909
+ stop = true;
910
+ status = tmp_status;
911
+ break;
912
+
913
+ case WEVT_NOT_MODIFIED:
914
+ internal_fatal(true, "this should never be returned here");
915
+ break;
916
+ }
917
+
918
+ if(stop)
919
+ break;
920
+ }
921
+ buffer_json_array_close(wb); // _channels
922
+
923
+ // release the files
924
+ for(size_t f = 0; f < files_used ;f++)
925
+ dictionary_acquired_item_release(wevt_sources, file_items[f]);
926
+
927
+ wevt_closelog6(log);
928
+
929
+ switch (status) {
930
+ case WEVT_OK:
931
+ if(lqs->rq.if_modified_since && !lqs->c.rows_useful)
932
+ return rrd_call_function_error(wb, "no useful logs, not modified", HTTP_RESP_NOT_MODIFIED);
933
+ break;
934
+
935
+ case WEVT_TIMED_OUT:
936
+ case WEVT_NO_CHANNEL_MATCHED:
937
+ break;
938
+
939
+ case WEVT_CANCELLED:
940
+ return rrd_call_function_error(wb, "client closed connection", HTTP_RESP_CLIENT_CLOSED_REQUEST);
941
+
942
+ case WEVT_NOT_MODIFIED:
943
+ return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
944
+
945
+ case WEVT_FAILED_TO_OPEN:
946
+ return rrd_call_function_error(wb, "failed to open event log", HTTP_RESP_INTERNAL_SERVER_ERROR);
947
+
948
+ case WEVT_FAILED_TO_SEEK:
949
+ return rrd_call_function_error(wb, "failed to execute event log query", HTTP_RESP_INTERNAL_SERVER_ERROR);
950
+
951
+ default:
952
+ return rrd_call_function_error(wb, "unknown status", HTTP_RESP_INTERNAL_SERVER_ERROR);
953
+ }
954
+
955
+ buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
956
+ buffer_json_member_add_boolean(wb, "partial", partial);
957
+ buffer_json_member_add_string(wb, "type", "table");
958
+
959
+ // build a message for the query
960
+ if(!lqs->rq.data_only) {
961
+ CLEAN_BUFFER *msg = buffer_create(0, NULL);
962
+ CLEAN_BUFFER *msg_description = buffer_create(0, NULL);
963
+ ND_LOG_FIELD_PRIORITY msg_priority = NDLP_INFO;
964
+
965
+ // if(!journal_files_completed_once()) {
966
+ // buffer_strcat(msg, "Journals are still being scanned. ");
967
+ // buffer_strcat(msg_description
968
+ // , "LIBRARY SCAN: The journal files are still being scanned, you are probably viewing incomplete data. ");
969
+ // msg_priority = NDLP_WARNING;
970
+ // }
971
+
972
+ if(partial) {
973
+ buffer_strcat(msg, "Query timed-out, incomplete data. ");
974
+ buffer_strcat(msg_description
975
+ , "QUERY TIMEOUT: The query timed out and may not include all the data of the selected window. ");
976
+ msg_priority = NDLP_WARNING;
977
+ }
978
+
979
+ // if(lqs->c.samples.estimated || lqs->c.samples.unsampled) {
980
+ // double percent = (double) (lqs->c.samples.sampled * 100.0 /
981
+ // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
982
+ // buffer_sprintf(msg, "%.2f%% real data", percent);
983
+ // buffer_sprintf(msg_description, "ACTUAL DATA: The filters counters reflect %0.2f%% of the data. ", percent);
984
+ // msg_priority = MIN(msg_priority, NDLP_NOTICE);
985
+ // }
986
+ //
987
+ // if(lqs->c.samples.unsampled) {
988
+ // double percent = (double) (lqs->c.samples.unsampled * 100.0 /
989
+ // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
990
+ // buffer_sprintf(msg, ", %.2f%% unsampled", percent);
991
+ // buffer_sprintf(msg_description
992
+ // , "UNSAMPLED DATA: %0.2f%% of the events exist and have been counted, but their values have not been evaluated, so they are not included in the filters counters. "
993
+ // , percent);
994
+ // msg_priority = MIN(msg_priority, NDLP_NOTICE);
995
+ // }
996
+ //
997
+ // if(lqs->c.samples.estimated) {
998
+ // double percent = (double) (lqs->c.samples.estimated * 100.0 /
999
+ // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1000
+ // buffer_sprintf(msg, ", %.2f%% estimated", percent);
1001
+ // buffer_sprintf(msg_description
1002
+ // , "ESTIMATED DATA: The query selected a large amount of data, so to avoid delaying too much, the presented data are estimated by %0.2f%%. "
1003
+ // , percent);
1004
+ // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1005
+ // }
1006
+
1007
+ buffer_json_member_add_object(wb, "message");
1008
+ if(buffer_tostring(msg)) {
1009
+ buffer_json_member_add_string(wb, "title", buffer_tostring(msg));
1010
+ buffer_json_member_add_string(wb, "description", buffer_tostring(msg_description));
1011
+ buffer_json_member_add_string(wb, "status", nd_log_id2priority(msg_priority));
1012
+ }
1013
+ // else send an empty object if there is nothing to tell
1014
+ buffer_json_object_close(wb); // message
1015
+ }
1016
+
1017
+ if(!lqs->rq.data_only) {
1018
+ buffer_json_member_add_time_t(wb, "update_every", 1);
1019
+ buffer_json_member_add_string(wb, "help", WEVT_FUNCTION_DESCRIPTION);
1020
+ }
1021
+
1022
+ if(!lqs->rq.data_only || lqs->rq.tail)
1023
+ buffer_json_member_add_uint64(wb, "last_modified", lqs->last_modified);
1024
+
1025
+ facets_sort_and_reorder_keys(facets);
1026
+ facets_report(facets, wb, used_hashes_registry);
1027
+
1028
+ wb->expires = now_realtime_sec() + (lqs->rq.data_only ? 3600 : 0);
1029
+ buffer_json_member_add_time_t(wb, "expires", wb->expires);
1030
+
1031
+ // if(lqs->rq.sampling) {
1032
+ // buffer_json_member_add_object(wb, "_sampling");
1033
+ // {
1034
+ // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples.sampled);
1035
+ // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples.unsampled);
1036
+ // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples.estimated);
1037
+ // }
1038
+ // buffer_json_object_close(wb); // _sampling
1039
+ // }
1040
+
1041
+ wb->content_type = CT_APPLICATION_JSON;
1042
+ wb->response_code = HTTP_RESP_OK;
1043
+ return wb->response_code;
1044
+}
1045
+
1046
+void function_windows_events(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
1047
+ BUFFER *payload, HTTP_ACCESS access __maybe_unused,
1048
+ const char *source __maybe_unused, void *data __maybe_unused) {
1049
+ bool have_slice = LQS_DEFAULT_SLICE_MODE;
1050
+
1051
+ LOGS_QUERY_STATUS tmp_fqs = {
1052
+ .facets = lqs_facets_create(
1053
+ LQS_DEFAULT_ITEMS_PER_QUERY,
1054
+ FACETS_OPTION_ALL_KEYS_FTS,
1055
+ WEVT_ALWAYS_VISIBLE_KEYS,
1056
+ WEVT_KEYS_INCLUDED_IN_FACETS,
1057
+ WEVT_KEYS_EXCLUDED_FROM_FACETS,
1058
+ have_slice),
1059
+
1060
+ .rq = LOGS_QUERY_REQUEST_DEFAULTS(transaction, have_slice, FACETS_ANCHOR_DIRECTION_BACKWARD),
1061
+
1062
+ .cancelled = cancelled,
1063
+ .stop_monotonic_ut = stop_monotonic_ut,
1064
+ };
1065
+ LOGS_QUERY_STATUS *lqs = &tmp_fqs;
1066
+
1067
+ CLEAN_BUFFER *wb = lqs_create_output_buffer();
1068
+
1069
+ // ------------------------------------------------------------------------
1070
+ // parse the parameters
1071
+
1072
+ if(lqs_request_parse_and_validate(lqs, wb, function, payload, have_slice, WEVT_FIELD_LEVEL)) {
1073
+ wevt_register_fields(lqs);
1074
+
1075
+ // ------------------------------------------------------------------------
1076
+ // add versions to the response
1077
+
1078
+ buffer_json_wevt_versions(wb);
1079
+
1080
+ // ------------------------------------------------------------------------
1081
+ // run the request
1082
+
1083
+ if (lqs->rq.info)
1084
+ lqs_info_response(wb, lqs->facets);
1085
+ else {
1086
+ wevt_master_query(wb, lqs);
1087
+ if (wb->response_code == HTTP_RESP_OK)
1088
+ buffer_json_finalize(wb);
1089
+ }
1090
+ }
1091
+
1092
+ netdata_mutex_lock(&stdout_mutex);
1093
+ pluginsd_function_result_to_stdout(transaction, wb);
1094
+ netdata_mutex_unlock(&stdout_mutex);
1095
+
1096
+ lqs_cleanup(lqs);
1097
+}
1098
+
1099
+int main(int argc __maybe_unused, char **argv __maybe_unused) {
1100
+ clocks_init();
1101
+ nd_thread_tag_set("wevt.plugin");
1102
+ nd_log_initialize_for_external_plugins("windows-events.plugin");
1103
+
1104
+ // ------------------------------------------------------------------------
1105
+ // initialization
1106
+
1107
+ wevt_sources_init();
1108
+
1109
+ // ------------------------------------------------------------------------
1110
+ // debug
1111
+
1112
+ if(argc >= 2 && strcmp(argv[argc - 1], "debug") == 0) {
1113
+ wevt_sources_scan();
1114
+
1115
+ struct {
1116
+ const char *func;
1117
+ } array[] = {
1118
+ { "windows-events after:1725866669 before:1725953069 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6,ClaDGnYSQE7,H106l8MXSSr,HREiMN.4Ahu data_only:false source:All" },
1119
+ //{ "windows-events after:1725650277 before:1725736677 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1120
+ //{ "windows-events after:1725650284 before:1725736684 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1121
+ //{ "windows-events after:1725650386 before:1725736786 anchor:1725652420809461 direction:forward last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 if_modified_since:1725736649011085 data_only:true delta:true tail:true source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1122
+ //{ "windows-events info after:1725650420 before:1725736820" },
1123
+ //{ "windows-events after:1725650420 before:1725736820 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1124
+ //{ "windows-events after:1725650430 before:1725736830 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1125
+ { NULL },
1126
+ };
1127
+
1128
+ for(int i = 0; array[i].func ;i++) {
1129
+ bool cancelled = false;
1130
+ usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
1131
+ //char buf[] = "windows-events after:-86400 before:0 direction:backward last:200 data_only:false slice:true source:all";
1132
+ function_windows_events("123", (char *)array[i].func, &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
1133
+ }
1134
+ printf("\n\nAll done!\n\n");
1135
+ fflush(stdout);
1136
+ exit(1);
1137
+ }
1138
+
1139
+ // ------------------------------------------------------------------------
1140
+ // the event loop for functions
1141
+
1142
+ struct functions_evloop_globals *wg =
1143
+ functions_evloop_init(WINDOWS_EVENTS_WORKER_THREADS, "WEVT", &stdout_mutex, &plugin_should_exit);
1144
+
1145
+ functions_evloop_add_function(wg,
1146
+ WEVT_FUNCTION_NAME,
1147
+ function_windows_events,
1148
+ WINDOWS_EVENTS_DEFAULT_TIMEOUT,
1149
+ NULL);
1150
+
1151
+ // ------------------------------------------------------------------------
1152
+ // register functions to netdata
1153
+
1154
+ netdata_mutex_lock(&stdout_mutex);
1155
+
1156
+ fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" "HTTP_ACCESS_FORMAT" %d\n",
1157
+ WEVT_FUNCTION_NAME, WINDOWS_EVENTS_DEFAULT_TIMEOUT, WEVT_FUNCTION_DESCRIPTION,
1158
+ (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
1159
+ RRDFUNCTIONS_PRIORITY_DEFAULT);
1160
+
1161
+ fflush(stdout);
1162
+ netdata_mutex_unlock(&stdout_mutex);
1163
+
1164
+ // ------------------------------------------------------------------------
1165
+
1166
+ const usec_t step_ut = 100 * USEC_PER_MS;
1167
+ usec_t send_newline_ut = 0;
1168
+ usec_t since_last_scan_ut = WINDOWS_EVENTS_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
1169
+ const bool tty = isatty(fileno(stdout)) == 1;
1170
+
1171
+ heartbeat_t hb;
1172
+ heartbeat_init(&hb);
1173
+ while(!plugin_should_exit) {
1174
+
1175
+ if(since_last_scan_ut > WINDOWS_EVENTS_SCAN_EVERY_USEC) {
1176
+ wevt_sources_scan();
1177
+ since_last_scan_ut = 0;
1178
+ }
1179
+
1180
+ usec_t dt_ut = heartbeat_next(&hb, step_ut);
1181
+ since_last_scan_ut += dt_ut;
1182
+ send_newline_ut += dt_ut;
1183
+
1184
+ if(!tty && send_newline_ut > USEC_PER_SEC) {
1185
+ send_newline_and_flush(&stdout_mutex);
1186
+ send_newline_ut = 0;
1187
+ }
1188
+ }
1189
+
1190
+ exit(0);
1191
+}
src/collectors/windows-events.plugin/windows-events.h
new
+28
@@ -0,0 +1,28 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WINDOWS_EVENTS_H
4
+#define NETDATA_WINDOWS_EVENTS_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+#include "collectors/all.h"
8
+#include <windows.h>
9
+#include <winevt.h>
10
+#include <wchar.h>
11
+
12
+typedef enum {
13
+ WEVT_NO_CHANNEL_MATCHED,
14
+ WEVT_FAILED_TO_OPEN,
15
+ WEVT_FAILED_TO_SEEK,
16
+ WEVT_TIMED_OUT,
17
+ WEVT_OK,
18
+ WEVT_NOT_MODIFIED,
19
+ WEVT_CANCELLED,
20
+} WEVT_QUERY_STATUS;
21
+
22
+#include "windows-events-unicode.h"
23
+#include "windows-events-query.h"
24
+#include "windows-events-sources.h"
25
+#include "windows-events-sid.h"
26
+#include "windows-events-xml.h"
27
+
28
+#endif //NETDATA_WINDOWS_EVENTS_H
src/libnetdata/buffer/buffer.c
+2
-2
@@ -247,7 +247,7 @@ void buffer_free(BUFFER *b) {
247
248
buffer_overflow_check(b);
249
250
- netdata_log_debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size);
250
+ netdata_log_debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", (size_t)b->size);
251
252
if(b->statistics)
253
__atomic_sub_fetch(b->statistics, b->size + sizeof(BUFFER) + sizeof(BUFFER_OVERFLOW_EOF) + 2, __ATOMIC_RELAXED);
@@ -269,7 +269,7 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
269
size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size;
270
if(optimal > increase) increase = optimal;
271
272
- netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + increase);
272
+ netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", (size_t)b->size, (size_t)(b->size + increase));
273
274
b->buffer = reallocz(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2);
275
b->size += increase;
src/libnetdata/buffer/buffer.h
+36
-23
@@ -39,14 +39,15 @@ typedef enum __attribute__ ((__packed__)) {
39
} BUFFER_JSON_OPTIONS;
40
41
typedef struct web_buffer {
42
- size_t size; // allocation size of buffer, in bytes
43
- size_t len; // current data length in buffer, in bytes
44
- char *buffer; // the buffer itself
42
+ uint32_t size; // allocation size of buffer, in bytes
43
+ uint32_t len; // current data length in buffer, in bytes
44
HTTP_CONTENT_TYPE content_type; // the content type of the data in the buffer
45
BUFFER_OPTIONS options; // options related to the content
46
+ uint16_t response_code;
47
time_t date; // the timestamp this content has been generated
48
time_t expires; // the timestamp this content expires
49
size_t *statistics;
50
+ char *buffer; // the buffer itself
51
52
struct {
53
char key_quote[BUFFER_QUOTE_MAX_SIZE + 1];
@@ -62,7 +63,7 @@ typedef struct web_buffer {
63
#define buffer_cacheable(wb) do { (wb)->options |= WB_CONTENT_CACHEABLE; if((wb)->options & WB_CONTENT_NO_CACHEABLE) (wb)->options &= ~WB_CONTENT_NO_CACHEABLE; } while(0)
64
#define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE) (wb)->options &= ~WB_CONTENT_CACHEABLE; (wb)->expires = 0; } while(0)
65
65
-#define buffer_strlen(wb) ((wb)->len)
66
+#define buffer_strlen(wb) (size_t)((wb)->len)
67
68
#define BUFFER_OVERFLOW_EOF "EOF"
69
@@ -488,42 +489,54 @@ static inline int print_netdata_double(char *dst, NETDATA_DOUBLE value) {
489
return (int)(d - dst);
490
}
491
491
-static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
492
- buffer_need_bytes(wb, 50);
493
-
494
- char *s = &wb->buffer[wb->len];
492
+static inline size_t print_uint64(char *dst, uint64_t value) {
493
+ char *s = dst;
494
char *d = print_uint64_reversed(s, value);
495
char_array_reverse(s, d - 1);
496
*d = '\0';
498
- wb->len += d - s;
499
-
500
- buffer_overflow_check(wb);
497
+ return d - s;
498
}
499
503
-static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
504
- buffer_need_bytes(wb, 50);
500
+static inline size_t print_int64(char *dst, int64_t value) {
501
+ size_t len = 0;
502
503
if(value < 0) {
507
- buffer_fast_strcat(wb, "-", 1);
504
+ *dst++ = '-';
505
value = -value;
506
+ len++;
507
}
508
511
- buffer_print_uint64(wb, (uint64_t)value);
509
+ return print_uint64(dst, value) + len;
510
+}
511
+
512
+static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
513
+ buffer_need_bytes(wb, 50);
514
+ wb->len += print_uint64(&wb->buffer[wb->len], value);
515
+ buffer_overflow_check(wb);
516
+}
517
518
+static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
519
+ buffer_need_bytes(wb, 50);
520
+ wb->len += print_int64(&wb->buffer[wb->len], value);
521
buffer_overflow_check(wb);
522
}
523
516
-static inline void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) {
517
- buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1);
524
+#define UINT64_HEX_LENGTH ((sizeof(HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 2 + 1)
525
+static inline size_t print_uint64_hex(char *dst, uint64_t value) {
526
+ char *d = dst;
527
519
- buffer_fast_strcat(wb, HEX_PREFIX, sizeof(HEX_PREFIX) - 1);
528
+ const char *s = HEX_PREFIX;
529
+ while(*s) *d++ = *s++;
530
521
- char *s = &wb->buffer[wb->len];
522
- char *d = print_uint64_hex_reversed(s, value);
523
- char_array_reverse(s, d - 1);
524
- *d = '\0';
525
- wb->len += d - s;
531
+ char *e = print_uint64_hex_reversed(d, value);
532
+ char_array_reverse(d, e - 1);
533
+ *e = '\0';
534
+ return e - dst;
535
+}
536
537
+static inline void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) {
538
+ buffer_need_bytes(wb, UINT64_HEX_LENGTH);
539
+ wb->len += print_uint64_hex(&wb->buffer[wb->len], value);
540
buffer_overflow_check(wb);
541
}
542
src/libnetdata/facets/facets.c
+9
@@ -1618,6 +1618,8 @@ FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const ch
1618
}
1619
1620
void facets_destroy(FACETS *facets) {
1621
+ if(!facets) return;
1622
+
1623
dictionary_destroy(facets->accepted_params);
1624
FACETS_KEYS_INDEX_DESTROY(facets);
1625
simple_pattern_free(facets->visible_keys);
@@ -1879,6 +1881,10 @@ void facets_add_key_value(FACETS *facets, const char *key, const char *value) {
1881
}
1882
1883
void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len) {
1884
+ if(!key || !*key || !key_len || !value || !*value || !value_len)
1885
+ // adding empty values, makes the rows unmatched
1886
+ return;
1887
+
1888
FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0);
1889
k->current_value.raw = value;
1890
k->current_value.raw_len = value_len;
@@ -2142,6 +2148,9 @@ void facets_rows_begin(FACETS *facets) {
2148
}
2149
2150
bool facets_row_finished(FACETS *facets, usec_t usec) {
2151
+// char buf[RFC3339_MAX_LENGTH];
2152
+// rfc3339_datetime_ut(buf, sizeof(buf), usec, 3, false);
2153
+
2154
facets->operations.rows.evaluated++;
2155
2156
if(unlikely((facets->query && facets->keys_filtered_by_query &&
src/libnetdata/facets/logs_query_status.h
new
+850
@@ -0,0 +1,850 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_LOGS_QUERY_STATUS_H
4
+#define NETDATA_LOGS_QUERY_STATUS_H
5
+
6
+#include "../libnetdata.h"
7
+
8
+#define LQS_PARAMETER_HELP "help"
9
+#define LQS_PARAMETER_AFTER "after"
10
+#define LQS_PARAMETER_BEFORE "before"
11
+#define LQS_PARAMETER_ANCHOR "anchor"
12
+#define LQS_PARAMETER_LAST "last"
13
+#define LQS_PARAMETER_QUERY "query"
14
+#define LQS_PARAMETER_FACETS "facets"
15
+#define LQS_PARAMETER_HISTOGRAM "histogram"
16
+#define LQS_PARAMETER_DIRECTION "direction"
17
+#define LQS_PARAMETER_IF_MODIFIED_SINCE "if_modified_since"
18
+#define LQS_PARAMETER_DATA_ONLY "data_only"
19
+#define LQS_PARAMETER_SOURCE "source"
20
+#define LQS_PARAMETER_INFO "info"
21
+#define LQS_PARAMETER_SLICE "slice"
22
+#define LQS_PARAMETER_DELTA "delta"
23
+#define LQS_PARAMETER_TAIL "tail"
24
+#define LQS_PARAMETER_SAMPLING "sampling"
25
+
26
+#define LQS_MAX_PARAMS 1000
27
+#define LQS_DEFAULT_QUERY_DURATION (1 * 3600)
28
+
29
+#undef LQS_SLICE_PARAMETER
30
+#if LQS_DEFAULT_SLICE_MODE == 1
31
+#define LQS_SLICE_PARAMETER 1
32
+#endif
33
+
34
+typedef struct {
35
+ const char *transaction;
36
+
37
+ FACET_KEY_OPTIONS default_facet; // the option to be used for internal fields.
38
+ // when the requests set facets, we disable all default facets,
39
+ // so that the UI has full control over them.
40
+
41
+ bool fields_are_ids; // POST works with field names, GET works with field hashes (IDs)
42
+ bool info; // the request is an INFO request, do not execute a query.
43
+
44
+ bool data_only; // return as fast as possible, with the requested amount of data,
45
+ // without scanning the entire duration.
46
+
47
+ bool slice; // apply native backend filters to slice the events database.
48
+ bool delta; // return incremental data for the histogram (used with data_only)
49
+ bool tail; // return NOT MODIFIED if no more data are available after the anchor given.
50
+
51
+ time_t after_s; // the starting timestamp of the query
52
+ time_t before_s; // the ending timestamp of the query
53
+ usec_t after_ut; // in microseconds
54
+ usec_t before_ut; // in microseconds
55
+
56
+ usec_t anchor; // the anchor to seek to
57
+ FACETS_ANCHOR_DIRECTION direction; // the direction based on the anchor (or the query timeframe)
58
+
59
+ usec_t if_modified_since; // the timestamp to check with tail == true
60
+
61
+ size_t entries; // the number of log events to return in a single response
62
+
63
+ const char *query; // full text search query string
64
+ const char *histogram; // the field to use for the histogram
65
+
66
+ SIMPLE_PATTERN *sources; // custom log sources to query
67
+ LQS_SOURCE_TYPE source_type; // pre-defined log sources to query
68
+
69
+ size_t filters; // the number of filters (facets selected) in the query
70
+ size_t sampling; // the number of log events to sample, when the query is too big
71
+
72
+ time_t now_s; // the timestamp the query was received
73
+ time_t expires_s; // the timestamp the response expires
74
+} LOGS_QUERY_REQUEST;
75
+
76
+#define LOGS_QUERY_REQUEST_DEFAULTS(function_transaction, default_slice, default_direction) \
77
+ (LOGS_QUERY_REQUEST) { \
78
+ .transaction = (function_transaction), \
79
+ .default_facet = FACET_KEY_OPTION_FACET, \
80
+ .info = false, \
81
+ .data_only = false, \
82
+ .slice = (default_slice), \
83
+ .delta = false, \
84
+ .tail = false, \
85
+ .after_s = 0, \
86
+ .before_s = 0, \
87
+ .anchor = 0, \
88
+ .if_modified_since = 0, \
89
+ .entries = 0, \
90
+ .direction = (default_direction), \
91
+ .query = NULL, \
92
+ .histogram = NULL, \
93
+ .sources = NULL, \
94
+ .source_type = LQS_SOURCE_TYPE_ALL, \
95
+ .filters = 0, \
96
+ .sampling = LQS_DEFAULT_ITEMS_SAMPLING, \
97
+}
98
+
99
+typedef struct {
100
+ FACETS *facets;
101
+
102
+ LOGS_QUERY_REQUEST rq;
103
+
104
+ bool *cancelled; // a pointer to the cancelling boolean
105
+ usec_t *stop_monotonic_ut;
106
+
107
+ struct {
108
+ usec_t start_ut;
109
+ usec_t stop_ut;
110
+ } anchor;
111
+
112
+ usec_t last_modified;
113
+
114
+ struct lqs_extension c;
115
+} LOGS_QUERY_STATUS;
116
+
117
+struct logs_query_data {
118
+ const char *transaction;
119
+ FACETS *facets;
120
+ LOGS_QUERY_REQUEST *rq;
121
+ BUFFER *wb;
122
+};
123
+
124
+static inline FACETS_ANCHOR_DIRECTION lgs_get_direction(const char *value) {
125
+ return strcasecmp(value, "forward") == 0 ? FACETS_ANCHOR_DIRECTION_FORWARD : FACETS_ANCHOR_DIRECTION_BACKWARD;
126
+}
127
+
128
+static inline void lqs_log_error(LOGS_QUERY_STATUS *lqs, const char *msg) {
129
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
130
+ "LOGS QUERY ERROR: %s, on query "
131
+ "timeframe [%"PRIu64" - %"PRIu64"], "
132
+ "anchor [%"PRIu64" - %"PRIu64"], "
133
+ "if_modified_since %"PRIu64", "
134
+ "data_only:%s, delta:%s, tail:%s, direction:%s"
135
+ , msg
136
+ , lqs->rq.after_ut
137
+ , lqs->rq.before_ut
138
+ , lqs->anchor.start_ut
139
+ , lqs->anchor.stop_ut
140
+ , lqs->rq.if_modified_since
141
+ , lqs->rq.data_only ? "true" : "false"
142
+ , lqs->rq.delta ? "true" : "false"
143
+ , lqs->rq.tail ? "tail" : "false"
144
+ , lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
145
+}
146
+
147
+static inline void lqs_function_help(LOGS_QUERY_STATUS *lqs, BUFFER *wb) {
148
+ buffer_reset(wb);
149
+ wb->content_type = CT_TEXT_PLAIN;
150
+ wb->response_code = HTTP_RESP_OK;
151
+
152
+ buffer_sprintf(wb,
153
+ "%s / %s\n"
154
+ "\n"
155
+ "%s\n"
156
+ "\n"
157
+ "The following parameters are supported:\n"
158
+ "\n"
159
+ , program_name
160
+ , LQS_FUNCTION_NAME
161
+ , LQS_FUNCTION_DESCRIPTION
162
+ );
163
+
164
+ buffer_sprintf(wb,
165
+ " " LQS_PARAMETER_HELP "\n"
166
+ " Shows this help message.\n"
167
+ "\n"
168
+ );
169
+
170
+ buffer_sprintf(wb,
171
+ " " LQS_PARAMETER_INFO "\n"
172
+ " Request initial configuration information about the plugin.\n"
173
+ " The key entity returned is the required_params array, which includes\n"
174
+ " all the available log sources.\n"
175
+ " When `" LQS_PARAMETER_INFO "` is requested, all other parameters are ignored.\n"
176
+ "\n"
177
+ );
178
+
179
+ buffer_sprintf(wb,
180
+ " " LQS_PARAMETER_DATA_ONLY ":true or " LQS_PARAMETER_DATA_ONLY ":false\n"
181
+ " Quickly respond with data requested, without generating a\n"
182
+ " `histogram`, `facets` counters and `items`.\n"
183
+ "\n"
184
+ );
185
+
186
+ buffer_sprintf(wb,
187
+ " " LQS_PARAMETER_DELTA ":true or " LQS_PARAMETER_DELTA ":false\n"
188
+ " When doing data only queries, include deltas for histogram, facets and items.\n"
189
+ "\n"
190
+ );
191
+
192
+ buffer_sprintf(wb,
193
+ " " LQS_PARAMETER_TAIL ":true or " LQS_PARAMETER_TAIL ":false\n"
194
+ " When doing data only queries, respond with the newest messages,\n"
195
+ " and up to the anchor, but calculate deltas (if requested) for\n"
196
+ " the duration [anchor - before].\n"
197
+ "\n"
198
+ );
199
+
200
+#ifdef LQS_SLICE_PARAMETER
201
+ buffer_sprintf(wb,
202
+ " " LQS_PARAMETER_SLICE ":true or " LQS_PARAMETER_SLICE ":false\n"
203
+ " When it is turned on, the plugin is is slicing the logs database,\n"
204
+ " utilizing the underlying available indexes.\n"
205
+ " When it is off, all filtering is done by the plugin.\n"
206
+ " The default is: %s\n"
207
+ "\n"
208
+ , lqs->rq.slice ? "true" : "false"
209
+ );
210
+#endif
211
+ buffer_sprintf(wb,
212
+ " " LQS_PARAMETER_SOURCE ":SOURCE\n"
213
+ " Query only the specified log sources.\n"
214
+ " Do an `" LQS_PARAMETER_INFO "` query to find the sources.\n"
215
+ "\n"
216
+ );
217
+
218
+ buffer_sprintf(wb,
219
+ " " LQS_PARAMETER_BEFORE ":TIMESTAMP_IN_SECONDS\n"
220
+ " Absolute or relative (to now) timestamp in seconds, to start the query.\n"
221
+ " The query is always executed from the most recent to the oldest log entry.\n"
222
+ " If not given the default is: now.\n"
223
+ "\n"
224
+ );
225
+
226
+ buffer_sprintf(wb,
227
+ " " LQS_PARAMETER_AFTER ":TIMESTAMP_IN_SECONDS\n"
228
+ " Absolute or relative (to `before`) timestamp in seconds, to end the query.\n"
229
+ " If not given, the default is %d.\n"
230
+ "\n"
231
+ , -LQS_DEFAULT_QUERY_DURATION
232
+ );
233
+
234
+ buffer_sprintf(wb,
235
+ " " LQS_PARAMETER_LAST ":ITEMS\n"
236
+ " The number of items to return.\n"
237
+ " The default is %zu.\n"
238
+ "\n"
239
+ , lqs->rq.entries
240
+ );
241
+
242
+ buffer_sprintf(wb,
243
+ " " LQS_PARAMETER_SAMPLING ":ITEMS\n"
244
+ " The number of log entries to sample to estimate facets counters and histogram.\n"
245
+ " The default is %zu.\n"
246
+ "\n"
247
+ , lqs->rq.sampling
248
+ );
249
+
250
+ buffer_sprintf(wb,
251
+ " " LQS_PARAMETER_ANCHOR ":TIMESTAMP_IN_MICROSECONDS\n"
252
+ " Return items relative to this timestamp.\n"
253
+ " The exact items to be returned depend on the query `" LQS_PARAMETER_DIRECTION "`.\n"
254
+ "\n"
255
+ );
256
+
257
+ buffer_sprintf(wb,
258
+ " " LQS_PARAMETER_DIRECTION ":forward or " LQS_PARAMETER_DIRECTION ":backward\n"
259
+ " When set to `backward` (default) the items returned are the newest before the\n"
260
+ " `" LQS_PARAMETER_ANCHOR "`, (or `" LQS_PARAMETER_BEFORE "` if `" LQS_PARAMETER_ANCHOR "` is not set)\n"
261
+ " When set to `forward` the items returned are the oldest after the\n"
262
+ " `" LQS_PARAMETER_ANCHOR "`, (or `" LQS_PARAMETER_AFTER "` if `" LQS_PARAMETER_ANCHOR "` is not set)\n"
263
+ " The default is: %s\n"
264
+ "\n"
265
+ , lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"
266
+ );
267
+
268
+ buffer_sprintf(wb,
269
+ " " LQS_PARAMETER_QUERY ":SIMPLE_PATTERN\n"
270
+ " Do a full text search to find the log entries matching the pattern given.\n"
271
+ " The plugin is searching for matches on all fields of the database.\n"
272
+ "\n"
273
+ );
274
+
275
+ buffer_sprintf(wb,
276
+ " " LQS_PARAMETER_IF_MODIFIED_SINCE ":TIMESTAMP_IN_MICROSECONDS\n"
277
+ " Each successful response, includes a `last_modified` field.\n"
278
+ " By providing the timestamp to the `" LQS_PARAMETER_IF_MODIFIED_SINCE "` parameter,\n"
279
+ " the plugin will return 200 with a successful response, or 304 if the source has not\n"
280
+ " been modified since that timestamp.\n"
281
+ "\n"
282
+ );
283
+
284
+ buffer_sprintf(wb,
285
+ " " LQS_PARAMETER_HISTOGRAM ":facet_id\n"
286
+ " Use the given `facet_id` for the histogram.\n"
287
+ " This parameter is ignored in `" LQS_PARAMETER_DATA_ONLY "` mode.\n"
288
+ "\n"
289
+ );
290
+
291
+ buffer_sprintf(wb,
292
+ " " LQS_PARAMETER_FACETS ":facet_id1,facet_id2,facet_id3,...\n"
293
+ " Add the given facets to the list of fields for which analysis is required.\n"
294
+ " The plugin will offer both a histogram and facet value counters for its values.\n"
295
+ " This parameter is ignored in `" LQS_PARAMETER_DATA_ONLY "` mode.\n"
296
+ "\n"
297
+ );
298
+
299
+ buffer_sprintf(wb,
300
+ " facet_id:value_id1,value_id2,value_id3,...\n"
301
+ " Apply filters to the query, based on the facet IDs returned.\n"
302
+ " Each `facet_id` can be given once, but multiple `facet_ids` can be given.\n"
303
+ "\n"
304
+ );
305
+}
306
+
307
+static inline bool lqs_request_parse_json_payload(json_object *jobj, const char *path, void *data, BUFFER *error) {
308
+ struct logs_query_data *qd = data;
309
+ LOGS_QUERY_REQUEST *rq = qd->rq;
310
+ BUFFER *wb = qd->wb;
311
+ FACETS *facets = qd->facets;
312
+ // const char *transaction = qd->transaction;
313
+
314
+ buffer_flush(error);
315
+
316
+ JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_INFO, rq->info, error, false);
317
+ JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_DELTA, rq->delta, error, false);
318
+ JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_TAIL, rq->tail, error, false);
319
+ JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_SLICE, rq->slice, error, false);
320
+ JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_DATA_ONLY, rq->data_only, error, false);
321
+ JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_SAMPLING, rq->sampling, error, false);
322
+ JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_AFTER, rq->after_s, error, false);
323
+ JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_BEFORE, rq->before_s, error, false);
324
+ JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_IF_MODIFIED_SINCE, rq->if_modified_since, error, false);
325
+ JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_ANCHOR, rq->anchor, error, false);
326
+ JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_LAST, rq->entries, error, false);
327
+ JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_DIRECTION, lgs_get_direction, rq->direction, error, false);
328
+ JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_QUERY, rq->query, error, false);
329
+ JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_HISTOGRAM, rq->histogram, error, false);
330
+
331
+ json_object *sources;
332
+ if (json_object_object_get_ex(jobj, LQS_PARAMETER_SOURCE, &sources)) {
333
+ if (json_object_get_type(sources) != json_type_array) {
334
+ buffer_sprintf(error, "member '%s' is not an array", LQS_PARAMETER_SOURCE);
335
+ return false;
336
+ }
337
+
338
+ buffer_json_member_add_array(wb, LQS_PARAMETER_SOURCE);
339
+
340
+ CLEAN_BUFFER *sources_list = buffer_create(0, NULL);
341
+
342
+ rq->source_type = LQS_SOURCE_TYPE_NONE;
343
+
344
+ size_t sources_len = json_object_array_length(sources);
345
+ for (size_t i = 0; i < sources_len; i++) {
346
+ json_object *src = json_object_array_get_idx(sources, i);
347
+
348
+ if (json_object_get_type(src) != json_type_string) {
349
+ buffer_sprintf(error, "sources array item %zu is not a string", i);
350
+ return false;
351
+ }
352
+
353
+ const char *value = json_object_get_string(src);
354
+ buffer_json_add_array_item_string(wb, value);
355
+
356
+ LQS_SOURCE_TYPE t = LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value);
357
+ if(t != LQS_SOURCE_TYPE_NONE) {
358
+ rq->source_type |= t;
359
+ value = NULL;
360
+ }
361
+ else {
362
+ // else, match the source, whatever it is
363
+ if(buffer_strlen(sources_list))
364
+ buffer_putc(sources_list, '|');
365
+
366
+ buffer_strcat(sources_list, value);
367
+ }
368
+ }
369
+
370
+ if(buffer_strlen(sources_list)) {
371
+ simple_pattern_free(rq->sources);
372
+ rq->sources = simple_pattern_create(buffer_tostring(sources_list), "|", SIMPLE_PATTERN_EXACT, false);
373
+ }
374
+
375
+ buffer_json_array_close(wb); // source
376
+ }
377
+
378
+ json_object *fcts;
379
+ if (json_object_object_get_ex(jobj, LQS_PARAMETER_FACETS, &fcts)) {
380
+ if (json_object_get_type(sources) != json_type_array) {
381
+ buffer_sprintf(error, "member '%s' is not an array", LQS_PARAMETER_FACETS);
382
+ return false;
383
+ }
384
+
385
+ rq->default_facet = FACET_KEY_OPTION_NONE;
386
+ facets_reset_and_disable_all_facets(facets);
387
+
388
+ buffer_json_member_add_array(wb, LQS_PARAMETER_FACETS);
389
+
390
+ size_t facets_len = json_object_array_length(fcts);
391
+ for (size_t i = 0; i < facets_len; i++) {
392
+ json_object *fct = json_object_array_get_idx(fcts, i);
393
+
394
+ if (json_object_get_type(fct) != json_type_string) {
395
+ buffer_sprintf(error, "facets array item %zu is not a string", i);
396
+ return false;
397
+ }
398
+
399
+ const char *value = json_object_get_string(fct);
400
+ facets_register_facet(facets, value, FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS|FACET_KEY_OPTION_REORDER);
401
+ buffer_json_add_array_item_string(wb, value);
402
+ }
403
+
404
+ buffer_json_array_close(wb); // facets
405
+ }
406
+
407
+ json_object *selections;
408
+ if (json_object_object_get_ex(jobj, "selections", &selections)) {
409
+ if (json_object_get_type(selections) != json_type_object) {
410
+ buffer_sprintf(error, "member 'selections' is not an object");
411
+ return false;
412
+ }
413
+
414
+ buffer_json_member_add_object(wb, "selections");
415
+
416
+ json_object_object_foreach(selections, key, val) {
417
+ if (json_object_get_type(val) != json_type_array) {
418
+ buffer_sprintf(error, "selection '%s' is not an array", key);
419
+ return false;
420
+ }
421
+
422
+ buffer_json_member_add_array(wb, key);
423
+
424
+ size_t values_len = json_object_array_length(val);
425
+ for (size_t i = 0; i < values_len; i++) {
426
+ json_object *value_obj = json_object_array_get_idx(val, i);
427
+
428
+ if (json_object_get_type(value_obj) != json_type_string) {
429
+ buffer_sprintf(error, "selection '%s' array item %zu is not a string", key, i);
430
+ return false;
431
+ }
432
+
433
+ const char *value = json_object_get_string(value_obj);
434
+
435
+ // Call facets_register_facet_id_filter for each value
436
+ facets_register_facet_filter(
437
+ facets, key, value, FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);
438
+
439
+ buffer_json_add_array_item_string(wb, value);
440
+ rq->filters++;
441
+ }
442
+
443
+ buffer_json_array_close(wb); // key
444
+ }
445
+
446
+ buffer_json_object_close(wb); // selections
447
+ }
448
+
449
+ rq->fields_are_ids = false;
450
+ return true;
451
+}
452
+
453
+static inline bool lqs_request_parse_POST(LOGS_QUERY_STATUS *lqs, BUFFER *wb, BUFFER *payload, const char *transaction) {
454
+ FACETS *facets = lqs->facets;
455
+ LOGS_QUERY_REQUEST *rq = &lqs->rq;
456
+
457
+ struct logs_query_data qd = {
458
+ .transaction = transaction,
459
+ .facets = facets,
460
+ .rq = rq,
461
+ .wb = wb,
462
+ };
463
+
464
+ int code;
465
+ CLEAN_JSON_OBJECT *jobj =
466
+ json_parse_function_payload_or_error(wb, payload, &code, lqs_request_parse_json_payload, &qd);
467
+ wb->response_code = code;
468
+
469
+ return (jobj && code == HTTP_RESP_OK);
470
+}
471
+
472
+static inline bool lqs_request_parse_GET(LOGS_QUERY_STATUS *lqs, BUFFER *wb, char *function) {
473
+ FACETS *facets = lqs->facets;
474
+ LOGS_QUERY_REQUEST *rq = &lqs->rq;
475
+
476
+ buffer_json_member_add_object(wb, "_request");
477
+
478
+ char func_copy[strlen(function) + 1];
479
+ strcpy(func_copy, function);
480
+
481
+ char *words[LQS_MAX_PARAMS] = { NULL };
482
+ size_t num_words = quoted_strings_splitter_pluginsd(func_copy, words, LQS_MAX_PARAMS);
483
+ for(int i = 1; i < LQS_MAX_PARAMS;i++) {
484
+ char *keyword = get_word(words, num_words, i);
485
+ if(!keyword) break;
486
+
487
+ if(strcmp(keyword, LQS_PARAMETER_HELP) == 0) {
488
+ lqs_function_help(lqs, wb);
489
+ return false;
490
+ }
491
+ else if(strcmp(keyword, LQS_PARAMETER_INFO) == 0) {
492
+ rq->info = true;
493
+ }
494
+ else if(strncmp(keyword, LQS_PARAMETER_DELTA ":", sizeof(LQS_PARAMETER_DELTA ":") - 1) == 0) {
495
+ char *v = &keyword[sizeof(LQS_PARAMETER_DELTA ":") - 1];
496
+
497
+ if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
498
+ rq->delta = false;
499
+ else
500
+ rq->delta = true;
501
+ }
502
+ else if(strncmp(keyword, LQS_PARAMETER_TAIL ":", sizeof(LQS_PARAMETER_TAIL ":") - 1) == 0) {
503
+ char *v = &keyword[sizeof(LQS_PARAMETER_TAIL ":") - 1];
504
+
505
+ if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
506
+ rq->tail = false;
507
+ else
508
+ rq->tail = true;
509
+ }
510
+ else if(strncmp(keyword, LQS_PARAMETER_SAMPLING ":", sizeof(LQS_PARAMETER_SAMPLING ":") - 1) == 0) {
511
+ rq->sampling = str2ul(&keyword[sizeof(LQS_PARAMETER_SAMPLING ":") - 1]);
512
+ }
513
+ else if(strncmp(keyword, LQS_PARAMETER_DATA_ONLY ":", sizeof(LQS_PARAMETER_DATA_ONLY ":") - 1) == 0) {
514
+ char *v = &keyword[sizeof(LQS_PARAMETER_DATA_ONLY ":") - 1];
515
+
516
+ if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
517
+ rq->data_only = false;
518
+ else
519
+ rq->data_only = true;
520
+ }
521
+ else if(strncmp(keyword, LQS_PARAMETER_SLICE ":", sizeof(LQS_PARAMETER_SLICE ":") - 1) == 0) {
522
+ char *v = &keyword[sizeof(LQS_PARAMETER_SLICE ":") - 1];
523
+
524
+ if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0)
525
+ rq->slice = false;
526
+ else
527
+ rq->slice = true;
528
+ }
529
+ else if(strncmp(keyword, LQS_PARAMETER_SOURCE ":", sizeof(LQS_PARAMETER_SOURCE ":") - 1) == 0) {
530
+ const char *value = &keyword[sizeof(LQS_PARAMETER_SOURCE ":") - 1];
531
+
532
+ buffer_json_member_add_array(wb, LQS_PARAMETER_SOURCE);
533
+
534
+ CLEAN_BUFFER *sources_list = buffer_create(0, NULL);
535
+
536
+ rq->source_type = LQS_SOURCE_TYPE_NONE;
537
+ while(value) {
538
+ char *sep = strchr(value, ',');
539
+ if(sep)
540
+ *sep++ = '\0';
541
+
542
+ buffer_json_add_array_item_string(wb, value);
543
+
544
+ LQS_SOURCE_TYPE t = LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value);
545
+ if(t != LQS_SOURCE_TYPE_NONE) {
546
+ rq->source_type |= t;
547
+ value = NULL;
548
+ }
549
+ else {
550
+ // else, match the source, whatever it is
551
+ if(buffer_strlen(sources_list))
552
+ buffer_putc(sources_list, '|');
553
+
554
+ buffer_strcat(sources_list, value);
555
+ }
556
+
557
+ value = sep;
558
+ }
559
+
560
+ if(buffer_strlen(sources_list)) {
561
+ simple_pattern_free(rq->sources);
562
+ rq->sources = simple_pattern_create(buffer_tostring(sources_list), "|", SIMPLE_PATTERN_EXACT, false);
563
+ }
564
+
565
+ buffer_json_array_close(wb); // source
566
+ }
567
+ else if(strncmp(keyword, LQS_PARAMETER_AFTER ":", sizeof(LQS_PARAMETER_AFTER ":") - 1) == 0) {
568
+ rq->after_s = str2l(&keyword[sizeof(LQS_PARAMETER_AFTER ":") - 1]);
569
+ }
570
+ else if(strncmp(keyword, LQS_PARAMETER_BEFORE ":", sizeof(LQS_PARAMETER_BEFORE ":") - 1) == 0) {
571
+ rq->before_s = str2l(&keyword[sizeof(LQS_PARAMETER_BEFORE ":") - 1]);
572
+ }
573
+ else if(strncmp(keyword, LQS_PARAMETER_IF_MODIFIED_SINCE ":", sizeof(LQS_PARAMETER_IF_MODIFIED_SINCE ":") - 1) == 0) {
574
+ rq->if_modified_since = str2ull(&keyword[sizeof(LQS_PARAMETER_IF_MODIFIED_SINCE ":") - 1], NULL);
575
+ }
576
+ else if(strncmp(keyword, LQS_PARAMETER_ANCHOR ":", sizeof(LQS_PARAMETER_ANCHOR ":") - 1) == 0) {
577
+ rq->anchor = str2ull(&keyword[sizeof(LQS_PARAMETER_ANCHOR ":") - 1], NULL);
578
+ }
579
+ else if(strncmp(keyword, LQS_PARAMETER_DIRECTION ":", sizeof(LQS_PARAMETER_DIRECTION ":") - 1) == 0) {
580
+ rq->direction = lgs_get_direction(&keyword[sizeof(LQS_PARAMETER_DIRECTION ":") - 1]);
581
+ }
582
+ else if(strncmp(keyword, LQS_PARAMETER_LAST ":", sizeof(LQS_PARAMETER_LAST ":") - 1) == 0) {
583
+ rq->entries = str2ul(&keyword[sizeof(LQS_PARAMETER_LAST ":") - 1]);
584
+ }
585
+ else if(strncmp(keyword, LQS_PARAMETER_QUERY ":", sizeof(LQS_PARAMETER_QUERY ":") - 1) == 0) {
586
+ freez((void *)rq->query);
587
+ rq->query= strdupz(&keyword[sizeof(LQS_PARAMETER_QUERY ":") - 1]);
588
+ }
589
+ else if(strncmp(keyword, LQS_PARAMETER_HISTOGRAM ":", sizeof(LQS_PARAMETER_HISTOGRAM ":") - 1) == 0) {
590
+ freez((void *)rq->histogram);
591
+ rq->histogram = strdupz(&keyword[sizeof(LQS_PARAMETER_HISTOGRAM ":") - 1]);
592
+ }
593
+ else if(strncmp(keyword, LQS_PARAMETER_FACETS ":", sizeof(LQS_PARAMETER_FACETS ":") - 1) == 0) {
594
+ rq->default_facet = FACET_KEY_OPTION_NONE;
595
+ facets_reset_and_disable_all_facets(facets);
596
+
597
+ char *value = &keyword[sizeof(LQS_PARAMETER_FACETS ":") - 1];
598
+ if(*value) {
599
+ buffer_json_member_add_array(wb, LQS_PARAMETER_FACETS);
600
+
601
+ while(value) {
602
+ char *sep = strchr(value, ',');
603
+ if(sep)
604
+ *sep++ = '\0';
605
+
606
+ facets_register_facet_id(facets, value, FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS|FACET_KEY_OPTION_REORDER);
607
+ buffer_json_add_array_item_string(wb, value);
608
+
609
+ value = sep;
610
+ }
611
+
612
+ buffer_json_array_close(wb); // facets
613
+ }
614
+ }
615
+ else {
616
+ char *value = strchr(keyword, ':');
617
+ if(value) {
618
+ *value++ = '\0';
619
+
620
+ buffer_json_member_add_array(wb, keyword);
621
+
622
+ while(value) {
623
+ char *sep = strchr(value, ',');
624
+ if(sep)
625
+ *sep++ = '\0';
626
+
627
+ facets_register_facet_filter_id(
628
+ facets, keyword, value,
629
+ FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);
630
+
631
+ buffer_json_add_array_item_string(wb, value);
632
+ rq->filters++;
633
+
634
+ value = sep;
635
+ }
636
+
637
+ buffer_json_array_close(wb); // keyword
638
+ }
639
+ }
640
+ }
641
+
642
+ rq->fields_are_ids = true;
643
+ return true;
644
+}
645
+
646
+static inline void lqs_info_response(BUFFER *wb, FACETS *facets) {
647
+ // the buffer already has the request in it
648
+ // DO NOT FLUSH IT
649
+
650
+ buffer_json_member_add_uint64(wb, "v", 3);
651
+ facets_accepted_parameters_to_json_array(facets, wb, false);
652
+ buffer_json_member_add_array(wb, "required_params");
653
+ {
654
+ buffer_json_add_array_item_object(wb);
655
+ {
656
+ buffer_json_member_add_string(wb, "id", "source");
657
+ buffer_json_member_add_string(wb, "name", "source");
658
+ buffer_json_member_add_string(wb, "help", "Select the logs source to query");
659
+ buffer_json_member_add_string(wb, "type", "multiselect");
660
+ buffer_json_member_add_array(wb, "options");
661
+ {
662
+ LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb);
663
+ }
664
+ buffer_json_array_close(wb); // options array
665
+ }
666
+ buffer_json_object_close(wb); // required params object
667
+ }
668
+ buffer_json_array_close(wb); // required_params array
669
+
670
+ facets_table_config(wb);
671
+
672
+ buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
673
+ buffer_json_member_add_string(wb, "type", "table");
674
+ buffer_json_member_add_string(wb, "help", LQS_FUNCTION_DESCRIPTION);
675
+ buffer_json_finalize(wb);
676
+
677
+ wb->content_type = CT_APPLICATION_JSON;
678
+ wb->response_code = HTTP_RESP_OK;
679
+}
680
+
681
+static inline BUFFER *lqs_create_output_buffer(void) {
682
+ BUFFER *wb = buffer_create(0, NULL);
683
+ buffer_flush(wb);
684
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
685
+ return wb;
686
+}
687
+
688
+static inline FACETS *lqs_facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys, bool have_slice) {
689
+ FACETS *facets = facets_create(items_to_return, options,
690
+ visible_keys, facet_keys, non_facet_keys);
691
+
692
+ facets_accepted_param(facets, LQS_PARAMETER_INFO);
693
+ facets_accepted_param(facets, LQS_PARAMETER_SOURCE);
694
+ facets_accepted_param(facets, LQS_PARAMETER_AFTER);
695
+ facets_accepted_param(facets, LQS_PARAMETER_BEFORE);
696
+ facets_accepted_param(facets, LQS_PARAMETER_ANCHOR);
697
+ facets_accepted_param(facets, LQS_PARAMETER_DIRECTION);
698
+ facets_accepted_param(facets, LQS_PARAMETER_LAST);
699
+ facets_accepted_param(facets, LQS_PARAMETER_QUERY);
700
+ facets_accepted_param(facets, LQS_PARAMETER_FACETS);
701
+ facets_accepted_param(facets, LQS_PARAMETER_HISTOGRAM);
702
+ facets_accepted_param(facets, LQS_PARAMETER_IF_MODIFIED_SINCE);
703
+ facets_accepted_param(facets, LQS_PARAMETER_DATA_ONLY);
704
+ facets_accepted_param(facets, LQS_PARAMETER_DELTA);
705
+ facets_accepted_param(facets, LQS_PARAMETER_TAIL);
706
+ facets_accepted_param(facets, LQS_PARAMETER_SAMPLING);
707
+
708
+ if(have_slice)
709
+ facets_accepted_param(facets, LQS_PARAMETER_SLICE);
710
+
711
+ return facets;
712
+}
713
+
714
+static inline bool lqs_request_parse_and_validate(LOGS_QUERY_STATUS *lqs, BUFFER *wb, char *function, BUFFER *payload, bool have_slice, const char *default_histogram) {
715
+ LOGS_QUERY_REQUEST *rq = &lqs->rq;
716
+ FACETS *facets = lqs->facets;
717
+
718
+ if( (payload && !lqs_request_parse_POST(lqs, wb, payload, rq->transaction)) ||
719
+ (!payload && !lqs_request_parse_GET(lqs, wb, function)) )
720
+ return false;
721
+
722
+ // ----------------------------------------------------------------------------------------------------------------
723
+ // validate parameters
724
+
725
+ if(rq->query && !*rq->query) {
726
+ freez((void *)rq->query);
727
+ rq->query = NULL;
728
+ }
729
+
730
+ if(rq->histogram && !*rq->histogram) {
731
+ freez((void *)rq->histogram);
732
+ rq->histogram = NULL;
733
+ }
734
+
735
+ if(!rq->data_only)
736
+ rq->delta = false;
737
+
738
+ if(!rq->data_only || !rq->if_modified_since)
739
+ rq->tail = false;
740
+
741
+ rq->now_s = now_realtime_sec();
742
+ rq->expires_s = rq->now_s + 1;
743
+ wb->expires = rq->expires_s;
744
+
745
+ if(!rq->after_s && !rq->before_s) {
746
+ rq->before_s = rq->now_s;
747
+ rq->after_s = rq->before_s - LQS_DEFAULT_QUERY_DURATION;
748
+ }
749
+ else
750
+ rrdr_relative_window_to_absolute(&rq->after_s, &rq->before_s, rq->now_s);
751
+
752
+ if(rq->after_s > rq->before_s) {
753
+ time_t tmp = rq->after_s;
754
+ rq->after_s = rq->before_s;
755
+ rq->before_s = tmp;
756
+ }
757
+
758
+ if(rq->after_s == rq->before_s)
759
+ rq->after_s = rq->before_s - LQS_DEFAULT_QUERY_DURATION;
760
+
761
+ rq->after_ut = rq->after_s * USEC_PER_SEC;
762
+ rq->before_ut = (rq->before_s * USEC_PER_SEC) + USEC_PER_SEC - 1;
763
+
764
+ if(!rq->entries)
765
+ rq->entries = LQS_DEFAULT_ITEMS_PER_QUERY;
766
+
767
+ // ----------------------------------------------------------------------------------------------------------------
768
+ // validate the anchor
769
+
770
+ lqs->last_modified = 0;
771
+ lqs->anchor.start_ut = lqs->rq.anchor;
772
+ lqs->anchor.stop_ut = 0;
773
+
774
+ if(lqs->anchor.start_ut && lqs->rq.tail) {
775
+ // a tail request
776
+ // we need the top X entries from BEFORE
777
+ // but, we need to calculate the facets and the
778
+ // histogram up to the anchor
779
+ lqs->rq.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
780
+ lqs->anchor.start_ut = 0;
781
+ lqs->anchor.stop_ut = lqs->rq.anchor;
782
+ }
783
+
784
+ if(lqs->rq.anchor && lqs->rq.anchor < lqs->rq.after_ut) {
785
+ lqs_log_error(lqs, "received anchor is too small for query timeframe, ignoring anchor");
786
+ lqs->rq.anchor = 0;
787
+ lqs->anchor.start_ut = 0;
788
+ lqs->anchor.stop_ut = 0;
789
+ lqs->rq.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
790
+ }
791
+ else if(lqs->rq.anchor > lqs->rq.before_ut) {
792
+ lqs_log_error(lqs, "received anchor is too big for query timeframe, ignoring anchor");
793
+ lqs->rq.anchor = 0;
794
+ lqs->anchor.start_ut = 0;
795
+ lqs->anchor.stop_ut = 0;
796
+ lqs->rq.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
797
+ }
798
+
799
+ facets_set_anchor(facets, lqs->anchor.start_ut, lqs->anchor.stop_ut, lqs->rq.direction);
800
+
801
+ facets_set_additional_options(facets,
802
+ ((lqs->rq.data_only) ? FACETS_OPTION_DATA_ONLY : 0) |
803
+ ((lqs->rq.delta) ? FACETS_OPTION_SHOW_DELTAS : 0));
804
+
805
+ facets_set_items(facets, lqs->rq.entries);
806
+ facets_set_query(facets, lqs->rq.query);
807
+
808
+ if(lqs->rq.slice && have_slice)
809
+ facets_enable_slice_mode(facets);
810
+ else
811
+ lqs->rq.slice = false;
812
+
813
+ if(lqs->rq.histogram) {
814
+ if(lqs->rq.fields_are_ids)
815
+ facets_set_timeframe_and_histogram_by_id(facets, lqs->rq.histogram, lqs->rq.after_ut, lqs->rq.before_ut);
816
+ else
817
+ facets_set_timeframe_and_histogram_by_name(facets, lqs->rq.histogram, lqs->rq.after_ut, lqs->rq.before_ut);
818
+ }
819
+ else if(default_histogram)
820
+ facets_set_timeframe_and_histogram_by_name(facets, default_histogram, lqs->rq.after_ut, lqs->rq.before_ut);
821
+
822
+ // complete the request object
823
+ buffer_json_member_add_boolean(wb, LQS_PARAMETER_INFO, lqs->rq.info);
824
+ buffer_json_member_add_boolean(wb, LQS_PARAMETER_SLICE, lqs->rq.slice);
825
+ buffer_json_member_add_boolean(wb, LQS_PARAMETER_DATA_ONLY, lqs->rq.data_only);
826
+ buffer_json_member_add_boolean(wb, LQS_PARAMETER_DELTA, lqs->rq.delta);
827
+ buffer_json_member_add_boolean(wb, LQS_PARAMETER_TAIL, lqs->rq.tail);
828
+ buffer_json_member_add_uint64(wb, LQS_PARAMETER_SAMPLING, lqs->rq.sampling);
829
+ buffer_json_member_add_uint64(wb, "source_type", lqs->rq.source_type);
830
+ buffer_json_member_add_uint64(wb, LQS_PARAMETER_AFTER, lqs->rq.after_ut / USEC_PER_SEC);
831
+ buffer_json_member_add_uint64(wb, LQS_PARAMETER_BEFORE, lqs->rq.before_ut / USEC_PER_SEC);
832
+ buffer_json_member_add_uint64(wb, "if_modified_since", lqs->rq.if_modified_since);
833
+ buffer_json_member_add_uint64(wb, LQS_PARAMETER_ANCHOR, lqs->rq.anchor);
834
+ buffer_json_member_add_string(wb, LQS_PARAMETER_DIRECTION, lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
835
+ buffer_json_member_add_uint64(wb, LQS_PARAMETER_LAST, lqs->rq.entries);
836
+ buffer_json_member_add_string(wb, LQS_PARAMETER_QUERY, lqs->rq.query);
837
+ buffer_json_member_add_string(wb, LQS_PARAMETER_HISTOGRAM, lqs->rq.histogram);
838
+ buffer_json_object_close(wb); // request
839
+
840
+ return true;
841
+}
842
+
843
+static inline void lqs_cleanup(LOGS_QUERY_STATUS *lqs) {
844
+ freez((void *)lqs->rq.query);
845
+ freez((void *)lqs->rq.histogram);
846
+ simple_pattern_free(lqs->rq.sources);
847
+ facets_destroy(lqs->facets);
848
+}
849
+
850
+#endif //NETDATA_LOGS_QUERY_STATUS_H
src/libnetdata/functions_evloop/functions_evloop.h
+6
-2
@@ -133,9 +133,13 @@ static inline void pluginsd_function_json_error_to_stdout(const char *transactio
133
fflush(stdout);
134
}
135
136
-static inline void pluginsd_function_result_to_stdout(const char *transaction, int code, const char *content_type, time_t expires, BUFFER *result) {
137
- pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires);
136
+static inline void pluginsd_function_result_to_stdout(const char *transaction, BUFFER *result) {
137
+ pluginsd_function_result_begin_to_stdout(transaction, result->response_code,
138
+ content_type_id2string(result->content_type),
139
+ result->expires);
140
+
141
fwrite(buffer_tostring(result), buffer_strlen(result), 1, stdout);
142
+
143
pluginsd_function_result_end_to_stdout();
144
fflush(stdout);
145
}
src/libnetdata/http/content_type.c
+2
-2
@@ -10,14 +10,14 @@ static struct {
10
const char *options;
11
} content_types[] = {
12
// primary - preferred during id-to-string conversions
13
- { .format = "text/html", CT_TEXT_HTML, true },
13
+ { .format = "application/json", CT_APPLICATION_JSON, true },
14
{ .format = "text/plain", CT_TEXT_PLAIN, true },
15
+ { .format = "text/html", CT_TEXT_HTML, true },
16
{ .format = "text/css", CT_TEXT_CSS, true },
17
{ .format = "text/yaml", CT_TEXT_YAML, true },
18
{ .format = "application/yaml", CT_APPLICATION_YAML, true },
19
{ .format = "text/xml", CT_TEXT_XML, true },
20
{ .format = "text/xsl", CT_TEXT_XSL, true },
20
- { .format = "application/json", CT_APPLICATION_JSON, true },
21
{ .format = "application/xml", CT_APPLICATION_XML, true },
22
{ .format = "application/javascript", CT_APPLICATION_X_JAVASCRIPT, true },
23
{ .format = "application/octet-stream", CT_APPLICATION_OCTET_STREAM, false },
src/libnetdata/json/json-c-parser-inline.c
+3
@@ -8,6 +8,9 @@ int rrd_call_function_error(BUFFER *wb, const char *msg, int code) {
8
buffer_json_member_add_int64(wb, "status", code);
9
buffer_json_member_add_string(wb, "error_message", msg);
10
buffer_json_finalize(wb);
11
+ wb->date = now_realtime_sec();
12
+ wb->expires = wb->date + 1;
13
+ wb->response_code = code;
14
return code;
15
}
16
src/libnetdata/os/os.h
+1
@@ -7,6 +7,7 @@
7
#include <sys/syscall.h>
8
#endif
9
10
+#include "timestamps.h"
11
#include "setproctitle.h"
12
#include "close_range.h"
13
#include "setresuid.h"
src/libnetdata/os/timestamps.c
new
+4
@@ -0,0 +1,4 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "libnetdata/libnetdata.h"
4
+#include "timestamps.h"
src/libnetdata/os/timestamps.h
new
+42
@@ -0,0 +1,42 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef LIBNETDATA_OS_TIMESTAMPS_H
4
+#define LIBNETDATA_OS_TIMESTAMPS_H
5
+
6
+// Windows file time starts on January 1, 1601, Unix epoch starts on January 1, 1970
7
+// Difference in 100-nanosecond intervals between these two dates is 116444736000000000ULL
8
+
9
+// Convert Windows file time (in 100-nanosecond intervals) to Unix epoch in nanoseconds
10
+#define os_windows_ulonglong_to_unix_epoch_ns(ft) (((uint64_t)(ft) - 116444736000000000ULL) * 100ULL)
11
+
12
+// Convert Unix epoch time (in nanoseconds) to Windows file time (in 100-nanosecond intervals)
13
+#define os_unix_epoch_ns_to_windows_ulonglong(ns) (((uint64_t)(ns) / 100ULL) + 116444736000000000ULL)
14
+
15
+#if defined(OS_WINDOWS)
16
+// Convert FILETIME to Unix epoch in nanoseconds
17
+#define os_filetime_to_unix_epoch_ns(ft) \
18
+ ((((uint64_t)(ft).dwHighDateTime << 32 | (ft).dwLowDateTime) - 116444736000000000ULL) * 100ULL)
19
+
20
+// Convert Unix epoch in nanoseconds to FILETIME (returns FILETIME)
21
+#define os_unix_epoch_ns_to_filetime(ns) \
22
+ ({ \
23
+ uint64_t temp = ((uint64_t)(ns) / 100ULL) + 116444736000000000ULL; \
24
+ FILETIME ft; \
25
+ ft.dwLowDateTime = (uint32_t)(temp & 0xFFFFFFFF); \
26
+ ft.dwHighDateTime = (uint32_t)(temp >> 32); \
27
+ ft; \
28
+ })
29
+
30
+// Convert Unix epoch in microseconds to FILETIME (returns FILETIME)
31
+#define os_unix_epoch_ut_to_filetime(ns) \
32
+ ({ \
33
+ uint64_t temp = ((uint64_t)(ns) * 10ULL) + 116444736000000000ULL; \
34
+ FILETIME ft; \
35
+ ft.dwLowDateTime = (uint32_t)(temp & 0xFFFFFFFF); \
36
+ ft.dwHighDateTime = (uint32_t)(temp >> 32); \
37
+ ft; \
38
+ })
39
+
40
+#endif //OS_WINDOWS
41
+
42
+#endif //LIBNETDATA_OS_TIMESTAMPS_H
src/libnetdata/parsers/duration.c
+3
-2
@@ -166,7 +166,7 @@ bool duration_parse(const char *duration, int64_t *result, const char *default_u
166
// --------------------------------------------------------------------------------------------------------------------
167
// generate a string to represent a duration
168
169
-ssize_t duration_snprintf(char *dst, size_t dst_size, int64_t value, const char *unit) {
169
+ssize_t duration_snprintf(char *dst, size_t dst_size, int64_t value, const char *unit, bool add_spaces) {
170
if (!dst || dst_size == 0) return -1;
171
if (dst_size == 1) {
172
dst[0] = '\0';
@@ -204,8 +204,9 @@ ssize_t duration_snprintf(char *dst, size_t dst_size, int64_t value, const char
204
205
int64_t unit_count = rounded / multiplier;
206
if (unit_count > 0) {
207
+ const char *space = (add_spaces && offset) ? " " : "";
208
int written = snprintfz(dst + offset, dst_size - offset,
208
- "%s%" PRIi64 "%s", sign, unit_count, units[i].unit);
209
+ "%s%s%" PRIi64 "%s", space, sign, unit_count, units[i].unit);
210
211
if (written < 0)
212
return -3;
src/libnetdata/parsers/duration.h
+8
-8
@@ -18,14 +18,14 @@ bool duration_parse(const char *duration, int64_t *result, const char *default_u
18
#define duration_parse_days(duration, days_ptr) duration_parse(duration, days_ptr, "d")
19
20
// duration (number to string)
21
-ssize_t duration_snprintf(char *dst, size_t dst_size, int64_t value, const char *unit);
22
-#define duration_snprintf_nsec_t(dst, dst_size, ns) duration_snprintf(dst, dst_size, ns, "ns")
23
-#define duration_snprintf_usec_t(dst, dst_size, us) duration_snprintf(dst, dst_size, us, "us")
24
-#define duration_snprintf_msec_t(dst, dst_size, ms) duration_snprintf(dst, dst_size, ms, "ms")
25
-#define duration_snprintf_time_t(dst, dst_size, secs) duration_snprintf(dst, dst_size, secs, "s")
26
-#define duration_snprintf_mins(dst, dst_size, mins) duration_snprintf(dst, dst_size, mins, "m")
27
-#define duration_snprintf_hours(dst, dst_size, hours) duration_snprintf(dst, dst_size, hours, "h")
28
-#define duration_snprintf_days(dst, dst_size, days) duration_snprintf(dst, dst_size, days, "d")
21
+ssize_t duration_snprintf(char *dst, size_t dst_size, int64_t value, const char *unit, bool add_spaces);
22
+#define duration_snprintf_nsec_t(dst, dst_size, ns) duration_snprintf(dst, dst_size, ns, "ns", false)
23
+#define duration_snprintf_usec_t(dst, dst_size, us) duration_snprintf(dst, dst_size, us, "us", false)
24
+#define duration_snprintf_msec_t(dst, dst_size, ms) duration_snprintf(dst, dst_size, ms, "ms", false)
25
+#define duration_snprintf_time_t(dst, dst_size, secs) duration_snprintf(dst, dst_size, secs, "s", false)
26
+#define duration_snprintf_mins(dst, dst_size, mins) duration_snprintf(dst, dst_size, mins, "m", false)
27
+#define duration_snprintf_hours(dst, dst_size, hours) duration_snprintf(dst, dst_size, hours, "h", false)
28
+#define duration_snprintf_days(dst, dst_size, days) duration_snprintf(dst, dst_size, days, "d", false)
29
30
bool duration_parse_seconds(const char *str, int *result);
31
src/libnetdata/parsers/size.c
+11
-4
@@ -148,7 +148,7 @@ bool size_parse(const char *size_str, uint64_t *result, const char *default_unit
148
// --------------------------------------------------------------------------------------------------------------------
149
// generate a string to represent a size
150
151
-ssize_t size_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit) {
151
+ssize_t size_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit, bool accurate) {
152
if (!dst || dst_size == 0) return -1;
153
if (dst_size == 1) {
154
dst[0] = '\0';
@@ -178,9 +178,16 @@ ssize_t size_snprintf(char *dst, size_t dst_size, uint64_t value, const char *un
178
179
uint64_t reversed_bytes = (uint64_t)(converted * (double)su->multiplier);
180
181
- if (reversed_bytes == bytes)
182
- // no precision loss, this is good to use
183
- su_best = su;
181
+ if(accurate) {
182
+ // no precision loss is required
183
+ if (reversed_bytes == bytes)
184
+ // no precision loss, this is good to use
185
+ su_best = su;
186
+ }
187
+ else {
188
+ if(converted > 1.0)
189
+ su_best = su;
190
+ }
191
}
192
193
double converted = size_round_to_resolution_dbl2(bytes, su_best->multiplier);
src/libnetdata/parsers/size.h
+5
-5
@@ -11,10 +11,10 @@ bool size_parse(const char *size_str, uint64_t *result, const char *default_unit
11
#define size_parse_mb(size_str, mb) size_parse(size_str, mb, "MiB")
12
#define size_parse_gb(size_str, gb) size_parse(size_str, gb, "GiB")
13
14
-ssize_t size_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit);
15
-#define size_snprintf_bytes(dst, dst_size, value) size_snprintf(dst, dst_size, value, "B")
16
-#define size_snprintf_kb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "KiB")
17
-#define size_snprintf_mb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "MiB")
18
-#define size_snprintf_gb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "GiB")
14
+ssize_t size_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit, bool accurate);
15
+#define size_snprintf_bytes(dst, dst_size, value) size_snprintf(dst, dst_size, value, "B", true)
16
+#define size_snprintf_kb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "KiB", true)
17
+#define size_snprintf_mb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "MiB", true)
18
+#define size_snprintf_gb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "GiB", true)
19
20
#endif //LIBNETDATA_PARSERS_SIZE_H
src/libnetdata/simple_hashtable.h
+2
-2
@@ -520,13 +520,13 @@ static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HAS
520
return SIMPLE_HASHTABLE_SLOT_DATA(sl);
521
}
522
523
-static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
523
+static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len) {
524
XXH64_hash_t hash = XXH3_64bits((void *)key, key_len);
525
SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true);
526
return SIMPLE_HASHTABLE_SLOT_DATA(sl);
527
}
528
529
-static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
529
+static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len) {
530
XXH64_hash_t hash = XXH3_64bits((void *)key, key_len);
531
SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true);
532
return simple_hashtable_del_slot_named(ht, sl);
src/libnetdata/spawn_server/spawn_server_posix.c
+53
-13
@@ -219,6 +219,57 @@ int spawn_server_exec_kill(SPAWN_SERVER *server, SPAWN_INSTANCE *si) {
219
return spawn_server_exec_wait(server, si);
220
}
221
222
+static int spawn_server_waitpid(SPAWN_INSTANCE *si) {
223
+ int status;
224
+ pid_t pid;
225
+
226
+ pid = waitpid(si->child_pid, &status, 0);
227
+
228
+ if(pid != si->child_pid) {
229
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
230
+ "SPAWN PARENT: failed to wait for pid %d: %s",
231
+ si->child_pid, si->cmdline);
232
+
233
+ return -1;
234
+ }
235
+
236
+ errno_clear();
237
+
238
+ if(WIFEXITED(status)) {
239
+ if(WEXITSTATUS(status))
240
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
241
+ "SPAWN SERVER: child with pid %d (request %zu) exited with exit code %d: %s",
242
+ pid, si->request_id, WEXITSTATUS(status), si->cmdline);
243
+ }
244
+ else if(WIFSIGNALED(status)) {
245
+ if(WCOREDUMP(status))
246
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
247
+ "SPAWN SERVER: child with pid %d (request %zu) coredump'd due to signal %d: %s",
248
+ pid, si->request_id, WTERMSIG(status), si->cmdline);
249
+ else
250
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
251
+ "SPAWN SERVER: child with pid %d (request %zu) killed by signal %d: %s",
252
+ pid, si->request_id, WTERMSIG(status), si->cmdline);
253
+ }
254
+ else if(WIFSTOPPED(status)) {
255
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
256
+ "SPAWN SERVER: child with pid %d (request %zu) stopped due to signal %d: %s",
257
+ pid, si->request_id, WSTOPSIG(status), si->cmdline);
258
+ }
259
+ else if(WIFCONTINUED(status)) {
260
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
261
+ "SPAWN SERVER: child with pid %d (request %zu) continued due to signal %d: %s",
262
+ pid, si->request_id, SIGCONT, si->cmdline);
263
+ }
264
+ else {
265
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
266
+ "SPAWN SERVER: child with pid %d (request %zu) reports unhandled status: %s",
267
+ pid, si->request_id, si->cmdline);
268
+ }
269
+
270
+ return status;
271
+}
272
+
273
int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) {
274
if (!si) return -1;
275
@@ -229,19 +280,8 @@ int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
280
// Wait for the process to exit
281
int status = __atomic_load_n(&si->waitpid_status, __ATOMIC_RELAXED);
282
bool exited = __atomic_load_n(&si->exited, __ATOMIC_RELAXED);
232
- if(!exited) {
233
- if(waitpid(si->child_pid, &status, 0) != si->child_pid) {
234
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
235
- "SPAWN PARENT: failed to wait for pid %d: %s",
236
- si->child_pid, si->cmdline);
237
- status = -1;
238
- }
239
- else {
240
- nd_log(NDLS_COLLECTORS, NDLP_INFO,
241
- "SPAWN PARENT: child with pid %d exited with status %d (waitpid): %s",
242
- si->child_pid, status, si->cmdline);
243
- }
244
- }
283
+ if(!exited)
284
+ status = spawn_server_waitpid(si);
285
else
286
nd_log(NDLS_COLLECTORS, NDLP_INFO,
287
"SPAWN PARENT: child with pid %d exited with status %d (sighandler): %s",
src/web/api/functions/function-bearer_get_token.c
+2
-2
@@ -13,7 +13,7 @@ struct bearer_token_request {
13
STRING *client_name;
14
};
15
16
-static bool parse_json_payload(json_object *jobj, const char *path, void *data, BUFFER *error) {
16
+static bool bearer_parse_json_payload(json_object *jobj, const char *path, void *data, BUFFER *error) {
17
struct bearer_token_request *rq = data;
18
JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "claim_id", rq->claim_id, error, true);
19
JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "machine_guid", rq->machine_guid, error, true);
@@ -32,7 +32,7 @@ int function_bearer_get_token(BUFFER *wb, const char *function __maybe_unused, B
32
33
int code;
34
struct bearer_token_request rq = { 0 };
35
- CLEAN_JSON_OBJECT *jobj = json_parse_function_payload_or_error(wb, payload, &code, parse_json_payload, &rq);
35
+ CLEAN_JSON_OBJECT *jobj = json_parse_function_payload_or_error(wb, payload, &code, bearer_parse_json_payload, &rq);
36
if(!jobj || code != HTTP_RESP_OK) {
37
string_freez(rq.client_name);
38
return code;
src/web/server/web_client.c
+10
-5
@@ -1426,7 +1426,7 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1426
buffer_flush(w->url_as_received);
1427
buffer_strcat(w->url_as_received, "too big request");
1428
1429
- netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, w->response.data->len);
1429
+ netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, (size_t)w->response.data->len);
1430
1431
size_t len = w->response.data->len;
1432
buffer_flush(w->response.data);
@@ -1506,14 +1506,18 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1506
break;
1507
1508
case HTTP_REQUEST_MODE_OPTIONS:
1509
- netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
1509
+ netdata_log_debug(D_WEB_CLIENT,
1510
+ "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.",
1511
+ w->id, (size_t)w->response.data->len);
1512
break;
1513
1514
case HTTP_REQUEST_MODE_POST:
1515
case HTTP_REQUEST_MODE_GET:
1516
case HTTP_REQUEST_MODE_PUT:
1517
case HTTP_REQUEST_MODE_DELETE:
1516
- netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
1518
+ netdata_log_debug(D_WEB_CLIENT,
1519
+ "%llu: Done preparing the response. Sending data (%zu bytes) to client.",
1520
+ w->id, (size_t)w->response.data->len);
1521
break;
1522
1523
case HTTP_REQUEST_MODE_FILECOPY:
@@ -1620,8 +1624,9 @@ ssize_t web_client_send_deflate(struct web_client *w)
1624
// when using compression,
1625
// w->response.sent is the amount of bytes passed through compression
1626
1623
- netdata_log_debug(D_DEFLATE, "%llu: web_client_send_deflate(): w->response.data->len = %zu, w->response.sent = %zu, w->response.zhave = %zu, w->response.zsent = %zu, w->response.zstream.avail_in = %u, w->response.zstream.avail_out = %u, w->response.zstream.total_in = %lu, w->response.zstream.total_out = %lu.",
1624
- w->id, w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out);
1627
+ netdata_log_debug(D_DEFLATE,
1628
+ "%llu: web_client_send_deflate(): w->response.data->len = %zu, w->response.sent = %zu, w->response.zhave = %zu, w->response.zsent = %zu, w->response.zstream.avail_in = %u, w->response.zstream.avail_out = %u, w->response.zstream.total_in = %lu, w->response.zstream.total_out = %lu.",
1629
+ w->id, (size_t)w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out);
1630
1631
if(w->response.data->len - w->response.sent == 0 && w->response.zstream.avail_in == 0 && w->response.zhave == w->response.zsent && w->response.zstream.avail_out != 0) {
1632
// there is nothing to send