@cryptotaxi247 / netdata-1 / commits / f6a175aeb

Windows Events Improvements 1 (#18528)

* extract more information for the XML to form the message * support slicing for EventID and Level * the fastest it can get * remove obsolete member * stop service Netdata before starting Netdata * do not reuse buffers * convert newlines to unix style * send progress based on the number of log events * rework on opcodes, keywords and task to allow hard-coded values * complete messages; facets annotated from XML * working with field and provider caches * cleanup * query the event log in batch mode * extract keywords from publishers * detect closed sockets on non-Linux systems * unified event log processing * work on caching, improving accuracy * optimization on keywords * caching improvements

Costa Tsaousis committed Sep 16, 2024 at 15:40 UTC f6a175aeb156756f040da0813fedfcb1c52981fd
27 files changed +4221 -2653
CMakeLists.txt
+4
@@ -1411,6 +1411,10 @@ set(WINDOWS_EVENTS_PLUGIN_FILES
1411 src/collectors/windows-events.plugin/windows-events-unicode.h
1412 src/collectors/windows-events.plugin/windows-events-xml.c
1413 src/collectors/windows-events.plugin/windows-events-xml.h
1414 + src/collectors/windows-events.plugin/windows-events-publishers.c
1415 + src/collectors/windows-events.plugin/windows-events-publishers.h
1416 + src/collectors/windows-events.plugin/windows-events-fields-cache.c
1417 + src/collectors/windows-events.plugin/windows-events-fields-cache.h
1418 )
1419
1420 set(WINDOWS_PLUGIN_FILES
packaging/utils/compile-on-windows.sh
+4 -1
@@ -3,7 +3,7 @@
3 # On MSYS2, install these dependencies to build netdata:
4 install_dependencies() {
5 pacman -S \
6 - git cmake ninja base-devel msys2-devel \
6 + git cmake ninja clang base-devel msys2-devel \
7 libyaml-devel libzstd-devel libutil-linux libutil-linux-devel \
8 mingw-w64-x86_64-toolchain mingw-w64-ucrt-x86_64-toolchain \
9 mingw64/mingw-w64-x86_64-mold ucrt64/mingw-w64-ucrt-x86_64-mold \
@@ -75,6 +75,9 @@ ninja -v -C "${build}" install || ninja -v -C "${build}" -j 1
75 #echo "Compile with:"
76 #echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"
77
78 +echo "Stopping service Netdata"
79 +sc stop "Netdata" || echo "Failed"
80 +
81 echo "starting netdata..."
82 # enable JIT debug with gdb
83 export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"
src/collectors/systemd-journal.plugin/systemd-journal.c
+14 -9
@@ -336,10 +336,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
336 LOGS_QUERY_STATUS *fqs) {
337
338 usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
339 -
340 - usec_t start_ut = ((fqs->rq.data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->rq.before_ut) + anchor_delta;
341 - usec_t stop_ut = (fqs->rq.data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->rq.after_ut;
342 - bool stop_when_full = (fqs->rq.data_only && !fqs->anchor.stop_ut);
339 + lqs_query_timeframe(fqs, anchor_delta);
340 + usec_t start_ut = fqs->query.start_ut;
341 + usec_t stop_ut = fqs->query.stop_ut;
342 + bool stop_when_full = fqs->query.stop_when_full;
343
344 fqs->c.query_file.start_ut = start_ut;
345 fqs->c.query_file.stop_ut = stop_ut;
@@ -451,10 +451,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
451 LOGS_QUERY_STATUS *fqs) {
452
453 usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
454 -
455 - usec_t start_ut = (fqs->rq.data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->rq.after_ut;
456 - usec_t stop_ut = ((fqs->rq.data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->rq.before_ut) + anchor_delta;
457 - bool stop_when_full = (fqs->rq.data_only && !fqs->anchor.stop_ut);
454 + lqs_query_timeframe(fqs, anchor_delta);
455 + usec_t start_ut = fqs->query.start_ut;
456 + usec_t stop_ut = fqs->query.stop_ut;
457 + bool stop_when_full = fqs->query.stop_when_full;
458
459 fqs->c.query_file.start_ut = start_ut;
460 fqs->c.query_file.stop_ut = stop_ut;
@@ -747,8 +747,13 @@ static int netdata_systemd_journal_query(BUFFER *wb, LOGS_QUERY_STATUS *lqs) {
747
748 lqs->c.files_matched = files_used;
749
750 - if(lqs->rq.if_modified_since && !files_are_newer)
750 + if(lqs->rq.if_modified_since && !files_are_newer) {
751 + // release the files
752 + for(size_t f = 0; f < files_used ;f++)
753 + dictionary_acquired_item_release(journal_files_registry, file_items[f]);
754 +
755 return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
756 + }
757
758 // sort the files, so that they are optimal for facets
759 if(files_used >= 2) {
src/collectors/windows-events.plugin/windows-events-fields-cache.c new
+158
@@ -0,0 +1,158 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "windows-events-fields-cache.h"
4 +
5 +typedef struct field_key {
6 + uint64_t value;
7 + ND_UUID provider;
8 +} WEVT_FIELD_KEY;
9 +
10 +typedef struct field_value {
11 + WEVT_FIELD_KEY key;
12 + uint32_t name_size;
13 + char name[];
14 +} WEVT_FIELD_VALUE;
15 +
16 +#define SIMPLE_HASHTABLE_NAME _FIELDS_CACHE
17 +#define SIMPLE_HASHTABLE_VALUE_TYPE WEVT_FIELD_VALUE
18 +#define SIMPLE_HASHTABLE_KEY_TYPE WEVT_FIELD_KEY
19 +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION field_cache_value_to_key
20 +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION field_cache_cache_compar
21 +#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
22 +#include "libnetdata/simple_hashtable.h"
23 +
24 +static inline WEVT_FIELD_KEY *field_cache_value_to_key(WEVT_FIELD_VALUE *p) {
25 + return &p->key;
26 +}
27 +
28 +static inline bool field_cache_cache_compar(WEVT_FIELD_KEY *a, WEVT_FIELD_KEY *b) {
29 + return memcmp(a, b, sizeof(WEVT_FIELD_KEY)) == 0;
30 +}
31 +
32 +struct ht {
33 + SPINLOCK spinlock;
34 + size_t allocations;
35 + size_t bytes;
36 + struct simple_hashtable_FIELDS_CACHE ht;
37 +};
38 +
39 +static struct {
40 + bool initialized;
41 + struct ht ht[WEVT_FIELD_TYPE_MAX];
42 +} fdc = {
43 + .initialized = false,
44 +};
45 +
46 +void field_cache_init(void) {
47 + for(size_t type = 0; type < WEVT_FIELD_TYPE_MAX ; type++) {
48 + spinlock_init(&fdc.ht[type].spinlock);
49 + simple_hashtable_init_FIELDS_CACHE(&fdc.ht[type].ht, 10000);
50 + }
51 +}
52 +
53 +static inline bool should_zero_provider(WEVT_FIELD_TYPE type, uint64_t value) {
54 + switch(type) {
55 + case WEVT_FIELD_TYPE_LEVEL:
56 + return !is_valid_publisher_level(value, true);
57 +
58 + case WEVT_FIELD_TYPE_KEYWORDS:
59 + return !is_valid_publisher_keywords(value, true);
60 +
61 + case WEVT_FIELD_TYPE_OPCODE:
62 + return !is_valid_publisher_opcode(value, true);
63 +
64 + case WEVT_FIELD_TYPE_TASK:
65 + return !is_valid_publisher_task(value, true);
66 +
67 + default:
68 + return false;
69 + }
70 +}
71 +
72 +bool field_cache_get(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *dst) {
73 + fatal_assert(type < WEVT_FIELD_TYPE_MAX);
74 +
75 + struct ht *ht = &fdc.ht[type];
76 +
77 + WEVT_FIELD_KEY t = {
78 + .value = value,
79 + .provider = should_zero_provider(type, value) ? UUID_ZERO : *uuid,
80 + };
81 + XXH64_hash_t hash = XXH3_64bits(&t, sizeof(t));
82 +
83 + spinlock_lock(&ht->spinlock);
84 + SIMPLE_HASHTABLE_SLOT_FIELDS_CACHE *slot = simple_hashtable_get_slot_FIELDS_CACHE(&ht->ht, hash, &t, true);
85 + WEVT_FIELD_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot);
86 + spinlock_unlock(&ht->spinlock);
87 +
88 + if(v) {
89 + txt_utf8_resize(dst, v->name_size, false);
90 + memcpy(dst->data, v->name, v->name_size);
91 + dst->used = v->name_size;
92 + dst->src = TXT_SOURCE_FIELD_CACHE;
93 + return true;
94 + }
95 +
96 + return false;
97 +}
98 +
99 +static WEVT_FIELD_VALUE *wevt_create_cache_entry(WEVT_FIELD_KEY *t, TXT_UTF8 *name, size_t *bytes) {
100 + *bytes = sizeof(WEVT_FIELD_VALUE) + name->used;
101 + WEVT_FIELD_VALUE *v = callocz(1, *bytes);
102 + v->key = *t;
103 + memcpy(v->name, name->data, name->used);
104 + v->name_size = name->used;
105 + return v;
106 +}
107 +
108 +//static bool is_numeric(const char *s) {
109 +// while(*s) {
110 +// if(!isdigit((uint8_t)*s++))
111 +// return false;
112 +// }
113 +//
114 +// return true;
115 +//}
116 +
117 +void field_cache_set(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *name) {
118 + fatal_assert(type < WEVT_FIELD_TYPE_MAX);
119 +
120 + struct ht *ht = &fdc.ht[type];
121 +
122 + WEVT_FIELD_KEY t = {
123 + .value = value,
124 + .provider = should_zero_provider(type, value) ? UUID_ZERO : *uuid,
125 + };
126 + XXH64_hash_t hash = XXH3_64bits(&t, sizeof(t));
127 +
128 + spinlock_lock(&ht->spinlock);
129 + SIMPLE_HASHTABLE_SLOT_FIELDS_CACHE *slot = simple_hashtable_get_slot_FIELDS_CACHE(&ht->ht, hash, &t, true);
130 + WEVT_FIELD_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot);
131 + if(!v) {
132 + size_t bytes;
133 + v = wevt_create_cache_entry(&t, name, &bytes);
134 + simple_hashtable_set_slot_FIELDS_CACHE(&ht->ht, slot, hash, v);
135 +
136 + ht->allocations++;
137 + ht->bytes += bytes;
138 + }
139 +// else {
140 +// if((v->name_size == 1 && name->used > 0) || is_numeric(v->name)) {
141 +// size_t bytes;
142 +// WEVT_FIELD_VALUE *nv = wevt_create_cache_entry(&t, name, &bytes);
143 +// simple_hashtable_set_slot_FIELDS_CACHE(&ht->ht, slot, hash, nv);
144 +// ht->bytes += name->used;
145 +// ht->bytes -= v->name_size;
146 +// freez(v);
147 +// }
148 +// else if(name->used > 2 && !is_numeric(name->data) && (v->name_size != name->used || strcasecmp(v->name, name->data) != 0)) {
149 +// const char *a = v->name;
150 +// const char *b = name->data;
151 +// int x = 0;
152 +// x++;
153 +// }
154 +// }
155 +
156 + spinlock_unlock(&ht->spinlock);
157 +}
158 +
src/collectors/windows-events.plugin/windows-events-fields-cache.h new
+22
@@ -0,0 +1,22 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WINDOWS_EVENTS_FIELDS_CACHE_H
4 +#define NETDATA_WINDOWS_EVENTS_FIELDS_CACHE_H
5 +
6 +#include "windows-events.h"
7 +
8 +typedef enum __attribute__((packed)) {
9 + WEVT_FIELD_TYPE_LEVEL = 0,
10 + WEVT_FIELD_TYPE_OPCODE,
11 + WEVT_FIELD_TYPE_KEYWORDS,
12 + WEVT_FIELD_TYPE_TASK,
13 +
14 + // terminator
15 + WEVT_FIELD_TYPE_MAX,
16 +} WEVT_FIELD_TYPE;
17 +
18 +void field_cache_init(void);
19 +bool field_cache_get(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *dst);
20 +void field_cache_set(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *name);
21 +
22 +#endif //NETDATA_WINDOWS_EVENTS_FIELDS_CACHE_H
src/collectors/windows-events.plugin/windows-events-publishers.c new
+569
@@ -0,0 +1,569 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "windows-events-publishers.h"
4 +
5 +#define MAX_OPEN_HANDLES_PER_PUBLISHER 5
6 +
7 +struct publisher;
8 +
9 +// typedef as PROVIDER_META_HANDLE in include file
10 +struct provider_meta_handle {
11 + pid_t owner; // the owner of the handle, or zero
12 + uint32_t locks; // the number of locks the owner has on this handle
13 + EVT_HANDLE hMetadata; // the handle
14 + struct publisher *publisher; // a pointer back to the publisher
15 +
16 + // double linked list
17 + PROVIDER_META_HANDLE *prev;
18 + PROVIDER_META_HANDLE *next;
19 +};
20 +
21 +struct provider_data {
22 + uint64_t value; // the mask of the keyword
23 + XXH64_hash_t hash; // the hash of the name
24 + uint32_t len; // the length of the name
25 + char *name; // the name of the keyword in UTF-8
26 +};
27 +
28 +struct provider_list {
29 + uint64_t min, max;
30 + bool exceeds_data_type; // true when the manifest values exceed the capacity of the EvtXXX() API
31 + uint32_t total; // the number of entries in the array
32 + struct provider_data *array; // the array of entries, sorted (for binary search)
33 +};
34 +
35 +typedef struct publisher {
36 + ND_UUID uuid; // the Provider GUID
37 + const char *name; // the Provider name (UTF-8)
38 + uint32_t total_handles; // the number of handles allocated
39 + uint32_t available_handles; // the number of available handles
40 + uint32_t deleted_handles; // the number of deleted handles
41 + PROVIDER_META_HANDLE *handles; // a double linked list of all the handles
42 +
43 + struct provider_list keywords;
44 + struct provider_list tasks;
45 + struct provider_list opcodes;
46 + struct provider_list levels;
47 +} PUBLISHER;
48 +
49 +// A hashtable implementation for publishers
50 +// using the provider GUID as key and PUBLISHER as value
51 +#define SIMPLE_HASHTABLE_NAME _PROVIDER_GUID
52 +#define SIMPLE_HASHTABLE_VALUE_TYPE PUBLISHER
53 +#define SIMPLE_HASHTABLE_KEY_TYPE ND_UUID
54 +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION publisher_value_to_key
55 +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION publisher_cache_compar
56 +#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
57 +#include "libnetdata/simple_hashtable.h"
58 +
59 +static struct {
60 + SPINLOCK spinlock;
61 + uint32_t total_publishers;
62 + uint32_t total_handles;
63 + uint32_t deleted_handles;
64 + struct simple_hashtable_PROVIDER_GUID hashtable;
65 + ARAL *aral_publishers;
66 + ARAL *aral_handles;
67 +} pbc = {
68 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
69 +};
70 +
71 +static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *unicode, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
72 +
73 +static inline ND_UUID *publisher_value_to_key(PUBLISHER *p) {
74 + return &p->uuid;
75 +}
76 +
77 +static inline bool publisher_cache_compar(ND_UUID *a, ND_UUID *b) {
78 + return UUIDeq(*a, *b);
79 +}
80 +
81 +void publisher_cache_init(void) {
82 + simple_hashtable_init_PROVIDER_GUID(&pbc.hashtable, 100000);
83 + pbc.aral_publishers = aral_create("wevt_publishers", sizeof(PUBLISHER), 0, 4096, NULL, NULL, NULL, false, true);
84 + pbc.aral_handles = aral_create("wevt_handles", sizeof(PROVIDER_META_HANDLE), 0, 4096, NULL, NULL, NULL, false, true);
85 +}
86 +
87 +PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
88 + if(!providerName || !providerName[0] || UUIDiszero(uuid))
89 + return NULL;
90 +
91 + // XXH64_hash_t hash = XXH3_64bits(&uuid, sizeof(uuid));
92 + uint64_t hash = uuid.parts.low64 + uuid.parts.hig64;
93 +
94 + spinlock_lock(&pbc.spinlock);
95 +
96 + SIMPLE_HASHTABLE_SLOT_PROVIDER_GUID *slot =
97 + simple_hashtable_get_slot_PROVIDER_GUID(&pbc.hashtable, hash, &uuid, true);
98 +
99 + bool load_it = false;
100 + PUBLISHER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
101 + if(!p) {
102 + p = aral_callocz(pbc.aral_publishers);
103 + p->uuid = uuid;
104 + simple_hashtable_set_slot_PROVIDER_GUID(&pbc.hashtable, slot, hash, p);
105 + load_it = true;
106 + pbc.total_publishers++;
107 + }
108 +
109 + pid_t me = gettid_cached();
110 + PROVIDER_META_HANDLE *h;
111 + for(h = p->handles; h ;h = h->next) {
112 + // find the first that is mine,
113 + // or the first not owned by anyone
114 + if(!h->owner || h->owner == me)
115 + break;
116 + }
117 +
118 + if(!h) {
119 + h = aral_callocz(pbc.aral_handles);
120 + h->publisher = p;
121 + h->hMetadata = EvtOpenPublisherMetadata(
122 + NULL, // Local machine
123 + providerName, // Provider name
124 + NULL, // Log file path (NULL for default)
125 + 0, // Locale (0 for default locale)
126 + 0 // Flags
127 + );
128 +
129 + // we put it at the beginning of the list
130 + // to find it first if the same owner needs more locks on it
131 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(p->handles, h, prev, next);
132 + pbc.total_handles++;
133 + p->total_handles++;
134 + p->available_handles++;
135 + }
136 +
137 + if(!h->owner) {
138 + fatal_assert(p->available_handles > 0);
139 + p->available_handles--;
140 + h->owner = me;
141 + }
142 +
143 + h->locks++;
144 +
145 + if(load_it) {
146 + WEVT_VARIANT content = { 0 };
147 + WEVT_VARIANT property = { 0 };
148 + TXT_UNICODE unicode = { 0 };
149 +
150 + publisher_load_list(h, &content, &property, &unicode, &p->keywords, EvtPublisherMetadataKeywords);
151 + publisher_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
152 + publisher_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
153 + publisher_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
154 +
155 + txt_unicode_cleanup(&unicode);
156 + wevt_variant_cleanup(&content);
157 + wevt_variant_cleanup(&property);
158 + }
159 +
160 + spinlock_unlock(&pbc.spinlock);
161 +
162 + return h;
163 +}
164 +
165 +EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h) {
166 + return h ? h->hMetadata : NULL;
167 +}
168 +
169 +PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h) {
170 + if(h) h->locks++;
171 + return h;
172 +}
173 +
174 +void publisher_release(PROVIDER_META_HANDLE *h) {
175 + if(!h) return;
176 + pid_t me = gettid_cached();
177 + fatal_assert(h->owner == me);
178 + fatal_assert(h->locks > 0);
179 + if(--h->locks == 0) {
180 + PUBLISHER *p = h->publisher;
181 +
182 + spinlock_lock(&pbc.spinlock);
183 + h->owner = 0;
184 +
185 + if(++p->available_handles > MAX_OPEN_HANDLES_PER_PUBLISHER) {
186 + // there are multiple handles on this publisher
187 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
188 +
189 + if(h->hMetadata)
190 + EvtClose(h->hMetadata);
191 +
192 + aral_freez(pbc.aral_handles, h);
193 +
194 + pbc.total_handles--;
195 + p->total_handles--;
196 +
197 + pbc.deleted_handles++;
198 + p->deleted_handles++;
199 +
200 + p->available_handles--;
201 + }
202 + else if(h->next) {
203 + // it is not the last, put it at the end
204 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
205 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(p->handles, h, prev, next);
206 + }
207 +
208 + spinlock_unlock(&pbc.spinlock);
209 + }
210 +}
211 +
212 +// --------------------------------------------------------------------------------------------------------------------
213 +// load publisher lists
214 +
215 +static bool wevt_get_property_from_array(WEVT_VARIANT *property, EVT_HANDLE handle, DWORD dwIndex, EVT_PUBLISHER_METADATA_PROPERTY_ID PropertyId) {
216 + DWORD used = 0;
217 +
218 + if (!EvtGetObjectArrayProperty(handle, PropertyId, dwIndex, 0, property->size, property->data, &used)) {
219 + DWORD status = GetLastError();
220 + if (status != ERROR_INSUFFICIENT_BUFFER) {
221 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetObjectArrayProperty() failed");
222 + return false;
223 + }
224 +
225 + wevt_variant_resize(property, used);
226 + if (!EvtGetObjectArrayProperty(handle, PropertyId, dwIndex, 0, property->size, property->data, &used)) {
227 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetObjectArrayProperty() failed");
228 + return false;
229 + }
230 + }
231 +
232 + property->used = used;
233 + return true;
234 +}
235 +
236 +// Comparison function for ascending order (for Levels, Opcodes, Tasks)
237 +static int compare_ascending(const void *a, const void *b) {
238 + struct provider_data *d1 = (struct provider_data *)a;
239 + struct provider_data *d2 = (struct provider_data *)b;
240 +
241 + if (d1->value < d2->value) return -1;
242 + if (d1->value > d2->value) return 1;
243 + return 0;
244 +}
245 +
246 +//// Comparison function for descending order (for Keywords)
247 +//static int compare_descending(const void *a, const void *b) {
248 +// struct provider_data *d1 = (struct provider_data *)a;
249 +// struct provider_data *d2 = (struct provider_data *)b;
250 +//
251 +// if (d1->value > d2->value) return -1;
252 +// if (d1->value < d2->value) return 1;
253 +// return 0;
254 +//}
255 +
256 +static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *unicode, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
257 + if(!h || !h->hMetadata) return;
258 +
259 + EVT_PUBLISHER_METADATA_PROPERTY_ID name_id, message_id, value_id;
260 + uint8_t value_bits = 32;
261 + int (*compare_func)(const void *, const void *);
262 + bool (*is_valid)(uint64_t, bool);
263 +
264 + switch(property_id) {
265 + case EvtPublisherMetadataLevels:
266 + name_id = EvtPublisherMetadataLevelName;
267 + message_id = EvtPublisherMetadataLevelMessageID;
268 + value_id = EvtPublisherMetadataLevelValue;
269 + value_bits = 32;
270 + compare_func = compare_ascending;
271 + is_valid = is_valid_publisher_level;
272 + break;
273 +
274 + case EvtPublisherMetadataOpcodes:
275 + name_id = EvtPublisherMetadataOpcodeName;
276 + message_id = EvtPublisherMetadataOpcodeMessageID;
277 + value_id = EvtPublisherMetadataOpcodeValue;
278 + value_bits = 32;
279 + is_valid = is_valid_publisher_opcode;
280 + compare_func = compare_ascending;
281 + break;
282 +
283 + case EvtPublisherMetadataTasks:
284 + name_id = EvtPublisherMetadataTaskName;
285 + message_id = EvtPublisherMetadataTaskMessageID;
286 + value_id = EvtPublisherMetadataTaskValue;
287 + value_bits = 32;
288 + is_valid = is_valid_publisher_task;
289 + compare_func = compare_ascending;
290 + break;
291 +
292 + case EvtPublisherMetadataKeywords:
293 + name_id = EvtPublisherMetadataKeywordName;
294 + message_id = EvtPublisherMetadataKeywordMessageID;
295 + value_id = EvtPublisherMetadataKeywordValue;
296 + value_bits = 64;
297 + is_valid = is_valid_publisher_keywords;
298 + compare_func = NULL;
299 + break;
300 +
301 + default:
302 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Internal Error: Can't handle property id %u", property_id);
303 + return;
304 + }
305 +
306 + EVT_HANDLE hMetadata = h->hMetadata;
307 + EVT_HANDLE hArray = NULL;
308 + DWORD bufferUsed = 0;
309 + DWORD itemCount = 0;
310 +
311 + // Get the metadata array for the list (e.g., opcodes, tasks, or levels)
312 + if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
313 + DWORD status = GetLastError();
314 + if (status != ERROR_INSUFFICIENT_BUFFER) {
315 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
316 + goto cleanup;
317 + }
318 + }
319 +
320 + wevt_variant_resize(content, bufferUsed);
321 + if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
322 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
323 + goto cleanup;
324 + }
325 +
326 + // Get the number of items (e.g., levels, tasks, or opcodes)
327 + hArray = content->data->EvtHandleVal;
328 + if (!EvtGetObjectArraySize(hArray, &itemCount)) {
329 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetObjectArraySize() failed");
330 + goto cleanup;
331 + }
332 +
333 + if (itemCount == 0) {
334 + l->total = 0;
335 + l->array = NULL;
336 + goto cleanup;
337 + }
338 +
339 + // Allocate memory for the list items
340 + l->array = callocz(itemCount, sizeof(struct provider_data));
341 + l->total = itemCount;
342 +
343 + uint64_t min = UINT64_MAX, max = 0;
344 +
345 + // Iterate over the list and populate the entries
346 + for (DWORD i = 0; i < itemCount; ++i) {
347 + struct provider_data *d = &l->array[i];
348 +
349 + // Get the value (e.g., opcode, task, or level)
350 + if (wevt_get_property_from_array(property, hArray, i, value_id)) {
351 + switch(value_bits) {
352 + case 64:
353 + d->value = wevt_field_get_uint64(property->data);
354 + break;
355 +
356 + case 32:
357 + d->value = wevt_field_get_uint32(property->data);
358 + break;
359 + }
360 +
361 + if(d->value < min)
362 + min = d->value;
363 +
364 + if(d->value > max)
365 + max = d->value;
366 +
367 + if(!is_valid(d->value, false))
368 + l->exceeds_data_type = true;
369 + }
370 +
371 + // Get the message ID
372 + if (wevt_get_property_from_array(property, hArray, i, message_id)) {
373 + uint32_t messageID = wevt_field_get_uint32(property->data);
374 +
375 + if (messageID != (uint32_t)-1) {
376 + if (wevt_get_message_unicode(unicode, hMetadata, NULL, messageID, EvtFormatMessageId)) {
377 + size_t len;
378 + d->name = unicode2utf8_strdupz(unicode->data, &len);
379 + d->len = len;
380 + }
381 + }
382 + }
383 +
384 + // Get the name if the message is missing
385 + if (!d->name && wevt_get_property_from_array(property, hArray, i, name_id)) {
386 + fatal_assert(property->data->Type == EvtVarTypeString);
387 + size_t len;
388 + d->name = unicode2utf8_strdupz(property->data->StringVal, &len);
389 + d->len = len;
390 + }
391 +
392 + // Calculate the hash for the name
393 + if (d->name)
394 + d->hash = XXH3_64bits(d->name, d->len);
395 + }
396 +
397 + l->min = min;
398 + l->max = max;
399 +
400 + if(itemCount > 1 && compare_func != NULL) {
401 + // Sort the array based on the value (ascending for all except keywords, descending for keywords)
402 + qsort(l->array, itemCount, sizeof(struct provider_data), compare_func);
403 + }
404 +
405 +cleanup:
406 + if (hArray)
407 + EvtClose(hArray);
408 +}
409 +
410 +// --------------------------------------------------------------------------------------------------------------------
411 +// lookup functions
412 +
413 +// lookup bitmap metdata (returns a comma separated list of strings)
414 +static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
415 + if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
416 + return false;
417 +
418 + dst->used = 0;
419 +
420 + for(size_t k = 0; value && k < l->total; k++) {
421 + struct provider_data *d = &l->array[k];
422 +
423 + if(d->value && (value & d->value) == d->value && d->name && d->len) {
424 + const char *s = d->name;
425 + size_t slen = d->len;
426 +
427 + // remove the mask from the value
428 + value &= ~(d->value);
429 +
430 + txt_utf8_resize(dst, dst->used + slen + 2 + 1, true);
431 +
432 + if(dst->used) {
433 + // add a comma and a space
434 + dst->data[dst->used++] = ',';
435 + dst->data[dst->used++] = ' ';
436 + }
437 +
438 + memcpy(&dst->data[dst->used], s, slen);
439 + dst->used += slen;
440 + dst->src = TXT_SOURCE_PUBLISHER;
441 + }
442 + }
443 +
444 + if(dst->used) {
445 + txt_utf8_resize(dst, dst->used + 1, true);
446 + dst->data[dst->used++] = 0;
447 + }
448 +
449 + fatal_assert(dst->used <= dst->size);
450 +
451 + return (dst->used > 0);
452 +}
453 +
454 +//// lookup a single value (returns its string)
455 +//static bool publisher_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
456 +// if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
457 +// return false;
458 +//
459 +// dst->used = 0;
460 +//
461 +// for(size_t k = 0; k < l->total; k++) {
462 +// struct provider_data *d = &l->array[k];
463 +//
464 +// if(d->value == value && d->name && d->len) {
465 +// const char *s = d->name;
466 +// size_t slen = d->len;
467 +//
468 +// txt_utf8_resize(dst, slen + 1, false);
469 +//
470 +// memcpy(dst->data, s, slen);
471 +// dst->used = slen;
472 +// dst->src = TXT_SOURCE_PUBLISHER;
473 +//
474 +// break;
475 +// }
476 +// }
477 +//
478 +// if(dst->used) {
479 +// txt_utf8_resize(dst, dst->used + 1, true);
480 +// dst->data[dst->used++] = 0;
481 +// }
482 +//
483 +// fatal_assert(dst->used <= dst->size);
484 +//
485 +// return (dst->used > 0);
486 +//}
487 +
488 +static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
489 + if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
490 + return false;
491 +
492 + // if(l->total < 3) return publisher_value_metadata_linear(dst, l, value);
493 +
494 + dst->used = 0;
495 +
496 + size_t left = 0;
497 + size_t right = l->total - 1;
498 +
499 + // Binary search within bounds
500 + while (left <= right) {
501 + size_t mid = left + (right - left) / 2;
502 + struct provider_data *d = &l->array[mid];
503 +
504 + if (d->value == value) {
505 + // Value found, now check if it has a valid name and length
506 + if (d->name && d->len) {
507 + const char *s = d->name;
508 + size_t slen = d->len;
509 +
510 + txt_utf8_resize(dst, slen + 1, false);
511 + memcpy(dst->data, s, slen);
512 + dst->used = slen;
513 + dst->data[dst->used++] = 0;
514 + dst->src = TXT_SOURCE_PUBLISHER;
515 + }
516 + break;
517 + }
518 +
519 + if (d->value < value)
520 + left = mid + 1;
521 + else {
522 + if (mid == 0) break;
523 + right = mid - 1;
524 + }
525 + }
526 +
527 + fatal_assert(dst->used <= dst->size);
528 +
529 + return (dst->used > 0);
530 +}
531 +
532 +// --------------------------------------------------------------------------------------------------------------------
533 +// public API to lookup metadata
534 +
535 +bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h) {
536 + return h && !h->publisher->keywords.exceeds_data_type;
537 +}
538 +
539 +bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h) {
540 + return h && !h->publisher->tasks.exceeds_data_type;
541 +}
542 +
543 +bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h) {
544 + return h && !h->publisher->levels.exceeds_data_type;
545 +}
546 +
547 +bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
548 + return h && !h->publisher->opcodes.exceeds_data_type;
549 +}
550 +
551 +bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
552 + if(!h) return false;
553 + return publisher_bitmap_metadata(dst, &h->publisher->keywords, value);
554 +}
555 +
556 +bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
557 + if(!h) return false;
558 + return publisher_value_metadata(dst, &h->publisher->levels, value);
559 +}
560 +
561 +bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
562 + if(!h) return false;
563 + return publisher_value_metadata(dst, &h->publisher->tasks, value);
564 +}
565 +
566 +bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
567 + if(!h) return false;
568 + return publisher_value_metadata(dst, &h->publisher->opcodes, value);
569 +}
src/collectors/windows-events.plugin/windows-events-publishers.h new
+28
@@ -0,0 +1,28 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
4 +#define NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
5 +
6 +#include "windows-events.h"
7 +
8 +struct provider_meta_handle;
9 +typedef struct provider_meta_handle PROVIDER_META_HANDLE;
10 +
11 +PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName);
12 +void publisher_release(PROVIDER_META_HANDLE *h);
13 +EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h);
14 +PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h);
15 +
16 +void publisher_cache_init(void);
17 +
18 +bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h);
19 +bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h);
20 +bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h);
21 +bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h);
22 +
23 +bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
24 +bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
25 +bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
26 +bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
27 +
28 +#endif //NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
src/collectors/windows-events.plugin/windows-events-query.c
+693 -675
@@ -1,675 +1,693 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "windows-events-query.h"
4 -
5 -static uint64_t wevt_log_file_size(const wchar_t *channel);
6 -
7 -#define FIELD_CHANNEL (0)
8 -#define FIELD_PROVIDER_NAME (1)
9 -#define FIELD_EVENT_SOURCE_NAME (2)
10 -#define FIELD_PROVIDER_GUID (3)
11 -#define FIELD_RECORD_NUMBER (4)
12 -#define FIELD_EVENT_ID (5)
13 -#define FIELD_LEVEL (6)
14 -#define FIELD_KEYWORDS (7)
15 -#define FIELD_TIME_CREATED (8)
16 -#define FIELD_COMPUTER_NAME (9)
17 -#define FIELD_USER_ID (10)
18 -#define FIELD_CORRELATION_ACTIVITY_ID (11)
19 -#define FIELD_OPCODE (12)
20 -#define FIELD_VERSION (13)
21 -#define FIELD_TASK (14)
22 -#define FIELD_PROCESS_ID (15)
23 -#define FIELD_THREAD_ID (16)
24 -#define FIELD_EVENT_DATA (17)
25 -
26 -// These are the fields we extract from the logs
27 -static const wchar_t *RENDER_ITEMS[] = {
28 - L"/Event/System/Channel",
29 - L"/Event/System/Provider/@Name",
30 - L"/Event/System/Provider/@EventSourceName",
31 - L"/Event/System/Provider/@Guid",
32 - L"/Event/System/EventRecordID",
33 - L"/Event/System/EventID",
34 - L"/Event/System/Level",
35 - L"/Event/System/Keywords",
36 - L"/Event/System/TimeCreated/@SystemTime",
37 - L"/Event/System/Computer",
38 - L"/Event/System/Security/@UserID",
39 - L"/Event/System/Correlation/@ActivityID",
40 - L"/Event/System/Opcode",
41 - L"/Event/System/Version",
42 - L"/Event/System/Task",
43 - L"/Event/System/Execution/@ProcessID",
44 - L"/Event/System/Execution/@ThreadID",
45 - L"/Event/EventData/Data"
46 -};
47 -
48 -static bool wevt_GUID_to_ND_UUID(ND_UUID *nd_uuid, const GUID *guid) {
49 - if(guid && sizeof(GUID) == sizeof(ND_UUID)) {
50 - memcpy(nd_uuid->uuid, guid, sizeof(ND_UUID));
51 - return true;
52 - }
53 - else {
54 - *nd_uuid = UUID_ZERO;
55 - return false;
56 - }
57 -}
58 -
59 -static uint64_t wevt_get_unsigned_by_type(WEVT_LOG *log, size_t field) {
60 - switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
61 - case EvtVarTypeHexInt64: return log->ops.content.data[field].UInt64Val;
62 - case EvtVarTypeHexInt32: return log->ops.content.data[field].UInt32Val;
63 - case EvtVarTypeUInt64: return log->ops.content.data[field].UInt64Val;
64 - case EvtVarTypeUInt32: return log->ops.content.data[field].UInt32Val;
65 - case EvtVarTypeUInt16: return log->ops.content.data[field].UInt16Val;
66 - case EvtVarTypeInt64: return ABS(log->ops.content.data[field].Int64Val);
67 - case EvtVarTypeInt32: return ABS(log->ops.content.data[field].Int32Val);
68 - case EvtVarTypeByte: return log->ops.content.data[field].ByteVal;
69 - case EvtVarTypeInt16: return ABS(log->ops.content.data[field].Int16Val);
70 - case EvtVarTypeSByte: return ABS(log->ops.content.data[field].SByteVal);
71 - case EvtVarTypeSingle: return ABS(log->ops.content.data[field].SingleVal);
72 - case EvtVarTypeDouble: return ABS(log->ops.content.data[field].DoubleVal);
73 - case EvtVarTypeBoolean: return log->ops.content.data[field].BooleanVal ? 1 : 0;
74 - case EvtVarTypeSizeT: return log->ops.content.data[field].SizeTVal;
75 - default: return 0;
76 - }
77 -}
78 -
79 -static uint64_t wevt_get_filetime_to_ns_by_type(WEVT_LOG *log, size_t field) {
80 - switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
81 - case EvtVarTypeFileTime:
82 - case EvtVarTypeUInt64:
83 - return os_windows_ulonglong_to_unix_epoch_ns(log->ops.content.data[field].FileTimeVal);
84 -
85 - default: return 0;
86 - }
87 -}
88 -
89 -static bool wevt_get_uuid_by_type(WEVT_LOG *log, size_t field, ND_UUID *dst) {
90 - switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
91 - case EvtVarTypeGuid:
92 - return wevt_GUID_to_ND_UUID(dst, log->ops.content.data[field].GuidVal);
93 -
94 - default:
95 - return wevt_GUID_to_ND_UUID(dst, NULL);
96 - }
97 -}
98 -
99 -static void wevt_empty_utf8(TXT_UTF8 *dst) {
100 - txt_utf8_resize(dst, 1);
101 - dst->data[0] = '\0';
102 - dst->used = 1;
103 -}
104 -
105 -static bool wevt_get_message_utf8(WEVT_LOG *log, EVT_HANDLE event_handle, TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS what) {
106 - DWORD size = 0;
107 -
108 - if(!log->ops.unicode.data) {
109 - EvtFormatMessage(NULL, event_handle, 0, 0, NULL, what, 0, NULL, &size);
110 - if(!size) {
111 - // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() to get message size failed.");
112 - goto cleanup;
113 - }
114 - txt_unicode_resize(&log->ops.unicode, size);
115 - }
116 -
117 - // First, try to get the message using the existing buffer
118 - if (!EvtFormatMessage(NULL, event_handle, 0, 0, NULL, what, log->ops.unicode.size, log->ops.unicode.data, &size) || !log->ops.unicode.data) {
119 - if (log->ops.unicode.data && GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
120 - // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() failed.");
121 - goto cleanup;
122 - }
123 -
124 - // Try again with the resized buffer
125 - txt_unicode_resize(&log->ops.unicode, size);
126 - if (!EvtFormatMessage(NULL, event_handle, 0, 0, NULL, what, log->ops.unicode.size, log->ops.unicode.data, &size)) {
127 - // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() failed after resizing buffer.");
128 - goto cleanup;
129 - }
130 - }
131 -
132 - // make sure it is null terminated
133 - if(size <= log->ops.unicode.size)
134 - log->ops.unicode.data[size - 1] = 0;
135 - else
136 - log->ops.unicode.data[log->ops.unicode.size - 1] = 0;
137 -
138 - // unfortunately we have to calculate the length every time
139 - // the size returned may not be the length of the unicode string
140 - log->ops.unicode.used = wcslen(log->ops.unicode.data) + 1;
141 -
142 - return wevt_str_unicode_to_utf8(dst, &log->ops.unicode);
143 -
144 -cleanup:
145 - wevt_empty_utf8(dst);
146 - return false;
147 -}
148 -
149 -static bool wevt_get_utf8_by_type(WEVT_LOG *log, size_t field, TXT_UTF8 *dst) {
150 - switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
151 - case EvtVarTypeString:
152 - return wevt_str_wchar_to_utf8(dst, log->ops.content.data[field].StringVal, -1);
153 -
154 - default:
155 - wevt_empty_utf8(dst);
156 - return false;
157 - }
158 -}
159 -
160 -static bool wevt_get_sid_by_type(WEVT_LOG *log, size_t field, TXT_UTF8 *dst) {
161 - switch(log->ops.content.data[field].Type & EVT_VARIANT_TYPE_MASK) {
162 - case EvtVarTypeSid:
163 - return wevt_convert_user_id_to_name(log->ops.content.data[field].SidVal, dst);
164 -
165 - default:
166 - wevt_empty_utf8(dst);
167 - return false;
168 - }
169 -}
170 -
171 -void wevt_field_to_buffer_append(BUFFER *wb, EVT_VARIANT *d, const char *prefix, const char *suffix) {
172 - buffer_strcat(wb, prefix);
173 -
174 - switch(d->Type & EVT_VARIANT_TYPE_MASK) {
175 - case EvtVarTypeNull:
176 - buffer_strcat(wb, "(null)");
177 - break;
178 -
179 - case EvtVarTypeString:
180 - buffer_sprintf(wb, "%ls", d->StringVal);
181 - break;
182 -
183 - case EvtVarTypeSByte:
184 - buffer_sprintf(wb, "%d", (int) d->SByteVal);
185 - break;
186 -
187 - case EvtVarTypeByte:
188 - buffer_sprintf(wb, "%u", (unsigned) d->ByteVal);
189 - break;
190 -
191 - case EvtVarTypeInt16:
192 - buffer_sprintf(wb, "%d", (int) d->Int16Val);
193 - break;
194 -
195 - case EvtVarTypeUInt16:
196 - buffer_sprintf(wb, "%u", (unsigned) d->UInt16Val);
197 - break;
198 -
199 - case EvtVarTypeInt32:
200 - buffer_sprintf(wb, "%d", (int) d->Int32Val);
201 - break;
202 -
203 - case EvtVarTypeUInt32:
204 - case EvtVarTypeHexInt32:
205 - buffer_sprintf(wb, "%u", (unsigned) d->UInt32Val);
206 - break;
207 -
208 - case EvtVarTypeInt64:
209 - buffer_sprintf(wb, "%" PRIi64, (uint64_t)d->Int64Val);
210 - break;
211 -
212 - case EvtVarTypeUInt64:
213 - case EvtVarTypeHexInt64:
214 - buffer_sprintf(wb, "%" PRIu64, (uint64_t)d->UInt64Val);
215 - break;
216 -
217 - case EvtVarTypeSizeT:
218 - buffer_sprintf(wb, "%zu", d->SizeTVal);
219 - break;
220 -
221 - case EvtVarTypeSysTime:
222 - buffer_sprintf(wb, "%04d-%02d-%02dT%02d:%02d:%02d.%03d (SystemTime)",
223 - d->SysTimeVal->wYear, d->SysTimeVal->wMonth, d->SysTimeVal->wDay,
224 - d->SysTimeVal->wHour, d->SysTimeVal->wMinute, d->SysTimeVal->wSecond,
225 - d->SysTimeVal->wMilliseconds);
226 - break;
227 -
228 - case EvtVarTypeGuid: {
229 - char uuid_str[UUID_COMPACT_STR_LEN];
230 - ND_UUID uuid;
231 - if (wevt_GUID_to_ND_UUID(&uuid, d->GuidVal)) {
232 - uuid_unparse_lower_compact(uuid.uuid, uuid_str);
233 - buffer_strcat(wb, uuid_str);
234 - buffer_strcat(wb, " (GUID)");
235 - }
236 - break;
237 - }
238 -
239 - case EvtVarTypeSingle:
240 - buffer_print_netdata_double(wb, d->SingleVal);
241 - break;
242 -
243 - case EvtVarTypeDouble:
244 - buffer_print_netdata_double(wb, d->DoubleVal);
245 - break;
246 -
247 - case EvtVarTypeBoolean:
248 - buffer_strcat(wb, d->BooleanVal ? "true" : "false");
249 - break;
250 -
251 - case EvtVarTypeFileTime: {
252 - nsec_t ns = os_windows_ulonglong_to_unix_epoch_ns(d->FileTimeVal);
253 - char buf[RFC3339_MAX_LENGTH];
254 - rfc3339_datetime_ut(buf, sizeof(buf), ns, 2, true);
255 - buffer_strcat(wb, buf);
256 - buffer_strcat(wb, " (FileTime)");
257 - break;
258 - }
259 -
260 - case EvtVarTypeBinary:
261 - buffer_strcat(wb, "(binary data)");
262 - break;
263 -
264 - case EvtVarTypeSid:
265 - buffer_strcat(wb, "(user id data)");
266 - break;
267 -
268 - case EvtVarTypeAnsiString:
269 - case EvtVarTypeEvtHandle:
270 - case EvtVarTypeEvtXml:
271 - default:
272 - buffer_sprintf(wb, "(unsupported data type: %u)", d->Type);
273 - break;
274 - }
275 -
276 - buffer_strcat(wb, suffix);
277 -}
278 -
279 -static bool wevt_format_summary_array_traversal(WEVT_LOG *log, size_t field, WEVT_EVENT *ev) {
280 - EVT_VARIANT *d = &log->ops.content.data[field];
281 -
282 - if (!log->ops.message)
283 - log->ops.message = buffer_create(0, NULL);
284 -
285 - BUFFER *wb = log->ops.message;
286 - buffer_flush(wb);
287 -
288 - // Check if there is an event description
289 - if (log->ops.event.data && *log->ops.event.data) {
290 - buffer_strcat(wb, log->ops.event.data);
291 - buffer_strcat(wb, "\nRelated data:\n");
292 - } else {
293 - buffer_sprintf(wb, "Event %" PRIu64 ", with the following related data:\n", (uint64_t)ev->event_id);
294 - }
295 -
296 - // Check if the field contains an array or a single value
297 - bool is_array = (d->Type & EVT_VARIANT_TYPE_ARRAY) != 0;
298 - EVT_VARIANT_TYPE base_type = (EVT_VARIANT_TYPE)(d->Type & EVT_VARIANT_TYPE_MASK);
299 -
300 - if (is_array) {
301 - DWORD count = d->Count; // Number of elements in the array
302 -
303 - for (DWORD i = 0; i < count; i++) {
304 - EVT_VARIANT array_element = {
305 - .Type = base_type,
306 - .Count = 0,
307 - };
308 -
309 - // Point the array element to the correct data
310 - switch (base_type) {
311 - case EvtVarTypeBoolean:
312 - array_element.BooleanVal = d->BooleanArr[i];
313 - break;
314 - case EvtVarTypeSByte:
315 - array_element.SByteVal = d->SByteArr[i];
316 - break;
317 - case EvtVarTypeByte:
318 - array_element.ByteVal = d->ByteArr[i];
319 - break;
320 - case EvtVarTypeInt16:
321 - array_element.Int16Val = d->Int16Arr[i];
322 - break;
323 - case EvtVarTypeUInt16:
324 - array_element.UInt16Val = d->UInt16Arr[i];
325 - break;
326 - case EvtVarTypeInt32:
327 - array_element.Int32Val = d->Int32Arr[i];
328 - break;
329 - case EvtVarTypeUInt32:
330 - array_element.UInt32Val = d->UInt32Arr[i];
331 - break;
332 - case EvtVarTypeInt64:
333 - array_element.Int64Val = d->Int64Arr[i];
334 - break;
335 - case EvtVarTypeUInt64:
336 - array_element.UInt64Val = d->UInt64Arr[i];
337 - break;
338 - case EvtVarTypeSingle:
339 - array_element.SingleVal = d->SingleArr[i];
340 - break;
341 - case EvtVarTypeDouble:
342 - array_element.DoubleVal = d->DoubleArr[i];
343 - break;
344 - case EvtVarTypeFileTime:
345 - array_element.FileTimeVal = ((ULONGLONG)d->FileTimeArr[i].dwLowDateTime |
346 - ((ULONGLONG)d->FileTimeArr[i].dwHighDateTime << 32));
347 - break;
348 - case EvtVarTypeSysTime:
349 - array_element.SysTimeVal = &d->SysTimeArr[i];
350 - break;
351 - case EvtVarTypeGuid:
352 - array_element.GuidVal = &d->GuidArr[i];
353 - break;
354 - case EvtVarTypeString:
355 - array_element.StringVal = d->StringArr[i];
356 - break;
357 - case EvtVarTypeAnsiString:
358 - array_element.AnsiStringVal = d->AnsiStringArr[i];
359 - break;
360 - case EvtVarTypeSid:
361 - array_element.SidVal = d->SidArr[i];
362 - break;
363 - case EvtVarTypeSizeT:
364 - array_element.SizeTVal = d->SizeTArr[i];
365 - break;
366 - case EvtVarTypeEvtXml:
367 - array_element.XmlVal = d->XmlValArr[i];
368 - break;
369 - default:
370 - buffer_sprintf(wb, " - Unsupported array type: %u\n", base_type);
371 - continue;
372 - }
373 -
374 - // Call the field appending function for each array element
375 - wevt_field_to_buffer_append(wb, &array_element, " - ", "\n");
376 - }
377 - } else {
378 - // Handle single values, pass the EVT_VARIANT directly
379 - wevt_field_to_buffer_append(wb, d, " - ", "\n");
380 - }
381 -
382 - return true;
383 -}
384 -
385 -static bool wevt_format_summary(WEVT_LOG *log, WEVT_EVENT *ev) {
386 - if (!log->ops.message)
387 - log->ops.message = buffer_create(0, NULL);
388 -
389 - BUFFER *wb = log->ops.message;
390 - buffer_flush(wb);
391 -
392 - // Check if there is an event description
393 - if (log->ops.event.data && *log->ops.event.data)
394 - buffer_strcat(wb, log->ops.event.data);
395 - else
396 - buffer_sprintf(wb, "Event %" PRIu64, (uint64_t) ev->event_id);
397 -
398 - const char *xml = log->ops.xml.data;
399 - const char *event_data_start = strstr(xml, "<EventData>");
400 - if(event_data_start)
401 - event_data_start = &event_data_start[11];
402 -
403 - const char *event_data_end = event_data_start ? strstr(xml, "</EventData>") : NULL;
404 -
405 - if(event_data_start && event_data_end) {
406 - buffer_strcat(wb, "\n\nRelated data:\n\n");
407 - // copy the event data block
408 - buffer_fast_strcat(wb, event_data_start, event_data_end - event_data_start);
409 - }
410 -
411 - return true;
412 -}
413 -
414 -bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
415 - DWORD size_required_next = 0, size = 0, bookmarkedCount = 0;
416 - EVT_HANDLE tmp_event_bookmark = NULL;
417 - bool ret = false;
418 -
419 - fatal_assert(log && log->event_query && log->render_context);
420 -
421 - if (!EvtNext(log->event_query, 1, &tmp_event_bookmark, INFINITE, 0, &size_required_next))
422 - goto cleanup; // no data available, return failure
423 -
424 - // obtain the information from selected events
425 - if (!EvtRender(log->render_context, tmp_event_bookmark, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &size, &bookmarkedCount)) {
426 - // information exceeds the allocated space
427 - if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
428 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed");
429 - goto cleanup;
430 - }
431 -
432 - freez(log->ops.content.data);
433 - log->ops.content.size = size;
434 - log->ops.content.data = (EVT_VARIANT *)mallocz(log->ops.content.size);
435 -
436 - if (!EvtRender(log->render_context, tmp_event_bookmark, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &size, &bookmarkedCount)) {
437 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, after size increase");
438 - goto cleanup;
439 - }
440 - }
441 - log->ops.content.len = size;
442 -
443 - ev->id = wevt_get_unsigned_by_type(log, FIELD_RECORD_NUMBER);
444 - ev->event_id = wevt_get_unsigned_by_type(log, FIELD_EVENT_ID);
445 - ev->level = wevt_get_unsigned_by_type(log, FIELD_LEVEL);
446 - ev->keyword = wevt_get_unsigned_by_type(log, FIELD_KEYWORDS);
447 - ev->created_ns = wevt_get_filetime_to_ns_by_type(log, FIELD_TIME_CREATED);
448 - ev->opcode = wevt_get_unsigned_by_type(log, FIELD_OPCODE);
449 - ev->version = wevt_get_unsigned_by_type(log, FIELD_VERSION);
450 - ev->task = wevt_get_unsigned_by_type(log, FIELD_TASK);
451 - ev->process_id = wevt_get_unsigned_by_type(log, FIELD_PROCESS_ID);
452 - ev->thread_id = wevt_get_unsigned_by_type(log, FIELD_THREAD_ID);
453 -
454 - if(full) {
455 - wevt_get_utf8_by_type(log, FIELD_CHANNEL, &log->ops.channel);
456 - wevt_get_utf8_by_type(log, FIELD_PROVIDER_NAME, &log->ops.provider);
457 - wevt_get_utf8_by_type(log, FIELD_EVENT_SOURCE_NAME, &log->ops.source);
458 - wevt_get_uuid_by_type(log, FIELD_PROVIDER_GUID, &ev->provider);
459 - wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.level, EvtFormatMessageLevel);
460 -
461 - if(ev->event_id)
462 - wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.event, EvtFormatMessageEvent);
463 - else
464 - wevt_empty_utf8(&log->ops.event);
465 -
466 - if(ev->keyword)
467 - wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.keyword, EvtFormatMessageKeyword);
468 - else
469 - wevt_empty_utf8(&log->ops.keyword);
470 -
471 - if(ev->opcode)
472 - wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.opcode, EvtFormatMessageOpcode);
473 - else
474 - wevt_empty_utf8(&log->ops.keyword);
475 -
476 - // ComputerName
477 - wevt_get_utf8_by_type(log, FIELD_COMPUTER_NAME, &log->ops.computer);
478 -
479 - // User
480 - wevt_get_sid_by_type(log, FIELD_USER_ID, &log->ops.user);
481 -
482 - // CorrelationActivityID
483 - wevt_get_uuid_by_type(log, FIELD_CORRELATION_ACTIVITY_ID, &ev->correlation_activity_id);
484 -
485 - // Full XML of the entire message
486 - wevt_get_message_utf8(log, tmp_event_bookmark, &log->ops.xml, EvtFormatMessageXml);
487 -
488 - // Format a text message for the users to see
489 - // wevt_format_summary(log, ev);
490 - }
491 -
492 - ret = true;
493 -
494 -cleanup:
495 - if (tmp_event_bookmark)
496 - EvtClose(tmp_event_bookmark);
497 -
498 - return ret;
499 -}
500 -
501 -void wevt_query_done(WEVT_LOG *log) {
502 - if (!log->event_query) return;
503 -
504 - EvtClose(log->event_query);
505 - log->event_query = NULL;
506 -}
507 -
508 -void wevt_closelog6(WEVT_LOG *log) {
509 - wevt_query_done(log);
510 -
511 - if (log->render_context)
512 - EvtClose(log->render_context);
513 -
514 - freez(log->ops.content.data);
515 - txt_unicode_cleanup(&log->ops.unicode);
516 - txt_utf8_cleanup(&log->ops.channel);
517 - txt_utf8_cleanup(&log->ops.provider);
518 - txt_utf8_cleanup(&log->ops.source);
519 - txt_utf8_cleanup(&log->ops.computer);
520 - txt_utf8_cleanup(&log->ops.event);
521 - txt_utf8_cleanup(&log->ops.user);
522 - txt_utf8_cleanup(&log->ops.opcode);
523 - txt_utf8_cleanup(&log->ops.level);
524 - txt_utf8_cleanup(&log->ops.keyword);
525 - txt_utf8_cleanup(&log->ops.xml);
526 - buffer_free(log->ops.message);
527 - freez(log);
528 -}
529 -
530 -bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention) {
531 - bool ret = false;
532 -
533 - // get the number of the oldest record in the log
534 - // "EvtGetLogInfo()" does not work properly with "EvtLogOldestRecordNumber"
535 - // we have to get it from the first EventRecordID
536 -
537 - // query the eventlog
538 - log->event_query = EvtQuery(NULL, channel, NULL, EvtQueryChannelPath);
539 - if (!log->event_query) {
540 - if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
541 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, channel '%s' not found, cannot open log", channel2utf8(channel));
542 - else
543 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() on channel '%s' failed, cannot open log", channel2utf8(channel));
544 -
545 - goto cleanup;
546 - }
547 -
548 - if (!wevt_get_next_event(log, &retention->first_event, false))
549 - goto cleanup;
550 -
551 - if (!retention->first_event.id) {
552 - // no data in the event log
553 - retention->first_event = retention->last_event = WEVT_EVENT_EMPTY;
554 - ret = true;
555 - goto cleanup;
556 - }
557 - EvtClose(log->event_query);
558 -
559 - log->event_query = EvtQuery(NULL, channel, NULL, EvtQueryChannelPath | EvtQueryReverseDirection);
560 - if (!log->event_query) {
561 - if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
562 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, channel '%s' not found", channel2utf8(channel));
563 - else
564 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() on channel '%s' failed", channel2utf8(channel));
565 -
566 - goto cleanup;
567 - }
568 -
569 - if (!wevt_get_next_event(log, &retention->last_event, false) || retention->last_event.id == 0) {
570 - // no data in eventlog
571 - retention->last_event = retention->first_event;
572 - }
573 - retention->last_event.id += 1; // we should read the last record
574 - ret = true;
575 -
576 -cleanup:
577 - wevt_query_done(log);
578 -
579 - if(ret) {
580 - retention->entries = retention->last_event.id - retention->first_event.id;
581 -
582 - if(retention->last_event.created_ns >= retention->first_event.created_ns)
583 - retention->duration_ns = retention->last_event.created_ns - retention->first_event.created_ns;
584 - else
585 - retention->duration_ns = retention->first_event.created_ns - retention->last_event.created_ns;
586 -
587 - retention->size_bytes = wevt_log_file_size(channel);
588 - }
589 - else
590 - memset(retention, 0, sizeof(*retention));
591 -
592 - return ret;
593 -}
594 -
595 -WEVT_LOG *wevt_openlog6(void) {
596 - size_t RENDER_ITEMS_count = (sizeof(RENDER_ITEMS) / sizeof(const wchar_t *));
597 -
598 - WEVT_LOG *log = callocz(1, sizeof(*log));
599 -
600 - // create the system render
601 - log->render_context = EvtCreateRenderContext(RENDER_ITEMS_count, RENDER_ITEMS, EvtRenderContextValues);
602 - if (!log->render_context) {
603 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtCreateRenderContext failed.");
604 - goto cleanup;
605 - }
606 -
607 -cleanup:
608 - return log;
609 -}
610 -
611 -static uint64_t wevt_log_file_size(const wchar_t *channel) {
612 - EVT_HANDLE hLog = NULL;
613 - EVT_VARIANT evtVariant;
614 - DWORD bufferUsed = 0;
615 - uint64_t file_size = 0;
616 -
617 - // Open the event log channel
618 - hLog = EvtOpenLog(NULL, channel, EvtOpenChannelPath);
619 - if (!hLog) {
620 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtOpenLog() on channel '%s' failed", channel2utf8(channel));
621 - goto cleanup;
622 - }
623 -
624 - // Get the file size of the log
625 - if (!EvtGetLogInfo(hLog, EvtLogFileSize, sizeof(evtVariant), &evtVariant, &bufferUsed)) {
626 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetLogInfo() on channel '%s' failed", channel2utf8(channel));
627 - goto cleanup;
628 - }
629 -
630 - // Extract the file size from the EVT_VARIANT structure
631 - file_size = evtVariant.UInt64Val;
632 -
633 -cleanup:
634 - if (hLog)
635 - EvtClose(hLog);
636 -
637 - return file_size;
638 -}
639 -
640 -EVT_HANDLE wevt_query(LPCWSTR channel, usec_t seek_to, bool backward) {
641 - // Convert microseconds to nanoseconds first (correct handling of precision)
642 - if(backward) seek_to += USEC_PER_MS; // for backward mode, add a millisecond to the seek time.
643 -
644 - // Convert the microseconds since Unix epoch to FILETIME (used in Windows APIs)
645 - FILETIME fileTime = os_unix_epoch_ut_to_filetime(seek_to);
646 -
647 - // Convert FILETIME to SYSTEMTIME for use in XPath
648 - SYSTEMTIME systemTime;
649 - if (!FileTimeToSystemTime(&fileTime, &systemTime)) {
650 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "FileTimeToSystemTime() failed");
651 - return NULL;
652 - }
653 -
654 - // Format SYSTEMTIME into ISO 8601 format (YYYY-MM-DDTHH:MM:SS.sssZ)
655 - static __thread WCHAR query[4096];
656 - swprintf(query, 4096,
657 - L"Event/System[TimeCreated[@SystemTime%ls\"%04d-%02d-%02dT%02d:%02d:%02d.%03dZ\"]]",
658 - backward ? L"<=" : L">=", // Use <= if backward, >= if forward
659 - systemTime.wYear, systemTime.wMonth, systemTime.wDay,
660 - systemTime.wHour, systemTime.wMinute, systemTime.wSecond, systemTime.wMilliseconds);
661 -
662 - // Execute the query
663 - EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (backward ? EvtQueryReverseDirection : EvtQueryForwardDirection));
664 - if (!hQuery) {
665 - wchar_t wbuf[1024];
666 - DWORD wbuf_used;
667 - EvtGetExtendedStatus(sizeof(wbuf), wbuf, &wbuf_used);
668 -
669 - char buf[1024];
670 - rfc3339_datetime_ut(buf, sizeof(buf), seek_to, 3, true);
671 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, seek to '%s', query: %s | extended info: %ls", buf, query2utf8(query), wbuf);
672 - }
673 -
674 - return hQuery;
675 -}
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "windows-events-query.h"
4 +
5 +static uint64_t wevt_log_file_size(const wchar_t *channel);
6 +
7 +#define FIELD_RECORD_NUMBER (0)
8 +#define FIELD_EVENT_ID (1)
9 +#define FIELD_LEVEL (2)
10 +#define FIELD_OPCODE (3)
11 +#define FIELD_KEYWORDS (4)
12 +#define FIELD_VERSION (5)
13 +#define FIELD_TASK (6)
14 +#define FIELD_PROCESS_ID (7)
15 +#define FIELD_THREAD_ID (8)
16 +#define FIELD_TIME_CREATED (9)
17 +#define FIELD_CHANNEL (10)
18 +#define FIELD_COMPUTER_NAME (11)
19 +#define FIELD_PROVIDER_NAME (12)
20 +#define FIELD_EVENT_SOURCE_NAME (13)
21 +#define FIELD_PROVIDER_GUID (14)
22 +#define FIELD_CORRELATION_ACTIVITY_ID (15)
23 +#define FIELD_USER_ID (16)
24 +
25 +// These are the fields we extract from the logs
26 +static const wchar_t *RENDER_ITEMS[] = {
27 + L"/Event/System/EventRecordID",
28 + L"/Event/System/EventID",
29 + L"/Event/System/Level",
30 + L"/Event/System/Opcode",
31 + L"/Event/System/Keywords",
32 + L"/Event/System/Version",
33 + L"/Event/System/Task",
34 + L"/Event/System/Execution/@ProcessID",
35 + L"/Event/System/Execution/@ThreadID",
36 + L"/Event/System/TimeCreated/@SystemTime",
37 + L"/Event/System/Channel",
38 + L"/Event/System/Computer",
39 + L"/Event/System/Provider/@Name",
40 + L"/Event/System/Provider/@EventSourceName",
41 + L"/Event/System/Provider/@Guid",
42 + L"/Event/System/Correlation/@ActivityID",
43 + L"/Event/System/Security/@UserID",
44 +};
45 +
46 +static const char *wevt_extended_status(void) {
47 + static __thread wchar_t wbuf[4096];
48 + static __thread char buf[4096];
49 + DWORD wbuf_used = 0;
50 +
51 + if(EvtGetExtendedStatus(sizeof(wbuf) / sizeof(wchar_t), wbuf, &wbuf_used) == ERROR_SUCCESS) {
52 + wbuf[sizeof(wbuf) / sizeof(wchar_t) - 1] = 0;
53 + unicode2utf8(buf, sizeof(buf), wbuf);
54 + }
55 + else
56 + buf[0] = '\0';
57 +
58 + // the EvtGetExtendedStatus() may be successful with an empty message
59 + if(!buf[0])
60 + strncpyz(buf, "no additional information", sizeof(buf) - 1);
61 +
62 + return buf;
63 +}
64 +
65 +bool wevt_get_message_unicode(TXT_UNICODE *unicode, EVT_HANDLE hMetadata, EVT_HANDLE bookmark, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) {
66 + unicode->used = 0;
67 +
68 + DWORD size = 0;
69 + if(!unicode->data) {
70 + EvtFormatMessage(hMetadata, bookmark, dwMessageId, 0, NULL, flags, 0, NULL, &size);
71 + if(!size) {
72 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() to get message size failed.");
73 + goto cleanup;
74 + }
75 + txt_unicode_resize(unicode, size);
76 + }
77 +
78 + // First, try to get the message using the existing buffer
79 + if (!EvtFormatMessage(hMetadata, bookmark, dwMessageId, 0, NULL, flags, unicode->size, unicode->data, &size) || !unicode->data) {
80 + if (unicode->data && GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
81 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() failed.");
82 + goto cleanup;
83 + }
84 +
85 + // Try again with the resized buffer
86 + txt_unicode_resize(unicode, size);
87 + if (!EvtFormatMessage(hMetadata, bookmark, dwMessageId, 0, NULL, flags, unicode->size, unicode->data, &size)) {
88 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() failed after resizing buffer.");
89 + goto cleanup;
90 + }
91 + }
92 +
93 + // make sure it is null terminated
94 + if(size <= unicode->size)
95 + unicode->data[size - 1] = 0;
96 + else
97 + unicode->data[unicode->size - 1] = 0;
98 +
99 + // unfortunately we have to calculate the length every time
100 + // the size returned may not be the length of the unicode string
101 + unicode->used = wcslen(unicode->data) + 1;
102 + return true;
103 +
104 +cleanup:
105 + unicode->used = 0;
106 + return false;
107 +}
108 +
109 +static bool wevt_get_field_from_events_log(
110 + WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE event_handle,
111 + TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags, const char *empty) {
112 +
113 + dst->src = TXT_SOURCE_EVENT_LOG;
114 +
115 + if(wevt_get_message_unicode(&log->ops.unicode, publisher_handle(p), event_handle, 0, flags))
116 + return wevt_str_unicode_to_utf8(dst, &log->ops.unicode);
117 +
118 + if(empty) {
119 + txt_utf8_resize(dst, 128, false);
120 + dst->used = snprintfz(dst->data, dst->size, "%s", empty);
121 + }
122 + else
123 + wevt_utf8_empty(dst);
124 +
125 + return false;
126 +}
127 +
128 +bool wevt_get_event_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE event_handle, TXT_UTF8 *dst) {
129 + return wevt_get_field_from_events_log(
130 + log, p, event_handle, dst, EvtFormatMessageEvent, "No event message for this event.");
131 +}
132 +
133 +bool wevt_get_xml_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE event_handle, TXT_UTF8 *dst) {
134 + return wevt_get_field_from_events_log(
135 + log, p, event_handle, dst, EvtFormatMessageXml, "No XML in this event entry.");
136 +}
137 +
138 +static inline void wevt_event_done(WEVT_LOG *log) {
139 + if (log->publisher) {
140 + publisher_release(log->publisher);
141 + log->publisher = NULL;
142 + }
143 +
144 + if (log->bookmark) {
145 + EvtClose(log->bookmark);
146 + log->bookmark = NULL;
147 + }
148 +
149 + log->ops.level.src = TXT_SOURCE_UNKNOWN;
150 + log->ops.keywords.src = TXT_SOURCE_UNKNOWN;
151 + log->ops.opcode.src = TXT_SOURCE_UNKNOWN;
152 + log->ops.task.src = TXT_SOURCE_UNKNOWN;
153 +}
154 +
155 +static void wevt_get_field_from_cache(
156 + WEVT_LOG *log, uint64_t value, PROVIDER_META_HANDLE *h,
157 + TXT_UTF8 *dst, const ND_UUID *provider,
158 + WEVT_FIELD_TYPE cache_type, EVT_FORMAT_MESSAGE_FLAGS flags) {
159 +
160 + if (field_cache_get(cache_type, provider, value, dst))
161 + return;
162 +
163 + wevt_get_field_from_events_log(log, h, log->bookmark, dst, flags, NULL);
164 + field_cache_set(cache_type, provider, value, dst);
165 +}
166 +
167 +static inline const char *wevt_level_hardcoded(uint64_t level, size_t *len) {
168 + switch(level) {
169 + case 0: *len = 4; return "None";
170 + case 1: *len = 8; return "Critical";
171 + case 2: *len = 5; return "Error";
172 + case 3: *len = 7; return "Warning";
173 + case 4: *len = 11; return "Information";
174 + case 5: *len = 7; return "Verbose";
175 + default: return NULL;
176 + }
177 +}
178 +
179 +static const char *wevt_level_default(uint64_t level, size_t *len) {
180 + static __thread char buf[6 + UINT64_MAX_LENGTH + 1];
181 + strcpy(buf, "Level ");
182 + *len = 6 + print_uint64(&buf[6], level);
183 + return buf;
184 +}
185 +
186 +static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
187 + TXT_UTF8 *dst = &log->ops.level;
188 + uint64_t value = ev->level;
189 +
190 + dst->used = 0; // empty our response
191 +
192 + EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageLevel;
193 + WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_LEVEL;
194 + bool is_publisher = is_valid_publisher_level(value, true);
195 +
196 + if(!is_publisher) {
197 + size_t len;
198 + const char *hardcoded = wevt_level_hardcoded(value, &len);
199 + if(hardcoded) {
200 + txt_utf8_resize(dst, len + 1, false);
201 + memcpy(dst->data, hardcoded, len + 1);
202 + dst->used = len + 1;
203 + dst->src = TXT_SOURCE_HARDCODED;
204 + }
205 + else {
206 + // since this is not a publisher value
207 + // we expect to get the system description of it
208 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
209 + }
210 + }
211 + else if (!publisher_get_level(dst, h, value)) {
212 + // not found in the manifest, get it from the cache
213 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
214 + }
215 +
216 + if(dst->used <= 1) {
217 + // an empty value, provide something meaningful
218 + size_t len = 0;
219 + const char *s = wevt_level_default(value, &len);
220 + txt_utf8_resize(dst, len + 1, false);
221 + memcpy(dst->data, s, len + 1);
222 + dst->used = len + 1;
223 + }
224 +}
225 +
226 +static const char *wevt_opcode_hardcoded(uint64_t level, size_t *len) {
227 + switch(level) {
228 + case 0: *len = 4; return "Info";
229 + case 1: *len = 5; return "Start";
230 + case 2: *len = 4; return "Stop";
231 + case 3: *len = 8; return "DC Start";
232 + case 4: *len = 7; return "DC Stop";
233 + case 5: *len = 9; return "Extension";
234 + case 6: *len = 5; return "Reply";
235 + case 7: *len = 6; return "Resume";
236 + case 8: *len = 7; return "Suspend";
237 + case 9: *len = 4; return "Send";
238 + case 240: *len = 7; return "Receive";
239 + default: *len = 0; return NULL;
240 + }
241 +}
242 +
243 +static const char *wevt_opcode_default(uint64_t opcode, size_t *len) {
244 + static __thread char buf[7 + UINT64_MAX_LENGTH + 1];
245 + strcpy(buf, "Opcode ");
246 + *len = 7 + print_uint64(&buf[7], opcode);
247 + return buf;
248 +}
249 +
250 +static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
251 + TXT_UTF8 *dst = &log->ops.opcode;
252 + uint64_t value = ev->opcode;
253 +
254 + dst->used = 0; // empty our response
255 +
256 + EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageOpcode;
257 + WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_OPCODE;
258 + bool is_publisher = is_valid_publisher_opcode(value, true);
259 +
260 + if(!is_publisher) {
261 + size_t len;
262 + const char *hardcoded = wevt_opcode_hardcoded(value, &len);
263 + if(hardcoded) {
264 + txt_utf8_resize(dst, len + 1, false);
265 + memcpy(dst->data, hardcoded, len + 1);
266 + dst->used = len + 1;
267 + dst->src = TXT_SOURCE_HARDCODED;
268 + }
269 + else {
270 + // since this is not a publisher value
271 + // we expect to get the system description of it
272 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
273 + }
274 + }
275 + else if (!publisher_get_opcode(dst, h, value)) {
276 + // not found in the manifest, get it from the cache
277 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
278 + }
279 +
280 + if(dst->used <= 1) {
281 + // an empty value, provide something meaningful
282 + size_t len = 0;
283 + const char *s = wevt_opcode_default(value, &len);
284 + txt_utf8_resize(dst, len + 1, false);
285 + memcpy(dst->data, s, len + 1);
286 + dst->used = len + 1;
287 + }
288 +}
289 +
290 +static const char *wevt_task_hardcoded(uint64_t task, size_t *len) {
291 + switch(task) {
292 + case 0: *len = 4; return "None";
293 + default: *len = 0; return NULL;
294 + }
295 +}
296 +
297 +static const char *wevt_task_default(uint64_t task, size_t *len) {
298 + static __thread char buf[5 + UINT64_MAX_LENGTH + 1];
299 + strcpy(buf, "Task ");
300 + *len = 5 + print_uint64(&buf[5], task);
301 + return buf;
302 +}
303 +
304 +static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
305 + TXT_UTF8 *dst = &log->ops.task;
306 + uint64_t value = ev->task;
307 +
308 + dst->used = 0; // empty our response
309 +
310 + EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageTask;
311 + WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_TASK;
312 + bool is_publisher = is_valid_publisher_task(value, true);
313 +
314 + if(!is_publisher) {
315 + size_t len;
316 + const char *hardcoded = wevt_task_hardcoded(value, &len);
317 + if(hardcoded) {
318 + txt_utf8_resize(dst, len + 1, false);
319 + memcpy(dst->data, hardcoded, len + 1);
320 + dst->used = len + 1;
321 + dst->src = TXT_SOURCE_HARDCODED;
322 + }
323 + else {
324 + // since this is not a publisher value
325 + // we expect to get the system description of it
326 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
327 + }
328 + }
329 + else if (!publisher_get_task(dst, h, value)) {
330 + // not found in the manifest, get it from the cache
331 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
332 + }
333 +
334 + if(dst->used <= 1) {
335 + // an empty value, provide something meaningful
336 + size_t len = 0;
337 + const char *s = wevt_task_default(value, &len);
338 + txt_utf8_resize(dst, len + 1, false);
339 + memcpy(dst->data, s, len + 1);
340 + dst->used = len + 1;
341 + }
342 +}
343 +
344 +static const char *wevt_keywords_hardcoded(uint64_t keywords, size_t *len) {
345 + switch(keywords) {
346 + case 0: *len = 4; return "None";
347 + default: *len = 0; return NULL;
348 + }
349 +}
350 +
351 +static const char *wevt_keywords_default(uint64_t task, size_t *len) {
352 + static __thread char buf[9 + UINT64_MAX_LENGTH + 1];
353 + strcpy(buf, "Keywords ");
354 + *len = 9 + print_uint64(&buf[9], task);
355 + return buf;
356 +}
357 +
358 +static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
359 + TXT_UTF8 *dst = &log->ops.keywords;
360 +
361 + // microsoft suggests the high 16 bits are for internal use
362 + uint64_t value = ev->keywords & 0x0000FFFFFFFFFFFF;
363 +
364 + dst->used = 0; // empty our response
365 +
366 + EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageKeyword;
367 + WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_KEYWORDS;
368 + bool is_publisher = is_valid_publisher_keywords(value, true);
369 +
370 + if(!is_publisher) {
371 + size_t len;
372 + const char *hardcoded = wevt_keywords_hardcoded(value, &len);
373 + if(hardcoded) {
374 + txt_utf8_resize(dst, len + 1, false);
375 + memcpy(dst->data, hardcoded, len + 1);
376 + dst->used = len + 1;
377 + dst->src = TXT_SOURCE_HARDCODED;
378 + }
379 + else {
380 + // since this is not a publisher value
381 + // we expect to get the system description of it
382 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
383 + }
384 + }
385 + else if (!publisher_get_keywords(dst, h, value)) {
386 + // not found in the manifest, get it from the cache
387 + wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
388 + }
389 +
390 + if(dst->used <= 1) {
391 + // an empty value, provide something meaningful
392 + size_t len = 0;
393 + const char *s = wevt_keywords_default(value, &len);
394 + txt_utf8_resize(dst, len + 1, false);
395 + memcpy(dst->data, s, len + 1);
396 + dst->used = len + 1;
397 + }
398 +}
399 +
400 +bool wevt_get_next_event_one(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
401 + bool ret = false;
402 +
403 + // obtain the information from selected events
404 + DWORD bytes_used = 0, property_count = 0;
405 + if (!EvtRender(log->render_context, log->bookmark, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
406 + // information exceeds the allocated space
407 + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
408 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, render_context: 0x%lx, bookmark: 0x%lx, content: 0x%lx, size: %zu, extended info: %s",
409 + (uintptr_t)log->render_context, (uintptr_t)log->bookmark, (uintptr_t)log->ops.content.data, log->ops.content.size, wevt_extended_status());
410 + goto cleanup;
411 + }
412 +
413 + wevt_variant_resize(&log->ops.content, bytes_used);
414 + if (!EvtRender(log->render_context, log->bookmark, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
415 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, after bytes_used increase, extended info: %s",
416 + wevt_extended_status());
417 + goto cleanup;
418 + }
419 + }
420 + log->ops.content.used = bytes_used;
421 +
422 + EVT_VARIANT *content = log->ops.content.data;
423 +
424 + ev->id = wevt_field_get_uint64(&content[FIELD_RECORD_NUMBER]);
425 + ev->event_id = wevt_field_get_uint16(&content[FIELD_EVENT_ID]);
426 + ev->level = wevt_field_get_uint8(&content[FIELD_LEVEL]);
427 + ev->opcode = wevt_field_get_uint8(&content[FIELD_OPCODE]);
428 + ev->keywords = wevt_field_get_uint64_hex(&content[FIELD_KEYWORDS]);
429 + ev->version = wevt_field_get_uint8(&content[FIELD_VERSION]);
430 + ev->task = wevt_field_get_uint16(&content[FIELD_TASK]);
431 + ev->process_id = wevt_field_get_uint32(&content[FIELD_PROCESS_ID]);
432 + ev->thread_id = wevt_field_get_uint32(&content[FIELD_THREAD_ID]);
433 + ev->created_ns = wevt_field_get_filetime_to_ns(&content[FIELD_TIME_CREATED]);
434 +
435 + if(full) {
436 + wevt_field_get_string_utf8(&content[FIELD_CHANNEL], &log->ops.channel);
437 + wevt_field_get_string_utf8(&content[FIELD_COMPUTER_NAME], &log->ops.computer);
438 + wevt_field_get_string_utf8(&content[FIELD_PROVIDER_NAME], &log->ops.provider);
439 + wevt_field_get_string_utf8(&content[FIELD_EVENT_SOURCE_NAME], &log->ops.source);
440 + wevt_get_uuid_by_type(&content[FIELD_PROVIDER_GUID], &ev->provider);
441 + wevt_get_uuid_by_type(&content[FIELD_CORRELATION_ACTIVITY_ID], &ev->correlation_activity_id);
442 + wevt_field_get_sid(&content[FIELD_USER_ID], &log->ops.user);
443 +
444 + PROVIDER_META_HANDLE *h = log->publisher =
445 + publisher_get(ev->provider, log->ops.content.data[FIELD_PROVIDER_NAME].StringVal);
446 +
447 + wevt_get_level(log, ev, h);
448 + wevt_get_task(log, ev, h);
449 + wevt_get_opcode(log, ev, h);
450 + wevt_get_keywords(log, ev, h);
451 + }
452 +
453 + ret = true;
454 +
455 +cleanup:
456 + return ret;
457 +}
458 +
459 +bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
460 + DWORD size = full ? BATCH_NEXT_EVENT : 1;
461 + DWORD max_failures = 10;
462 +
463 + fatal_assert(log && log->event_query && log->render_context);
464 +
465 + while(max_failures > 0) {
466 + if (log->batch.used >= log->batch.size) {
467 + log->batch.size = 0;
468 + log->batch.used = 0;
469 + DWORD err;
470 + if(!EvtNext(log->event_query, size, log->batch.bk, INFINITE, 0, &log->batch.size)) {
471 + err = GetLastError();
472 + if(err == ERROR_NO_MORE_ITEMS)
473 + return false; // no data available, return failure
474 + }
475 +
476 + if(!log->batch.size) {
477 + if(size == 1) {
478 + nd_log(NDLS_COLLECTORS, NDLP_ERR,
479 + "EvtNext() failed, event_query: 0x%lx, size: %zu, extended info: %s",
480 + (uintptr_t)log->event_query, (size_t)size, wevt_extended_status());
481 + return false;
482 + }
483 +
484 + // EvtNext() returns true when it can full the array
485 + // so, let's retry with a smaller array.
486 + size /= 2;
487 + if(size < 1) size = 1;
488 + continue;
489 + }
490 + }
491 +
492 + log->query_stats.event_count++;
493 + log->log_stats.event_count++;
494 +
495 + // cleanup any previous event data
496 + wevt_event_done(log);
497 +
498 + log->bookmark = log->batch.bk[log->batch.used];
499 + log->batch.bk[log->batch.used] = NULL;
500 + log->batch.used++;
501 +
502 + if(wevt_get_next_event_one(log, ev, full))
503 + return true;
504 + else {
505 + log->query_stats.failed_count++;
506 + log->log_stats.failed_count++;
507 + max_failures--;
508 + }
509 + }
510 +
511 + return false;
512 +}
513 +
514 +void wevt_query_done(WEVT_LOG *log) {
515 + // close the last working bookmark
516 + wevt_event_done(log);
517 +
518 + // close all batched bookmarks
519 + for(DWORD i = log->batch.used; i < log->batch.size ;i++) {
520 + if(log->batch.bk[i])
521 + EvtClose(log->batch.bk[i]);
522 +
523 + log->batch.bk[i] = NULL;
524 + }
525 + log->batch.used = 0;
526 + log->batch.size = 0;
527 +
528 + if (log->event_query) {
529 + EvtClose(log->event_query);
530 + log->event_query = NULL;
531 + }
532 +
533 + log->query_stats.event_count = 0;
534 + log->query_stats.failed_count = 0;
535 +}
536 +
537 +void wevt_closelog6(WEVT_LOG *log) {
538 + wevt_query_done(log);
539 +
540 + if (log->render_context)
541 + EvtClose(log->render_context);
542 +
543 + wevt_variant_cleanup(&log->ops.content);
544 + txt_unicode_cleanup(&log->ops.unicode);
545 + txt_utf8_cleanup(&log->ops.channel);
546 + txt_utf8_cleanup(&log->ops.provider);
547 + txt_utf8_cleanup(&log->ops.source);
548 + txt_utf8_cleanup(&log->ops.computer);
549 + txt_utf8_cleanup(&log->ops.user);
550 +
551 + txt_utf8_cleanup(&log->ops.event);
552 + txt_utf8_cleanup(&log->ops.level);
553 + txt_utf8_cleanup(&log->ops.keywords);
554 + txt_utf8_cleanup(&log->ops.opcode);
555 + txt_utf8_cleanup(&log->ops.task);
556 + txt_utf8_cleanup(&log->ops.xml);
557 + freez(log);
558 +}
559 +
560 +bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention) {
561 + bool ret = false;
562 +
563 + // get the number of the oldest record in the log
564 + // "EvtGetLogInfo()" does not work properly with "EvtLogOldestRecordNumber"
565 + // we have to get it from the first EventRecordID
566 +
567 + // query the eventlog
568 + log->event_query = EvtQuery(NULL, channel, NULL, EvtQueryChannelPath | EvtQueryForwardDirection);
569 + if (!log->event_query) {
570 + if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
571 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, cannot get retention, extended info: %s",
572 + channel2utf8(channel), wevt_extended_status());
573 + else
574 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, cannot get retention, extended info: %s",
575 + channel2utf8(channel), wevt_extended_status());
576 +
577 + goto cleanup;
578 + }
579 +
580 + if (!wevt_get_next_event(log, &retention->first_event, false))
581 + goto cleanup;
582 +
583 + if (!retention->first_event.id) {
584 + // no data in the event log
585 + retention->first_event = retention->last_event = WEVT_EVENT_EMPTY;
586 + ret = true;
587 + goto cleanup;
588 + }
589 + EvtClose(log->event_query);
590 +
591 + log->event_query = EvtQuery(NULL, channel, NULL, EvtQueryChannelPath | EvtQueryReverseDirection);
592 + if (!log->event_query) {
593 + if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
594 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, extended info: %s",
595 + channel2utf8(channel), wevt_extended_status());
596 + else
597 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, extended info: %s",
598 + channel2utf8(channel), wevt_extended_status());
599 +
600 + goto cleanup;
601 + }
602 +
603 + if (!wevt_get_next_event(log, &retention->last_event, false) || retention->last_event.id == 0) {
604 + // no data in eventlog
605 + retention->last_event = retention->first_event;
606 + }
607 + retention->last_event.id += 1; // we should read the last record
608 + ret = true;
609 +
610 +cleanup:
611 + wevt_query_done(log);
612 +
613 + if(ret) {
614 + retention->entries = retention->last_event.id - retention->first_event.id;
615 +
616 + if(retention->last_event.created_ns >= retention->first_event.created_ns)
617 + retention->duration_ns = retention->last_event.created_ns - retention->first_event.created_ns;
618 + else
619 + retention->duration_ns = retention->first_event.created_ns - retention->last_event.created_ns;
620 +
621 + retention->size_bytes = wevt_log_file_size(channel);
622 + }
623 + else
624 + memset(retention, 0, sizeof(*retention));
625 +
626 + return ret;
627 +}
628 +
629 +WEVT_LOG *wevt_openlog6(void) {
630 + size_t RENDER_ITEMS_count = (sizeof(RENDER_ITEMS) / sizeof(const wchar_t *));
631 +
632 + WEVT_LOG *log = callocz(1, sizeof(*log));
633 +
634 + // create the system render
635 + log->render_context = EvtCreateRenderContext(RENDER_ITEMS_count, RENDER_ITEMS, EvtRenderContextValues);
636 + if (!log->render_context) {
637 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtCreateRenderContext failed, extended info: %s", wevt_extended_status());
638 + freez(log);
639 + log = NULL;
640 + goto cleanup;
641 + }
642 +
643 +cleanup:
644 + return log;
645 +}
646 +
647 +static uint64_t wevt_log_file_size(const wchar_t *channel) {
648 + EVT_HANDLE hLog = NULL;
649 + EVT_VARIANT evtVariant;
650 + DWORD bufferUsed = 0;
651 + uint64_t file_size = 0;
652 +
653 + // Open the event log channel
654 + hLog = EvtOpenLog(NULL, channel, EvtOpenChannelPath);
655 + if (!hLog) {
656 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtOpenLog() on channel '%s' failed, extended info: %s",
657 + channel2utf8(channel), wevt_extended_status());
658 + goto cleanup;
659 + }
660 +
661 + // Get the file size of the log
662 + if (!EvtGetLogInfo(hLog, EvtLogFileSize, sizeof(evtVariant), &evtVariant, &bufferUsed)) {
663 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetLogInfo() on channel '%s' failed, extended info: %s",
664 + channel2utf8(channel), wevt_extended_status());
665 + goto cleanup;
666 + }
667 +
668 + // Extract the file size from the EVT_VARIANT structure
669 + file_size = evtVariant.UInt64Val;
670 +
671 +cleanup:
672 + if (hLog)
673 + EvtClose(hLog);
674 +
675 + return file_size;
676 +}
677 +
678 +bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction) {
679 + wevt_query_done(log);
680 + log->log_stats.queries_count++;
681 +
682 + EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (direction & (EvtQueryReverseDirection | EvtQueryForwardDirection)) | EvtQueryTolerateQueryErrors);
683 + if (!hQuery) {
684 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, query: %s | extended info: %s",
685 + query2utf8(query), wevt_extended_status());
686 +
687 + log->log_stats.queries_failed++;
688 + return false;
689 + }
690 +
691 + log->event_query = hQuery;
692 + return true;
693 +}
src/collectors/windows-events.plugin/windows-events-query.h
+247 -81
@@ -1,81 +1,247 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_WINDOWS_EVENTS_QUERY_H
4 -#define NETDATA_WINDOWS_EVENTS_QUERY_H
5 -
6 -#include "windows-events.h"
7 -
8 -typedef struct wevt_event {
9 - uint64_t id; // EventRecordId (unique and sequential per channel)
10 - uint16_t event_id; // This is the template that defines the message to be shown
11 - uint16_t opcode;
12 - uint8_t level; // The severity of event
13 - uint8_t version;
14 - uint16_t task;
15 - uint32_t process_id;
16 - uint32_t thread_id;
17 - uint64_t keyword; // Categorization of the event
18 - ND_UUID provider;
19 - ND_UUID correlation_activity_id;
20 - nsec_t created_ns;
21 -} WEVT_EVENT;
22 -
23 -#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
24 -
25 -typedef struct {
26 - WEVT_EVENT first_event;
27 - WEVT_EVENT last_event;
28 -
29 - uint64_t entries;
30 - nsec_t duration_ns;
31 - uint64_t size_bytes;
32 -} EVT_RETENTION;
33 -
34 -typedef struct wevt_log {
35 - EVT_HANDLE event_query;
36 - EVT_HANDLE render_context;
37 -
38 - struct {
39 - // temp buffer used for rendering event log messages
40 - // never use directly
41 - struct {
42 - EVT_VARIANT *data;
43 - size_t size;
44 - size_t len;
45 - } content;
46 -
47 - // temp buffer used for fetching and converting UNICODE and UTF-8
48 - // every string operation overwrites it, multiple times per event log entry
49 - // it can be used within any function, for its own purposes,
50 - // but never share between functions
51 - TXT_UNICODE unicode;
52 -
53 - // string attributes of the current event log entry
54 - // valid until another event if fetched
55 - TXT_UTF8 channel;
56 - TXT_UTF8 provider;
57 - TXT_UTF8 source;
58 - TXT_UTF8 computer;
59 - TXT_UTF8 event;
60 - TXT_UTF8 user;
61 - TXT_UTF8 opcode;
62 - TXT_UTF8 level;
63 - TXT_UTF8 keyword;
64 - TXT_UTF8 xml;
65 -
66 - BUFFER *message;
67 - } ops;
68 -
69 -} WEVT_LOG;
70 -
71 -WEVT_LOG *wevt_openlog6(void);
72 -void wevt_closelog6(WEVT_LOG *log);
73 -
74 -bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention);
75 -
76 -EVT_HANDLE wevt_query(LPCWSTR channel, usec_t seek_to, bool backward);
77 -void wevt_query_done(WEVT_LOG *log);
78 -
79 -bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
80 -
81 -#endif //NETDATA_WINDOWS_EVENTS_QUERY_H
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WINDOWS_EVENTS_QUERY_H
4 +#define NETDATA_WINDOWS_EVENTS_QUERY_H
5 +
6 +#include "windows-events.h"
7 +
8 +#define BATCH_NEXT_EVENT 500
9 +
10 +typedef struct wevt_event {
11 + uint64_t id; // EventRecordId (unique and sequential per channel)
12 + uint8_t version;
13 + uint8_t level; // The severity of event
14 + uint8_t opcode; // we receive this as 8bit, but publishers use 32bit
15 + uint16_t event_id; // This is the template that defines the message to be shown
16 + uint16_t task;
17 + uint32_t process_id;
18 + uint32_t thread_id;
19 + uint64_t keywords; // Categorization of the event
20 + ND_UUID provider;
21 + ND_UUID correlation_activity_id;
22 + nsec_t created_ns;
23 +} WEVT_EVENT;
24 +
25 +#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
26 +
27 +typedef struct {
28 + EVT_VARIANT *data;
29 + size_t size;
30 + size_t used;
31 +} WEVT_VARIANT;
32 +
33 +typedef struct {
34 + WEVT_EVENT first_event;
35 + WEVT_EVENT last_event;
36 +
37 + uint64_t entries;
38 + nsec_t duration_ns;
39 + uint64_t size_bytes;
40 +} EVT_RETENTION;
41 +
42 +struct provider_meta_handle;
43 +
44 +typedef struct wevt_log {
45 + struct {
46 + DWORD size;
47 + DWORD used;
48 + EVT_HANDLE bk[BATCH_NEXT_EVENT];
49 + } batch;
50 +
51 + EVT_HANDLE bookmark;
52 + EVT_HANDLE event_query;
53 + EVT_HANDLE render_context;
54 + struct provider_meta_handle *publisher;
55 +
56 + struct {
57 + // temp buffer used for rendering event log messages
58 + // never use directly
59 + WEVT_VARIANT content;
60 +
61 + // temp buffer used for fetching and converting UNICODE and UTF-8
62 + // every string operation overwrites it, multiple times per event log entry
63 + // it can be used within any function, for its own purposes,
64 + // but never share between functions
65 + TXT_UNICODE unicode;
66 +
67 + // string attributes of the current event log entry
68 + // valid until another event if fetched
69 +
70 + // IMPORTANT:
71 + // EVERY FIELD NEEDS ITS OWN BUFFER!
72 + // the way facets work, all the field value pointers need to be valid
73 + // until the entire row closes, so reusing a buffer for the same field
74 + // actually copies the same value to all fields using the same buffer.
75 +
76 + TXT_UTF8 channel;
77 + TXT_UTF8 provider;
78 + TXT_UTF8 source;
79 + TXT_UTF8 computer;
80 + TXT_UTF8 user;
81 +
82 + TXT_UTF8 event;
83 + TXT_UTF8 level;
84 + TXT_UTF8 keywords;
85 + TXT_UTF8 opcode;
86 + TXT_UTF8 task;
87 + TXT_UTF8 xml;
88 + } ops;
89 +
90 + struct {
91 + size_t event_count;
92 + size_t failed_count;
93 + } query_stats;
94 +
95 + struct {
96 + size_t queries_count;
97 + size_t queries_failed;
98 +
99 + size_t event_count;
100 + size_t failed_count;
101 + } log_stats;
102 +
103 +} WEVT_LOG;
104 +
105 +WEVT_LOG *wevt_openlog6(void);
106 +void wevt_closelog6(WEVT_LOG *log);
107 +
108 +bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention);
109 +
110 +bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction);
111 +void wevt_query_done(WEVT_LOG *log);
112 +
113 +bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
114 +
115 +bool wevt_get_message_unicode(TXT_UNICODE *unicode, EVT_HANDLE hMetadata, EVT_HANDLE bookmark, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
116 +
117 +struct provider_meta_handle;
118 +bool wevt_get_event_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE event_handle, TXT_UTF8 *dst);
119 +bool wevt_get_xml_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE event_handle, TXT_UTF8 *dst);
120 +
121 +static inline void wevt_variant_cleanup(WEVT_VARIANT *v) {
122 + freez(v->data);
123 +}
124 +
125 +static inline void wevt_variant_resize(WEVT_VARIANT *v, size_t required_size) {
126 + if(required_size < v->size)
127 + return;
128 +
129 + wevt_variant_cleanup(v);
130 + v->size = compute_new_size(v->size, required_size);
131 + v->data = mallocz(v->size);
132 +}
133 +
134 +static inline uint8_t wevt_field_get_uint8(EVT_VARIANT *ev) {
135 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeByte);
136 + return ev->ByteVal;
137 +}
138 +
139 +static inline uint16_t wevt_field_get_uint16(EVT_VARIANT *ev) {
140 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeUInt16);
141 + return ev->UInt16Val;
142 +}
143 +
144 +static inline uint32_t wevt_field_get_uint32(EVT_VARIANT *ev) {
145 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeUInt32);
146 + return ev->UInt32Val;
147 +}
148 +
149 +static inline uint64_t wevt_field_get_uint64(EVT_VARIANT *ev) {
150 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeUInt64);
151 + return ev->UInt64Val;
152 +}
153 +
154 +static inline uint64_t wevt_field_get_uint64_hex(EVT_VARIANT *ev) {
155 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeHexInt64);
156 + return ev->UInt64Val;
157 +}
158 +
159 +static inline bool wevt_field_get_string_utf8(EVT_VARIANT *ev, TXT_UTF8 *dst) {
160 + if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
161 + wevt_utf8_empty(dst);
162 + return false;
163 + }
164 +
165 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeString);
166 + return wevt_str_wchar_to_utf8(dst, ev->StringVal, -1);
167 +}
168 +
169 +bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
170 +
171 +static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst) {
172 + if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
173 + wevt_utf8_empty(dst);
174 + return false;
175 + }
176 +
177 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeSid);
178 + return wevt_convert_user_id_to_name(ev->SidVal, dst);
179 +}
180 +
181 +static inline uint64_t wevt_field_get_filetime_to_ns(EVT_VARIANT *ev) {
182 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeFileTime);
183 + return os_windows_ulonglong_to_unix_epoch_ns(ev->FileTimeVal);
184 +}
185 +
186 +static inline bool wevt_GUID_to_ND_UUID(ND_UUID *nd_uuid, const GUID *guid) {
187 + if(guid && sizeof(GUID) == sizeof(ND_UUID)) {
188 + memcpy(nd_uuid->uuid, guid, sizeof(ND_UUID));
189 + return true;
190 + }
191 + else {
192 + *nd_uuid = UUID_ZERO;
193 + return false;
194 + }
195 +}
196 +
197 +static inline bool wevt_get_uuid_by_type(EVT_VARIANT *ev, ND_UUID *dst) {
198 + if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
199 + wevt_GUID_to_ND_UUID(dst, NULL);
200 + return false;
201 + }
202 +
203 + fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeGuid);
204 + return wevt_GUID_to_ND_UUID(dst, ev->GuidVal);
205 +}
206 +
207 +// https://learn.microsoft.com/en-us/windows/win32/wes/defining-severity-levels
208 +static inline bool is_valid_publisher_level(uint64_t level, bool strict) {
209 + if(strict)
210 + // when checking if the name is publisher independent
211 + return level >= 16 && level <= 255;
212 + else
213 + // when checking acceptable values in publisher manifests
214 + return level <= 255;
215 +}
216 +
217 +// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
218 +static inline bool is_valid_publisher_opcode(uint64_t opcode, bool strict) {
219 + if(strict)
220 + // when checking if the name is publisher independent
221 + return opcode >= 10 && opcode <= 239;
222 + else
223 + // when checking acceptable values in publisher manifests
224 + return opcode <= 255;
225 +}
226 +
227 +// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
228 +static inline bool is_valid_publisher_task(uint64_t task, bool strict) {
229 + if(strict)
230 + // when checking if the name is publisher independent
231 + return task > 0 && task <= 0xFFFF;
232 + else
233 + // when checking acceptable values in publisher manifests
234 + return task <= 0xFFFF;
235 +}
236 +
237 +// https://learn.microsoft.com/en-us/windows/win32/wes/defining-keywords-used-to-classify-types-of-events
238 +static inline bool is_valid_publisher_keywords(uint64_t keyword, bool strict) {
239 + if(strict)
240 + // when checking if the name is publisher independent
241 + return keyword > 0 && keyword <= 0x0000FFFFFFFFFFFF;
242 + else
243 + // when checking acceptable values in publisher manifests
244 + return true;
245 +}
246 +
247 +#endif //NETDATA_WINDOWS_EVENTS_QUERY_H
src/collectors/windows-events.plugin/windows-events-sid.c
+118 -120
@@ -1,120 +1,118 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "windows-events-sid.h"
4 -#include <sddl.h>
5 -
6 -typedef struct {
7 - size_t len;
8 - uint8_t sid[];
9 -} SID_KEY;
10 -
11 -typedef struct {
12 - const char *user;
13 - size_t user_len;
14 -
15 - // this needs to be last, because of its variable size
16 - SID_KEY key;
17 -} SID_VALUE;
18 -
19 -#define SIMPLE_HASHTABLE_NAME _SID
20 -#define SIMPLE_HASHTABLE_VALUE_TYPE SID_VALUE
21 -#define SIMPLE_HASHTABLE_KEY_TYPE SID_KEY
22 -#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION sid_value_to_key
23 -#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION sid_cache_compar
24 -#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
25 -#include "libnetdata/simple_hashtable.h"
26 -
27 -static struct {
28 - SPINLOCK spinlock;
29 - bool initialized;
30 - struct simple_hashtable_SID hashtable;
31 -} sid_globals = {
32 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
33 - .hashtable = { 0 },
34 -};
35 -
36 -static inline SID_KEY *sid_value_to_key(SID_VALUE *s) {
37 - return &s->key;
38 -}
39 -
40 -static inline bool sid_cache_compar(SID_KEY *a, SID_KEY *b) {
41 - return a->len == b->len && memcmp(&a->sid, &b->sid, a->len) == 0;
42 -}
43 -
44 -static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
45 - if(found && found->user) {
46 - txt_utf8_resize(dst, found->user_len + 1);
47 - memcpy(dst->data, found->user, found->user_len + 1);
48 - dst->used = found->user_len + 1;
49 - return true;
50 - }
51 -
52 - txt_utf8_resize(dst, 1);
53 - dst->data[0] = '\0';
54 - dst->used = 1;
55 - return false;
56 -}
57 -
58 -static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
59 - static __thread wchar_t account_unicode[256];
60 - static __thread wchar_t domain_unicode[256];
61 - DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
62 - DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
63 - SID_NAME_USE sid_type;
64 -
65 - txt_utf8_resize(dst, 1024);
66 -
67 - if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
68 - const char *user = account2utf8(account_unicode);
69 - const char *domain = domain2utf8(domain_unicode);
70 - dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
71 - }
72 - else {
73 - wchar_t *sid_string = NULL;
74 - if (ConvertSidToStringSidW(sid, &sid_string)) {
75 - const char *user = account2utf8(sid_string);
76 - dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
77 - }
78 - else
79 - dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
80 - }
81 -}
82 -
83 -bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
84 - if(!sid || !IsValidSid(sid))
85 - return update_user(NULL, dst);
86 -
87 - size_t size = GetLengthSid(sid);
88 -
89 - size_t tmp_size = sizeof(SID_VALUE) + size;
90 - size_t tmp_key_size = sizeof(SID_KEY) + size;
91 - uint8_t buf[tmp_size];
92 - SID_VALUE *tmp = (SID_VALUE *)&buf;
93 - memcpy(&tmp->key.sid, sid, size);
94 - tmp->key.len = size;
95 -
96 - spinlock_lock(&sid_globals.spinlock);
97 - if(!sid_globals.initialized) {
98 - simple_hashtable_init_SID(&sid_globals.hashtable, 100);
99 - sid_globals.initialized = true;
100 - }
101 - SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
102 - spinlock_unlock(&sid_globals.spinlock);
103 - if(found) return update_user(found, dst);
104 -
105 - // allocate the SID_VALUE
106 - found = mallocz(tmp_size);
107 - memcpy(found, buf, tmp_size);
108 -
109 - // lookup the user
110 - lookup_user(sid, dst);
111 - found->user = strdupz(dst->data);
112 - found->user_len = dst->used - 1;
113 -
114 - // add it to the cache
115 - spinlock_lock(&sid_globals.spinlock);
116 - simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
117 - spinlock_unlock(&sid_globals.spinlock);
118 -
119 - return update_user(found, dst);
120 -}
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "windows-events-sid.h"
4 +#include <sddl.h>
5 +
6 +typedef struct {
7 + size_t len;
8 + uint8_t sid[];
9 +} SID_KEY;
10 +
11 +typedef struct {
12 + const char *user;
13 + size_t user_len;
14 +
15 + // this needs to be last, because of its variable size
16 + SID_KEY key;
17 +} SID_VALUE;
18 +
19 +#define SIMPLE_HASHTABLE_NAME _SID
20 +#define SIMPLE_HASHTABLE_VALUE_TYPE SID_VALUE
21 +#define SIMPLE_HASHTABLE_KEY_TYPE SID_KEY
22 +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION sid_value_to_key
23 +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION sid_cache_compar
24 +#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
25 +#include "libnetdata/simple_hashtable.h"
26 +
27 +static struct {
28 + SPINLOCK spinlock;
29 + struct simple_hashtable_SID hashtable;
30 +} sid_globals = {
31 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
32 +};
33 +
34 +static inline SID_KEY *sid_value_to_key(SID_VALUE *s) {
35 + return &s->key;
36 +}
37 +
38 +static inline bool sid_cache_compar(SID_KEY *a, SID_KEY *b) {
39 + return a->len == b->len && memcmp(&a->sid, &b->sid, a->len) == 0;
40 +}
41 +
42 +void sid_cache_init(void) {
43 + simple_hashtable_init_SID(&sid_globals.hashtable, 100);
44 +}
45 +
46 +static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
47 + if(found && found->user) {
48 + txt_utf8_resize(dst, found->user_len + 1, false);
49 + memcpy(dst->data, found->user, found->user_len + 1);
50 + dst->used = found->user_len + 1;
51 + return true;
52 + }
53 +
54 + txt_utf8_resize(dst, 1, false);
55 + dst->data[0] = '\0';
56 + dst->used = 1;
57 + return false;
58 +}
59 +
60 +static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
61 + static __thread wchar_t account_unicode[256];
62 + static __thread wchar_t domain_unicode[256];
63 + DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
64 + DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
65 + SID_NAME_USE sid_type;
66 +
67 + txt_utf8_resize(dst, 1024, false);
68 +
69 + if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
70 + const char *user = account2utf8(account_unicode);
71 + const char *domain = domain2utf8(domain_unicode);
72 + dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
73 + }
74 + else {
75 + wchar_t *sid_string = NULL;
76 + if (ConvertSidToStringSidW(sid, &sid_string)) {
77 + const char *user = account2utf8(sid_string);
78 + dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
79 + }
80 + else
81 + dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
82 + }
83 +}
84 +
85 +bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
86 + if(!sid || !IsValidSid(sid))
87 + return update_user(NULL, dst);
88 +
89 + size_t size = GetLengthSid(sid);
90 +
91 + size_t tmp_size = sizeof(SID_VALUE) + size;
92 + size_t tmp_key_size = sizeof(SID_KEY) + size;
93 + uint8_t buf[tmp_size];
94 + SID_VALUE *tmp = (SID_VALUE *)&buf;
95 + memcpy(&tmp->key.sid, sid, size);
96 + tmp->key.len = size;
97 +
98 + spinlock_lock(&sid_globals.spinlock);
99 + SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
100 + spinlock_unlock(&sid_globals.spinlock);
101 + if(found) return update_user(found, dst);
102 +
103 + // allocate the SID_VALUE
104 + found = mallocz(tmp_size);
105 + memcpy(found, buf, tmp_size);
106 +
107 + // lookup the user
108 + lookup_user(sid, dst);
109 + found->user = strdupz(dst->data);
110 + found->user_len = dst->used - 1;
111 +
112 + // add it to the cache
113 + spinlock_lock(&sid_globals.spinlock);
114 + simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
115 + spinlock_unlock(&sid_globals.spinlock);
116 +
117 + return update_user(found, dst);
118 +}
src/collectors/windows-events.plugin/windows-events-sid.h
+12 -11
@@ -1,11 +1,12 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_WINDOWS_EVENTS_SID_H
4 -#define NETDATA_WINDOWS_EVENTS_SID_H
5 -
6 -#include "windows-events.h"
7 -
8 -struct wevt_log;
9 -bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
10 -
11 -#endif //NETDATA_WINDOWS_EVENTS_SID_H
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WINDOWS_EVENTS_SID_H
4 +#define NETDATA_WINDOWS_EVENTS_SID_H
5 +
6 +#include "windows-events.h"
7 +
8 +struct wevt_log;
9 +bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
10 +void sid_cache_init(void);
11 +
12 +#endif //NETDATA_WINDOWS_EVENTS_SID_H
src/collectors/windows-events.plugin/windows-events-sources.c
+317 -267
@@ -1,267 +1,317 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "windows-events-sources.h"
4 -
5 -DICTIONARY *wevt_sources = NULL;
6 -DICTIONARY *used_hashes_registry = NULL;
7 -static usec_t wevt_session = 0;
8 -
9 -WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
10 - if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
11 - return WEVTS_ALL;
12 -
13 - return WEVTS_NONE;
14 -}
15 -
16 -void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
17 - LOGS_QUERY_SOURCE *src = value;
18 - freez((void *)src->fullname);
19 - string_freez(src->source);
20 -
21 - src->fullname = NULL;
22 - src->source = NULL;
23 -}
24 -
25 -static bool wevt_sources_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
26 - LOGS_QUERY_SOURCE *src_old = old_value;
27 - LOGS_QUERY_SOURCE *src_new = new_value;
28 -
29 - bool ret = false;
30 - if(src_new->last_scan_monotonic_ut > src_old->last_scan_monotonic_ut) {
31 - src_old->last_scan_monotonic_ut = src_new->last_scan_monotonic_ut;
32 -
33 - if (src_old->source != src_new->source) {
34 - string_freez(src_old->source);
35 - src_old->source = src_new->source;
36 - src_new->source = NULL;
37 - }
38 - src_old->source_type = src_new->source_type;
39 -
40 - src_old->msg_first_ut = src_new->msg_first_ut;
41 - src_old->msg_last_ut = src_new->msg_last_ut;
42 - src_old->msg_first_id = src_new->msg_first_id;
43 - src_old->msg_last_id = src_new->msg_last_id;
44 - src_old->entries = src_new->entries;
45 - src_old->size = src_new->size;
46 -
47 - ret = true;
48 - }
49 -
50 - freez((void *)src_new->fullname);
51 - string_freez(src_new->source);
52 - src_new->fullname = NULL;
53 - src_new->source = NULL;
54 -
55 - return ret;
56 -}
57 -
58 -void wevt_sources_init(void) {
59 - wevt_session = now_realtime_usec();
60 -
61 - used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
62 -
63 - wevt_sources = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
64 - NULL, sizeof(LOGS_QUERY_SOURCE));
65 -
66 - dictionary_register_delete_callback(wevt_sources, wevt_sources_del_cb, NULL);
67 - dictionary_register_conflict_callback(wevt_sources, wevt_sources_conflict_cb, NULL);
68 -}
69 -
70 -void buffer_json_wevt_versions(BUFFER *wb __maybe_unused) {
71 - buffer_json_member_add_object(wb, "versions");
72 - {
73 - buffer_json_member_add_uint64(wb, "sources",
74 - wevt_session + dictionary_version(wevt_sources));
75 - }
76 - buffer_json_object_close(wb);
77 -}
78 -
79 -// --------------------------------------------------------------------------------------------------------------------
80 -
81 -int wevt_sources_dict_items_backward_compar(const void *a, const void *b) {
82 - const DICTIONARY_ITEM **da = (const DICTIONARY_ITEM **)a, **db = (const DICTIONARY_ITEM **)b;
83 - LOGS_QUERY_SOURCE *sa = dictionary_acquired_item_value(*da);
84 - LOGS_QUERY_SOURCE *sb = dictionary_acquired_item_value(*db);
85 -
86 - // compare the last message timestamps
87 - if(sa->msg_last_ut < sb->msg_last_ut)
88 - return 1;
89 -
90 - if(sa->msg_last_ut > sb->msg_last_ut)
91 - return -1;
92 -
93 - // compare the first message timestamps
94 - if(sa->msg_first_ut < sb->msg_first_ut)
95 - return 1;
96 -
97 - if(sa->msg_first_ut > sb->msg_first_ut)
98 - return -1;
99 -
100 - return 0;
101 -}
102 -
103 -int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
104 - return -wevt_sources_dict_items_backward_compar(a, b);
105 -}
106 -
107 -// --------------------------------------------------------------------------------------------------------------------
108 -
109 -struct wevt_source {
110 - usec_t first_ut;
111 - usec_t last_ut;
112 - size_t count;
113 - uint64_t size;
114 -};
115 -
116 -static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) {
117 - const struct wevt_source *s = entry;
118 - BUFFER *wb = data;
119 -
120 - const char *name = dictionary_acquired_item_name(item);
121 -
122 - buffer_json_add_array_item_object(wb);
123 - {
124 - char size_for_humans[128];
125 - size_snprintf(size_for_humans, sizeof(size_for_humans), s->size, "B", false);
126 -
127 - char duration_for_humans[128];
128 - duration_snprintf(duration_for_humans, sizeof(duration_for_humans),
129 - (time_t)((s->last_ut - s->first_ut) / USEC_PER_SEC), "s", true);
130 -
131 - char info[1024];
132 - snprintfz(info, sizeof(info), "%zu channels, with a total size of %s, covering %s",
133 - s->count, size_for_humans, duration_for_humans);
134 -
135 - buffer_json_member_add_string(wb, "id", name);
136 - buffer_json_member_add_string(wb, "name", name);
137 - buffer_json_member_add_string(wb, "pill", size_for_humans);
138 - buffer_json_member_add_string(wb, "info", info);
139 - }
140 - buffer_json_object_close(wb); // options object
141 -
142 - return 1;
143 -}
144 -
145 -static bool wevt_source_merge_sizes(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value , void *data __maybe_unused) {
146 - struct wevt_source *old_v = old_value;
147 - const struct wevt_source *new_v = new_value;
148 -
149 - old_v->count += new_v->count;
150 - old_v->size += new_v->size;
151 -
152 - if(new_v->first_ut && new_v->first_ut < old_v->first_ut)
153 - old_v->first_ut = new_v->first_ut;
154 -
155 - if(new_v->last_ut && new_v->last_ut > old_v->last_ut)
156 - old_v->last_ut = new_v->last_ut;
157 -
158 - return false;
159 -}
160 -
161 -void wevt_sources_to_json_array(BUFFER *wb) {
162 - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_NAME_LINK_DONT_CLONE|DICT_OPTION_DONT_OVERWRITE_VALUE);
163 - dictionary_register_conflict_callback(dict, wevt_source_merge_sizes, NULL);
164 -
165 - struct wevt_source t = { 0 };
166 -
167 - LOGS_QUERY_SOURCE *src;
168 - dfe_start_read(wevt_sources, src) {
169 - t.first_ut = src->msg_first_ut;
170 - t.last_ut = src->msg_last_ut;
171 - t.count = 1;
172 - t.size = src->size;
173 -
174 - dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
175 -
176 - if(src->source)
177 - dictionary_set(dict, string2str(src->source), &t, sizeof(t));
178 - }
179 - dfe_done(jf);
180 -
181 - dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
182 -}
183 -
184 -void wevt_sources_scan(void) {
185 - static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
186 - LPWSTR channel = NULL;
187 - EVT_HANDLE hChannelEnum = NULL;
188 -
189 - if(spinlock_trylock(&spinlock)) {
190 - const usec_t now_monotonic_ut = now_monotonic_usec();
191 -
192 - DWORD dwChannelBufferSize = 0;
193 - DWORD dwChannelBufferUsed = 0;
194 - DWORD status = ERROR_SUCCESS;
195 -
196 - // Open a handle to enumerate the event channels
197 - hChannelEnum = EvtOpenChannelEnum(NULL, 0);
198 - if (!hChannelEnum) {
199 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "WINDOWS EVENTS: EvtOpenChannelEnum() failed with %" PRIu64 "\n",
200 - (uint64_t)GetLastError());
201 - goto cleanup;
202 - }
203 -
204 - WEVT_LOG *log = wevt_openlog6();
205 - if(!log) goto cleanup;
206 -
207 - while (true) {
208 - if (!EvtNextChannelPath(hChannelEnum, dwChannelBufferSize, channel, &dwChannelBufferUsed)) {
209 - status = GetLastError();
210 - if (status == ERROR_NO_MORE_ITEMS)
211 - break; // No more channels
212 - else if (status == ERROR_INSUFFICIENT_BUFFER) {
213 - dwChannelBufferSize = dwChannelBufferUsed;
214 - freez(channel);
215 - channel = mallocz(dwChannelBufferSize * sizeof(WCHAR));
216 - continue;
217 - } else {
218 - nd_log(NDLS_COLLECTORS, NDLP_ERR,
219 - "WINDOWS EVENTS: EvtNextChannelPath() failed\n");
220 - break;
221 - }
222 - }
223 -
224 - EVT_RETENTION retention;
225 - if(!wevt_channel_retention(log, channel, &retention))
226 - continue;
227 -
228 - const char *name = channel2utf8(channel);
229 - const char *fullname = strdupz(name);
230 - char *slash = strchr(name, '/');
231 - if(slash) *slash = '\0';
232 -
233 - LOGS_QUERY_SOURCE src = {
234 - .entries = retention.entries,
235 - .fullname = fullname,
236 - .fullname_len = strlen(fullname),
237 - .last_scan_monotonic_ut = now_monotonic_usec(),
238 - .msg_first_id = retention.first_event.id,
239 - .msg_last_id = retention.last_event.id,
240 - .msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC,
241 - .msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
242 - .size = retention.size_bytes,
243 - .source_type = WEVTS_ALL,
244 - .source = string_strdupz(name),
245 - };
246 -
247 - dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
248 - }
249 -
250 - wevt_closelog6(log);
251 -
252 - LOGS_QUERY_SOURCE *src;
253 - dfe_start_write(wevt_sources, src)
254 - {
255 - if(src->last_scan_monotonic_ut < now_monotonic_ut)
256 - dictionary_del(wevt_sources, src->fullname);
257 - }
258 - dfe_done(src);
259 - dictionary_garbage_collect(wevt_sources);
260 -
261 - spinlock_unlock(&spinlock);
262 - }
263 -
264 -cleanup:
265 - freez(channel);
266 - EvtClose(hChannelEnum);
267 -}
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "windows-events-sources.h"
4 +
5 +DICTIONARY *wevt_sources = NULL;
6 +DICTIONARY *used_hashes_registry = NULL;
7 +static usec_t wevt_session = 0;
8 +
9 +WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
10 + if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
11 + return WEVTS_ALL;
12 +
13 + if(strcmp(value, WEVT_SOURCE_ALL_ADMIN_NAME) == 0)
14 + return WEVTS_ADMIN;
15 +
16 + if(strcmp(value, WEVT_SOURCE_ALL_OPERATIONAL_NAME) == 0)
17 + return WEVTS_OPERATIONAL;
18 +
19 + if(strcmp(value, WEVT_SOURCE_ALL_ANALYTIC_NAME) == 0)
20 + return WEVTS_ANALYTIC;
21 +
22 + if(strcmp(value, WEVT_SOURCE_ALL_DEBUG_NAME) == 0)
23 + return WEVTS_DEBUG;
24 +
25 + if(strcmp(value, WEVT_SOURCE_ALL_WINDOWS_NAME) == 0)
26 + return WEVTS_WINDOWS;
27 +
28 + return WEVTS_NONE;
29 +}
30 +
31 +void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
32 + LOGS_QUERY_SOURCE *src = value;
33 + freez((void *)src->fullname);
34 + string_freez(src->source);
35 +
36 + src->fullname = NULL;
37 + src->source = NULL;
38 +}
39 +
40 +static bool wevt_sources_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
41 + LOGS_QUERY_SOURCE *src_old = old_value;
42 + LOGS_QUERY_SOURCE *src_new = new_value;
43 +
44 + bool ret = false;
45 + if(src_new->last_scan_monotonic_ut > src_old->last_scan_monotonic_ut) {
46 + src_old->last_scan_monotonic_ut = src_new->last_scan_monotonic_ut;
47 +
48 + if (src_old->source != src_new->source) {
49 + string_freez(src_old->source);
50 + src_old->source = src_new->source;
51 + src_new->source = NULL;
52 + }
53 + src_old->source_type = src_new->source_type;
54 +
55 + src_old->msg_first_ut = src_new->msg_first_ut;
56 + src_old->msg_last_ut = src_new->msg_last_ut;
57 + src_old->msg_first_id = src_new->msg_first_id;
58 + src_old->msg_last_id = src_new->msg_last_id;
59 + src_old->entries = src_new->entries;
60 + src_old->size = src_new->size;
61 +
62 + ret = true;
63 + }
64 +
65 + freez((void *)src_new->fullname);
66 + string_freez(src_new->source);
67 + src_new->fullname = NULL;
68 + src_new->source = NULL;
69 +
70 + return ret;
71 +}
72 +
73 +void wevt_sources_init(void) {
74 + wevt_session = now_realtime_usec();
75 +
76 + used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
77 +
78 + wevt_sources = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
79 + NULL, sizeof(LOGS_QUERY_SOURCE));
80 +
81 + dictionary_register_delete_callback(wevt_sources, wevt_sources_del_cb, NULL);
82 + dictionary_register_conflict_callback(wevt_sources, wevt_sources_conflict_cb, NULL);
83 +}
84 +
85 +void buffer_json_wevt_versions(BUFFER *wb __maybe_unused) {
86 + buffer_json_member_add_object(wb, "versions");
87 + {
88 + buffer_json_member_add_uint64(wb, "sources",
89 + wevt_session + dictionary_version(wevt_sources));
90 + }
91 + buffer_json_object_close(wb);
92 +}
93 +
94 +// --------------------------------------------------------------------------------------------------------------------
95 +
96 +int wevt_sources_dict_items_backward_compar(const void *a, const void *b) {
97 + const DICTIONARY_ITEM **da = (const DICTIONARY_ITEM **)a, **db = (const DICTIONARY_ITEM **)b;
98 + LOGS_QUERY_SOURCE *sa = dictionary_acquired_item_value(*da);
99 + LOGS_QUERY_SOURCE *sb = dictionary_acquired_item_value(*db);
100 +
101 + // compare the last message timestamps
102 + if(sa->msg_last_ut < sb->msg_last_ut)
103 + return 1;
104 +
105 + if(sa->msg_last_ut > sb->msg_last_ut)
106 + return -1;
107 +
108 + // compare the first message timestamps
109 + if(sa->msg_first_ut < sb->msg_first_ut)
110 + return 1;
111 +
112 + if(sa->msg_first_ut > sb->msg_first_ut)
113 + return -1;
114 +
115 + return 0;
116 +}
117 +
118 +int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
119 + return -wevt_sources_dict_items_backward_compar(a, b);
120 +}
121 +
122 +// --------------------------------------------------------------------------------------------------------------------
123 +
124 +struct wevt_source {
125 + usec_t first_ut;
126 + usec_t last_ut;
127 + size_t count;
128 + uint64_t size;
129 +};
130 +
131 +static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) {
132 + const struct wevt_source *s = entry;
133 + BUFFER *wb = data;
134 +
135 + const char *name = dictionary_acquired_item_name(item);
136 +
137 + buffer_json_add_array_item_object(wb);
138 + {
139 + char size_for_humans[128];
140 + size_snprintf(size_for_humans, sizeof(size_for_humans), s->size, "B", false);
141 +
142 + char duration_for_humans[128];
143 + duration_snprintf(duration_for_humans, sizeof(duration_for_humans),
144 + (time_t)((s->last_ut - s->first_ut) / USEC_PER_SEC), "s", true);
145 +
146 + char info[1024];
147 + snprintfz(info, sizeof(info), "%zu channel%s, with a total size of %s, covering %s",
148 + s->count, s->count > 1 ? "s":"", size_for_humans, duration_for_humans);
149 +
150 + buffer_json_member_add_string(wb, "id", name);
151 + buffer_json_member_add_string(wb, "name", name);
152 + buffer_json_member_add_string(wb, "pill", size_for_humans);
153 + buffer_json_member_add_string(wb, "info", info);
154 + }
155 + buffer_json_object_close(wb); // options object
156 +
157 + return 1;
158 +}
159 +
160 +static bool wevt_source_merge_sizes(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value , void *data __maybe_unused) {
161 + struct wevt_source *old_v = old_value;
162 + const struct wevt_source *new_v = new_value;
163 +
164 + old_v->count += new_v->count;
165 + old_v->size += new_v->size;
166 +
167 + if(new_v->first_ut && new_v->first_ut < old_v->first_ut)
168 + old_v->first_ut = new_v->first_ut;
169 +
170 + if(new_v->last_ut && new_v->last_ut > old_v->last_ut)
171 + old_v->last_ut = new_v->last_ut;
172 +
173 + return false;
174 +}
175 +
176 +void wevt_sources_to_json_array(BUFFER *wb) {
177 + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_NAME_LINK_DONT_CLONE|DICT_OPTION_DONT_OVERWRITE_VALUE);
178 + dictionary_register_conflict_callback(dict, wevt_source_merge_sizes, NULL);
179 +
180 + struct wevt_source t = { 0 };
181 +
182 + LOGS_QUERY_SOURCE *src;
183 + dfe_start_read(wevt_sources, src) {
184 + t.first_ut = src->msg_first_ut;
185 + t.last_ut = src->msg_last_ut;
186 + t.count = 1;
187 + t.size = src->size;
188 +
189 + dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
190 +
191 + if(src->source_type & WEVTS_ADMIN)
192 + dictionary_set(dict, WEVT_SOURCE_ALL_ADMIN_NAME, &t, sizeof(t));
193 +
194 + if(src->source_type & WEVTS_OPERATIONAL)
195 + dictionary_set(dict, WEVT_SOURCE_ALL_OPERATIONAL_NAME, &t, sizeof(t));
196 +
197 + if(src->source_type & WEVTS_ANALYTIC)
198 + dictionary_set(dict, WEVT_SOURCE_ALL_ANALYTIC_NAME, &t, sizeof(t));
199 +
200 + if(src->source_type & WEVTS_DEBUG)
201 + dictionary_set(dict, WEVT_SOURCE_ALL_DEBUG_NAME, &t, sizeof(t));
202 +
203 + if(src->source_type & WEVTS_WINDOWS)
204 + dictionary_set(dict, WEVT_SOURCE_ALL_WINDOWS_NAME, &t, sizeof(t));
205 +
206 + if(src->source)
207 + dictionary_set(dict, string2str(src->source), &t, sizeof(t));
208 + }
209 + dfe_done(jf);
210 +
211 + dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
212 +}
213 +
214 +void wevt_sources_scan(void) {
215 + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
216 + LPWSTR channel = NULL;
217 + EVT_HANDLE hChannelEnum = NULL;
218 +
219 + if(spinlock_trylock(&spinlock)) {
220 + const usec_t now_monotonic_ut = now_monotonic_usec();
221 +
222 + DWORD dwChannelBufferSize = 0;
223 + DWORD dwChannelBufferUsed = 0;
224 + DWORD status = ERROR_SUCCESS;
225 +
226 + // Open a handle to enumerate the event channels
227 + hChannelEnum = EvtOpenChannelEnum(NULL, 0);
228 + if (!hChannelEnum) {
229 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "WINDOWS EVENTS: EvtOpenChannelEnum() failed with %" PRIu64 "\n",
230 + (uint64_t)GetLastError());
231 + goto cleanup;
232 + }
233 +
234 + WEVT_LOG *log = wevt_openlog6();
235 + if(!log) goto cleanup;
236 +
237 + while (true) {
238 + if (!EvtNextChannelPath(hChannelEnum, dwChannelBufferSize, channel, &dwChannelBufferUsed)) {
239 + status = GetLastError();
240 + if (status == ERROR_NO_MORE_ITEMS)
241 + break; // No more channels
242 + else if (status == ERROR_INSUFFICIENT_BUFFER) {
243 + dwChannelBufferSize = dwChannelBufferUsed;
244 + freez(channel);
245 + channel = mallocz(dwChannelBufferSize * sizeof(WCHAR));
246 + continue;
247 + } else {
248 + nd_log(NDLS_COLLECTORS, NDLP_ERR,
249 + "WINDOWS EVENTS: EvtNextChannelPath() failed\n");
250 + break;
251 + }
252 + }
253 +
254 + EVT_RETENTION retention;
255 + if(!wevt_channel_retention(log, channel, &retention))
256 + continue;
257 +
258 + const char *name = channel2utf8(channel);
259 + const char *fullname = strdupz(name);
260 + char *slash = strchr(name, '/');
261 + WEVT_SOURCE_TYPE sources = WEVTS_ALL;
262 + if(slash) {
263 + *slash++ = '\0';
264 + if(strcasecmp(slash, "Admin") == 0)
265 + sources |= WEVTS_ADMIN;
266 + if(strcasecmp(slash, "Operational") == 0)
267 + sources |= WEVTS_OPERATIONAL;
268 + if(strcasecmp(slash, "Analytic") == 0)
269 + sources |= WEVTS_ANALYTIC;
270 + if(strcasecmp(slash, "Debug") == 0)
271 + sources |= WEVTS_DEBUG;
272 + }
273 +
274 + if(strcasecmp(name, "Application") == 0)
275 + sources |= WEVTS_WINDOWS;
276 + if(strcasecmp(name, "Security") == 0)
277 + sources |= WEVTS_WINDOWS;
278 + if(strcasecmp(name, "Setup") == 0)
279 + sources |= WEVTS_WINDOWS;
280 + if(strcasecmp(name, "System") == 0)
281 + sources |= WEVTS_WINDOWS;
282 +
283 + LOGS_QUERY_SOURCE src = {
284 + .entries = retention.entries,
285 + .fullname = fullname,
286 + .fullname_len = strlen(fullname),
287 + .last_scan_monotonic_ut = now_monotonic_usec(),
288 + .msg_first_id = retention.first_event.id,
289 + .msg_last_id = retention.last_event.id,
290 + .msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC,
291 + .msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
292 + .size = retention.size_bytes,
293 + .source_type = sources,
294 + .source = string_strdupz(name),
295 + };
296 +
297 + dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
298 + }
299 +
300 + wevt_closelog6(log);
301 +
302 + LOGS_QUERY_SOURCE *src;
303 + dfe_start_write(wevt_sources, src)
304 + {
305 + if(src->last_scan_monotonic_ut < now_monotonic_ut)
306 + dictionary_del(wevt_sources, src->fullname);
307 + }
308 + dfe_done(src);
309 + dictionary_garbage_collect(wevt_sources);
310 +
311 + spinlock_unlock(&spinlock);
312 + }
313 +
314 +cleanup:
315 + freez(channel);
316 + EvtClose(hChannelEnum);
317 +}
src/collectors/windows-events.plugin/windows-events-sources.h
+55 -45
@@ -1,45 +1,55 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_WINDOWS_EVENTS_SOURCES_H
4 -#define NETDATA_WINDOWS_EVENTS_SOURCES_H
5 -
6 -#include "windows-events.h"
7 -
8 -typedef enum {
9 - WEVTS_NONE = 0,
10 - WEVTS_ALL = (1 << 0),
11 -} WEVT_SOURCE_TYPE;
12 -
13 -typedef struct {
14 - const char *fullname;
15 - size_t fullname_len;
16 -
17 - STRING *source;
18 - WEVT_SOURCE_TYPE source_type;
19 - usec_t msg_first_ut;
20 - usec_t msg_last_ut;
21 - size_t size;
22 -
23 - usec_t last_scan_monotonic_ut;
24 -
25 - uint64_t msg_first_id;
26 - uint64_t msg_last_id;
27 - uint64_t entries;
28 -} LOGS_QUERY_SOURCE;
29 -
30 -extern DICTIONARY *wevt_sources;
31 -extern DICTIONARY *used_hashes_registry;
32 -
33 -#define WEVT_SOURCE_ALL_NAME "All"
34 -
35 -void wevt_sources_init(void);
36 -void wevt_sources_scan(void);
37 -void buffer_json_wevt_versions(BUFFER *wb);
38 -
39 -void wevt_sources_to_json_array(BUFFER *wb);
40 -WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value);
41 -
42 -int wevt_sources_dict_items_backward_compar(const void *a, const void *b);
43 -int wevt_sources_dict_items_forward_compar(const void *a, const void *b);
44 -
45 -#endif //NETDATA_WINDOWS_EVENTS_SOURCES_H
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WINDOWS_EVENTS_SOURCES_H
4 +#define NETDATA_WINDOWS_EVENTS_SOURCES_H
5 +
6 +#include "windows-events.h"
7 +
8 +typedef enum {
9 + WEVTS_NONE = 0,
10 + WEVTS_ALL = (1 << 0),
11 + WEVTS_ADMIN = (1 << 1),
12 + WEVTS_OPERATIONAL = (1 << 2),
13 + WEVTS_ANALYTIC = (1 << 3),
14 + WEVTS_DEBUG = (1 << 4),
15 + WEVTS_WINDOWS = (1 << 5),
16 +} WEVT_SOURCE_TYPE;
17 +
18 +typedef struct {
19 + const char *fullname;
20 + size_t fullname_len;
21 +
22 + STRING *source;
23 + WEVT_SOURCE_TYPE source_type;
24 + usec_t msg_first_ut;
25 + usec_t msg_last_ut;
26 + size_t size;
27 +
28 + usec_t last_scan_monotonic_ut;
29 +
30 + uint64_t msg_first_id;
31 + uint64_t msg_last_id;
32 + uint64_t entries;
33 +} LOGS_QUERY_SOURCE;
34 +
35 +extern DICTIONARY *wevt_sources;
36 +extern DICTIONARY *used_hashes_registry;
37 +
38 +#define WEVT_SOURCE_ALL_NAME "All"
39 +#define WEVT_SOURCE_ALL_ADMIN_NAME "All-Admin"
40 +#define WEVT_SOURCE_ALL_OPERATIONAL_NAME "All-Operational"
41 +#define WEVT_SOURCE_ALL_ANALYTIC_NAME "All-Analytic"
42 +#define WEVT_SOURCE_ALL_DEBUG_NAME "All-Debug"
43 +#define WEVT_SOURCE_ALL_WINDOWS_NAME "All-Windows"
44 +
45 +void wevt_sources_init(void);
46 +void wevt_sources_scan(void);
47 +void buffer_json_wevt_versions(BUFFER *wb);
48 +
49 +void wevt_sources_to_json_array(BUFFER *wb);
50 +WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value);
51 +
52 +int wevt_sources_dict_items_backward_compar(const void *a, const void *b);
53 +int wevt_sources_dict_items_forward_compar(const void *a, const void *b);
54 +
55 +#endif //NETDATA_WINDOWS_EVENTS_SOURCES_H
src/collectors/windows-events.plugin/windows-events-unicode.c
+139 -121
@@ -1,121 +1,139 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "windows-events-unicode.h"
4 -
5 -inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
6 - if (src) {
7 - // Convert from UTF-8 to wide char (UTF-16)
8 - if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
9 - wcsncpy(dst, L"[failed conv.]", dst_size - 1);
10 - }
11 - else
12 - wcsncpy(dst, L"[null]", dst_size - 1);
13 -}
14 -
15 -inline void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src) {
16 - if (src) {
17 - if(WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, (int)dst_size, NULL, NULL) == 0)
18 - strncpyz(dst, "[failed conv.]", dst_size - 1);
19 - }
20 - else
21 - strncpyz(dst, "[null]", dst_size - 1);
22 -}
23 -
24 -wchar_t *channel2unicode(const char *utf8str) {
25 - static __thread wchar_t buffer[1024];
26 - utf82unicode(buffer, sizeof(buffer) / sizeof(buffer[0]), utf8str);
27 - return buffer;
28 -}
29 -
30 -char *channel2utf8(const wchar_t *channel) {
31 - static __thread char buffer[1024];
32 - unicode2utf8(buffer, sizeof(buffer), channel);
33 - return buffer;
34 -}
35 -
36 -char *account2utf8(const wchar_t *user) {
37 - static __thread char buffer[1024];
38 - unicode2utf8(buffer, sizeof(buffer), user);
39 - return buffer;
40 -}
41 -
42 -char *domain2utf8(const wchar_t *domain) {
43 - static __thread char buffer[1024];
44 - unicode2utf8(buffer, sizeof(buffer), domain);
45 - return buffer;
46 -}
47 -
48 -char *query2utf8(const wchar_t *query) {
49 - static __thread char buffer[16384];
50 - unicode2utf8(buffer, sizeof(buffer), query);
51 - return buffer;
52 -}
53 -
54 -bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null) {
55 - if(!src || !src_len_with_null)
56 - goto cleanup;
57 -
58 - // make sure the input is null terminated at the exact point we need it
59 - // (otherwise, the output will not be null terminated either)
60 - fatal_assert(src_len_with_null == -1 || (src_len_with_null >= 1 && src[src_len_with_null - 1] == 0));
61 -
62 - // Try to convert using the existing buffer (if it exists, otherwise get the required buffer size)
63 - int size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
64 - if(size <= 0 || !utf8->data) {
65 - // we have to set a buffer, or increase it
66 -
67 - if(utf8->data) {
68 - // we need to increase it the buffer size
69 -
70 - if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
71 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed.");
72 - goto cleanup;
73 - }
74 -
75 - // we have to find the required buffer size
76 - size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, NULL, 0, NULL, NULL);
77 - if(size <= 0)
78 - goto cleanup;
79 - }
80 -
81 - // Retry conversion with the new buffer
82 - txt_utf8_resize(utf8, size);
83 - size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
84 - if (size <= 0) {
85 - nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed after resizing.");
86 - goto cleanup;
87 - }
88 - }
89 -
90 - // Make sure it is not zero padded at the end
91 - while(size >= 2 && utf8->data[size - 2] == 0)
92 - size--;
93 -
94 - utf8->used = (size_t)size;
95 -
96 - internal_fatal(strlen(utf8->data) + 1 != utf8->used,
97 - "Wrong UTF8 string length");
98 -
99 - return true;
100 -
101 -cleanup:
102 - txt_utf8_resize(utf8, 128);
103 - if(src)
104 - utf8->used = snprintfz(utf8->data, utf8->size, "[failed conv.]") + 1;
105 - else {
106 - utf8->data[0] = '\0';
107 - utf8->used = 1;
108 - }
109 -
110 - return false;
111 -}
112 -
113 -bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode) {
114 - fatal_assert(utf8 && ((utf8->data && utf8->size) || (!utf8->data && !utf8->size)));
115 - fatal_assert(unicode && ((unicode->data && unicode->size) || (!unicode->data && !unicode->size)));
116 -
117 - // pass the entire unicode size, including the null terminator
118 - // so that the resulting utf8 message will be null terminated too.
119 - return wevt_str_wchar_to_utf8(utf8, unicode->data, (int)unicode->used);
120 -}
121 -
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "windows-events-unicode.h"
4 +
5 +inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
6 + if (src) {
7 + // Convert from UTF-8 to wide char (UTF-16)
8 + if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
9 + wcsncpy(dst, L"[failed conv.]", dst_size - 1);
10 + }
11 + else
12 + wcsncpy(dst, L"[null]", dst_size - 1);
13 +}
14 +
15 +inline void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src) {
16 + if (src) {
17 + if(WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, (int)dst_size, NULL, NULL) == 0)
18 + strncpyz(dst, "[failed conv.]", dst_size - 1);
19 + }
20 + else
21 + strncpyz(dst, "[null]", dst_size - 1);
22 +}
23 +
24 +char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len) {
25 + int size = WideCharToMultiByte(CP_UTF8, 0, src, -1, NULL, 0, NULL, NULL);
26 + if (size > 0) {
27 + char *dst = mallocz(size);
28 + WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, size, NULL, NULL);
29 +
30 + if(utf8_len)
31 + *utf8_len = size - 1;
32 +
33 + return dst;
34 + }
35 +
36 + if(utf8_len)
37 + *utf8_len = 0;
38 +
39 + return NULL;
40 +}
41 +
42 +wchar_t *channel2unicode(const char *utf8str) {
43 + static __thread wchar_t buffer[1024];
44 + utf82unicode(buffer, sizeof(buffer) / sizeof(buffer[0]), utf8str);
45 + return buffer;
46 +}
47 +
48 +char *channel2utf8(const wchar_t *channel) {
49 + static __thread char buffer[1024];
50 + unicode2utf8(buffer, sizeof(buffer), channel);
51 + return buffer;
52 +}
53 +
54 +char *account2utf8(const wchar_t *user) {
55 + static __thread char buffer[1024];
56 + unicode2utf8(buffer, sizeof(buffer), user);
57 + return buffer;
58 +}
59 +
60 +char *domain2utf8(const wchar_t *domain) {
61 + static __thread char buffer[1024];
62 + unicode2utf8(buffer, sizeof(buffer), domain);
63 + return buffer;
64 +}
65 +
66 +char *query2utf8(const wchar_t *query) {
67 + static __thread char buffer[16384];
68 + unicode2utf8(buffer, sizeof(buffer), query);
69 + return buffer;
70 +}
71 +
72 +bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null) {
73 + if(!src || !src_len_with_null)
74 + goto cleanup;
75 +
76 + // make sure the input is null terminated at the exact point we need it
77 + // (otherwise, the output will not be null terminated either)
78 + fatal_assert(src_len_with_null == -1 || (src_len_with_null >= 1 && src[src_len_with_null - 1] == 0));
79 +
80 + // Try to convert using the existing buffer (if it exists, otherwise get the required buffer size)
81 + int size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
82 + if(size <= 0 || !utf8->data) {
83 + // we have to set a buffer, or increase it
84 +
85 + if(utf8->data) {
86 + // we need to increase it the buffer size
87 +
88 + if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
89 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed.");
90 + goto cleanup;
91 + }
92 +
93 + // we have to find the required buffer size
94 + size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, NULL, 0, NULL, NULL);
95 + if(size <= 0)
96 + goto cleanup;
97 + }
98 +
99 + // Retry conversion with the new buffer
100 + txt_utf8_resize(utf8, size, false);
101 + size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
102 + if (size <= 0) {
103 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed after resizing.");
104 + goto cleanup;
105 + }
106 + }
107 +
108 + // Make sure it is not zero padded at the end
109 + while(size >= 2 && utf8->data[size - 2] == 0)
110 + size--;
111 +
112 + utf8->used = (size_t)size;
113 +
114 + internal_fatal(strlen(utf8->data) + 1 != utf8->used,
115 + "Wrong UTF8 string length");
116 +
117 + return true;
118 +
119 +cleanup:
120 + txt_utf8_resize(utf8, 128, false);
121 + if(src)
122 + utf8->used = snprintfz(utf8->data, utf8->size, "[failed conv.]") + 1;
123 + else {
124 + utf8->data[0] = '\0';
125 + utf8->used = 1;
126 + }
127 +
128 + return false;
129 +}
130 +
131 +bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode) {
132 + fatal_assert(utf8 && ((utf8->data && utf8->size) || (!utf8->data && !utf8->size)));
133 + fatal_assert(unicode && ((unicode->data && unicode->size) || (!unicode->data && !unicode->size)));
134 +
135 + // pass the entire unicode size, including the null terminator
136 + // so that the resulting utf8 message will be null terminated too.
137 + return wevt_str_wchar_to_utf8(utf8, unicode->data, (int)unicode->used);
138 +}
139 +
src/collectors/windows-events.plugin/windows-events-unicode.h
+99 -72
@@ -1,72 +1,99 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_WINDOWS_EVENTS_UNICODE_H
4 -#define NETDATA_WINDOWS_EVENTS_UNICODE_H
5 -
6 -#include "libnetdata/libnetdata.h"
7 -#include <windows.h>
8 -#include <wchar.h>
9 -
10 -typedef struct {
11 - char *data;
12 - size_t size; // the allocated size of data buffer
13 - size_t used; // the used size of the data buffer (including null terminators, if any)
14 -} TXT_UTF8;
15 -
16 -typedef struct {
17 - wchar_t *data;
18 - size_t size; // the allocated size of data buffer
19 - size_t used; // the used size of the data buffer (including null terminators, if any)
20 -} TXT_UNICODE;
21 -
22 -static inline size_t compute_new_size(size_t old_size, size_t required_size) {
23 - size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048;
24 - size = (size / 2048) * 2048;
25 -
26 - if(size < old_size * 2)
27 - size = old_size * 2;
28 -
29 - return size;
30 -}
31 -
32 -static inline void txt_utf8_cleanup(TXT_UTF8 *utf8) {
33 - freez(utf8->data);
34 -}
35 -
36 -static inline void txt_utf8_resize(TXT_UTF8 *utf8, size_t required_size) {
37 - if(required_size < utf8->size)
38 - return;
39 -
40 - txt_utf8_cleanup(utf8);
41 - utf8->size = compute_new_size(utf8->size, required_size);
42 - utf8->data = mallocz(utf8->size);
43 -}
44 -
45 -static inline void txt_unicode_cleanup(TXT_UNICODE *unicode) {
46 - freez(unicode->data);
47 -}
48 -
49 -static inline void txt_unicode_resize(TXT_UNICODE *unicode, size_t required_size) {
50 - if(required_size < unicode->size)
51 - return;
52 -
53 - txt_unicode_cleanup(unicode);
54 - unicode->size = compute_new_size(unicode->size, required_size);
55 - unicode->data = mallocz(unicode->size * sizeof(wchar_t));
56 -}
57 -
58 -bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode);
59 -bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null);
60 -
61 -void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src);
62 -void utf82unicode(wchar_t *dst, size_t dst_size, const char *src);
63 -
64 -char *account2utf8(const wchar_t *user);
65 -char *domain2utf8(const wchar_t *domain);
66 -
67 -char *channel2utf8(const wchar_t *channel);
68 -wchar_t *channel2unicode(const char *utf8str);
69 -
70 -char *query2utf8(const wchar_t *query);
71 -
72 -#endif //NETDATA_WINDOWS_EVENTS_UNICODE_H
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WINDOWS_EVENTS_UNICODE_H
4 +#define NETDATA_WINDOWS_EVENTS_UNICODE_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include <windows.h>
8 +#include <wchar.h>
9 +
10 +typedef enum __attribute__((packed)) {
11 + TXT_SOURCE_UNKNOWN = 0,
12 + TXT_SOURCE_PUBLISHER,
13 + TXT_SOURCE_FIELD_CACHE,
14 + TXT_SOURCE_EVENT_LOG,
15 + TXT_SOURCE_HARDCODED,
16 +
17 + // terminator
18 + TXT_SOURCE_MAX,
19 +} TXT_SOURCE;
20 +
21 +typedef struct {
22 + char *data;
23 + size_t size; // the allocated size of data buffer
24 + size_t used; // the used size of the data buffer (including null terminators, if any)
25 + TXT_SOURCE src;
26 +} TXT_UTF8;
27 +
28 +typedef struct {
29 + wchar_t *data;
30 + size_t size; // the allocated size of data buffer
31 + size_t used; // the used size of the data buffer (including null terminators, if any)
32 +} TXT_UNICODE;
33 +
34 +static inline size_t compute_new_size(size_t old_size, size_t required_size) {
35 + size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048;
36 + size = (size / 2048) * 2048;
37 +
38 + if(size < old_size * 2)
39 + size = old_size * 2;
40 +
41 + return size;
42 +}
43 +
44 +static inline void txt_utf8_cleanup(TXT_UTF8 *utf8) {
45 + freez(utf8->data);
46 +}
47 +
48 +static inline void txt_utf8_resize(TXT_UTF8 *utf8, size_t required_size, bool keep) {
49 + if(required_size < utf8->size)
50 + return;
51 +
52 + if(keep) {
53 + size_t new_size = compute_new_size(utf8->size, required_size);
54 + utf8->data = reallocz(utf8->data, new_size);
55 + utf8->size = new_size;
56 + }
57 + else {
58 + txt_utf8_cleanup(utf8);
59 + utf8->size = compute_new_size(utf8->size, required_size);
60 + utf8->data = mallocz(utf8->size);
61 + }
62 +}
63 +
64 +static inline void txt_unicode_cleanup(TXT_UNICODE *unicode) {
65 + freez(unicode->data);
66 +}
67 +
68 +static inline void txt_unicode_resize(TXT_UNICODE *unicode, size_t required_size) {
69 + if(required_size < unicode->size)
70 + return;
71 +
72 + txt_unicode_cleanup(unicode);
73 + unicode->size = compute_new_size(unicode->size, required_size);
74 + unicode->data = mallocz(unicode->size * sizeof(wchar_t));
75 +}
76 +
77 +bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode);
78 +bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null);
79 +
80 +void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src);
81 +void utf82unicode(wchar_t *dst, size_t dst_size, const char *src);
82 +
83 +char *account2utf8(const wchar_t *user);
84 +char *domain2utf8(const wchar_t *domain);
85 +
86 +char *channel2utf8(const wchar_t *channel);
87 +wchar_t *channel2unicode(const char *utf8str);
88 +
89 +char *query2utf8(const wchar_t *query);
90 +
91 +char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len);
92 +
93 +static inline void wevt_utf8_empty(TXT_UTF8 *dst) {
94 + txt_utf8_resize(dst, 1, false);
95 + dst->data[0] = '\0';
96 + dst->used = 1;
97 +}
98 +
99 +#endif //NETDATA_WINDOWS_EVENTS_UNICODE_H
src/collectors/windows-events.plugin/windows-events-xml.c
+144 -11
@@ -32,15 +32,20 @@ const char *parse_value_and_closing_tag(BUFFER *buffer, const char *xml, const c
32 const char *start = xml;
33 bool has_subnodes = false;
34
35 + // const char *tag_start = NULL, *tag_end = NULL;
36 while (xml < end) {
37 if(*xml == '<') {
38 if(xml + 1 < end && *(xml + 1) == '/') {
39 // a closing tag
40 xml += 2;
41
42 +// tag_start = xml;
43 +
44 while(xml < end && *xml != '>')
45 xml++;
46
47 +// tag_end = xml;
48 +
49 if(xml < end && *xml == '>')
50 xml++;
51
@@ -56,7 +61,7 @@ const char *parse_value_and_closing_tag(BUFFER *buffer, const char *xml, const c
61 // an opening tag
62 buffer_fast_strcat(buffer, start, xml - start);
63 xml = start = parse_node(buffer, xml, end, level + 1);
59 - while(xml < end && isspace(*xml))
64 + while(xml < end && isspace((uint8_t)*xml))
65 xml++;
66 has_subnodes = true;
67 }
@@ -101,7 +106,7 @@ const char *parse_field_value(BUFFER *buffer, const char *xml, const char *end)
106
107 // Parse a field name and return the next position to parse
108 const char *parse_field(BUFFER *buffer, const char *xml, const char *end) {
104 - while(isspace(*xml) && xml < end) xml++;
109 + while(isspace((uint8_t)*xml) && xml < end) xml++;
110
111 const char *start = xml;
112
@@ -136,16 +141,18 @@ static inline const char *parse_node(BUFFER *buffer, const char *xml, const char
141 buffer_add_xml_indent(buffer, level);
142
143 // skip spaces before the tag name
139 - while(xml < end && isspace(*xml)) xml++;
144 + while(xml < end && isspace((uint8_t)*xml)) xml++;
145
146 // Parse the tag name
147 +// const char *tag_start = xml, *tag_end = NULL;
148 while (xml < end && *xml != '>' && *xml != '/') {
149 xml++;
150
145 - if(xml < end && isspace(*xml)) {
151 + if(xml < end && isspace((uint8_t)*xml)) {
152 xml++;
153 +// tag_end = xml;
154
148 - while(xml < end && isspace(*xml))
155 + while(xml < end && isspace((uint8_t)*xml))
156 xml++;
157
158 if(xml < end && *xml == '/') {
@@ -168,7 +175,7 @@ static inline const char *parse_node(BUFFER *buffer, const char *xml, const char
175 else {
176 buffer_fast_strcat(buffer, start, xml - start);
177 xml = start = parse_field(buffer, xml, end);
171 - while(xml < end && isspace(*xml))
178 + while(xml < end && isspace((uint8_t)*xml))
179 xml++;
180 }
181 }
@@ -193,12 +200,9 @@ static inline const char *parse_node(BUFFER *buffer, const char *xml, const char
200 return append_the_rest(buffer, start, end);
201 }
202
196 -// Main pretty-print XML function
197 -void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len) {
198 - const char *end = xml + xml_len;
199 -
203 +static inline void buffer_pretty_print_xml_object(BUFFER *buffer, const char *xml, const char *end) {
204 while(xml < end) {
201 - while(xml < end && isspace(*xml))
205 + while(xml < end && isspace((uint8_t)*xml))
206 xml++;
207
208 if(xml < end && *xml == '<')
@@ -209,3 +213,132 @@ void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len) {
213 }
214 }
215 }
216 +
217 +void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len) {
218 + const char *end = xml + xml_len;
219 + buffer_pretty_print_xml_object(buffer, xml, end);
220 +}
221 +
222 +// --------------------------------------------------------------------------------------------------------------------
223 +
224 +bool buffer_extract_and_print_xml_with_cb(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[],
225 + void (*cb)(BUFFER *, const char *, const char *, const char *)) {
226 + if(!keys || !*keys[0]) {
227 + buffer_pretty_print_xml(buffer, xml, xml_len);
228 + return true;
229 + }
230 +
231 + const char *start = xml, *end = NULL;
232 + for(size_t k = 0; keys[k] ; k++) {
233 + if(!*keys[k]) continue;
234 +
235 + size_t klen = strlen(keys[k]);
236 + char tag_open[klen + 2];
237 + tag_open[0] = '<';
238 + strcpy(&tag_open[1], keys[k]);
239 + tag_open[klen + 1] = '\0';
240 +
241 + const char *new_start = strstr(start, tag_open);
242 + if(!new_start)
243 + return false;
244 +
245 + start = new_start + klen + 1;
246 +
247 + if(*start != '>' && !isspace((uint8_t)*start))
248 + return false;
249 +
250 + if(*start != '>') {
251 + start = strchr(start, '>');
252 + if(!start) return false;
253 + }
254 + start++; // skip the >
255 +
256 + char tag_close[klen + 4];
257 + tag_close[0] = '<';
258 + tag_close[1] = '/';
259 + strcpy(&tag_close[2], keys[k]);
260 + tag_close[klen + 2] = '>';
261 + tag_close[klen + 3] = '\0';
262 +
263 + const char *new_end = strstr(start, tag_close);
264 + if(!new_end || (end && new_end > end))
265 + return false;
266 +
267 + end = new_end;
268 + }
269 +
270 + if(!start || !end || start == end)
271 + return false;
272 +
273 + cb(buffer, prefix, start, end);
274 + return true;
275 +}
276 +
277 +static void print_xml_cb(BUFFER *buffer, const char *prefix, const char *start, const char *end) {
278 + if(prefix)
279 + buffer_strcat(buffer, prefix);
280 +
281 + buffer_pretty_print_xml_object(buffer, start, end);
282 +}
283 +
284 +bool buffer_extract_and_print_xml(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]) {
285 + return buffer_extract_and_print_xml_with_cb(
286 + buffer, xml, xml_len,
287 + prefix, keys,
288 + print_xml_cb);
289 +}
290 +
291 +static void print_value_cb(BUFFER *buffer, const char *prefix, const char *start, const char *end) {
292 + if(prefix)
293 + buffer_strcat(buffer, prefix);
294 +
295 + buffer_need_bytes(buffer, end - start + 1);
296 +
297 + char *started = &buffer->buffer[buffer->len];
298 + char *d = started;
299 + const char *s = start;
300 +
301 + while(s < end && s) {
302 + if(*s == '&' && s + 3 < end) {
303 + if(*(s + 1) == '#') {
304 + if(s + 4 < end && *(s + 2) == '1' && *(s + 4) == ';') {
305 + if (*(s + 3) == '0') {
306 + s += 5;
307 + *d++ = '\n';
308 + continue;
309 + } else if (*(s + 3) == '3') {
310 + s += 5;
311 + // *d++ = '\r';
312 + continue;
313 + }
314 + } else if (*(s + 2) == '9' && *(s + 3) == ';') {
315 + s += 4;
316 + *d++ = '\t';
317 + continue;
318 + }
319 + }
320 + else if(s + 3 < end && *(s + 2) == 't' && *(s + 3) == ';') {
321 + if(*(s + 1) == 'l') {
322 + s += 4;
323 + *d++ = '<';
324 + continue;
325 + }
326 + else if(*(s + 1) == 'g') {
327 + s += 4;
328 + *d++ = '>';
329 + continue;
330 + }
331 + }
332 + }
333 + *d++ = *s++;
334 + }
335 + *d = '\0';
336 + buffer->len += d - started;
337 +}
338 +
339 +bool buffer_xml_extract_and_print_value(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]) {
340 + return buffer_extract_and_print_xml_with_cb(
341 + buffer, xml, xml_len,
342 + prefix, keys,
343 + print_value_cb);
344 +}
src/collectors/windows-events.plugin/windows-events-xml.h
+2
@@ -6,5 +6,7 @@
6 #include "libnetdata/libnetdata.h"
7
8 void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len);
9 +bool buffer_extract_and_print_xml(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]);
10 +bool buffer_xml_extract_and_print_value(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]);
11
12 #endif //WINDOWS_EVENTS_XML_H
src/collectors/windows-events.plugin/windows-events.c
+1373 -1191
@@ -1,1191 +1,1373 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "libnetdata/libnetdata.h"
4 -#include "libnetdata/required_dummies.h"
5 -
6 -#include "windows-events.h"
7 -
8 -#define WINDOWS_EVENTS_WORKER_THREADS 5
9 -#define WINDOWS_EVENTS_DEFAULT_TIMEOUT 600
10 -#define WINDOWS_EVENTS_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
11 -#define WINDOWS_EVENTS_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
12 -
13 -#define FUNCTION_PROGRESS_UPDATE_ROWS(rows_read, rows) __atomic_fetch_add(&(rows_read), rows, __ATOMIC_RELAXED)
14 -#define FUNCTION_PROGRESS_UPDATE_BYTES(bytes_read, bytes) __atomic_fetch_add(&(bytes_read), bytes, __ATOMIC_RELAXED)
15 -#define FUNCTION_PROGRESS_EVERY_ROWS (1ULL << 13)
16 -#define FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS (1ULL << 7)
17 -#define ANCHOR_DELTA_UT (10 * USEC_PER_SEC)
18 -
19 -netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER;
20 -static bool plugin_should_exit = false;
21 -
22 -#define WEVT_FUNCTION_DESCRIPTION "View, search and analyze the Microsoft Windows Events log."
23 -#define WEVT_FUNCTION_NAME "windows-events"
24 -
25 -// functions needed by LQS
26 -
27 -// structures needed by LQS
28 -struct lqs_extension {
29 - struct {
30 - usec_t start_ut;
31 - usec_t stop_ut;
32 - usec_t first_msg_ut;
33 -
34 - uint64_t first_msg_seqnum;
35 - } query_file;
36 -
37 - // struct {
38 - // uint32_t enable_after_samples;
39 - // uint32_t slots;
40 - // uint32_t sampled;
41 - // uint32_t unsampled;
42 - // uint32_t estimated;
43 - // } samples;
44 -
45 - // struct {
46 - // uint32_t enable_after_samples;
47 - // uint32_t every;
48 - // uint32_t skipped;
49 - // uint32_t recalibrate;
50 - // uint32_t sampled;
51 - // uint32_t unsampled;
52 - // uint32_t estimated;
53 - // } samples_per_file;
54 -
55 - // struct {
56 - // usec_t start_ut;
57 - // usec_t end_ut;
58 - // usec_t step_ut;
59 - // uint32_t enable_after_samples;
60 - // uint32_t sampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
61 - // uint32_t unsampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
62 - // } samples_per_time_slot;
63 -
64 - // per file progress info
65 - // size_t cached_count;
66 -
67 - // progress statistics
68 - usec_t matches_setup_ut;
69 - size_t rows_useful;
70 - size_t rows_read;
71 - size_t bytes_read;
72 - size_t files_matched;
73 - size_t file_working;
74 -};
75 -
76 -// prepare LQS
77 -#define LQS_DEFAULT_SLICE_MODE 0
78 -#define LQS_FUNCTION_NAME WEVT_FUNCTION_NAME
79 -#define LQS_FUNCTION_DESCRIPTION WEVT_FUNCTION_DESCRIPTION
80 -#define LQS_DEFAULT_ITEMS_PER_QUERY 200
81 -#define LQS_DEFAULT_ITEMS_SAMPLING 1000000
82 -#define LQS_SOURCE_TYPE WEVT_SOURCE_TYPE
83 -#define LQS_SOURCE_TYPE_ALL WEVTS_ALL
84 -#define LQS_SOURCE_TYPE_NONE WEVTS_NONE
85 -#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) wevt_internal_source_type(value)
86 -#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) wevt_sources_to_json_array(wb)
87 -#include "libnetdata/facets/logs_query_status.h"
88 -
89 -#define WEVT_FIELD_COMPUTER "Computer"
90 -#define WEVT_FIELD_CHANNEL "Channel"
91 -#define WEVT_FIELD_PROVIDER "Provider"
92 -#define WEVT_FIELD_SOURCE "Source"
93 -#define WEVT_FIELD_EVENTRECORDID "EventRecordID"
94 -#define WEVT_FIELD_EVENTID "EventID"
95 -#define WEVT_FIELD_LEVELID "LevelID"
96 -#define WEVT_FIELD_LEVEL "Level"
97 -#define WEVT_FIELD_KEYWORDID "KeywordID"
98 -#define WEVT_FIELD_KEYWORD "Keyword"
99 -#define WEVT_FIELD_OPCODEID "OpcodeID"
100 -#define WEVT_FIELD_OPCODE "Opcode"
101 -#define WEVT_FIELD_USER "User"
102 -#define WEVT_FIELD_EVENT "Event"
103 -#define WEVT_FIELD_TASK "Task"
104 -#define WEVT_FIELD_PROCESSID "ProcessID"
105 -#define WEVT_FIELD_THREADID "ThreadID"
106 -#define WEVT_FIELD_XML "XML"
107 -#define WEVT_FIELD_MESSAGE "Message"
108 -
109 -#define WEVT_ALWAYS_VISIBLE_KEYS NULL
110 -
111 -#define WEVT_KEYS_EXCLUDED_FROM_FACETS \
112 - "|" WEVT_FIELD_EVENT \
113 - "|" WEVT_FIELD_MESSAGE \
114 - "|" WEVT_FIELD_XML \
115 - ""
116 -
117 -#define WEVT_KEYS_INCLUDED_IN_FACETS \
118 - "|" WEVT_FIELD_PROVIDER \
119 - "|" WEVT_FIELD_SOURCE \
120 - "|" WEVT_FIELD_CHANNEL \
121 - "|" WEVT_FIELD_LEVEL \
122 - "|" WEVT_FIELD_KEYWORD \
123 - "|" WEVT_FIELD_OPCODE \
124 - "|" WEVT_FIELD_COMPUTER \
125 - "|" WEVT_FIELD_USER \
126 - ""
127 -
128 -static inline WEVT_QUERY_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) {
129 - if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
130 - internal_error(true, "Function has been cancelled");
131 - return WEVT_CANCELLED;
132 - }
133 -
134 - if(now_monotonic_usec() > __atomic_load_n(stop_monotonic_ut, __ATOMIC_RELAXED)) {
135 - internal_error(true, "Function timed out");
136 - return WEVT_TIMED_OUT;
137 - }
138 -
139 - return WEVT_OK;
140 -}
141 -
142 -FACET_ROW_SEVERITY wevt_levelid_to_facet_severity(FACETS *facets __maybe_unused, FACET_ROW *row, void *data __maybe_unused) {
143 - FACET_ROW_KEY_VALUE *levelid_rkv = dictionary_get(row->dict, WEVT_FIELD_LEVELID);
144 - if(!levelid_rkv || levelid_rkv->empty)
145 - return FACET_ROW_SEVERITY_NORMAL;
146 -
147 - int windows_event_level = str2i(buffer_tostring(levelid_rkv->wb));
148 -
149 - switch (windows_event_level) {
150 - case 5: // Verbose
151 - return FACET_ROW_SEVERITY_DEBUG;
152 - case 4: // Information
153 - return FACET_ROW_SEVERITY_NORMAL;
154 - case 3: // Warning
155 - return FACET_ROW_SEVERITY_WARNING;
156 - case 2: // Error
157 - return FACET_ROW_SEVERITY_CRITICAL;
158 - case 1: // Critical
159 - return FACET_ROW_SEVERITY_CRITICAL;
160 - default: // Any unhandled or special levels
161 - return FACET_ROW_SEVERITY_NORMAL;
162 - }
163 -}
164 -
165 -void wevt_render_message(
166 - FACETS *facets __maybe_unused,
167 - BUFFER *json_array,
168 - FACET_ROW_KEY_VALUE *rkv,
169 - FACET_ROW *row,
170 - void *data __maybe_unused) {
171 -
172 - buffer_flush(rkv->wb);
173 -
174 - const FACET_ROW_KEY_VALUE *event_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENT);
175 - if(!event_rkv || !buffer_strlen(event_rkv->wb)) {
176 - const FACET_ROW_KEY_VALUE *event_id_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENTID);
177 - if(event_id_rkv && buffer_strlen(event_id_rkv->wb)) {
178 - buffer_fast_strcat(rkv->wb, "EventID ", 8);
179 - buffer_fast_strcat(rkv->wb, buffer_tostring(event_id_rkv->wb), buffer_strlen(event_id_rkv->wb));
180 - }
181 - else
182 - buffer_strcat(rkv->wb, "Unknown EventID ");
183 -
184 - const FACET_ROW_KEY_VALUE *provider_rkv = dictionary_get(row->dict, WEVT_FIELD_PROVIDER);
185 - if(provider_rkv && buffer_strlen(provider_rkv->wb)) {
186 - buffer_fast_strcat(rkv->wb, " of ", 4);
187 - buffer_fast_strcat(rkv->wb, buffer_tostring(provider_rkv->wb), buffer_strlen(provider_rkv->wb));
188 - buffer_putc(rkv->wb, '.');
189 - }
190 - else
191 - buffer_strcat(rkv->wb, "of unknown Provider.");
192 - }
193 - else
194 - buffer_fast_strcat(rkv->wb, buffer_tostring(event_rkv->wb), buffer_strlen(event_rkv->wb));
195 -
196 - const FACET_ROW_KEY_VALUE *xml_rkv = dictionary_get(row->dict, WEVT_FIELD_XML);
197 - if(xml_rkv && buffer_strlen(xml_rkv->wb)) {
198 - const char *xml = buffer_tostring(xml_rkv->wb);
199 -
200 - bool added_event_data = false, added_user_data = false;
201 -
202 - const char *data_start = strstr(xml, "<EventData>");
203 - if(data_start) {
204 - data_start = &data_start[11];
205 - const char *data_end = strstr(data_start, "</EventData>");
206 -
207 - if(data_start && data_end) {
208 - buffer_fast_strcat(rkv->wb, "\n\nRelated event data:\n", 22);
209 - // copy the event data block
210 - buffer_pretty_print_xml(rkv->wb, data_start, data_end - data_start);
211 - added_event_data = true;
212 - }
213 - }
214 -
215 - data_start = strstr(xml, "<UserData>");
216 - if(data_start) {
217 - data_start = &data_start[10];
218 - const char *data_end = strstr(data_start, "</UserData>");
219 -
220 - if(data_start && data_end) {
221 - buffer_fast_strcat(rkv->wb, "\n\nRelated user data:\n", 21);
222 - // copy the event data block
223 - buffer_pretty_print_xml(rkv->wb, data_start, data_end - data_start);
224 - added_user_data = true;
225 - }
226 - }
227 -
228 - if(!added_event_data && !added_user_data)
229 - buffer_strcat(rkv->wb, " Without any related data.");
230 - }
231 -
232 - buffer_json_add_array_item_string(json_array, buffer_tostring(rkv->wb));
233 -}
234 -
235 -static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
236 - FACETS *facets = lqs->facets;
237 - LOGS_QUERY_REQUEST *rq = &lqs->rq;
238 -
239 - facets_register_row_severity(facets, wevt_levelid_to_facet_severity, NULL);
240 -
241 - facets_register_key_name(
242 - facets, WEVT_FIELD_COMPUTER,
243 - rq->default_facet | FACET_KEY_OPTION_VISIBLE);
244 -
245 - facets_register_key_name(
246 - facets, WEVT_FIELD_CHANNEL,
247 - rq->default_facet |
248 - FACET_KEY_OPTION_FTS);
249 -
250 - facets_register_key_name(
251 - facets, WEVT_FIELD_PROVIDER,
252 - rq->default_facet |
253 - FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
254 -
255 - facets_register_key_name(
256 - facets, WEVT_FIELD_SOURCE,
257 - rq->default_facet |
258 - FACET_KEY_OPTION_FTS);
259 -
260 - facets_register_key_name(
261 - facets, WEVT_FIELD_EVENTID,
262 - rq->default_facet |
263 - FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
264 -
265 - facets_register_key_name(
266 - facets, WEVT_FIELD_LEVELID,
267 - rq->default_facet);
268 -
269 - facets_register_key_name(
270 - facets, WEVT_FIELD_LEVEL,
271 - rq->default_facet | FACET_KEY_OPTION_FTS);
272 -
273 - facets_register_key_name(
274 - facets, WEVT_FIELD_KEYWORDID,
275 - rq->default_facet | FACET_KEY_OPTION_FTS);
276 -
277 - facets_register_key_name(
278 - facets, WEVT_FIELD_KEYWORD,
279 - rq->default_facet | FACET_KEY_OPTION_FTS);
280 -
281 - facets_register_key_name(
282 - facets, WEVT_FIELD_OPCODEID,
283 - rq->default_facet | FACET_KEY_OPTION_FTS);
284 -
285 - facets_register_key_name(
286 - facets, WEVT_FIELD_OPCODE,
287 - rq->default_facet | FACET_KEY_OPTION_FTS);
288 -
289 - facets_register_key_name(
290 - facets, WEVT_FIELD_USER,
291 - rq->default_facet | FACET_KEY_OPTION_FTS);
292 -
293 - facets_register_key_name(
294 - facets, WEVT_FIELD_TASK,
295 - rq->default_facet | FACET_KEY_OPTION_FTS);
296 -
297 - facets_register_key_name(
298 - facets, WEVT_FIELD_PROCESSID,
299 - rq->default_facet | FACET_KEY_OPTION_FTS);
300 -
301 - facets_register_key_name(
302 - facets, WEVT_FIELD_THREADID,
303 - rq->default_facet | FACET_KEY_OPTION_FTS);
304 -
305 - facets_register_key_name(
306 - facets, WEVT_FIELD_EVENT,
307 - FACET_KEY_OPTION_NEVER_FACET |
308 - FACET_KEY_OPTION_FTS);
309 -
310 - facets_register_key_name(
311 - facets, WEVT_FIELD_TASK,
312 - rq->default_facet | FACET_KEY_OPTION_FTS);
313 -
314 - facets_register_key_name(
315 - facets, WEVT_FIELD_PROCESSID,
316 - rq->default_facet | FACET_KEY_OPTION_FTS);
317 -
318 - facets_register_key_name(
319 - facets, WEVT_FIELD_THREADID,
320 - rq->default_facet | FACET_KEY_OPTION_FTS);
321 -
322 - facets_register_key_name(
323 - facets, WEVT_FIELD_XML,
324 - FACET_KEY_OPTION_NEVER_FACET |
325 - FACET_KEY_OPTION_FTS);
326 -
327 - // facets_register_key_name(
328 - // facets, WEVT_FIELD_MESSAGE,
329 - // FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT |
330 - // FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
331 -
332 - facets_register_dynamic_key_name(
333 - facets, WEVT_FIELD_MESSAGE,
334 - FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE,
335 - wevt_render_message, NULL);
336 -}
337 -
338 -static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUERY_SOURCE *src, usec_t *msg_ut __maybe_unused, WEVT_EVENT *e) {
339 - size_t len, bytes = log->ops.content.len;
340 -
341 - bytes += log->ops.provider.used * 2; // unicode is double
342 - facets_add_key_value_length(facets,
343 - WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
344 - log->ops.provider.data, log->ops.provider.used - 1);
345 -
346 - bytes += log->ops.source.used * 2;
347 - facets_add_key_value_length(facets,
348 - WEVT_FIELD_SOURCE, sizeof(WEVT_FIELD_SOURCE) - 1,
349 - log->ops.source.data, log->ops.source.used - 1);
350 -
351 - if(log->ops.channel.used > 1) {
352 - bytes += log->ops.channel.used * 2;
353 - facets_add_key_value_length(facets,
354 - WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
355 - log->ops.channel.data, log->ops.channel.used - 1);
356 - }
357 - else {
358 - bytes += src->fullname_len * 2;
359 - facets_add_key_value_length(facets,
360 - WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
361 - src->fullname, src->fullname_len);
362 - }
363 -
364 - static __thread char id_str[24];
365 - len = print_uint64(id_str, e->id);
366 - bytes += len;
367 - facets_add_key_value_length(facets,
368 - WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
369 - id_str, len);
370 -
371 - static __thread char level_id_str[24];
372 - len = print_uint64(level_id_str, e->level);
373 - bytes += len;
374 - facets_add_key_value_length(facets,
375 - WEVT_FIELD_LEVELID, sizeof(WEVT_FIELD_LEVELID) - 1,
376 - level_id_str, len);
377 -
378 - bytes += log->ops.level.used * 2;
379 - facets_add_key_value_length(facets,
380 - WEVT_FIELD_LEVEL, sizeof(WEVT_FIELD_LEVEL) - 1,
381 - log->ops.level.data, log->ops.level.used - 1);
382 -
383 - if(e->keyword) {
384 - static __thread char keyword_id_str[24];
385 - len = print_uint64_hex(keyword_id_str, e->keyword);
386 - bytes += len;
387 - facets_add_key_value_length(facets,
388 - WEVT_FIELD_KEYWORDID, sizeof(WEVT_FIELD_KEYWORDID) - 1,
389 - keyword_id_str, len);
390 -
391 - bytes += log->ops.keyword.used * 2;
392 - facets_add_key_value_length(facets,
393 - WEVT_FIELD_KEYWORD, sizeof(WEVT_FIELD_KEYWORD) - 1,
394 - log->ops.keyword.data, log->ops.keyword.used - 1);
395 - }
396 -
397 - bytes += log->ops.computer.used * 2;
398 - facets_add_key_value_length(facets,
399 - WEVT_FIELD_COMPUTER, sizeof(WEVT_FIELD_COMPUTER) - 1,
400 - log->ops.computer.data, log->ops.computer.used - 1);
401 -
402 - if(e->opcode) {
403 - static __thread char opcode_id_str[24];
404 - len = print_uint64(opcode_id_str, e->opcode);
405 - bytes += len;
406 - facets_add_key_value_length(facets,
407 - WEVT_FIELD_OPCODEID, sizeof(WEVT_FIELD_OPCODEID) - 1,
408 - opcode_id_str, len);
409 -
410 - bytes += log->ops.opcode.used * 2;
411 - facets_add_key_value_length(facets,
412 - WEVT_FIELD_OPCODE, sizeof(WEVT_FIELD_OPCODE) - 1,
413 - log->ops.opcode.data, log->ops.opcode.used - 1);
414 - }
415 -
416 - bytes += log->ops.user.used * 2;
417 - facets_add_key_value_length(facets,
418 - WEVT_FIELD_USER, sizeof(WEVT_FIELD_USER) - 1,
419 - log->ops.user.data, log->ops.user.used - 1);
420 -
421 - if(e->event_id) {
422 - static __thread char event_id_str[24];
423 - len = print_uint64(event_id_str, e->event_id);
424 - bytes += len;
425 - facets_add_key_value_length(facets,
426 - WEVT_FIELD_EVENTID, sizeof(WEVT_FIELD_EVENTID) - 1,
427 - event_id_str, len);
428 -
429 - bytes += log->ops.event.used * 2;
430 - facets_add_key_value_length(facets,
431 - WEVT_FIELD_EVENT, sizeof(WEVT_FIELD_EVENT) - 1,
432 - log->ops.event.data, log->ops.event.used - 1);
433 - }
434 -
435 - if(e->task) {
436 - static __thread char task_str[24];
437 - len = print_uint64(task_str, e->task);
438 - bytes += len;
439 - facets_add_key_value_length(facets,
440 - WEVT_FIELD_TASK, sizeof(WEVT_FIELD_TASK) - 1,
441 - task_str, len);
442 - }
443 -
444 - if(e->process_id) {
445 - static __thread char process_id_str[24];
446 - len = print_uint64(process_id_str, e->process_id);
447 - bytes += len;
448 - facets_add_key_value_length(facets,
449 - WEVT_FIELD_PROCESSID, sizeof(WEVT_FIELD_PROCESSID) - 1,
450 - process_id_str, len);
451 - }
452 -
453 - if(e->thread_id) {
454 - static __thread char thread_id_str[24];
455 - len = print_uint64(thread_id_str, e->thread_id);
456 - bytes += len;
457 - facets_add_key_value_length(facets,
458 - WEVT_FIELD_THREADID, sizeof(WEVT_FIELD_THREADID) - 1,
459 - thread_id_str, len);
460 - }
461 -
462 - bytes += log->ops.xml.used * 2;
463 - facets_add_key_value_length(facets,
464 - WEVT_FIELD_XML, sizeof(WEVT_FIELD_XML) - 1,
465 - log->ops.xml.data, log->ops.xml.used - 1);
466 -
467 - // facets_add_key_value_length(facets,
468 - // WEVT_FIELD_MESSAGE, sizeof(WEVT_FIELD_MESSAGE) - 1,
469 - // buffer_tostring(log->ops.message), buffer_strlen(log->ops.message));
470 -
471 - return bytes;
472 -}
473 -
474 -static WEVT_QUERY_STATUS wevt_query_backward(
475 - WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
476 - LOGS_QUERY_SOURCE *src,
477 - LOGS_QUERY_STATUS *lqs)
478 -{
479 -
480 - usec_t anchor_delta = ANCHOR_DELTA_UT;
481 -
482 - usec_t start_ut = ((lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.before_ut) + anchor_delta;
483 - usec_t stop_ut = (lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.after_ut;
484 - bool stop_when_full = (lqs->rq.data_only && !lqs->anchor.stop_ut);
485 -
486 - lqs->c.query_file.start_ut = start_ut;
487 - lqs->c.query_file.stop_ut = stop_ut;
488 -
489 - log->event_query = wevt_query(channel2unicode(src->fullname), start_ut, true);
490 - if(!log->event_query)
491 - return WEVT_FAILED_TO_SEEK;
492 -
493 - size_t errors_no_timestamp = 0;
494 - usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
495 - usec_t first_msg_ut = 0; // the first message we got from the db
496 - size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
497 - size_t bytes = 0, last_bytes = 0;
498 -
499 - usec_t last_usec_from = 0;
500 - usec_t last_usec_to = 0;
501 -
502 - WEVT_QUERY_STATUS status = WEVT_OK;
503 -
504 - facets_rows_begin(facets);
505 - WEVT_EVENT e;
506 - while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
507 - usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
508 -
509 - if(unlikely(!msg_ut)) {
510 - errors_no_timestamp++;
511 - continue;
512 - }
513 -
514 - if (unlikely(msg_ut > start_ut))
515 - continue;
516 -
517 - if (unlikely(msg_ut < stop_ut))
518 - break;
519 -
520 - if(unlikely(msg_ut > latest_msg_ut))
521 - latest_msg_ut = msg_ut;
522 -
523 - if(unlikely(!first_msg_ut)) {
524 - first_msg_ut = msg_ut;
525 - lqs->c.query_file.first_msg_ut = msg_ut;
526 - }
527 -
528 -// sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
529 -// FACETS_ANCHOR_DIRECTION_BACKWARD,
530 -// facets_row_candidate_to_keep(facets, msg_ut));
531 -//
532 -// if(sample == SAMPLING_FULL) {
533 - bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
534 -
535 - // make sure each line gets a unique timestamp
536 - if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
537 - msg_ut = --last_usec_from;
538 - else
539 - last_usec_from = last_usec_to = msg_ut;
540 -
541 - if(facets_row_finished(facets, msg_ut))
542 - rows_useful++;
543 -
544 - row_counter++;
545 - if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
546 - stop_when_full &&
547 - facets_rows(facets) >= lqs->rq.entries)) {
548 - // stop the data only query
549 - usec_t oldest = facets_row_oldest_ut(facets);
550 - if(oldest && msg_ut < (oldest - anchor_delta))
551 - break;
552 - }
553 -
554 - if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
555 - FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
556 - last_row_counter = row_counter;
557 -
558 - FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
559 - last_bytes = bytes;
560 -
561 - status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
562 - }
563 -// }
564 -// else if(sample == SAMPLING_SKIP_FIELDS)
565 -// facets_row_finished_unsampled(facets, msg_ut);
566 -// else {
567 -// sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_BACKWARD);
568 -// break;
569 -// }
570 - }
571 -
572 - FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
573 - FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
574 -
575 - lqs->c.rows_useful += rows_useful;
576 -
577 - if(errors_no_timestamp)
578 - netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
579 -
580 - if(latest_msg_ut > lqs->last_modified)
581 - lqs->last_modified = latest_msg_ut;
582 -
583 - wevt_query_done(log);
584 -
585 - return status;
586 -}
587 -
588 -static WEVT_QUERY_STATUS wevt_query_forward(
589 - WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
590 - LOGS_QUERY_SOURCE *src,
591 - LOGS_QUERY_STATUS *lqs)
592 -{
593 - usec_t anchor_delta = ANCHOR_DELTA_UT;
594 -
595 - usec_t start_ut = (lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.after_ut;
596 - usec_t stop_ut = ((lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.before_ut) + anchor_delta;
597 - bool stop_when_full = (lqs->rq.data_only && !lqs->anchor.stop_ut);
598 -
599 - lqs->c.query_file.start_ut = start_ut;
600 - lqs->c.query_file.stop_ut = stop_ut;
601 -
602 - log->event_query = wevt_query(channel2unicode(src->fullname), start_ut, false);
603 - if(!log->event_query)
604 - return WEVT_FAILED_TO_SEEK;
605 -
606 - size_t errors_no_timestamp = 0;
607 - usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
608 - usec_t first_msg_ut = 0; // the first message we got from the db
609 - size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
610 - size_t bytes = 0, last_bytes = 0;
611 -
612 - usec_t last_usec_from = 0;
613 - usec_t last_usec_to = 0;
614 -
615 - WEVT_QUERY_STATUS status = WEVT_OK;
616 -
617 - facets_rows_begin(facets);
618 - WEVT_EVENT e;
619 - while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
620 - usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
621 -
622 - if(unlikely(!msg_ut)) {
623 - errors_no_timestamp++;
624 - continue;
625 - }
626 -
627 - if (unlikely(msg_ut < start_ut))
628 - continue;
629 -
630 - if (unlikely(msg_ut > stop_ut))
631 - break;
632 -
633 - if(likely(msg_ut > latest_msg_ut))
634 - latest_msg_ut = msg_ut;
635 -
636 - if(unlikely(!first_msg_ut)) {
637 - first_msg_ut = msg_ut;
638 - lqs->c.query_file.first_msg_ut = msg_ut;
639 - }
640 -
641 -// sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
642 -// FACETS_ANCHOR_DIRECTION_FORWARD,
643 -// facets_row_candidate_to_keep(facets, msg_ut));
644 -//
645 -// if(sample == SAMPLING_FULL) {
646 - bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
647 -
648 - // make sure each line gets a unique timestamp
649 - if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
650 - msg_ut = ++last_usec_to;
651 - else
652 - last_usec_from = last_usec_to = msg_ut;
653 -
654 - if(facets_row_finished(facets, msg_ut))
655 - rows_useful++;
656 -
657 - row_counter++;
658 - if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
659 - stop_when_full &&
660 - facets_rows(facets) >= lqs->rq.entries)) {
661 - // stop the data only query
662 - usec_t newest = facets_row_newest_ut(facets);
663 - if(newest && msg_ut > (newest + anchor_delta))
664 - break;
665 - }
666 -
667 - if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
668 - FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
669 - last_row_counter = row_counter;
670 -
671 - FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
672 - last_bytes = bytes;
673 -
674 - status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
675 - }
676 -// }
677 -// else if(sample == SAMPLING_SKIP_FIELDS)
678 -// facets_row_finished_unsampled(facets, msg_ut);
679 -// else {
680 -// sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_FORWARD);
681 -// break;
682 -// }
683 - }
684 -
685 - FUNCTION_PROGRESS_UPDATE_ROWS(lqs->c.rows_read, row_counter - last_row_counter);
686 - FUNCTION_PROGRESS_UPDATE_BYTES(lqs->c.bytes_read, bytes - last_bytes);
687 -
688 - lqs->c.rows_useful += rows_useful;
689 -
690 - if(errors_no_timestamp)
691 - netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
692 -
693 - if(latest_msg_ut > lqs->last_modified)
694 - lqs->last_modified = latest_msg_ut;
695 -
696 - wevt_query_done(log);
697 -
698 - return status;
699 -}
700 -
701 -static WEVT_QUERY_STATUS wevt_query_one_channel(
702 - WEVT_LOG *log,
703 - BUFFER *wb, FACETS *facets,
704 - LOGS_QUERY_SOURCE *src,
705 - LOGS_QUERY_STATUS *lqs) {
706 -
707 - errno_clear();
708 -
709 - WEVT_QUERY_STATUS status;
710 - bool matches_filters = true;
711 -
712 - // if(lqs->rq.slice) {
713 - // usec_t started = now_monotonic_usec();
714 - //
715 - // matches_filters = netdata_systemd_filtering_by_journal(j, facets, lqs) || !lqs->rq.filters;
716 - // usec_t ended = now_monotonic_usec();
717 - //
718 - // lqs->c.matches_setup_ut += (ended - started);
719 - // }
720 -
721 - if(matches_filters) {
722 - if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD)
723 - status = wevt_query_forward(log, wb, facets, src, lqs);
724 - else
725 - status = wevt_query_backward(log, wb, facets, src, lqs);
726 - }
727 - else
728 - status = WEVT_NO_CHANNEL_MATCHED;
729 -
730 - return status;
731 -}
732 -
733 -static bool source_is_mine(LOGS_QUERY_SOURCE *src, LOGS_QUERY_STATUS *lqs) {
734 - if((lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) || (src->source_type & lqs->rq.source_type) ||
735 - (lqs->rq.sources && simple_pattern_matches(lqs->rq.sources, string2str(src->source)))) {
736 -
737 - if(!src->msg_last_ut)
738 - // the file is not scanned yet, or the timestamps have not been updated,
739 - // so we don't know if it can contribute or not - let's add it.
740 - return true;
741 -
742 - usec_t anchor_delta = 0;
743 - usec_t first_ut = src->msg_first_ut - anchor_delta;
744 - usec_t last_ut = src->msg_last_ut + anchor_delta;
745 -
746 - if(last_ut >= lqs->rq.after_ut && first_ut <= lqs->rq.before_ut)
747 - return true;
748 - }
749 -
750 - return false;
751 -}
752 -
753 -static int wevt_master_query(BUFFER *wb __maybe_unused, LOGS_QUERY_STATUS *lqs __maybe_unused) {
754 - // make sure the sources list is updated
755 - wevt_sources_scan();
756 -
757 - FACETS *facets = lqs->facets;
758 -
759 - WEVT_QUERY_STATUS status = WEVT_NO_CHANNEL_MATCHED;
760 -
761 - lqs->c.files_matched = 0;
762 - lqs->c.file_working = 0;
763 - lqs->c.rows_useful = 0;
764 - lqs->c.rows_read = 0;
765 - lqs->c.bytes_read = 0;
766 -
767 - size_t files_used = 0;
768 - size_t files_max = dictionary_entries(wevt_sources);
769 - const DICTIONARY_ITEM *file_items[files_max];
770 -
771 - // count the files
772 - bool files_are_newer = false;
773 - LOGS_QUERY_SOURCE *src;
774 - dfe_start_read(wevt_sources, src) {
775 - if(!source_is_mine(src, lqs))
776 - continue;
777 -
778 - file_items[files_used++] = dictionary_acquired_item_dup(wevt_sources, src_dfe.item);
779 -
780 - if(src->msg_last_ut > lqs->rq.if_modified_since)
781 - files_are_newer = true;
782 - }
783 - dfe_done(jf);
784 -
785 - lqs->c.files_matched = files_used;
786 -
787 - if(lqs->rq.if_modified_since && !files_are_newer)
788 - return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
789 -
790 - // sort the files, so that they are optimal for facets
791 - if(files_used >= 2) {
792 - if (lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
793 - qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
794 - wevt_sources_dict_items_backward_compar);
795 - else
796 - qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
797 - wevt_sources_dict_items_forward_compar);
798 - }
799 -
800 - bool partial = false;
801 - usec_t query_started_ut = now_monotonic_usec();
802 - usec_t started_ut = query_started_ut;
803 - usec_t ended_ut = started_ut;
804 - usec_t duration_ut = 0, max_duration_ut = 0;
805 - usec_t progress_duration_ut = 0;
806 -
807 - WEVT_LOG *log = wevt_openlog6();
808 - if(!log) {
809 - netdata_log_error("WEVT: cannot windows event log");
810 - return rrd_call_function_error(wb, "cannot open windows events log", HTTP_RESP_INTERNAL_SERVER_ERROR);
811 - }
812 -
813 - // sampling_query_init(lqs, facets);
814 -
815 - buffer_json_member_add_array(wb, "_channels");
816 - for(size_t f = 0; f < files_used ;f++) {
817 - const char *filename = dictionary_acquired_item_name(file_items[f]);
818 - src = dictionary_acquired_item_value(file_items[f]);
819 -
820 - if(!source_is_mine(src, lqs))
821 - continue;
822 -
823 - started_ut = ended_ut;
824 -
825 - // do not even try to do the query if we expect it to pass the timeout
826 - if(ended_ut + max_duration_ut * 3 >= *lqs->stop_monotonic_ut) {
827 - partial = true;
828 - status = WEVT_TIMED_OUT;
829 - break;
830 - }
831 -
832 - lqs->c.file_working++;
833 -
834 - size_t rows_useful = lqs->c.rows_useful;
835 - size_t rows_read = lqs->c.rows_read;
836 - size_t bytes_read = lqs->c.bytes_read;
837 - size_t matches_setup_ut = lqs->c.matches_setup_ut;
838 -
839 - // sampling_file_init(lqs, src);
840 -
841 - WEVT_QUERY_STATUS tmp_status = wevt_query_one_channel(log, wb, facets, src, lqs);
842 -
843 - rows_useful = lqs->c.rows_useful - rows_useful;
844 - rows_read = lqs->c.rows_read - rows_read;
845 - bytes_read = lqs->c.bytes_read - bytes_read;
846 - matches_setup_ut = lqs->c.matches_setup_ut - matches_setup_ut;
847 -
848 - ended_ut = now_monotonic_usec();
849 - duration_ut = ended_ut - started_ut;
850 -
851 - if(duration_ut > max_duration_ut)
852 - max_duration_ut = duration_ut;
853 -
854 - progress_duration_ut += duration_ut;
855 - if(progress_duration_ut >= WINDOWS_EVENTS_PROGRESS_EVERY_UT) {
856 - progress_duration_ut = 0;
857 - netdata_mutex_lock(&stdout_mutex);
858 - pluginsd_function_progress_to_stdout(lqs->rq.transaction, f + 1, files_used);
859 - netdata_mutex_unlock(&stdout_mutex);
860 - }
861 -
862 - buffer_json_add_array_item_object(wb); // channel source
863 - {
864 - // information about the file
865 - buffer_json_member_add_string(wb, "_name", filename);
866 - buffer_json_member_add_uint64(wb, "_source_type", src->source_type);
867 - buffer_json_member_add_string(wb, "_source", string2str(src->source));
868 - buffer_json_member_add_uint64(wb, "_msg_first_ut", src->msg_first_ut);
869 - buffer_json_member_add_uint64(wb, "_msg_last_ut", src->msg_last_ut);
870 -
871 - // information about the current use of the file
872 - buffer_json_member_add_uint64(wb, "duration_ut", ended_ut - started_ut);
873 - buffer_json_member_add_uint64(wb, "rows_read", rows_read);
874 - buffer_json_member_add_uint64(wb, "rows_useful", rows_useful);
875 - buffer_json_member_add_double(wb, "rows_per_second", (double) rows_read / (double) duration_ut * (double) USEC_PER_SEC);
876 - buffer_json_member_add_uint64(wb, "bytes_read", bytes_read);
877 - buffer_json_member_add_double(wb, "bytes_per_second", (double) bytes_read / (double) duration_ut * (double) USEC_PER_SEC);
878 - buffer_json_member_add_uint64(wb, "duration_matches_ut", matches_setup_ut);
879 -
880 - // if(lqs->rq.sampling) {
881 - // buffer_json_member_add_object(wb, "_sampling");
882 - // {
883 - // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples_per_file.sampled);
884 - // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples_per_file.unsampled);
885 - // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples_per_file.estimated);
886 - // }
887 - // buffer_json_object_close(wb); // _sampling
888 - // }
889 - }
890 - buffer_json_object_close(wb); // channel source
891 -
892 - bool stop = false;
893 - switch(tmp_status) {
894 - case WEVT_OK:
895 - case WEVT_NO_CHANNEL_MATCHED:
896 - status = (status == WEVT_OK) ? WEVT_OK : tmp_status;
897 - break;
898 -
899 - case WEVT_FAILED_TO_OPEN:
900 - case WEVT_FAILED_TO_SEEK:
901 - partial = true;
902 - if(status == WEVT_NO_CHANNEL_MATCHED)
903 - status = tmp_status;
904 - break;
905 -
906 - case WEVT_CANCELLED:
907 - case WEVT_TIMED_OUT:
908 - partial = true;
909 - stop = true;
910 - status = tmp_status;
911 - break;
912 -
913 - case WEVT_NOT_MODIFIED:
914 - internal_fatal(true, "this should never be returned here");
915 - break;
916 - }
917 -
918 - if(stop)
919 - break;
920 - }
921 - buffer_json_array_close(wb); // _channels
922 -
923 - // release the files
924 - for(size_t f = 0; f < files_used ;f++)
925 - dictionary_acquired_item_release(wevt_sources, file_items[f]);
926 -
927 - wevt_closelog6(log);
928 -
929 - switch (status) {
930 - case WEVT_OK:
931 - if(lqs->rq.if_modified_since && !lqs->c.rows_useful)
932 - return rrd_call_function_error(wb, "no useful logs, not modified", HTTP_RESP_NOT_MODIFIED);
933 - break;
934 -
935 - case WEVT_TIMED_OUT:
936 - case WEVT_NO_CHANNEL_MATCHED:
937 - break;
938 -
939 - case WEVT_CANCELLED:
940 - return rrd_call_function_error(wb, "client closed connection", HTTP_RESP_CLIENT_CLOSED_REQUEST);
941 -
942 - case WEVT_NOT_MODIFIED:
943 - return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
944 -
945 - case WEVT_FAILED_TO_OPEN:
946 - return rrd_call_function_error(wb, "failed to open event log", HTTP_RESP_INTERNAL_SERVER_ERROR);
947 -
948 - case WEVT_FAILED_TO_SEEK:
949 - return rrd_call_function_error(wb, "failed to execute event log query", HTTP_RESP_INTERNAL_SERVER_ERROR);
950 -
951 - default:
952 - return rrd_call_function_error(wb, "unknown status", HTTP_RESP_INTERNAL_SERVER_ERROR);
953 - }
954 -
955 - buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
956 - buffer_json_member_add_boolean(wb, "partial", partial);
957 - buffer_json_member_add_string(wb, "type", "table");
958 -
959 - // build a message for the query
960 - if(!lqs->rq.data_only) {
961 - CLEAN_BUFFER *msg = buffer_create(0, NULL);
962 - CLEAN_BUFFER *msg_description = buffer_create(0, NULL);
963 - ND_LOG_FIELD_PRIORITY msg_priority = NDLP_INFO;
964 -
965 - // if(!journal_files_completed_once()) {
966 - // buffer_strcat(msg, "Journals are still being scanned. ");
967 - // buffer_strcat(msg_description
968 - // , "LIBRARY SCAN: The journal files are still being scanned, you are probably viewing incomplete data. ");
969 - // msg_priority = NDLP_WARNING;
970 - // }
971 -
972 - if(partial) {
973 - buffer_strcat(msg, "Query timed-out, incomplete data. ");
974 - buffer_strcat(msg_description
975 - , "QUERY TIMEOUT: The query timed out and may not include all the data of the selected window. ");
976 - msg_priority = NDLP_WARNING;
977 - }
978 -
979 - // if(lqs->c.samples.estimated || lqs->c.samples.unsampled) {
980 - // double percent = (double) (lqs->c.samples.sampled * 100.0 /
981 - // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
982 - // buffer_sprintf(msg, "%.2f%% real data", percent);
983 - // buffer_sprintf(msg_description, "ACTUAL DATA: The filters counters reflect %0.2f%% of the data. ", percent);
984 - // msg_priority = MIN(msg_priority, NDLP_NOTICE);
985 - // }
986 - //
987 - // if(lqs->c.samples.unsampled) {
988 - // double percent = (double) (lqs->c.samples.unsampled * 100.0 /
989 - // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
990 - // buffer_sprintf(msg, ", %.2f%% unsampled", percent);
991 - // buffer_sprintf(msg_description
992 - // , "UNSAMPLED DATA: %0.2f%% of the events exist and have been counted, but their values have not been evaluated, so they are not included in the filters counters. "
993 - // , percent);
994 - // msg_priority = MIN(msg_priority, NDLP_NOTICE);
995 - // }
996 - //
997 - // if(lqs->c.samples.estimated) {
998 - // double percent = (double) (lqs->c.samples.estimated * 100.0 /
999 - // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1000 - // buffer_sprintf(msg, ", %.2f%% estimated", percent);
1001 - // buffer_sprintf(msg_description
1002 - // , "ESTIMATED DATA: The query selected a large amount of data, so to avoid delaying too much, the presented data are estimated by %0.2f%%. "
1003 - // , percent);
1004 - // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1005 - // }
1006 -
1007 - buffer_json_member_add_object(wb, "message");
1008 - if(buffer_tostring(msg)) {
1009 - buffer_json_member_add_string(wb, "title", buffer_tostring(msg));
1010 - buffer_json_member_add_string(wb, "description", buffer_tostring(msg_description));
1011 - buffer_json_member_add_string(wb, "status", nd_log_id2priority(msg_priority));
1012 - }
1013 - // else send an empty object if there is nothing to tell
1014 - buffer_json_object_close(wb); // message
1015 - }
1016 -
1017 - if(!lqs->rq.data_only) {
1018 - buffer_json_member_add_time_t(wb, "update_every", 1);
1019 - buffer_json_member_add_string(wb, "help", WEVT_FUNCTION_DESCRIPTION);
1020 - }
1021 -
1022 - if(!lqs->rq.data_only || lqs->rq.tail)
1023 - buffer_json_member_add_uint64(wb, "last_modified", lqs->last_modified);
1024 -
1025 - facets_sort_and_reorder_keys(facets);
1026 - facets_report(facets, wb, used_hashes_registry);
1027 -
1028 - wb->expires = now_realtime_sec() + (lqs->rq.data_only ? 3600 : 0);
1029 - buffer_json_member_add_time_t(wb, "expires", wb->expires);
1030 -
1031 - // if(lqs->rq.sampling) {
1032 - // buffer_json_member_add_object(wb, "_sampling");
1033 - // {
1034 - // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples.sampled);
1035 - // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples.unsampled);
1036 - // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples.estimated);
1037 - // }
1038 - // buffer_json_object_close(wb); // _sampling
1039 - // }
1040 -
1041 - wb->content_type = CT_APPLICATION_JSON;
1042 - wb->response_code = HTTP_RESP_OK;
1043 - return wb->response_code;
1044 -}
1045 -
1046 -void function_windows_events(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
1047 - BUFFER *payload, HTTP_ACCESS access __maybe_unused,
1048 - const char *source __maybe_unused, void *data __maybe_unused) {
1049 - bool have_slice = LQS_DEFAULT_SLICE_MODE;
1050 -
1051 - LOGS_QUERY_STATUS tmp_fqs = {
1052 - .facets = lqs_facets_create(
1053 - LQS_DEFAULT_ITEMS_PER_QUERY,
1054 - FACETS_OPTION_ALL_KEYS_FTS,
1055 - WEVT_ALWAYS_VISIBLE_KEYS,
1056 - WEVT_KEYS_INCLUDED_IN_FACETS,
1057 - WEVT_KEYS_EXCLUDED_FROM_FACETS,
1058 - have_slice),
1059 -
1060 - .rq = LOGS_QUERY_REQUEST_DEFAULTS(transaction, have_slice, FACETS_ANCHOR_DIRECTION_BACKWARD),
1061 -
1062 - .cancelled = cancelled,
1063 - .stop_monotonic_ut = stop_monotonic_ut,
1064 - };
1065 - LOGS_QUERY_STATUS *lqs = &tmp_fqs;
1066 -
1067 - CLEAN_BUFFER *wb = lqs_create_output_buffer();
1068 -
1069 - // ------------------------------------------------------------------------
1070 - // parse the parameters
1071 -
1072 - if(lqs_request_parse_and_validate(lqs, wb, function, payload, have_slice, WEVT_FIELD_LEVEL)) {
1073 - wevt_register_fields(lqs);
1074 -
1075 - // ------------------------------------------------------------------------
1076 - // add versions to the response
1077 -
1078 - buffer_json_wevt_versions(wb);
1079 -
1080 - // ------------------------------------------------------------------------
1081 - // run the request
1082 -
1083 - if (lqs->rq.info)
1084 - lqs_info_response(wb, lqs->facets);
1085 - else {
1086 - wevt_master_query(wb, lqs);
1087 - if (wb->response_code == HTTP_RESP_OK)
1088 - buffer_json_finalize(wb);
1089 - }
1090 - }
1091 -
1092 - netdata_mutex_lock(&stdout_mutex);
1093 - pluginsd_function_result_to_stdout(transaction, wb);
1094 - netdata_mutex_unlock(&stdout_mutex);
1095 -
1096 - lqs_cleanup(lqs);
1097 -}
1098 -
1099 -int main(int argc __maybe_unused, char **argv __maybe_unused) {
1100 - clocks_init();
1101 - nd_thread_tag_set("wevt.plugin");
1102 - nd_log_initialize_for_external_plugins("windows-events.plugin");
1103 -
1104 - // ------------------------------------------------------------------------
1105 - // initialization
1106 -
1107 - wevt_sources_init();
1108 -
1109 - // ------------------------------------------------------------------------
1110 - // debug
1111 -
1112 - if(argc >= 2 && strcmp(argv[argc - 1], "debug") == 0) {
1113 - wevt_sources_scan();
1114 -
1115 - struct {
1116 - const char *func;
1117 - } array[] = {
1118 - { "windows-events after:1725866669 before:1725953069 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6,ClaDGnYSQE7,H106l8MXSSr,HREiMN.4Ahu data_only:false source:All" },
1119 - //{ "windows-events after:1725650277 before:1725736677 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1120 - //{ "windows-events after:1725650284 before:1725736684 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1121 - //{ "windows-events after:1725650386 before:1725736786 anchor:1725652420809461 direction:forward last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 if_modified_since:1725736649011085 data_only:true delta:true tail:true source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1122 - //{ "windows-events info after:1725650420 before:1725736820" },
1123 - //{ "windows-events after:1725650420 before:1725736820 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1124 - //{ "windows-events after:1725650430 before:1725736830 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1125 - { NULL },
1126 - };
1127 -
1128 - for(int i = 0; array[i].func ;i++) {
1129 - bool cancelled = false;
1130 - usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
1131 - //char buf[] = "windows-events after:-86400 before:0 direction:backward last:200 data_only:false slice:true source:all";
1132 - function_windows_events("123", (char *)array[i].func, &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
1133 - }
1134 - printf("\n\nAll done!\n\n");
1135 - fflush(stdout);
1136 - exit(1);
1137 - }
1138 -
1139 - // ------------------------------------------------------------------------
1140 - // the event loop for functions
1141 -
1142 - struct functions_evloop_globals *wg =
1143 - functions_evloop_init(WINDOWS_EVENTS_WORKER_THREADS, "WEVT", &stdout_mutex, &plugin_should_exit);
1144 -
1145 - functions_evloop_add_function(wg,
1146 - WEVT_FUNCTION_NAME,
1147 - function_windows_events,
1148 - WINDOWS_EVENTS_DEFAULT_TIMEOUT,
1149 - NULL);
1150 -
1151 - // ------------------------------------------------------------------------
1152 - // register functions to netdata
1153 -
1154 - netdata_mutex_lock(&stdout_mutex);
1155 -
1156 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" "HTTP_ACCESS_FORMAT" %d\n",
1157 - WEVT_FUNCTION_NAME, WINDOWS_EVENTS_DEFAULT_TIMEOUT, WEVT_FUNCTION_DESCRIPTION,
1158 - (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
1159 - RRDFUNCTIONS_PRIORITY_DEFAULT);
1160 -
1161 - fflush(stdout);
1162 - netdata_mutex_unlock(&stdout_mutex);
1163 -
1164 - // ------------------------------------------------------------------------
1165 -
1166 - const usec_t step_ut = 100 * USEC_PER_MS;
1167 - usec_t send_newline_ut = 0;
1168 - usec_t since_last_scan_ut = WINDOWS_EVENTS_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
1169 - const bool tty = isatty(fileno(stdout)) == 1;
1170 -
1171 - heartbeat_t hb;
1172 - heartbeat_init(&hb);
1173 - while(!plugin_should_exit) {
1174 -
1175 - if(since_last_scan_ut > WINDOWS_EVENTS_SCAN_EVERY_USEC) {
1176 - wevt_sources_scan();
1177 - since_last_scan_ut = 0;
1178 - }
1179 -
1180 - usec_t dt_ut = heartbeat_next(&hb, step_ut);
1181 - since_last_scan_ut += dt_ut;
1182 - send_newline_ut += dt_ut;
1183 -
1184 - if(!tty && send_newline_ut > USEC_PER_SEC) {
1185 - send_newline_and_flush(&stdout_mutex);
1186 - send_newline_ut = 0;
1187 - }
1188 - }
1189 -
1190 - exit(0);
1191 -}
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "libnetdata/libnetdata.h"
4 +#include "libnetdata/required_dummies.h"
5 +
6 +#include "windows-events.h"
7 +
8 +#define WINDOWS_EVENTS_WORKER_THREADS 5
9 +#define WINDOWS_EVENTS_DEFAULT_TIMEOUT 600
10 +#define WINDOWS_EVENTS_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
11 +#define WINDOWS_EVENTS_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
12 +
13 +#define FUNCTION_PROGRESS_EVERY_ROWS (2000)
14 +#define FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS (1000)
15 +#define ANCHOR_DELTA_UT (10 * USEC_PER_SEC)
16 +
17 +netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER;
18 +static bool plugin_should_exit = false;
19 +
20 +#define WEVT_FUNCTION_DESCRIPTION "View, search and analyze the Microsoft Windows Events log."
21 +#define WEVT_FUNCTION_NAME "windows-events"
22 +
23 +// functions needed by LQS
24 +
25 +// structures needed by LQS
26 +struct lqs_extension {
27 + wchar_t *query;
28 +
29 + struct {
30 + struct {
31 + size_t completed;
32 + size_t total;
33 + } queries;
34 +
35 + struct {
36 + size_t current_query_total;
37 + size_t completed;
38 + size_t total;
39 + } entries;
40 +
41 + usec_t last_ut;
42 + } progress;
43 +
44 + // struct {
45 + // usec_t start_ut;
46 + // usec_t stop_ut;
47 + // usec_t first_msg_ut;
48 + //
49 + // uint64_t first_msg_seqnum;
50 + // } query_file;
51 +
52 + // struct {
53 + // uint32_t enable_after_samples;
54 + // uint32_t slots;
55 + // uint32_t sampled;
56 + // uint32_t unsampled;
57 + // uint32_t estimated;
58 + // } samples;
59 +
60 + // struct {
61 + // uint32_t enable_after_samples;
62 + // uint32_t every;
63 + // uint32_t skipped;
64 + // uint32_t recalibrate;
65 + // uint32_t sampled;
66 + // uint32_t unsampled;
67 + // uint32_t estimated;
68 + // } samples_per_file;
69 +
70 + // struct {
71 + // usec_t start_ut;
72 + // usec_t end_ut;
73 + // usec_t step_ut;
74 + // uint32_t enable_after_samples;
75 + // uint32_t sampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
76 + // uint32_t unsampled[SYSTEMD_JOURNAL_SAMPLING_SLOTS];
77 + // } samples_per_time_slot;
78 +
79 + // per file progress info
80 + // size_t cached_count;
81 +
82 + // progress statistics
83 + usec_t matches_setup_ut;
84 + size_t rows_useful;
85 + size_t rows_read;
86 + size_t bytes_read;
87 + size_t files_matched;
88 + size_t file_working;
89 +};
90 +
91 +// prepare LQS
92 +#define LQS_DEFAULT_SLICE_MODE 0
93 +#define LQS_FUNCTION_NAME WEVT_FUNCTION_NAME
94 +#define LQS_FUNCTION_DESCRIPTION WEVT_FUNCTION_DESCRIPTION
95 +#define LQS_DEFAULT_ITEMS_PER_QUERY 200
96 +#define LQS_DEFAULT_ITEMS_SAMPLING 1000000
97 +#define LQS_SOURCE_TYPE WEVT_SOURCE_TYPE
98 +#define LQS_SOURCE_TYPE_ALL WEVTS_ALL
99 +#define LQS_SOURCE_TYPE_NONE WEVTS_NONE
100 +#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) wevt_internal_source_type(value)
101 +#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) wevt_sources_to_json_array(wb)
102 +#include "libnetdata/facets/logs_query_status.h"
103 +
104 +#define WEVT_FIELD_COMPUTER "Computer"
105 +#define WEVT_FIELD_CHANNEL "Channel"
106 +#define WEVT_FIELD_PROVIDER "Provider"
107 +#define WEVT_FIELD_SOURCE "Source"
108 +#define WEVT_FIELD_EVENTRECORDID "EventRecordID"
109 +#define WEVT_FIELD_EVENTID "EventID"
110 +#define WEVT_FIELD_LEVEL "Level"
111 +#define WEVT_FIELD_KEYWORDS "Keywords"
112 +#define WEVT_FIELD_OPCODE "Opcode"
113 +#define WEVT_FIELD_USER "User"
114 +#define WEVT_FIELD_TASK "Task"
115 +#define WEVT_FIELD_PROCESSID "ProcessID"
116 +#define WEVT_FIELD_THREADID "ThreadID"
117 +#define WEVT_FIELD_XML "XML"
118 +#define WEVT_FIELD_MESSAGE "Message"
119 +
120 +#define WEVT_ALWAYS_VISIBLE_KEYS NULL
121 +
122 +#define WEVT_KEYS_EXCLUDED_FROM_FACETS \
123 + "|" WEVT_FIELD_MESSAGE \
124 + "|" WEVT_FIELD_XML \
125 + ""
126 +
127 +#define WEVT_KEYS_INCLUDED_IN_FACETS \
128 + "|" WEVT_FIELD_PROVIDER \
129 + "|" WEVT_FIELD_SOURCE \
130 + "|" WEVT_FIELD_CHANNEL \
131 + "|" WEVT_FIELD_LEVEL \
132 + "|" WEVT_FIELD_KEYWORDS \
133 + "|" WEVT_FIELD_OPCODE \
134 + "|" WEVT_FIELD_TASK \
135 + "|" WEVT_FIELD_COMPUTER \
136 + "|" WEVT_FIELD_USER \
137 + ""
138 +
139 +static inline WEVT_QUERY_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) {
140 + if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
141 + nd_log(NDLS_COLLECTORS, NDLP_INFO, "Function has been cancelled");
142 + return WEVT_CANCELLED;
143 + }
144 +
145 + if(now_monotonic_usec() > __atomic_load_n(stop_monotonic_ut, __ATOMIC_RELAXED)) {
146 + internal_error(true, "Function timed out");
147 + return WEVT_TIMED_OUT;
148 + }
149 +
150 + return WEVT_OK;
151 +}
152 +
153 +FACET_ROW_SEVERITY wevt_levelid_to_facet_severity(FACETS *facets __maybe_unused, FACET_ROW *row, void *data __maybe_unused) {
154 + FACET_ROW_KEY_VALUE *levelid_rkv = dictionary_get(row->dict, WEVT_FIELD_LEVEL);
155 + if(!levelid_rkv || levelid_rkv->empty)
156 + return FACET_ROW_SEVERITY_NORMAL;
157 +
158 + int windows_event_level = str2i(buffer_tostring(levelid_rkv->wb));
159 +
160 + switch (windows_event_level) {
161 + case 5: // Verbose
162 + return FACET_ROW_SEVERITY_DEBUG;
163 + default:
164 + case 4: // Information
165 + return FACET_ROW_SEVERITY_NORMAL;
166 + case 3: // Warning
167 + return FACET_ROW_SEVERITY_WARNING;
168 + case 2: // Error
169 + case 1: // Critical
170 + return FACET_ROW_SEVERITY_CRITICAL;
171 + }
172 +}
173 +
174 +struct wevt_bin_data {
175 + WEVT_LOG *log;
176 + EVT_HANDLE bookmark;
177 + PROVIDER_META_HANDLE *publisher;
178 +};
179 +
180 +static void wevt_cleanup_bin_data(void *data) {
181 + struct wevt_bin_data *d = data;
182 +
183 + if(d->bookmark)
184 + EvtClose(d->bookmark);
185 +
186 + publisher_release(d->publisher);
187 + freez(d);
188 +}
189 +
190 +static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets) {
191 + struct wevt_bin_data *d = mallocz(sizeof(struct wevt_bin_data));
192 +
193 + d->log = log;
194 +
195 + // take the bookmark
196 + d->bookmark = log->bookmark; log->bookmark = NULL;
197 +
198 + // dup the publisher
199 + d->publisher = publisher_dup(log->publisher);
200 +
201 + facets_row_bin_data_set(facets, wevt_cleanup_bin_data, d);
202 +}
203 +
204 +static void wevt_render_xml(
205 + FACETS *facets,
206 + BUFFER *json_array,
207 + FACET_ROW_KEY_VALUE *rkv __maybe_unused,
208 + FACET_ROW *row,
209 + void *data __maybe_unused) {
210 +
211 + struct wevt_bin_data *d = facets_row_bin_data_get(facets, row);
212 + if(!d) {
213 + buffer_json_add_array_item_string(json_array, "Failed to get row BIN DATA from facets");
214 + return;
215 + }
216 +
217 + wevt_get_xml_utf8(d->log, d->publisher, d->bookmark, &d->log->ops.xml);
218 + buffer_json_add_array_item_string(json_array, d->log->ops.xml.data);
219 +}
220 +
221 +static void wevt_render_message(
222 + FACETS *facets,
223 + BUFFER *json_array,
224 + FACET_ROW_KEY_VALUE *rkv __maybe_unused,
225 + FACET_ROW *row,
226 + void *data __maybe_unused) {
227 +
228 + struct wevt_bin_data *d = facets_row_bin_data_get(facets, row);
229 + if(!d) {
230 + buffer_json_add_array_item_string(json_array, "Failed to get row BIN DATA from facets");
231 + return;
232 + }
233 +
234 + wevt_get_event_utf8(d->log, d->publisher, d->bookmark, &d->log->ops.event);
235 + buffer_json_add_array_item_string(json_array, d->log->ops.event.data);
236 +}
237 +
238 +static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
239 + FACETS *facets = lqs->facets;
240 + LOGS_QUERY_REQUEST *rq = &lqs->rq;
241 +
242 + facets_register_row_severity(facets, wevt_levelid_to_facet_severity, NULL);
243 +
244 + facets_register_key_name(
245 + facets, WEVT_FIELD_COMPUTER,
246 + rq->default_facet | FACET_KEY_OPTION_VISIBLE);
247 +
248 + facets_register_key_name(
249 + facets, WEVT_FIELD_CHANNEL,
250 + rq->default_facet |
251 + FACET_KEY_OPTION_FTS);
252 +
253 + facets_register_key_name(
254 + facets, WEVT_FIELD_PROVIDER,
255 + rq->default_facet |
256 + FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
257 +
258 + facets_register_key_name(
259 + facets, WEVT_FIELD_SOURCE,
260 + rq->default_facet |
261 + FACET_KEY_OPTION_FTS);
262 +
263 + facets_register_key_name(
264 + facets, WEVT_FIELD_USER,
265 + rq->default_facet | FACET_KEY_OPTION_FTS);
266 +
267 + facets_register_key_name(
268 + facets, WEVT_FIELD_EVENTID,
269 + rq->default_facet |
270 + FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
271 +
272 + facets_register_key_name(
273 + facets, WEVT_FIELD_LEVEL,
274 + rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_EXPANDED_FILTER);
275 +
276 + facets_register_key_name(
277 + facets, WEVT_FIELD_LEVEL "ID",
278 + FACET_KEY_OPTION_NONE);
279 +
280 + facets_register_key_name(
281 + facets, WEVT_FIELD_PROCESSID,
282 + rq->default_facet | FACET_KEY_OPTION_FTS);
283 +
284 + facets_register_key_name(
285 + facets, WEVT_FIELD_THREADID,
286 + rq->default_facet | FACET_KEY_OPTION_FTS);
287 +
288 + facets_register_key_name(
289 + facets, WEVT_FIELD_TASK,
290 + rq->default_facet | FACET_KEY_OPTION_FTS);
291 +
292 + facets_register_key_name(
293 + facets, WEVT_FIELD_PROCESSID,
294 + rq->default_facet | FACET_KEY_OPTION_FTS);
295 +
296 + facets_register_key_name(
297 + facets, WEVT_FIELD_THREADID,
298 + rq->default_facet | FACET_KEY_OPTION_FTS);
299 +
300 + facets_register_key_name(
301 + facets, WEVT_FIELD_OPCODE,
302 + rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_VISIBLE);
303 +
304 + facets_register_key_name(
305 + facets, WEVT_FIELD_OPCODE "ID",
306 + FACET_KEY_OPTION_NONE);
307 +
308 + facets_register_key_name(
309 + facets, WEVT_FIELD_TASK,
310 + rq->default_facet | FACET_KEY_OPTION_FTS);
311 +
312 + facets_register_key_name(
313 + facets, WEVT_FIELD_TASK "ID",
314 + FACET_KEY_OPTION_NONE);
315 +
316 + facets_register_key_name(
317 + facets, WEVT_FIELD_KEYWORDS,
318 + rq->default_facet | FACET_KEY_OPTION_FTS);
319 +
320 + facets_register_key_name(
321 + facets, WEVT_FIELD_KEYWORDS "ID",
322 + FACET_KEY_OPTION_NONE);
323 +
324 + facets_register_dynamic_key_name(
325 + facets, WEVT_FIELD_MESSAGE,
326 + FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE,
327 + wevt_render_message, NULL);
328 +
329 + facets_register_dynamic_key_name(
330 + facets, WEVT_FIELD_XML,
331 + FACET_KEY_OPTION_NEVER_FACET,
332 + wevt_render_xml, NULL);
333 +
334 +#ifdef NETDATA_INTERNAL_CHECKS
335 + facets_register_key_name(
336 + facets, "z_level_source",
337 + rq->default_facet);
338 +
339 + facets_register_key_name(
340 + facets, "z_keywords_source",
341 + rq->default_facet);
342 +
343 + facets_register_key_name(
344 + facets, "z_opcode_source",
345 + rq->default_facet);
346 +
347 + facets_register_key_name(
348 + facets, "z_task_source",
349 + rq->default_facet);
350 +#endif
351 +}
352 +
353 +#ifdef NETDATA_INTERNAL_CHECKS
354 +static const char *source_to_str(TXT_UTF8 *txt) {
355 + switch(txt->src) {
356 + default:
357 + case TXT_SOURCE_UNKNOWN:
358 + return "unknown";
359 +
360 + case TXT_SOURCE_EVENT_LOG:
361 + return "event-log";
362 +
363 + case TXT_SOURCE_PUBLISHER:
364 + return "publisher";
365 +
366 + case TXT_SOURCE_FIELD_CACHE:
367 + return "fields-cache";
368 +
369 + case TXT_SOURCE_HARDCODED:
370 + return "hardcoded";
371 + }
372 +}
373 +#endif
374 +
375 +static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUERY_SOURCE *src, usec_t *msg_ut __maybe_unused, WEVT_EVENT *e) {
376 + size_t len, bytes = log->ops.content.used;
377 +
378 + if(log->ops.provider.used > 1) {
379 + bytes += log->ops.provider.used * 2; // unicode is double
380 + facets_add_key_value_length(
381 + facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
382 + log->ops.provider.data, log->ops.provider.used - 1);
383 + }
384 +
385 + if(log->ops.source.used > 1) {
386 + bytes += log->ops.source.used * 2;
387 + facets_add_key_value_length(
388 + facets, WEVT_FIELD_SOURCE, sizeof(WEVT_FIELD_SOURCE) - 1,
389 + log->ops.source.data, log->ops.source.used - 1);
390 + }
391 +
392 + if(log->ops.channel.used > 1) {
393 + bytes += log->ops.channel.used * 2;
394 + facets_add_key_value_length(
395 + facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
396 + log->ops.channel.data, log->ops.channel.used - 1);
397 + }
398 + else {
399 + bytes += src->fullname_len * 2;
400 + facets_add_key_value_length(
401 + facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
402 + src->fullname, src->fullname_len);
403 + }
404 +
405 + {
406 + static __thread char event_record_id_str[UINT64_MAX_LENGTH];
407 + len = print_uint64(event_record_id_str, e->id);
408 + bytes += len;
409 + facets_add_key_value_length(
410 + facets, WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
411 + event_record_id_str, len);
412 + }
413 +
414 + if(log->ops.level.used > 1) {
415 + bytes += log->ops.level.used * 2;
416 + facets_add_key_value_length(
417 + facets, WEVT_FIELD_LEVEL, sizeof(WEVT_FIELD_LEVEL) - 1,
418 + log->ops.level.data, log->ops.level.used - 1);
419 + }
420 +
421 + if(log->ops.computer.used > 1) {
422 + bytes += log->ops.computer.used * 2;
423 + facets_add_key_value_length(
424 + facets, WEVT_FIELD_COMPUTER, sizeof(WEVT_FIELD_COMPUTER) - 1,
425 + log->ops.computer.data, log->ops.computer.used - 1);
426 + }
427 +
428 + if(log->ops.opcode.used > 1) {
429 + bytes += log->ops.opcode.used * 2;
430 + facets_add_key_value_length(
431 + facets, WEVT_FIELD_OPCODE, sizeof(WEVT_FIELD_OPCODE) - 1,
432 + log->ops.opcode.data, log->ops.opcode.used - 1);
433 + }
434 +
435 + if(log->ops.keywords.used > 1) {
436 + bytes += log->ops.keywords.used * 2;
437 + facets_add_key_value_length(
438 + facets, WEVT_FIELD_KEYWORDS, sizeof(WEVT_FIELD_KEYWORDS) - 1,
439 + log->ops.keywords.data, log->ops.keywords.used - 1);
440 + }
441 +
442 + if(log->ops.task.used > 1) {
443 + bytes += log->ops.task.used * 2;
444 + facets_add_key_value_length(
445 + facets, WEVT_FIELD_TASK, sizeof(WEVT_FIELD_TASK) - 1,
446 + log->ops.task.data, log->ops.task.used - 1);
447 + }
448 +
449 + if(log->ops.user.used > 1) {
450 + bytes += log->ops.user.used * 2;
451 + facets_add_key_value_length(
452 + facets, WEVT_FIELD_USER, sizeof(WEVT_FIELD_USER) - 1,
453 + log->ops.user.data, log->ops.user.used - 1);
454 + }
455 +
456 + {
457 + static __thread char event_id_str[UINT64_MAX_LENGTH];
458 + len = print_uint64(event_id_str, e->event_id);
459 + bytes += len;
460 + facets_add_key_value_length(
461 + facets, WEVT_FIELD_EVENTID, sizeof(WEVT_FIELD_EVENTID) - 1,
462 + event_id_str, len);
463 + }
464 +
465 + if(e->process_id) {
466 + static __thread char process_id_str[UINT64_MAX_LENGTH];
467 + len = print_uint64(process_id_str, e->process_id);
468 + bytes += len;
469 + facets_add_key_value_length(
470 + facets, WEVT_FIELD_PROCESSID, sizeof(WEVT_FIELD_PROCESSID) - 1,
471 + process_id_str, len);
472 + }
473 +
474 + if(e->thread_id) {
475 + static __thread char thread_id_str[UINT64_MAX_LENGTH];
476 + len = print_uint64(thread_id_str, e->thread_id);
477 + bytes += len;
478 + facets_add_key_value_length(
479 + facets, WEVT_FIELD_THREADID, sizeof(WEVT_FIELD_THREADID) - 1,
480 + thread_id_str, len);
481 + }
482 +
483 + {
484 + static __thread char str[UINT64_MAX_LENGTH];
485 + len = print_uint64(str, e->level);
486 + bytes += len;
487 + facets_add_key_value_length(
488 + facets, WEVT_FIELD_LEVEL "ID", sizeof(WEVT_FIELD_LEVEL) + 2 - 1, str, len);
489 + }
490 +
491 + {
492 + static __thread char str[UINT64_HEX_MAX_LENGTH];
493 + len = print_uint64_hex_full(str, e->keywords);
494 + bytes += len;
495 + facets_add_key_value_length(
496 + facets, WEVT_FIELD_KEYWORDS "ID", sizeof(WEVT_FIELD_KEYWORDS) + 2 - 1, str, len);
497 + }
498 +
499 + {
500 + static __thread char str[UINT64_MAX_LENGTH];
501 + len = print_uint64(str, e->opcode);
502 + bytes += len;
503 + facets_add_key_value_length(
504 + facets, WEVT_FIELD_OPCODE "ID", sizeof(WEVT_FIELD_OPCODE) + 2 - 1, str, len);
505 + }
506 +
507 + {
508 + static __thread char str[UINT64_MAX_LENGTH];
509 + len = print_uint64(str, e->task);
510 + bytes += len;
511 + facets_add_key_value_length(
512 + facets, WEVT_FIELD_TASK "ID", sizeof(WEVT_FIELD_TASK) + 2 - 1, str, len);
513 + }
514 +
515 +// bytes += log->ops.xml.used * 2;
516 +// facets_add_key_value_length(
517 +// facets, WEVT_FIELD_XML, sizeof(WEVT_FIELD_XML) - 1,
518 +// log->ops.xml.data, log->ops.xml.used - 1);
519 +
520 + wevt_facets_register_bin_data(log, facets);
521 +
522 +#ifdef NETDATA_INTERNAL_CHECKS
523 + facets_add_key_value(facets, "z_level_source", source_to_str(&log->ops.level));
524 + facets_add_key_value(facets, "z_keywords_source", source_to_str(&log->ops.keywords));
525 + facets_add_key_value(facets, "z_opcode_source", source_to_str(&log->ops.opcode));
526 + facets_add_key_value(facets, "z_task_source", source_to_str(&log->ops.task));
527 +#endif
528 +
529 + return bytes;
530 +}
531 +
532 +static void send_progress_update(LOGS_QUERY_STATUS *lqs, size_t current_row_counter, bool flush_current_file) {
533 + usec_t now_ut = now_monotonic_usec();
534 +
535 + if(current_row_counter > lqs->c.progress.entries.current_query_total) {
536 + lqs->c.progress.entries.total += current_row_counter - lqs->c.progress.entries.current_query_total;
537 + lqs->c.progress.entries.current_query_total = current_row_counter;
538 + }
539 +
540 + if(flush_current_file) {
541 + lqs->c.progress.entries.total += current_row_counter;
542 + lqs->c.progress.entries.total -= lqs->c.progress.entries.current_query_total;
543 + lqs->c.progress.entries.completed += current_row_counter;
544 + lqs->c.progress.entries.current_query_total = 0;
545 + }
546 +
547 + size_t completed = lqs->c.progress.entries.completed + current_row_counter;
548 + if(completed > lqs->c.progress.entries.total)
549 + lqs->c.progress.entries.total = completed;
550 +
551 + usec_t progress_duration_ut = now_ut - lqs->c.progress.last_ut;
552 + if(progress_duration_ut >= WINDOWS_EVENTS_PROGRESS_EVERY_UT) {
553 + lqs->c.progress.last_ut = now_ut;
554 +
555 + netdata_mutex_lock(&stdout_mutex);
556 + pluginsd_function_progress_to_stdout(lqs->rq.transaction, completed, lqs->c.progress.entries.total);
557 + netdata_mutex_unlock(&stdout_mutex);
558 + }
559 +}
560 +
561 +static WEVT_QUERY_STATUS wevt_query_backward(
562 + WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
563 + LOGS_QUERY_SOURCE *src,
564 + LOGS_QUERY_STATUS *lqs)
565 +{
566 + usec_t start_ut = lqs->query.start_ut;
567 + usec_t stop_ut = lqs->query.stop_ut;
568 + bool stop_when_full = lqs->query.stop_when_full;
569 +
570 +// lqs->c.query_file.start_ut = start_ut;
571 +// lqs->c.query_file.stop_ut = stop_ut;
572 +
573 + if(!wevt_query(log, channel2unicode(src->fullname), lqs->c.query, EvtQueryReverseDirection))
574 + return WEVT_FAILED_TO_SEEK;
575 +
576 + size_t errors_no_timestamp = 0;
577 + usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
578 + usec_t first_msg_ut = 0; // the first message we got from the db
579 + size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
580 + size_t bytes = 0, last_bytes = 0;
581 +
582 + usec_t last_usec_from = 0;
583 + usec_t last_usec_to = 0;
584 +
585 + WEVT_QUERY_STATUS status = WEVT_OK;
586 +
587 + facets_rows_begin(facets);
588 + WEVT_EVENT e;
589 + while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
590 + usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
591 +
592 + if(unlikely(!msg_ut)) {
593 + errors_no_timestamp++;
594 + continue;
595 + }
596 +
597 + if (unlikely(msg_ut > start_ut))
598 + continue;
599 +
600 + if (unlikely(msg_ut < stop_ut))
601 + break;
602 +
603 + if(unlikely(msg_ut > latest_msg_ut))
604 + latest_msg_ut = msg_ut;
605 +
606 + if(unlikely(!first_msg_ut)) {
607 + first_msg_ut = msg_ut;
608 + // lqs->c.query_file.first_msg_ut = msg_ut;
609 + }
610 +
611 +// sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
612 +// FACETS_ANCHOR_DIRECTION_BACKWARD,
613 +// facets_row_candidate_to_keep(facets, msg_ut));
614 +//
615 +// if(sample == SAMPLING_FULL) {
616 + bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
617 +
618 + // make sure each line gets a unique timestamp
619 + if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
620 + msg_ut = --last_usec_from;
621 + else
622 + last_usec_from = last_usec_to = msg_ut;
623 +
624 + if(facets_row_finished(facets, msg_ut))
625 + rows_useful++;
626 +
627 + row_counter++;
628 + if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
629 + stop_when_full &&
630 + facets_rows(facets) >= lqs->rq.entries)) {
631 + // stop the data only query
632 + usec_t oldest = facets_row_oldest_ut(facets);
633 + if(oldest && msg_ut < (oldest - lqs->anchor.delta_ut))
634 + break;
635 + }
636 +
637 + if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
638 + status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
639 +
640 + if(status == WEVT_OK) {
641 + lqs->c.rows_read += row_counter - last_row_counter;
642 + last_row_counter = row_counter;
643 +
644 + lqs->c.bytes_read += bytes - last_bytes;
645 + last_bytes = bytes;
646 +
647 + send_progress_update(lqs, row_counter, false);
648 + }
649 + }
650 +// }
651 +// else if(sample == SAMPLING_SKIP_FIELDS)
652 +// facets_row_finished_unsampled(facets, msg_ut);
653 +// else {
654 +// sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_BACKWARD);
655 +// break;
656 +// }
657 + }
658 +
659 + send_progress_update(lqs, row_counter, true);
660 + lqs->c.rows_read += row_counter - last_row_counter;
661 + lqs->c.bytes_read += bytes - last_bytes;
662 + lqs->c.rows_useful += rows_useful;
663 +
664 + if(errors_no_timestamp)
665 + netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
666 +
667 + if(latest_msg_ut > lqs->last_modified)
668 + lqs->last_modified = latest_msg_ut;
669 +
670 + wevt_query_done(log);
671 +
672 + return status;
673 +}
674 +
675 +static WEVT_QUERY_STATUS wevt_query_forward(
676 + WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
677 + LOGS_QUERY_SOURCE *src,
678 + LOGS_QUERY_STATUS *lqs)
679 +{
680 + usec_t start_ut = lqs->query.start_ut;
681 + usec_t stop_ut = lqs->query.stop_ut;
682 + bool stop_when_full = lqs->query.stop_when_full;
683 +
684 +// lqs->c.query_file.start_ut = start_ut;
685 +// lqs->c.query_file.stop_ut = stop_ut;
686 +
687 + if(!wevt_query(log, channel2unicode(src->fullname), lqs->c.query, EvtQueryForwardDirection))
688 + return WEVT_FAILED_TO_SEEK;
689 +
690 + size_t errors_no_timestamp = 0;
691 + usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
692 + usec_t first_msg_ut = 0; // the first message we got from the db
693 + size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
694 + size_t bytes = 0, last_bytes = 0;
695 +
696 + usec_t last_usec_from = 0;
697 + usec_t last_usec_to = 0;
698 +
699 + WEVT_QUERY_STATUS status = WEVT_OK;
700 +
701 + facets_rows_begin(facets);
702 + WEVT_EVENT e;
703 + while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
704 + usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
705 +
706 + if(unlikely(!msg_ut)) {
707 + errors_no_timestamp++;
708 + continue;
709 + }
710 +
711 + if (unlikely(msg_ut < start_ut))
712 + continue;
713 +
714 + if (unlikely(msg_ut > stop_ut))
715 + break;
716 +
717 + if(likely(msg_ut > latest_msg_ut))
718 + latest_msg_ut = msg_ut;
719 +
720 + if(unlikely(!first_msg_ut)) {
721 + first_msg_ut = msg_ut;
722 + // lqs->c.query_file.first_msg_ut = msg_ut;
723 + }
724 +
725 +// sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
726 +// FACETS_ANCHOR_DIRECTION_FORWARD,
727 +// facets_row_candidate_to_keep(facets, msg_ut));
728 +//
729 +// if(sample == SAMPLING_FULL) {
730 + bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
731 +
732 + // make sure each line gets a unique timestamp
733 + if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
734 + msg_ut = ++last_usec_to;
735 + else
736 + last_usec_from = last_usec_to = msg_ut;
737 +
738 + if(facets_row_finished(facets, msg_ut))
739 + rows_useful++;
740 +
741 + row_counter++;
742 + if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
743 + stop_when_full &&
744 + facets_rows(facets) >= lqs->rq.entries)) {
745 + // stop the data only query
746 + usec_t newest = facets_row_newest_ut(facets);
747 + if(newest && msg_ut > (newest + lqs->anchor.delta_ut))
748 + break;
749 + }
750 +
751 + if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
752 + status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
753 +
754 + if(status == WEVT_OK) {
755 + lqs->c.rows_read += row_counter - last_row_counter;
756 + last_row_counter = row_counter;
757 +
758 + lqs->c.bytes_read += bytes - last_bytes;
759 + last_bytes = bytes;
760 +
761 + send_progress_update(lqs, row_counter, false);
762 + }
763 + }
764 +// }
765 +// else if(sample == SAMPLING_SKIP_FIELDS)
766 +// facets_row_finished_unsampled(facets, msg_ut);
767 +// else {
768 +// sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_FORWARD);
769 +// break;
770 +// }
771 + }
772 +
773 + send_progress_update(lqs, row_counter, true);
774 + lqs->c.rows_read += row_counter - last_row_counter;
775 + lqs->c.bytes_read += bytes - last_bytes;
776 + lqs->c.rows_useful += rows_useful;
777 +
778 + if(errors_no_timestamp)
779 + netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
780 +
781 + if(latest_msg_ut > lqs->last_modified)
782 + lqs->last_modified = latest_msg_ut;
783 +
784 + wevt_query_done(log);
785 +
786 + return status;
787 +}
788 +
789 +static WEVT_QUERY_STATUS wevt_query_one_channel(
790 + WEVT_LOG *log,
791 + BUFFER *wb, FACETS *facets,
792 + LOGS_QUERY_SOURCE *src,
793 + LOGS_QUERY_STATUS *lqs) {
794 +
795 + errno_clear();
796 +
797 + WEVT_QUERY_STATUS status;
798 + if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD)
799 + status = wevt_query_forward(log, wb, facets, src, lqs);
800 + else
801 + status = wevt_query_backward(log, wb, facets, src, lqs);
802 +
803 + return status;
804 +}
805 +
806 +static bool source_is_mine(LOGS_QUERY_SOURCE *src, LOGS_QUERY_STATUS *lqs) {
807 + if((lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) || (src->source_type & lqs->rq.source_type) ||
808 + (lqs->rq.sources && simple_pattern_matches(lqs->rq.sources, string2str(src->source)))) {
809 +
810 + if(!src->msg_last_ut)
811 + // the file is not scanned yet, or the timestamps have not been updated,
812 + // so we don't know if it can contribute or not - let's add it.
813 + return true;
814 +
815 + usec_t anchor_delta = ANCHOR_DELTA_UT;
816 + usec_t first_ut = src->msg_first_ut - anchor_delta;
817 + usec_t last_ut = src->msg_last_ut + anchor_delta;
818 +
819 + if(last_ut >= lqs->rq.after_ut && first_ut <= lqs->rq.before_ut)
820 + return true;
821 + }
822 +
823 + return false;
824 +}
825 +
826 +typedef struct static_utf8_8k {
827 + char buffer[8192];
828 + size_t size;
829 + size_t len;
830 +} STATIC_BUF_8K;
831 +
832 +typedef struct static_unicode_16k {
833 + wchar_t buffer[16384];
834 + size_t size;
835 + size_t len;
836 +} STATIC_UNI_16K;
837 +
838 +static bool wevt_foreach_selected_value_cb(FACETS *facets __maybe_unused, size_t id, const char *key, const char *value, void *data) {
839 + STATIC_BUF_8K *b = data;
840 +
841 + b->len += snprintfz(&b->buffer[b->len], b->size - b->len,
842 + "%s%s=%s",
843 + id ? " or " : "", key, value);
844 +
845 + return b->len < b->size;
846 +}
847 +
848 +static wchar_t *wevt_generate_query(LOGS_QUERY_STATUS *lqs, BUFFER *wb) {
849 + static __thread STATIC_UNI_16K q = {
850 + .size = sizeof(q.buffer) / sizeof(wchar_t),
851 + .len = 0,
852 + };
853 + static __thread STATIC_BUF_8K b = {
854 + .size = sizeof(b.buffer) / sizeof(char),
855 + .len = 0,
856 + };
857 +
858 + lqs_query_timeframe(lqs, ANCHOR_DELTA_UT);
859 +
860 + usec_t seek_to = lqs->query.start_ut;
861 + if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
862 + // windows events queries are limited to millisecond resolution
863 + // so, in order not to lose data, we have to add
864 + // a millisecond when the direction is backward
865 + seek_to += USEC_PER_MS;
866 +
867 + // Convert the microseconds since Unix epoch to FILETIME (used in Windows APIs)
868 + FILETIME fileTime = os_unix_epoch_ut_to_filetime(seek_to);
869 +
870 + // Convert FILETIME to SYSTEMTIME for use in XPath
871 + SYSTEMTIME systemTime;
872 + if (!FileTimeToSystemTime(&fileTime, &systemTime)) {
873 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "FileTimeToSystemTime() failed");
874 + return NULL;
875 + }
876 +
877 + // Format SYSTEMTIME into ISO 8601 format (YYYY-MM-DDTHH:MM:SS.sssZ)
878 + q.len = swprintf(q.buffer, q.size,
879 + L"Event/System[TimeCreated[@SystemTime%ls\"%04d-%02d-%02dT%02d:%02d:%02d.%03dZ\"]",
880 + lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD ? L"<=" : L">=",
881 + systemTime.wYear, systemTime.wMonth, systemTime.wDay,
882 + systemTime.wHour, systemTime.wMinute, systemTime.wSecond, systemTime.wMilliseconds);
883 +
884 +
885 + if(lqs->rq.slice) {
886 + b.len = snprintf(b.buffer, b.size, " and (");
887 + if (facets_foreach_selected_value_in_key(
888 + lqs->facets,
889 + WEVT_FIELD_LEVEL,
890 + sizeof(WEVT_FIELD_LEVEL) - 1,
891 + used_hashes_registry,
892 + wevt_foreach_selected_value_cb,
893 + &b)) {
894 + b.len += snprintf(&b.buffer[b.len], b.size - b.len, ")");
895 + if (b.len < b.size) {
896 + utf82unicode(&q.buffer[q.len], q.size - q.len, b.buffer);
897 + q.len = wcslen(q.buffer);
898 + }
899 + }
900 +
901 + b.len = snprintf(b.buffer, b.size, " and (");
902 + if (facets_foreach_selected_value_in_key(
903 + lqs->facets,
904 + WEVT_FIELD_EVENTID,
905 + sizeof(WEVT_FIELD_EVENTID) - 1,
906 + used_hashes_registry,
907 + wevt_foreach_selected_value_cb,
908 + &b)) {
909 + b.len += snprintf(&b.buffer[b.len], b.size - b.len, ")");
910 + if (b.len < b.size) {
911 + utf82unicode(&q.buffer[q.len], q.size - q.len, b.buffer);
912 + q.len = wcslen(q.buffer);
913 + }
914 + }
915 + }
916 +
917 + q.len += swprintf(&q.buffer[q.len], q.size - q.len, L"]");
918 +
919 + buffer_json_member_add_string(wb, "_query", channel2utf8(q.buffer));
920 +
921 + return q.buffer;
922 +}
923 +
924 +static int wevt_master_query(BUFFER *wb __maybe_unused, LOGS_QUERY_STATUS *lqs __maybe_unused) {
925 + // make sure the sources list is updated
926 + wevt_sources_scan();
927 +
928 + lqs->c.query = wevt_generate_query(lqs, wb);
929 + if(!lqs->c.query)
930 + return rrd_call_function_error(wb, "failed to generate XPath query", HTTP_RESP_INTERNAL_SERVER_ERROR);
931 +
932 + FACETS *facets = lqs->facets;
933 +
934 + WEVT_QUERY_STATUS status = WEVT_NO_CHANNEL_MATCHED;
935 +
936 + lqs->c.files_matched = 0;
937 + lqs->c.file_working = 0;
938 + lqs->c.rows_useful = 0;
939 + lqs->c.rows_read = 0;
940 + lqs->c.bytes_read = 0;
941 +
942 + size_t files_used = 0;
943 + size_t files_max = dictionary_entries(wevt_sources);
944 + const DICTIONARY_ITEM *file_items[files_max];
945 +
946 + // count the files
947 + bool files_are_newer = false;
948 + LOGS_QUERY_SOURCE *src;
949 + dfe_start_read(wevt_sources, src) {
950 + if(!source_is_mine(src, lqs))
951 + continue;
952 +
953 + file_items[files_used++] = dictionary_acquired_item_dup(wevt_sources, src_dfe.item);
954 +
955 + if(src->msg_last_ut > lqs->rq.if_modified_since)
956 + files_are_newer = true;
957 +
958 + lqs->c.progress.entries.total += src->entries;
959 + }
960 + dfe_done(jf);
961 +
962 + lqs->c.files_matched = files_used;
963 +
964 + if(lqs->rq.if_modified_since && !files_are_newer) {
965 + // release the files
966 + for(size_t f = 0; f < files_used ;f++)
967 + dictionary_acquired_item_release(wevt_sources, file_items[f]);
968 +
969 + return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
970 + }
971 +
972 + // sort the files, so that they are optimal for facets
973 + if(files_used >= 2) {
974 + if (lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
975 + qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
976 + wevt_sources_dict_items_backward_compar);
977 + else
978 + qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
979 + wevt_sources_dict_items_forward_compar);
980 + }
981 +
982 + bool partial = false;
983 + usec_t query_started_ut = now_monotonic_usec();
984 + usec_t started_ut = query_started_ut;
985 + usec_t ended_ut = started_ut;
986 + usec_t duration_ut, max_duration_ut = 0;
987 +
988 + WEVT_LOG *log = wevt_openlog6();
989 + if(!log) {
990 + // release the files
991 + for(size_t f = 0; f < files_used ;f++)
992 + dictionary_acquired_item_release(wevt_sources, file_items[f]);
993 +
994 + netdata_log_error("WINDOWS EVENTS: cannot open windows event log");
995 + return rrd_call_function_error(wb, "cannot open windows events log", HTTP_RESP_INTERNAL_SERVER_ERROR);
996 + }
997 +
998 + // sampling_query_init(lqs, facets);
999 +
1000 + buffer_json_member_add_array(wb, "_channels");
1001 + for(size_t f = 0; f < files_used ;f++) {
1002 + const char *fullname = dictionary_acquired_item_name(file_items[f]);
1003 + src = dictionary_acquired_item_value(file_items[f]);
1004 +
1005 + if(!source_is_mine(src, lqs))
1006 + continue;
1007 +
1008 + started_ut = ended_ut;
1009 +
1010 + // do not even try to do the query if we expect it to pass the timeout
1011 + if(ended_ut + max_duration_ut * 3 >= *lqs->stop_monotonic_ut) {
1012 + partial = true;
1013 + status = WEVT_TIMED_OUT;
1014 + break;
1015 + }
1016 +
1017 + lqs->c.file_working++;
1018 +
1019 + size_t rows_useful = lqs->c.rows_useful;
1020 + size_t rows_read = lqs->c.rows_read;
1021 + size_t bytes_read = lqs->c.bytes_read;
1022 + size_t matches_setup_ut = lqs->c.matches_setup_ut;
1023 +
1024 + // sampling_file_init(lqs, src);
1025 +
1026 + lqs->c.progress.entries.current_query_total = src->entries;
1027 + WEVT_QUERY_STATUS tmp_status = wevt_query_one_channel(log, wb, facets, src, lqs);
1028 +
1029 + rows_useful = lqs->c.rows_useful - rows_useful;
1030 + rows_read = lqs->c.rows_read - rows_read;
1031 + bytes_read = lqs->c.bytes_read - bytes_read;
1032 + matches_setup_ut = lqs->c.matches_setup_ut - matches_setup_ut;
1033 +
1034 + ended_ut = now_monotonic_usec();
1035 + duration_ut = ended_ut - started_ut;
1036 +
1037 + if(duration_ut > max_duration_ut)
1038 + max_duration_ut = duration_ut;
1039 +
1040 + buffer_json_add_array_item_object(wb); // channel source
1041 + {
1042 + // information about the file
1043 + buffer_json_member_add_string(wb, "_name", fullname);
1044 + buffer_json_member_add_uint64(wb, "_source_type", src->source_type);
1045 + buffer_json_member_add_string(wb, "_source", string2str(src->source));
1046 + buffer_json_member_add_uint64(wb, "_msg_first_ut", src->msg_first_ut);
1047 + buffer_json_member_add_uint64(wb, "_msg_last_ut", src->msg_last_ut);
1048 +
1049 + // information about the current use of the file
1050 + buffer_json_member_add_uint64(wb, "duration_ut", ended_ut - started_ut);
1051 + buffer_json_member_add_uint64(wb, "rows_read", rows_read);
1052 + buffer_json_member_add_uint64(wb, "rows_useful", rows_useful);
1053 + buffer_json_member_add_double(wb, "rows_per_second", (double) rows_read / (double) duration_ut * (double) USEC_PER_SEC);
1054 + buffer_json_member_add_uint64(wb, "bytes_read", bytes_read);
1055 + buffer_json_member_add_double(wb, "bytes_per_second", (double) bytes_read / (double) duration_ut * (double) USEC_PER_SEC);
1056 + buffer_json_member_add_uint64(wb, "duration_matches_ut", matches_setup_ut);
1057 +
1058 + // if(lqs->rq.sampling) {
1059 + // buffer_json_member_add_object(wb, "_sampling");
1060 + // {
1061 + // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples_per_file.sampled);
1062 + // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples_per_file.unsampled);
1063 + // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples_per_file.estimated);
1064 + // }
1065 + // buffer_json_object_close(wb); // _sampling
1066 + // }
1067 + }
1068 + buffer_json_object_close(wb); // channel source
1069 +
1070 + bool stop = false;
1071 + switch(tmp_status) {
1072 + case WEVT_OK:
1073 + case WEVT_NO_CHANNEL_MATCHED:
1074 + status = (status == WEVT_OK) ? WEVT_OK : tmp_status;
1075 + break;
1076 +
1077 + case WEVT_FAILED_TO_OPEN:
1078 + case WEVT_FAILED_TO_SEEK:
1079 + partial = true;
1080 + if(status == WEVT_NO_CHANNEL_MATCHED)
1081 + status = tmp_status;
1082 + break;
1083 +
1084 + case WEVT_CANCELLED:
1085 + case WEVT_TIMED_OUT:
1086 + partial = true;
1087 + stop = true;
1088 + status = tmp_status;
1089 + break;
1090 +
1091 + case WEVT_NOT_MODIFIED:
1092 + internal_fatal(true, "this should never be returned here");
1093 + break;
1094 + }
1095 +
1096 + if(stop)
1097 + break;
1098 + }
1099 + buffer_json_array_close(wb); // _channels
1100 +
1101 + // release the files
1102 + for(size_t f = 0; f < files_used ;f++)
1103 + dictionary_acquired_item_release(wevt_sources, file_items[f]);
1104 +
1105 + switch (status) {
1106 + case WEVT_OK:
1107 + if(lqs->rq.if_modified_since && !lqs->c.rows_useful)
1108 + return rrd_call_function_error(wb, "no useful logs, not modified", HTTP_RESP_NOT_MODIFIED);
1109 + break;
1110 +
1111 + case WEVT_TIMED_OUT:
1112 + case WEVT_NO_CHANNEL_MATCHED:
1113 + break;
1114 +
1115 + case WEVT_CANCELLED:
1116 + return rrd_call_function_error(wb, "client closed connection", HTTP_RESP_CLIENT_CLOSED_REQUEST);
1117 +
1118 + case WEVT_NOT_MODIFIED:
1119 + return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
1120 +
1121 + case WEVT_FAILED_TO_OPEN:
1122 + return rrd_call_function_error(wb, "failed to open event log", HTTP_RESP_INTERNAL_SERVER_ERROR);
1123 +
1124 + case WEVT_FAILED_TO_SEEK:
1125 + return rrd_call_function_error(wb, "failed to execute event log query", HTTP_RESP_INTERNAL_SERVER_ERROR);
1126 +
1127 + default:
1128 + return rrd_call_function_error(wb, "unknown status", HTTP_RESP_INTERNAL_SERVER_ERROR);
1129 + }
1130 +
1131 + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
1132 + buffer_json_member_add_boolean(wb, "partial", partial);
1133 + buffer_json_member_add_string(wb, "type", "table");
1134 +
1135 + // build a message for the query
1136 + if(!lqs->rq.data_only) {
1137 + CLEAN_BUFFER *msg = buffer_create(0, NULL);
1138 + CLEAN_BUFFER *msg_description = buffer_create(0, NULL);
1139 + ND_LOG_FIELD_PRIORITY msg_priority = NDLP_INFO;
1140 +
1141 + // if(!journal_files_completed_once()) {
1142 + // buffer_strcat(msg, "Journals are still being scanned. ");
1143 + // buffer_strcat(msg_description
1144 + // , "LIBRARY SCAN: The journal files are still being scanned, you are probably viewing incomplete data. ");
1145 + // msg_priority = NDLP_WARNING;
1146 + // }
1147 +
1148 + if(partial) {
1149 + buffer_strcat(msg, "Query timed-out, incomplete data. ");
1150 + buffer_strcat(msg_description
1151 + , "QUERY TIMEOUT: The query timed out and may not include all the data of the selected window. ");
1152 + msg_priority = NDLP_WARNING;
1153 + }
1154 +
1155 + // if(lqs->c.samples.estimated || lqs->c.samples.unsampled) {
1156 + // double percent = (double) (lqs->c.samples.sampled * 100.0 /
1157 + // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1158 + // buffer_sprintf(msg, "%.2f%% real data", percent);
1159 + // buffer_sprintf(msg_description, "ACTUAL DATA: The filters counters reflect %0.2f%% of the data. ", percent);
1160 + // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1161 + // }
1162 + //
1163 + // if(lqs->c.samples.unsampled) {
1164 + // double percent = (double) (lqs->c.samples.unsampled * 100.0 /
1165 + // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1166 + // buffer_sprintf(msg, ", %.2f%% unsampled", percent);
1167 + // buffer_sprintf(msg_description
1168 + // , "UNSAMPLED DATA: %0.2f%% of the events exist and have been counted, but their values have not been evaluated, so they are not included in the filters counters. "
1169 + // , percent);
1170 + // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1171 + // }
1172 + //
1173 + // if(lqs->c.samples.estimated) {
1174 + // double percent = (double) (lqs->c.samples.estimated * 100.0 /
1175 + // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1176 + // buffer_sprintf(msg, ", %.2f%% estimated", percent);
1177 + // buffer_sprintf(msg_description
1178 + // , "ESTIMATED DATA: The query selected a large amount of data, so to avoid delaying too much, the presented data are estimated by %0.2f%%. "
1179 + // , percent);
1180 + // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1181 + // }
1182 +
1183 + buffer_json_member_add_object(wb, "message");
1184 + if(buffer_tostring(msg)) {
1185 + buffer_json_member_add_string(wb, "title", buffer_tostring(msg));
1186 + buffer_json_member_add_string(wb, "description", buffer_tostring(msg_description));
1187 + buffer_json_member_add_string(wb, "status", nd_log_id2priority(msg_priority));
1188 + }
1189 + // else send an empty object if there is nothing to tell
1190 + buffer_json_object_close(wb); // message
1191 + }
1192 +
1193 + if(!lqs->rq.data_only) {
1194 + buffer_json_member_add_time_t(wb, "update_every", 1);
1195 + buffer_json_member_add_string(wb, "help", WEVT_FUNCTION_DESCRIPTION);
1196 + }
1197 +
1198 + if(!lqs->rq.data_only || lqs->rq.tail)
1199 + buffer_json_member_add_uint64(wb, "last_modified", lqs->last_modified);
1200 +
1201 + facets_sort_and_reorder_keys(facets);
1202 + facets_report(facets, wb, used_hashes_registry);
1203 +
1204 + wb->expires = now_realtime_sec() + (lqs->rq.data_only ? 3600 : 0);
1205 + buffer_json_member_add_time_t(wb, "expires", wb->expires);
1206 +
1207 + // if(lqs->rq.sampling) {
1208 + // buffer_json_member_add_object(wb, "_sampling");
1209 + // {
1210 + // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples.sampled);
1211 + // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples.unsampled);
1212 + // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples.estimated);
1213 + // }
1214 + // buffer_json_object_close(wb); // _sampling
1215 + // }
1216 +
1217 + wevt_closelog6(log);
1218 +
1219 + wb->content_type = CT_APPLICATION_JSON;
1220 + wb->response_code = HTTP_RESP_OK;
1221 + return wb->response_code;
1222 +}
1223 +
1224 +void function_windows_events(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
1225 + BUFFER *payload, HTTP_ACCESS access __maybe_unused,
1226 + const char *source __maybe_unused, void *data __maybe_unused) {
1227 + bool have_slice = LQS_DEFAULT_SLICE_MODE;
1228 +
1229 + LOGS_QUERY_STATUS tmp_fqs = {
1230 + .facets = lqs_facets_create(
1231 + LQS_DEFAULT_ITEMS_PER_QUERY,
1232 + FACETS_OPTION_ALL_KEYS_FTS,
1233 + WEVT_ALWAYS_VISIBLE_KEYS,
1234 + WEVT_KEYS_INCLUDED_IN_FACETS,
1235 + WEVT_KEYS_EXCLUDED_FROM_FACETS,
1236 + have_slice),
1237 +
1238 + .rq = LOGS_QUERY_REQUEST_DEFAULTS(transaction, have_slice, FACETS_ANCHOR_DIRECTION_BACKWARD),
1239 +
1240 + .cancelled = cancelled,
1241 + .stop_monotonic_ut = stop_monotonic_ut,
1242 + };
1243 + LOGS_QUERY_STATUS *lqs = &tmp_fqs;
1244 +
1245 + CLEAN_BUFFER *wb = lqs_create_output_buffer();
1246 +
1247 + // ------------------------------------------------------------------------
1248 + // parse the parameters
1249 +
1250 + if(lqs_request_parse_and_validate(lqs, wb, function, payload, have_slice, WEVT_FIELD_LEVEL)) {
1251 + wevt_register_fields(lqs);
1252 +
1253 + // ------------------------------------------------------------------------
1254 + // add versions to the response
1255 +
1256 + buffer_json_wevt_versions(wb);
1257 +
1258 + // ------------------------------------------------------------------------
1259 + // run the request
1260 +
1261 + if (lqs->rq.info)
1262 + lqs_info_response(wb, lqs->facets);
1263 + else {
1264 + wevt_master_query(wb, lqs);
1265 + if (wb->response_code == HTTP_RESP_OK)
1266 + buffer_json_finalize(wb);
1267 + }
1268 + }
1269 +
1270 + netdata_mutex_lock(&stdout_mutex);
1271 + pluginsd_function_result_to_stdout(transaction, wb);
1272 + netdata_mutex_unlock(&stdout_mutex);
1273 +
1274 + lqs_cleanup(lqs);
1275 +}
1276 +
1277 +int main(int argc __maybe_unused, char **argv __maybe_unused) {
1278 + clocks_init();
1279 + nd_thread_tag_set("wevt.plugin");
1280 + nd_log_initialize_for_external_plugins("windows-events.plugin");
1281 +
1282 + // ------------------------------------------------------------------------
1283 + // initialization
1284 +
1285 + wevt_sources_init();
1286 + publisher_cache_init();
1287 + sid_cache_init();
1288 + field_cache_init();
1289 +
1290 + // ------------------------------------------------------------------------
1291 + // debug
1292 +
1293 + if(argc >= 2 && strcmp(argv[argc - 1], "debug") == 0) {
1294 + wevt_sources_scan();
1295 +
1296 + struct {
1297 + const char *func;
1298 + } array[] = {
1299 + //{ "windows-events after:-86400 before:0 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,HytMJ9kj82B,JM3OPW3kHn6,H106l8MXSSr,HREiMN.4Ahu,ClaDGnYSQE7,ApYltST_icg,PtkRm91M0En data_only:false slice:true source:All" },
1300 + //{ "windows-events after:1726055370 before:1726056270 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6,ClaDGnYSQE7,H106l8MXSSr,HREiMN.4Ahu data_only:false source:All HytMJ9kj82B:BlC24d5JBBV,PtVoyIuX.MU,HMj1B38kHTv KeCITtVD5AD:PY1JtCeWwSe,O9kz5J37nNl,JZoJURadhDb" },
1301 + { "windows-events after:1725738894 before:0 last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,HytMJ9kj82B,JM3OPW3kHn6,H106l8MXSSr,HREiMN.4Ahu,ClaDGnYSQE7,ApYltST_icg,PtkRm91M0En source:All" },
1302 + // { "windows-events after:1725636012 before:1726240812 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,HytMJ9kj82B,JM3OPW3kHn6,H106l8MXSSr,HREiMN.4Ahu,ClaDGnYSQE7,ApYltST_icg,PtkRm91M0En data_only:false source:All PtkRm91M0En:LDzHbP5libb" },
1303 + //{ "windows-events after:1725650386 before:1725736786 anchor:1725652420809461 direction:forward last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 if_modified_since:1725736649011085 data_only:true delta:true tail:true source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1304 + //{ "windows-events info after:1725650420 before:1725736820" },
1305 + //{ "windows-events after:1725650420 before:1725736820 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1306 + //{ "windows-events after:1725650430 before:1725736830 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1307 + { NULL },
1308 + };
1309 +
1310 + for(int i = 0; array[i].func ;i++) {
1311 + bool cancelled = false;
1312 + usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
1313 + //char buf[] = "windows-events after:-86400 before:0 direction:backward last:200 data_only:false slice:true source:all";
1314 + function_windows_events("123", (char *)array[i].func, &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
1315 + }
1316 + printf("\n\nAll done!\n\n");
1317 + fflush(stdout);
1318 + exit(1);
1319 + }
1320 +
1321 + // ------------------------------------------------------------------------
1322 + // the event loop for functions
1323 +
1324 + struct functions_evloop_globals *wg =
1325 + functions_evloop_init(WINDOWS_EVENTS_WORKER_THREADS, "WEVT", &stdout_mutex, &plugin_should_exit);
1326 +
1327 + functions_evloop_add_function(wg,
1328 + WEVT_FUNCTION_NAME,
1329 + function_windows_events,
1330 + WINDOWS_EVENTS_DEFAULT_TIMEOUT,
1331 + NULL);
1332 +
1333 + // ------------------------------------------------------------------------
1334 + // register functions to netdata
1335 +
1336 + netdata_mutex_lock(&stdout_mutex);
1337 +
1338 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" "HTTP_ACCESS_FORMAT" %d\n",
1339 + WEVT_FUNCTION_NAME, WINDOWS_EVENTS_DEFAULT_TIMEOUT, WEVT_FUNCTION_DESCRIPTION,
1340 + (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
1341 + RRDFUNCTIONS_PRIORITY_DEFAULT);
1342 +
1343 + fflush(stdout);
1344 + netdata_mutex_unlock(&stdout_mutex);
1345 +
1346 + // ------------------------------------------------------------------------
1347 +
1348 + const usec_t step_ut = 100 * USEC_PER_MS;
1349 + usec_t send_newline_ut = 0;
1350 + usec_t since_last_scan_ut = WINDOWS_EVENTS_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
1351 + const bool tty = isatty(fileno(stdout)) == 1;
1352 +
1353 + heartbeat_t hb;
1354 + heartbeat_init(&hb);
1355 + while(!plugin_should_exit) {
1356 +
1357 + if(since_last_scan_ut > WINDOWS_EVENTS_SCAN_EVERY_USEC) {
1358 + wevt_sources_scan();
1359 + since_last_scan_ut = 0;
1360 + }
1361 +
1362 + usec_t dt_ut = heartbeat_next(&hb, step_ut);
1363 + since_last_scan_ut += dt_ut;
1364 + send_newline_ut += dt_ut;
1365 +
1366 + if(!tty && send_newline_ut > USEC_PER_SEC) {
1367 + send_newline_and_flush(&stdout_mutex);
1368 + send_newline_ut = 0;
1369 + }
1370 + }
1371 +
1372 + exit(0);
1373 +}
src/collectors/windows-events.plugin/windows-events.h
+30 -28
@@ -1,28 +1,30 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_WINDOWS_EVENTS_H
4 -#define NETDATA_WINDOWS_EVENTS_H
5 -
6 -#include "libnetdata/libnetdata.h"
7 -#include "collectors/all.h"
8 -#include <windows.h>
9 -#include <winevt.h>
10 -#include <wchar.h>
11 -
12 -typedef enum {
13 - WEVT_NO_CHANNEL_MATCHED,
14 - WEVT_FAILED_TO_OPEN,
15 - WEVT_FAILED_TO_SEEK,
16 - WEVT_TIMED_OUT,
17 - WEVT_OK,
18 - WEVT_NOT_MODIFIED,
19 - WEVT_CANCELLED,
20 -} WEVT_QUERY_STATUS;
21 -
22 -#include "windows-events-unicode.h"
23 -#include "windows-events-query.h"
24 -#include "windows-events-sources.h"
25 -#include "windows-events-sid.h"
26 -#include "windows-events-xml.h"
27 -
28 -#endif //NETDATA_WINDOWS_EVENTS_H
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WINDOWS_EVENTS_H
4 +#define NETDATA_WINDOWS_EVENTS_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "collectors/all.h"
8 +#include <windows.h>
9 +#include <winevt.h>
10 +#include <wchar.h>
11 +
12 +typedef enum {
13 + WEVT_NO_CHANNEL_MATCHED,
14 + WEVT_FAILED_TO_OPEN,
15 + WEVT_FAILED_TO_SEEK,
16 + WEVT_TIMED_OUT,
17 + WEVT_OK,
18 + WEVT_NOT_MODIFIED,
19 + WEVT_CANCELLED,
20 +} WEVT_QUERY_STATUS;
21 +
22 +#include "windows-events-unicode.h"
23 +#include "windows-events-query.h"
24 +#include "windows-events-sources.h"
25 +#include "windows-events-sid.h"
26 +#include "windows-events-xml.h"
27 +#include "windows-events-publishers.h"
28 +#include "windows-events-fields-cache.h"
29 +
30 +#endif //NETDATA_WINDOWS_EVENTS_H
src/libnetdata/buffer/buffer.h
+39 -6
@@ -414,6 +414,16 @@ static inline char *print_uint64_hex_reversed(char *dst, uint64_t value) {
414 #endif
415 }
416
417 +static inline char *print_uint64_hex_reversed_full(char *dst, uint64_t value) {
418 + char *d = dst;
419 + for(size_t c = 0; c < sizeof(uint64_t) * 2; c++) {
420 + *d++ = hex_digits[value & 0xf];
421 + value >>= 4;
422 + }
423 +
424 + return d;
425 +}
426 +
427 static inline char *print_uint64_base64_reversed(char *dst, uint64_t value) {
428 char *d = dst;
429 do *d++ = base64_digits[value & 63]; while ((value >>= 6));
@@ -509,6 +519,7 @@ static inline size_t print_int64(char *dst, int64_t value) {
519 return print_uint64(dst, value) + len;
520 }
521
522 +#define UINT64_MAX_LENGTH (24) // 21 should be enough
523 static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
524 buffer_need_bytes(wb, 50);
525 wb->len += print_uint64(&wb->buffer[wb->len], value);
@@ -521,7 +532,7 @@ static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
532 buffer_overflow_check(wb);
533 }
534
524 -#define UINT64_HEX_LENGTH ((sizeof(HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 2 + 1)
535 +#define UINT64_HEX_MAX_LENGTH ((sizeof(HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
536 static inline size_t print_uint64_hex(char *dst, uint64_t value) {
537 char *d = dst;
538
@@ -534,14 +545,33 @@ static inline size_t print_uint64_hex(char *dst, uint64_t value) {
545 return e - dst;
546 }
547
548 +static inline size_t print_uint64_hex_full(char *dst, uint64_t value) {
549 + char *d = dst;
550 +
551 + const char *s = HEX_PREFIX;
552 + while(*s) *d++ = *s++;
553 +
554 + char *e = print_uint64_hex_reversed_full(d, value);
555 + char_array_reverse(d, e - 1);
556 + *e = '\0';
557 + return e - dst;
558 +}
559 +
560 static inline void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) {
538 - buffer_need_bytes(wb, UINT64_HEX_LENGTH);
561 + buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH);
562 wb->len += print_uint64_hex(&wb->buffer[wb->len], value);
563 buffer_overflow_check(wb);
564 }
565
566 +static inline void buffer_print_uint64_hex_full(BUFFER *wb, uint64_t value) {
567 + buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH);
568 + wb->len += print_uint64_hex_full(&wb->buffer[wb->len], value);
569 + buffer_overflow_check(wb);
570 +}
571 +
572 +#define UINT64_B64_MAX_LENGTH ((sizeof(IEEE754_UINT64_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
573 static inline void buffer_print_uint64_base64(BUFFER *wb, uint64_t value) {
544 - buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1);
574 + buffer_need_bytes(wb, UINT64_B64_MAX_LENGTH);
575
576 buffer_fast_strcat(wb, IEEE754_UINT64_B64_PREFIX, sizeof(IEEE754_UINT64_B64_PREFIX) - 1);
577
@@ -580,8 +610,9 @@ static inline void buffer_print_int64_base64(BUFFER *wb, int64_t value) {
610 buffer_overflow_check(wb);
611 }
612
613 +#define DOUBLE_MAX_LENGTH (512) // 318 should be enough, including null
614 static inline void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value) {
584 - buffer_need_bytes(wb, 512 + 2);
615 + buffer_need_bytes(wb, DOUBLE_MAX_LENGTH);
616
617 if(isnan(value) || isinf(value)) {
618 buffer_fast_strcat(wb, "null", 4);
@@ -597,8 +628,9 @@ static inline void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value)
628 buffer_overflow_check(wb);
629 }
630
631 +#define DOUBLE_HEX_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
632 static inline void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE value) {
601 - buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1);
633 + buffer_need_bytes(wb, DOUBLE_HEX_MAX_LENGTH);
634
635 uint64_t *ptr = (uint64_t *) (&value);
636 buffer_fast_strcat(wb, IEEE754_DOUBLE_HEX_PREFIX, sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1);
@@ -612,8 +644,9 @@ static inline void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE va
644 buffer_overflow_check(wb);
645 }
646
647 +#define DOUBLE_B64_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
648 static inline void buffer_print_netdata_double_base64(BUFFER *wb, NETDATA_DOUBLE value) {
616 - buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1);
649 + buffer_need_bytes(wb, DOUBLE_B64_MAX_LENGTH);
650
651 uint64_t *ptr = (uint64_t *) (&value);
652 buffer_fast_strcat(wb, IEEE754_DOUBLE_B64_PREFIX, sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1);
src/libnetdata/facets/facets.c
+81 -1
@@ -256,6 +256,7 @@ struct facets {
256 } keys_in_row;
257
258 FACET_ROW *base; // double linked list of the selected facets rows
259 + FACET_ROW_BIN_DATA bin_data;
260
261 uint32_t items_to_return;
262 uint32_t max_items_to_return;
@@ -329,6 +330,10 @@ struct facets {
330 struct {
331 size_t searches;
332 } fts;
333 +
334 + struct {
335 + size_t bin_data_inflight;
336 + };
337 } operations;
338
339 struct {
@@ -576,7 +581,7 @@ static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, const char *
581 .hash = hash,
582 .selected = true,
583 .name = name,
579 - .name_len = 0,
584 + .name_len = name ? strlen(name) : 0,
585 };
586 FACET_VALUE_ADD_TO_INDEX(k, &tv);
587 }
@@ -644,6 +649,35 @@ bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, s
649 return (v && v->selected) ? true : false;
650 }
651
652 +bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_t key_length, DICTIONARY *used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void *data) {
653 + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
654 + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
655 + if(!k || k->default_selected_for_values)
656 + return false;
657 +
658 + size_t selected = 0;
659 + for(FACET_VALUE *v = k->values.ll; v ;v = v->next) {
660 + if(!v->selected) continue;
661 +
662 + const char *value = v->name;
663 + if(!value) {
664 + if(used_hashes_registry) {
665 + char hash_str[FACET_STRING_HASH_SIZE];
666 + facets_hash_to_str(v->hash, hash_str);
667 + value = dictionary_get(used_hashes_registry, hash_str);
668 + }
669 +
670 + if(!value)
671 + return false;
672 + }
673 +
674 + if(!cb(facets, selected++, k->name, value, data))
675 + return false;
676 + }
677 +
678 + return selected > 0;
679 +}
680 +
681 void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) {
682 FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
683 FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
@@ -1591,6 +1625,35 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
1625 return false;
1626 }
1627
1628 +// ----------------------------------------------------------------------------
1629 +// bin_data management
1630 +
1631 +static inline void facets_row_bin_data_cleanup(FACETS *facets, FACET_ROW_BIN_DATA *bin_data) {
1632 + if(!bin_data->data)
1633 + return;
1634 +
1635 + bin_data->cleanup_cb(bin_data->data);
1636 + *bin_data = FACET_ROW_BIN_DATA_EMPTY;
1637 +
1638 + fatal_assert(facets->operations.bin_data_inflight > 0);
1639 + facets->operations.bin_data_inflight--;
1640 +}
1641 +
1642 +void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data) {
1643 + // in case the caller tries to register bin_data multiple times
1644 + // for the same row.
1645 + facets_row_bin_data_cleanup(facets, &facets->bin_data);
1646 +
1647 + // set the new values
1648 + facets->bin_data.cleanup_cb = cleanup_cb;
1649 + facets->bin_data.data = data;
1650 + facets->operations.bin_data_inflight++;
1651 +}
1652 +
1653 +void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row) {
1654 + return row->bin_data.data;
1655 +}
1656 +
1657 // ----------------------------------------------------------------------------
1658
1659 FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
@@ -1633,6 +1696,13 @@ void facets_destroy(FACETS *facets) {
1696 facets_row_free(facets, r);
1697 }
1698
1699 + // in case the caller did not call facets_row_finished()
1700 + // on the last row.
1701 + facets_row_bin_data_cleanup(facets, &facets->bin_data);
1702 +
1703 + // make sure we didn't lose any data
1704 + fatal_assert(facets->operations.bin_data_inflight == 0);
1705 +
1706 freez(facets->histogram.chart);
1707 freez(facets);
1708 }
@@ -1930,7 +2000,9 @@ static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __ma
2000 // FACET_ROW management
2001
2002 static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) {
2003 + facets_row_bin_data_cleanup(facets, &row->bin_data);
2004 dictionary_destroy(row->dict);
2005 + row->dict = NULL;
2006 freez(row);
2007 }
2008
@@ -1940,6 +2012,7 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into
2012 if(into) {
2013 row = into;
2014 facets->operations.rows.reused++;
2015 + facets_row_bin_data_cleanup(facets, &row->bin_data);
2016 }
2017 else {
2018 row = callocz(1, sizeof(FACET_ROW));
@@ -1950,6 +2023,11 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into
2023 facets->operations.rows.created++;
2024 }
2025
2026 + // copy the bin_data to the row
2027 + // and forget about them in facets
2028 + row->bin_data = facets->bin_data;
2029 + facets->bin_data = FACET_ROW_BIN_DATA_EMPTY;
2030 +
2031 row->severity = facets->current_row.severity;
2032 row->usec = usec;
2033
@@ -2134,6 +2212,8 @@ static void facets_reset_keys_with_value_and_row(FACETS *facets) {
2212 facets->current_row.keys_matched_by_query_positive = 0;
2213 facets->current_row.keys_matched_by_query_negative = 0;
2214 facets->keys_in_row.used = 0;
2215 +
2216 + facets_row_bin_data_cleanup(facets, &facets->bin_data);
2217 }
2218
2219 void facets_rows_begin(FACETS *facets) {
src/libnetdata/facets/facets.h
+14
@@ -52,10 +52,18 @@ typedef struct facet_row_key_value {
52 BUFFER *wb;
53 } FACET_ROW_KEY_VALUE;
54
55 +typedef struct facet_row_bin_data {
56 + void (*cleanup_cb)(void *data);
57 + void *data;
58 +} FACET_ROW_BIN_DATA;
59 +
60 +#define FACET_ROW_BIN_DATA_EMPTY (FACET_ROW_BIN_DATA){.data = NULL, .cleanup_cb = NULL}
61 +
62 typedef struct facet_row {
63 usec_t usec;
64 DICTIONARY *dict;
65 FACET_ROW_SEVERITY severity;
66 + FACET_ROW_BIN_DATA bin_data;
67 struct facet_row *prev, *next;
68 } FACET_ROW;
69
@@ -132,4 +140,10 @@ void facets_table_config(BUFFER *wb);
140
141 const char *facets_severity_to_string(FACET_ROW_SEVERITY severity);
142
143 +typedef bool (*facets_foreach_selected_value_in_key_t)(FACETS *facets, size_t id, const char *key, const char *value, void *data);
144 +bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_t key_length, DICTIONARY *used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void *data);
145 +
146 +void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data);
147 +void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row);
148 +
149 #endif
src/libnetdata/facets/logs_query_status.h
+22
@@ -107,8 +107,15 @@ typedef struct {
107 struct {
108 usec_t start_ut;
109 usec_t stop_ut;
110 + usec_t delta_ut;
111 } anchor;
112
113 + struct {
114 + usec_t start_ut;
115 + usec_t stop_ut;
116 + bool stop_when_full;
117 + } query;
118 +
119 usec_t last_modified;
120
121 struct lqs_extension c;
@@ -144,6 +151,21 @@ static inline void lqs_log_error(LOGS_QUERY_STATUS *lqs, const char *msg) {
151 , lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
152 }
153
154 +static inline void lqs_query_timeframe(LOGS_QUERY_STATUS *lqs, usec_t anchor_delta_ut) {
155 + lqs->anchor.delta_ut = anchor_delta_ut;
156 +
157 + if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
158 + lqs->query.start_ut = (lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.after_ut;
159 + lqs->query.stop_ut = ((lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.before_ut) + lqs->anchor.delta_ut;
160 + }
161 + else {
162 + lqs->query.start_ut = ((lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.before_ut) + lqs->anchor.delta_ut;
163 + lqs->query.stop_ut = (lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.after_ut;
164 + }
165 +
166 + lqs->query.stop_when_full = (lqs->rq.data_only && !lqs->anchor.stop_ut);
167 +}
168 +
169 static inline void lqs_function_help(LOGS_QUERY_STATUS *lqs, BUFFER *wb) {
170 buffer_reset(wb);
171 wb->content_type = CT_TEXT_PLAIN;
src/libnetdata/socket/socket.c
+30 -10
@@ -119,22 +119,17 @@ bool fd_is_socket(int fd) {
119 return true;
120 }
121
122 -bool sock_has_output_error(int fd) {
123 - if(fd < 0) {
124 - //internal_error(true, "invalid socket %d", fd);
125 - return false;
126 - }
122 +#ifdef POLLRDHUP
123 +bool is_socket_closed(int fd) {
124 + if(fd < 0)
125 + return true;
126
127 // if(!fd_is_socket(fd)) {
128 // //internal_error(true, "fd %d is not a socket", fd);
129 // return false;
130 // }
131
133 - short int errors = POLLERR | POLLHUP | POLLNVAL;
134 -
135 -#ifdef POLLRDHUP
136 - errors |= POLLRDHUP;
137 -#endif
132 + short int errors = POLLERR | POLLHUP | POLLNVAL | POLLRDHUP;
133
134 struct pollfd pfd = {
135 .fd = fd,
@@ -149,6 +144,31 @@ bool sock_has_output_error(int fd) {
144
145 return ((pfd.revents & errors) || !(pfd.revents & POLLOUT));
146 }
147 +#else
148 +bool is_socket_closed(int fd) {
149 + if(fd < 0)
150 + return true;
151 +
152 + char buffer;
153 + ssize_t result = recv(fd, &buffer, 1, MSG_PEEK | MSG_DONTWAIT);
154 + if (result == 0) {
155 + // Connection closed
156 + return true;
157 + }
158 + else if (result < 0) {
159 + if (errno == EAGAIN || errno == EWOULDBLOCK) {
160 + // No data available, but socket is still open
161 + return false;
162 + } else {
163 + // An error occurred
164 + return true;
165 + }
166 + }
167 +
168 + // Data is available, socket is open
169 + return false;
170 +}
171 +#endif
172
173 int sock_setnonblock(int fd) {
174 int flags;
src/libnetdata/socket/socket.h
+1 -1
@@ -44,7 +44,7 @@ ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int fla
44 int wait_on_socket_or_cancel_with_timeout(NETDATA_SSL *ssl, int fd, int timeout_ms, short int poll_events, short int *revents);
45
46 bool fd_is_socket(int fd);
47 -bool sock_has_output_error(int fd);
47 +bool is_socket_closed(int fd);
48
49 int sock_setnonblock(int fd);
50 int sock_delnonblock(int fd);
src/web/api/v3/api_v3_settings.c
+1 -1
@@ -219,7 +219,7 @@ static inline bool is_settings_file_valid(char *file) {
219 return false;
220
221 while(*s) {
222 - if(!isalnum(*s) && *s != '-' && *s != '_')
222 + if(!isalnum((uint8_t)*s) && *s != '-' && *s != '_')
223 return false;
224 s++;
225 }
src/web/api/web_api.c
+5 -2
@@ -121,10 +121,13 @@ RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {
121 bool web_client_interrupt_callback(void *data) {
122 struct web_client *w = data;
123
124 + bool ret;
125 if(w->interrupt.callback)
125 - return w->interrupt.callback(w, w->interrupt.callback_data);
126 + ret = w->interrupt.callback(w, w->interrupt.callback_data);
127 + else
128 + ret = is_socket_closed(w->ofd);
129
127 - return sock_has_output_error(w->ofd);
130 + return ret;
131 }
132
133 void nd_web_api_init(void) {