@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
c 1,411 lines 51.8 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "libnetdata/libnetdata.h"
4
5 #include "windows-events.h"
6
7 netdata_mutex_t stdout_mutex;
8
9 static void __attribute__((constructor)) init_mutex(void) {
10 netdata_mutex_init(&stdout_mutex);
11 }
12
13 static void __attribute__((destructor)) destroy_mutex(void) {
14 netdata_mutex_destroy(&stdout_mutex);
15 }
16
17 static bool plugin_should_exit = false;
18
19 #define WEVT_ALWAYS_VISIBLE_KEYS NULL
20
21 #define WEVT_KEYS_EXCLUDED_FROM_FACETS \
22 "|" WEVT_FIELD_MESSAGE \
23 "|" WEVT_FIELD_XML \
24 ""
25
26 #define WEVT_KEYS_INCLUDED_IN_FACETS \
27 "|" WEVT_FIELD_COMPUTER \
28 "|" WEVT_FIELD_PROVIDER \
29 "|" WEVT_FIELD_LEVEL \
30 "|" WEVT_FIELD_KEYWORDS \
31 "|" WEVT_FIELD_OPCODE \
32 "|" WEVT_FIELD_TASK \
33 "|" WEVT_FIELD_ACCOUNT \
34 "|" WEVT_FIELD_DOMAIN \
35 "|" WEVT_FIELD_SID \
36 ""
37
38 #define query_has_fts(lqs) ((lqs)->rq.query != NULL)
39
40 static inline WEVT_QUERY_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) {
41 if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
42 nd_log(NDLS_COLLECTORS, NDLP_INFO, "Function has been cancelled");
43 return WEVT_CANCELLED;
44 }
45
46 if(now_monotonic_usec() > __atomic_load_n(stop_monotonic_ut, __ATOMIC_RELAXED)) {
47 internal_error(true, "Function timed out");
48 return WEVT_TIMED_OUT;
49 }
50
51 return WEVT_OK;
52 }
53
54 FACET_ROW_SEVERITY wevt_levelid_to_facet_severity(FACETS *facets __maybe_unused, FACET_ROW *row, void *data __maybe_unused) {
55 FACET_ROW_KEY_VALUE *levelid_rkv = dictionary_get(row->dict, WEVT_FIELD_LEVEL "ID");
56 if(!levelid_rkv || levelid_rkv->empty)
57 return FACET_ROW_SEVERITY_NORMAL;
58
59 int windows_event_level = str2i(buffer_tostring(levelid_rkv->wb));
60
61 switch (windows_event_level) {
62 case WEVT_LEVEL_VERBOSE:
63 return FACET_ROW_SEVERITY_DEBUG;
64
65 default:
66 case WEVT_LEVEL_INFORMATION:
67 return FACET_ROW_SEVERITY_NORMAL;
68
69 case WEVT_LEVEL_WARNING:
70 return FACET_ROW_SEVERITY_WARNING;
71
72 case WEVT_LEVEL_ERROR:
73 case WEVT_LEVEL_CRITICAL:
74 return FACET_ROW_SEVERITY_CRITICAL;
75 }
76 }
77
78 struct wevt_bin_data {
79 bool rendered;
80 WEVT_EVENT ev;
81 WEVT_LOG *log;
82 EVT_HANDLE hEvent;
83 PROVIDER_META_HANDLE *provider;
84 };
85
86 static void wevt_cleanup_bin_data(void *data) {
87 struct wevt_bin_data *d = data;
88
89 if(d->hEvent)
90 EvtClose(d->hEvent);
91
92 provider_release(d->provider);
93 freez(d);
94 }
95
96 static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets, WEVT_EVENT *ev) {
97 struct wevt_bin_data *d = mallocz(sizeof(struct wevt_bin_data));
98
99 #ifdef NETDATA_INTERNAL_CHECKS
100 internal_fatal(strcmp(log->ops.provider.data, provider_get_name(log->provider)) != 0,
101 "Provider name mismatch in data!");
102
103 internal_fatal(!UUIDeq(ev->provider, provider_get_uuid(log->provider)),
104 "Provider UUID mismatch in data!");
105 #endif
106
107 d->ev = *ev;
108 d->log = log;
109 d->rendered = false;
110
111 // take the bookmark
112 d->hEvent = log->hEvent; log->hEvent = NULL;
113
114 // dup the provider
115 d->provider = provider_dup(log->provider);
116
117 facets_row_bin_data_set(facets, wevt_cleanup_bin_data, d);
118 }
119
120 static void wevt_lazy_loading_event_and_xml(struct wevt_bin_data *d, FACET_ROW *row __maybe_unused) {
121 if(d->rendered) return;
122
123 #ifdef NETDATA_INTERNAL_CHECKS
124 const FACET_ROW_KEY_VALUE *provider_rkv = dictionary_get(row->dict, WEVT_FIELD_PROVIDER);
125 internal_fatal(!provider_rkv || strcmp(buffer_tostring(provider_rkv->wb), provider_get_name(d->provider)) != 0,
126 "Provider of row does not match the bin data associated with it");
127
128 uint64_t event_record_id = UINT64_MAX;
129 const FACET_ROW_KEY_VALUE *event_record_id_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENTRECORDID);
130 if(event_record_id_rkv)
131 event_record_id = str2uint64_t(buffer_tostring(event_record_id_rkv->wb), NULL);
132 internal_fatal(event_record_id != d->ev.id,
133 "Event Record ID of row does not match the bin data associated with it");
134 #endif
135
136 // the message needs the xml
137 EvtFormatMessage_Xml_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.xml);
138 EvtFormatMessage_Event_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.event);
139 d->rendered = true;
140 }
141
142 static void wevt_lazy_load_xml(
143 FACETS *facets,
144 BUFFER *json_array,
145 FACET_ROW_KEY_VALUE *rkv __maybe_unused,
146 FACET_ROW *row,
147 void *data __maybe_unused) {
148
149 struct wevt_bin_data *d = facets_row_bin_data_get(facets, row);
150 if(!d) {
151 buffer_json_add_array_item_string(json_array, "Failed to get row BIN DATA from facets");
152 return;
153 }
154
155 wevt_lazy_loading_event_and_xml(d, row);
156 buffer_json_add_array_item_string(json_array, d->log->ops.xml.data);
157 }
158
159 static void wevt_lazy_load_message(
160 FACETS *facets,
161 BUFFER *json_array,
162 FACET_ROW_KEY_VALUE *rkv __maybe_unused,
163 FACET_ROW *row,
164 void *data __maybe_unused) {
165
166 struct wevt_bin_data *d = facets_row_bin_data_get(facets, row);
167 if(!d) {
168 buffer_json_add_array_item_string(json_array, "Failed to get row BIN DATA from facets");
169 return;
170 }
171
172 wevt_lazy_loading_event_and_xml(d, row);
173
174 if(d->log->ops.event.used <= 1) {
175 TXT_UTF8 *xml = &d->log->ops.xml;
176
177 buffer_flush(rkv->wb);
178
179 bool added_message = false;
180 if(xml->used > 1) {
181 const char *message_path[] = {
182 "RenderingInfo",
183 "Message",
184 NULL};
185
186 added_message = buffer_xml_extract_and_print_value(
187 rkv->wb,
188 xml->data, xml->used - 1,
189 NULL,
190 message_path);
191 }
192
193 if(!added_message) {
194 const FACET_ROW_KEY_VALUE *event_id_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENTID);
195 if (event_id_rkv && buffer_strlen(event_id_rkv->wb)) {
196 buffer_fast_strcat(rkv->wb, "Event ", 6);
197 buffer_fast_strcat(rkv->wb, buffer_tostring(event_id_rkv->wb), buffer_strlen(event_id_rkv->wb));
198 } else
199 buffer_strcat(rkv->wb, "Unknown Event ");
200
201 const FACET_ROW_KEY_VALUE *provider_rkv = dictionary_get(row->dict, WEVT_FIELD_PROVIDER);
202 if (provider_rkv && buffer_strlen(provider_rkv->wb)) {
203 buffer_fast_strcat(rkv->wb, " of ", 4);
204 buffer_fast_strcat(rkv->wb, buffer_tostring(provider_rkv->wb), buffer_strlen(provider_rkv->wb));
205 buffer_putc(rkv->wb, '.');
206 } else
207 buffer_strcat(rkv->wb, "of unknown Provider.");
208 }
209
210 if(xml->used > 1) {
211 const char *event_path[] = {
212 "EventData",
213 NULL
214 };
215 bool added_event_data = buffer_extract_and_print_xml(
216 rkv->wb,
217 xml->data, xml->used - 1,
218 "\n\nRelated event data:\n",
219 event_path);
220
221 const char *user_path[] = {
222 "UserData",
223 NULL
224 };
225 bool added_user_data = buffer_extract_and_print_xml(
226 rkv->wb,
227 xml->data, xml->used - 1,
228 "\n\nRelated user data:\n",
229 user_path);
230
231 if(!added_event_data && !added_user_data)
232 buffer_strcat(rkv->wb, " Without any related data.");
233 }
234
235 buffer_json_add_array_item_string(json_array, buffer_tostring(rkv->wb));
236 }
237 else
238 buffer_json_add_array_item_string(json_array, d->log->ops.event.data);
239 }
240
241 static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
242 // the order of the fields here, controls the order of the fields at the table presented
243
244 FACETS *facets = lqs->facets;
245 LOGS_QUERY_REQUEST *rq = &lqs->rq;
246
247 facets_register_row_severity(facets, wevt_levelid_to_facet_severity, NULL);
248
249 facets_register_key_name(
250 facets, WEVT_FIELD_COMPUTER,
251 rq->default_facet | FACET_KEY_OPTION_VISIBLE);
252
253 facets_register_key_name(
254 facets, WEVT_FIELD_CHANNEL,
255 rq->default_facet | FACET_KEY_OPTION_FTS);
256
257 facets_register_key_name(
258 facets, WEVT_FIELD_PROVIDER,
259 rq->default_facet | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
260
261 facets_register_key_name(
262 facets, WEVT_FIELD_ACCOUNT,
263 rq->default_facet | FACET_KEY_OPTION_FTS);
264
265 facets_register_key_name(
266 facets, WEVT_FIELD_DOMAIN,
267 rq->default_facet | FACET_KEY_OPTION_FTS);
268
269 facets_register_key_name(
270 facets, WEVT_FIELD_SID,
271 rq->default_facet | FACET_KEY_OPTION_FTS);
272
273 facets_register_key_name(
274 facets, WEVT_FIELD_EVENTID,
275 rq->default_facet |
276 FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
277
278 facets_register_key_name(
279 facets, WEVT_FIELD_EVENTS_API,
280 rq->default_facet |
281 FACET_KEY_OPTION_FTS);
282
283 facets_register_key_name(
284 facets, WEVT_FIELD_LEVEL,
285 rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_EXPANDED_FILTER);
286
287 facets_register_key_name(
288 facets, WEVT_FIELD_LEVEL "ID",
289 FACET_KEY_OPTION_NONE);
290
291 facets_register_key_name(
292 facets, WEVT_FIELD_PROCESSID,
293 FACET_KEY_OPTION_FTS);
294
295 facets_register_key_name(
296 facets, WEVT_FIELD_THREADID,
297 FACET_KEY_OPTION_FTS);
298
299 facets_register_key_name(
300 facets, WEVT_FIELD_TASK,
301 rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_VISIBLE);
302
303 facets_register_key_name(
304 facets, WEVT_FIELD_TASK "ID",
305 FACET_KEY_OPTION_NONE);
306
307 facets_register_key_name(
308 facets, WEVT_FIELD_OPCODE,
309 rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_VISIBLE);
310
311 facets_register_key_name(
312 facets, WEVT_FIELD_OPCODE "ID",
313 FACET_KEY_OPTION_NONE);
314
315 facets_register_key_name(
316 facets, WEVT_FIELD_KEYWORDS,
317 rq->default_facet | FACET_KEY_OPTION_FTS);
318
319 facets_register_key_name(
320 facets, WEVT_FIELD_KEYWORDS "ID",
321 FACET_KEY_OPTION_NONE);
322
323 facets_register_dynamic_key_name(
324 facets,
325 WEVT_FIELD_MESSAGE,
326 FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE,
327 wevt_lazy_load_message,
328 NULL);
329
330 facets_register_dynamic_key_name(
331 facets,
332 WEVT_FIELD_XML,
333 FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_PRETTY_XML,
334 wevt_lazy_load_xml,
335 NULL);
336
337 if(query_has_fts(lqs)) {
338 facets_register_key_name(
339 facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN,
340 FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
341
342 facets_register_key_name(
343 facets, WEVT_FIELD_EVENT_XML_HIDDEN,
344 FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
345
346 facets_register_key_name(
347 facets, WEVT_FIELD_EVENT_DATA_HIDDEN,
348 FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
349 }
350
351 #ifdef NETDATA_INTERNAL_CHECKS
352 facets_register_key_name(
353 facets, "z_level_source",
354 rq->default_facet);
355
356 facets_register_key_name(
357 facets, "z_keywords_source",
358 rq->default_facet);
359
360 facets_register_key_name(
361 facets, "z_opcode_source",
362 rq->default_facet);
363
364 facets_register_key_name(
365 facets, "z_task_source",
366 rq->default_facet);
367 #endif
368 }
369
370 #ifdef NETDATA_INTERNAL_CHECKS
371 static const char *source_to_str(TXT_UTF8 *txt) {
372 switch(txt->src) {
373 default:
374 case TXT_SOURCE_UNKNOWN:
375 return "unknown";
376
377 case TXT_SOURCE_EVENT_LOG:
378 return "event-log";
379
380 case TXT_SOURCE_PROVIDER:
381 return "provider";
382
383 case TXT_SOURCE_FIELD_CACHE:
384 return "fields-cache";
385
386 case TXT_SOURCE_HARDCODED:
387 return "hardcoded";
388 }
389 }
390 #endif
391
392 static const char *events_api_to_str(WEVT_PROVIDER_PLATFORM platform) {
393 switch(platform) {
394 case WEVT_PLATFORM_WEL:
395 return "Windows Event Log";
396
397 case WEVT_PLATFORM_ETW:
398 return "Event Tracing for Windows";
399
400 case WEVT_PLATFORM_TL:
401 return "TraceLogging";
402
403 default:
404 return "Unknown";
405 }
406 }
407
408 static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUERY_SOURCE *src, usec_t *msg_ut __maybe_unused, WEVT_EVENT *ev) {
409 static __thread char uuid_str[UUID_STR_LEN];
410
411 size_t len, bytes = log->ops.raw.system.used + log->ops.raw.user.used;
412
413 if(!UUIDiszero(ev->provider)) {
414 uuid_unparse_lower(ev->provider.uuid, uuid_str);
415 facets_add_key_value_length(
416 facets, WEVT_FIELD_PROVIDER_GUID, sizeof(WEVT_FIELD_PROVIDER_GUID) - 1,
417 uuid_str, sizeof(uuid_str) - 1);
418 }
419
420 if(!UUIDiszero(ev->activity_id)) {
421 uuid_unparse_lower(ev->activity_id.uuid, uuid_str);
422 facets_add_key_value_length(
423 facets, WEVT_FIELD_ACTIVITY_ID, sizeof(WEVT_FIELD_ACTIVITY_ID) - 1,
424 uuid_str, sizeof(uuid_str) - 1);
425 }
426
427 if(!UUIDiszero(ev->related_activity_id)) {
428 uuid_unparse_lower(ev->related_activity_id.uuid, uuid_str);
429 facets_add_key_value_length(
430 facets, WEVT_FIELD_RELATED_ACTIVITY_ID, sizeof(WEVT_FIELD_RELATED_ACTIVITY_ID) - 1,
431 uuid_str, sizeof(uuid_str) - 1);
432 }
433
434 if(ev->qualifiers) {
435 static __thread char qualifiers[UINT64_HEX_MAX_LENGTH];
436 len = print_uint64_hex(qualifiers, ev->qualifiers);
437 bytes += len;
438 facets_add_key_value_length(
439 facets, WEVT_FIELD_QUALIFIERS, sizeof(WEVT_FIELD_QUALIFIERS) - 1,
440 qualifiers, len);
441 }
442
443 {
444 static __thread char event_record_id_str[UINT64_MAX_LENGTH];
445 len = print_uint64(event_record_id_str, ev->id);
446 bytes += len;
447 facets_add_key_value_length(
448 facets, WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
449 event_record_id_str, len);
450 }
451
452 if(ev->version) {
453 static __thread char version[UINT64_MAX_LENGTH];
454 len = print_uint64(version, ev->version);
455 bytes += len;
456 facets_add_key_value_length(
457 facets, WEVT_FIELD_VERSION, sizeof(WEVT_FIELD_VERSION) - 1,
458 version, len);
459 }
460
461 if(log->ops.provider.used > 1) {
462 bytes += log->ops.provider.used * 2; // unicode is double
463 facets_add_key_value_length(
464 facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
465 log->ops.provider.data, log->ops.provider.used - 1);
466 }
467
468 if(log->ops.channel.used > 1) {
469 bytes += log->ops.channel.used * 2;
470 facets_add_key_value_length(
471 facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
472 log->ops.channel.data, log->ops.channel.used - 1);
473 }
474 else {
475 bytes += src->fullname_len * 2;
476 facets_add_key_value_length(
477 facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
478 src->fullname, src->fullname_len);
479 }
480
481 if(log->ops.level.used > 1) {
482 bytes += log->ops.level.used * 2;
483 facets_add_key_value_length(
484 facets, WEVT_FIELD_LEVEL, sizeof(WEVT_FIELD_LEVEL) - 1,
485 log->ops.level.data, log->ops.level.used - 1);
486 }
487
488 if(log->ops.computer.used > 1) {
489 bytes += log->ops.computer.used * 2;
490 facets_add_key_value_length(
491 facets, WEVT_FIELD_COMPUTER, sizeof(WEVT_FIELD_COMPUTER) - 1,
492 log->ops.computer.data, log->ops.computer.used - 1);
493 }
494
495 if(log->ops.opcode.used > 1) {
496 bytes += log->ops.opcode.used * 2;
497 facets_add_key_value_length(
498 facets, WEVT_FIELD_OPCODE, sizeof(WEVT_FIELD_OPCODE) - 1,
499 log->ops.opcode.data, log->ops.opcode.used - 1);
500 }
501
502 if(log->ops.keywords.used > 1) {
503 bytes += log->ops.keywords.used * 2;
504 facets_add_key_value_length(
505 facets, WEVT_FIELD_KEYWORDS, sizeof(WEVT_FIELD_KEYWORDS) - 1,
506 log->ops.keywords.data, log->ops.keywords.used - 1);
507 }
508
509 if(log->ops.task.used > 1) {
510 bytes += log->ops.task.used * 2;
511 facets_add_key_value_length(
512 facets, WEVT_FIELD_TASK, sizeof(WEVT_FIELD_TASK) - 1,
513 log->ops.task.data, log->ops.task.used - 1);
514 }
515
516 if(log->ops.account.used > 1) {
517 bytes += log->ops.account.used * 2;
518 facets_add_key_value_length(
519 facets,
520 WEVT_FIELD_ACCOUNT, sizeof(WEVT_FIELD_ACCOUNT) - 1,
521 log->ops.account.data, log->ops.account.used - 1);
522 }
523
524 if(log->ops.domain.used > 1) {
525 bytes += log->ops.domain.used * 2;
526 facets_add_key_value_length(
527 facets,
528 WEVT_FIELD_DOMAIN, sizeof(WEVT_FIELD_DOMAIN) - 1,
529 log->ops.domain.data, log->ops.domain.used - 1);
530 }
531
532 if(log->ops.sid.used > 1) {
533 bytes += log->ops.sid.used * 2;
534 facets_add_key_value_length(
535 facets,
536 WEVT_FIELD_SID, sizeof(WEVT_FIELD_SID) - 1,
537 log->ops.sid.data, log->ops.sid.used - 1);
538 }
539
540 {
541 static __thread char event_id_str[UINT64_MAX_LENGTH];
542 len = print_uint64(event_id_str, ev->event_id);
543 bytes += len;
544 facets_add_key_value_length(
545 facets, WEVT_FIELD_EVENTID, sizeof(WEVT_FIELD_EVENTID) - 1,
546 event_id_str, len);
547 }
548
549 {
550 const char *s = events_api_to_str(ev->platform);
551 facets_add_key_value_length(
552 facets, WEVT_FIELD_EVENTS_API, sizeof(WEVT_FIELD_EVENTS_API) - 1, s, strlen(s));
553 }
554
555 if(ev->process_id) {
556 static __thread char process_id_str[UINT64_MAX_LENGTH];
557 len = print_uint64(process_id_str, ev->process_id);
558 bytes += len;
559 facets_add_key_value_length(
560 facets, WEVT_FIELD_PROCESSID, sizeof(WEVT_FIELD_PROCESSID) - 1,
561 process_id_str, len);
562 }
563
564 if(ev->thread_id) {
565 static __thread char thread_id_str[UINT64_MAX_LENGTH];
566 len = print_uint64(thread_id_str, ev->thread_id);
567 bytes += len;
568 facets_add_key_value_length(
569 facets, WEVT_FIELD_THREADID, sizeof(WEVT_FIELD_THREADID) - 1,
570 thread_id_str, len);
571 }
572
573 {
574 static __thread char str[UINT64_MAX_LENGTH];
575 len = print_uint64(str, ev->level);
576 bytes += len;
577 facets_add_key_value_length(
578 facets, WEVT_FIELD_LEVEL "ID", sizeof(WEVT_FIELD_LEVEL) + 2 - 1, str, len);
579 }
580
581 {
582 static __thread char str[UINT64_HEX_MAX_LENGTH];
583 len = print_uint64_hex_full(str, ev->keywords);
584 bytes += len;
585 facets_add_key_value_length(
586 facets, WEVT_FIELD_KEYWORDS "ID", sizeof(WEVT_FIELD_KEYWORDS) + 2 - 1, str, len);
587 }
588
589 {
590 static __thread char str[UINT64_MAX_LENGTH];
591 len = print_uint64(str, ev->opcode);
592 bytes += len;
593 facets_add_key_value_length(
594 facets, WEVT_FIELD_OPCODE "ID", sizeof(WEVT_FIELD_OPCODE) + 2 - 1, str, len);
595 }
596
597 {
598 static __thread char str[UINT64_MAX_LENGTH];
599 len = print_uint64(str, ev->task);
600 bytes += len;
601 facets_add_key_value_length(
602 facets, WEVT_FIELD_TASK "ID", sizeof(WEVT_FIELD_TASK) + 2 - 1, str, len);
603 }
604
605 if(log->type & WEVT_QUERY_EVENT_DATA) {
606 // the query has full text-search
607 if(log->ops.event.used > 1) {
608 bytes += log->ops.event.used;
609 facets_add_key_value_length(
610 facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN, sizeof(WEVT_FIELD_EVENT_MESSAGE_HIDDEN) - 1,
611 log->ops.event.data, log->ops.event.used - 1);
612 }
613
614 if(log->ops.xml.used > 1) {
615 bytes += log->ops.xml.used;
616 facets_add_key_value_length(
617 facets, WEVT_FIELD_EVENT_XML_HIDDEN, sizeof(WEVT_FIELD_EVENT_XML_HIDDEN) - 1,
618 log->ops.xml.data, log->ops.xml.used - 1);
619 }
620
621 if(log->ops.event_data->len) {
622 bytes += log->ops.event_data->len;
623 facets_add_key_value_length(
624 facets, WEVT_FIELD_EVENT_DATA_HIDDEN, sizeof(WEVT_FIELD_EVENT_DATA_HIDDEN) - 1,
625 buffer_tostring(log->ops.event_data), buffer_strlen(log->ops.event_data));
626 }
627 }
628
629 wevt_facets_register_bin_data(log, facets, ev);
630
631 #ifdef NETDATA_INTERNAL_CHECKS
632 facets_add_key_value(facets, "z_level_source", source_to_str(&log->ops.level));
633 facets_add_key_value(facets, "z_keywords_source", source_to_str(&log->ops.keywords));
634 facets_add_key_value(facets, "z_opcode_source", source_to_str(&log->ops.opcode));
635 facets_add_key_value(facets, "z_task_source", source_to_str(&log->ops.task));
636 #endif
637
638 return bytes;
639 }
640
641 static void send_progress_update(LOGS_QUERY_STATUS *lqs, size_t current_row_counter, bool flush_current_file) {
642 usec_t now_ut = now_monotonic_usec();
643
644 if(current_row_counter > lqs->c.progress.entries.current_query_total) {
645 lqs->c.progress.entries.total += current_row_counter - lqs->c.progress.entries.current_query_total;
646 lqs->c.progress.entries.current_query_total = current_row_counter;
647 }
648
649 if(flush_current_file) {
650 lqs->c.progress.entries.total += current_row_counter;
651 lqs->c.progress.entries.total -= lqs->c.progress.entries.current_query_total;
652 lqs->c.progress.entries.completed += current_row_counter;
653 lqs->c.progress.entries.current_query_total = 0;
654 }
655
656 size_t completed = lqs->c.progress.entries.completed + current_row_counter;
657 if(completed > lqs->c.progress.entries.total)
658 lqs->c.progress.entries.total = completed;
659
660 usec_t progress_duration_ut = now_ut - lqs->c.progress.last_ut;
661 if(progress_duration_ut >= WINDOWS_EVENTS_PROGRESS_EVERY_UT) {
662 lqs->c.progress.last_ut = now_ut;
663
664 netdata_mutex_lock(&stdout_mutex);
665 pluginsd_function_progress_to_stdout(lqs->rq.transaction, completed, lqs->c.progress.entries.total);
666 netdata_mutex_unlock(&stdout_mutex);
667 }
668 }
669
670 static WEVT_QUERY_STATUS wevt_query_backward(
671 WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
672 LOGS_QUERY_SOURCE *src,
673 LOGS_QUERY_STATUS *lqs)
674 {
675 usec_t start_ut = lqs->query.start_ut;
676 usec_t stop_ut = lqs->query.stop_ut;
677 bool stop_when_full = lqs->query.stop_when_full;
678
679 // lqs->c.query_file.start_ut = start_ut;
680 // lqs->c.query_file.stop_ut = stop_ut;
681
682 if(!wevt_query(log, channel2unicode(src->fullname), lqs->c.query, EvtQueryReverseDirection))
683 return WEVT_FAILED_TO_SEEK;
684
685 size_t errors_no_timestamp = 0;
686 usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
687 usec_t first_msg_ut = 0; // the first message we got from the db
688 size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
689 size_t bytes = 0, last_bytes = 0;
690
691 usec_t last_usec_from = 0;
692 usec_t last_usec_to = 0;
693
694 WEVT_QUERY_STATUS status = WEVT_OK;
695
696 facets_rows_begin(facets);
697 WEVT_EVENT e;
698 while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
699 usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
700
701 if(unlikely(!msg_ut)) {
702 errors_no_timestamp++;
703 continue;
704 }
705
706 if (unlikely(msg_ut > start_ut))
707 continue;
708
709 if (unlikely(msg_ut < stop_ut))
710 break;
711
712 if(unlikely(msg_ut > latest_msg_ut))
713 latest_msg_ut = msg_ut;
714
715 if(unlikely(!first_msg_ut)) {
716 first_msg_ut = msg_ut;
717 // lqs->c.query_file.first_msg_ut = msg_ut;
718 }
719
720 // sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
721 // FACETS_ANCHOR_DIRECTION_BACKWARD,
722 // facets_row_candidate_to_keep(facets, msg_ut));
723 //
724 // if(sample == SAMPLING_FULL) {
725 bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
726
727 // make sure each line gets a unique timestamp
728 if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
729 msg_ut = --last_usec_from;
730 else
731 last_usec_from = last_usec_to = msg_ut;
732
733 if(facets_row_finished(facets, msg_ut))
734 rows_useful++;
735
736 row_counter++;
737 if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
738 stop_when_full &&
739 facets_rows(facets) >= lqs->rq.entries)) {
740 // stop the data only query
741 usec_t oldest = facets_row_oldest_ut(facets);
742 if(oldest && msg_ut < (oldest - lqs->anchor.delta_ut))
743 break;
744 }
745
746 if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
747 status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
748
749 if(status == WEVT_OK) {
750 lqs->c.rows_read += row_counter - last_row_counter;
751 last_row_counter = row_counter;
752
753 lqs->c.bytes_read += bytes - last_bytes;
754 last_bytes = bytes;
755
756 send_progress_update(lqs, row_counter, false);
757 }
758 }
759 // }
760 // else if(sample == SAMPLING_SKIP_FIELDS)
761 // facets_row_finished_unsampled(facets, msg_ut);
762 // else {
763 // sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_BACKWARD);
764 // break;
765 // }
766 }
767
768 send_progress_update(lqs, row_counter, true);
769 lqs->c.rows_read += row_counter - last_row_counter;
770 lqs->c.bytes_read += bytes - last_bytes;
771 lqs->c.rows_useful += rows_useful;
772
773 if(errors_no_timestamp)
774 netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
775
776 if(latest_msg_ut > lqs->last_modified)
777 lqs->last_modified = latest_msg_ut;
778
779 wevt_query_done(log);
780
781 return status;
782 }
783
784 static WEVT_QUERY_STATUS wevt_query_forward(
785 WEVT_LOG *log, BUFFER *wb __maybe_unused, FACETS *facets,
786 LOGS_QUERY_SOURCE *src,
787 LOGS_QUERY_STATUS *lqs)
788 {
789 usec_t start_ut = lqs->query.start_ut;
790 usec_t stop_ut = lqs->query.stop_ut;
791 bool stop_when_full = lqs->query.stop_when_full;
792
793 // lqs->c.query_file.start_ut = start_ut;
794 // lqs->c.query_file.stop_ut = stop_ut;
795
796 if(!wevt_query(log, channel2unicode(src->fullname), lqs->c.query, EvtQueryForwardDirection))
797 return WEVT_FAILED_TO_SEEK;
798
799 size_t errors_no_timestamp = 0;
800 usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
801 usec_t first_msg_ut = 0; // the first message we got from the db
802 size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
803 size_t bytes = 0, last_bytes = 0;
804
805 usec_t last_usec_from = 0;
806 usec_t last_usec_to = 0;
807
808 WEVT_QUERY_STATUS status = WEVT_OK;
809
810 facets_rows_begin(facets);
811 WEVT_EVENT e;
812 while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
813 usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
814
815 if(unlikely(!msg_ut)) {
816 errors_no_timestamp++;
817 continue;
818 }
819
820 if (unlikely(msg_ut < start_ut))
821 continue;
822
823 if (unlikely(msg_ut > stop_ut))
824 break;
825
826 if(likely(msg_ut > latest_msg_ut))
827 latest_msg_ut = msg_ut;
828
829 if(unlikely(!first_msg_ut)) {
830 first_msg_ut = msg_ut;
831 // lqs->c.query_file.first_msg_ut = msg_ut;
832 }
833
834 // sampling_t sample = is_row_in_sample(log, lqs, src, msg_ut,
835 // FACETS_ANCHOR_DIRECTION_FORWARD,
836 // facets_row_candidate_to_keep(facets, msg_ut));
837 //
838 // if(sample == SAMPLING_FULL) {
839 bytes += wevt_process_event(log, facets, src, &msg_ut, &e);
840
841 // make sure each line gets a unique timestamp
842 if(unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
843 msg_ut = ++last_usec_to;
844 else
845 last_usec_from = last_usec_to = msg_ut;
846
847 if(facets_row_finished(facets, msg_ut))
848 rows_useful++;
849
850 row_counter++;
851 if(unlikely((row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 &&
852 stop_when_full &&
853 facets_rows(facets) >= lqs->rq.entries)) {
854 // stop the data only query
855 usec_t newest = facets_row_newest_ut(facets);
856 if(newest && msg_ut > (newest + lqs->anchor.delta_ut))
857 break;
858 }
859
860 if(unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
861 status = check_stop(lqs->cancelled, lqs->stop_monotonic_ut);
862
863 if(status == WEVT_OK) {
864 lqs->c.rows_read += row_counter - last_row_counter;
865 last_row_counter = row_counter;
866
867 lqs->c.bytes_read += bytes - last_bytes;
868 last_bytes = bytes;
869
870 send_progress_update(lqs, row_counter, false);
871 }
872 }
873 // }
874 // else if(sample == SAMPLING_SKIP_FIELDS)
875 // facets_row_finished_unsampled(facets, msg_ut);
876 // else {
877 // sampling_update_running_query_file_estimates(facets, log, lqs, src, msg_ut, FACETS_ANCHOR_DIRECTION_FORWARD);
878 // break;
879 // }
880 }
881
882 send_progress_update(lqs, row_counter, true);
883 lqs->c.rows_read += row_counter - last_row_counter;
884 lqs->c.bytes_read += bytes - last_bytes;
885 lqs->c.rows_useful += rows_useful;
886
887 if(errors_no_timestamp)
888 netdata_log_error("WINDOWS-EVENTS: %zu events did not have timestamps", errors_no_timestamp);
889
890 if(latest_msg_ut > lqs->last_modified)
891 lqs->last_modified = latest_msg_ut;
892
893 wevt_query_done(log);
894
895 return status;
896 }
897
898 static WEVT_QUERY_STATUS wevt_query_one_channel(
899 WEVT_LOG *log,
900 BUFFER *wb, FACETS *facets,
901 LOGS_QUERY_SOURCE *src,
902 LOGS_QUERY_STATUS *lqs) {
903
904 errno_clear();
905
906 WEVT_QUERY_STATUS status;
907 if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD)
908 status = wevt_query_forward(log, wb, facets, src, lqs);
909 else
910 status = wevt_query_backward(log, wb, facets, src, lqs);
911
912 return status;
913 }
914
915 static bool source_is_mine(LOGS_QUERY_SOURCE *src, LOGS_QUERY_STATUS *lqs) {
916 if(
917 // no source is requested
918 (lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) ||
919
920 // matches our internal source types
921 (src->source_type & lqs->rq.source_type) ||
922
923 // matches the source name
924 (lqs->rq.sources && src->source && simple_pattern_matches(lqs->rq.sources, string2str(src->source))) ||
925
926 // matches the provider (providers start with a special prefix to avoid mix and match)
927 (lqs->rq.sources && src->provider && simple_pattern_matches(lqs->rq.sources, string2str(src->provider)))
928
929 ) {
930
931 if(!src->msg_last_ut)
932 // the file is not scanned yet, or the timestamps have not been updated,
933 // so we don't know if it can contribute or not - let's add it.
934 return true;
935
936 usec_t anchor_delta = ANCHOR_DELTA_UT;
937 usec_t first_ut = src->msg_first_ut - anchor_delta;
938 usec_t last_ut = src->msg_last_ut + anchor_delta;
939
940 if(last_ut >= lqs->rq.after_ut && first_ut <= lqs->rq.before_ut)
941 return true;
942 }
943
944 return false;
945 }
946
947 static int wevt_master_query(BUFFER *wb __maybe_unused, LOGS_QUERY_STATUS *lqs __maybe_unused) {
948 // make sure the sources list is updated
949 wevt_sources_scan();
950
951 lqs->c.query = wevt_generate_query_no_xpath(lqs, wb);
952 if(!lqs->c.query)
953 return rrd_call_function_error(wb, "failed to generate query", HTTP_RESP_INTERNAL_SERVER_ERROR);
954
955 FACETS *facets = lqs->facets;
956
957 WEVT_QUERY_STATUS status = WEVT_NO_CHANNEL_MATCHED;
958
959 lqs->c.files_matched = 0;
960 lqs->c.file_working = 0;
961 lqs->c.rows_useful = 0;
962 lqs->c.rows_read = 0;
963 lqs->c.bytes_read = 0;
964
965 size_t files_used = 0;
966 size_t files_max = dictionary_entries(wevt_sources);
967 const DICTIONARY_ITEM *file_items[files_max];
968
969 // count the files
970 bool files_are_newer = false;
971 LOGS_QUERY_SOURCE *src;
972 dfe_start_read(wevt_sources, src) {
973 if(!source_is_mine(src, lqs))
974 continue;
975
976 file_items[files_used++] = dictionary_acquired_item_dup(wevt_sources, src_dfe.item);
977
978 if(src->msg_last_ut > lqs->rq.if_modified_since)
979 files_are_newer = true;
980
981 lqs->c.progress.entries.total += src->entries;
982 }
983 dfe_done(jf);
984
985 lqs->c.files_matched = files_used;
986
987 if(lqs->rq.if_modified_since && !files_are_newer) {
988 // release the files
989 for(size_t f = 0; f < files_used ;f++)
990 dictionary_acquired_item_release(wevt_sources, file_items[f]);
991
992 return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
993 }
994
995 // sort the files, so that they are optimal for facets
996 if(files_used >= 2) {
997 if (lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
998 qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
999 wevt_sources_dict_items_backward_compar);
1000 else
1001 qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
1002 wevt_sources_dict_items_forward_compar);
1003 }
1004
1005 bool partial = false;
1006 usec_t query_started_ut = now_monotonic_usec();
1007 usec_t started_ut = query_started_ut;
1008 usec_t ended_ut = started_ut;
1009 usec_t duration_ut, max_duration_ut = 0;
1010
1011 WEVT_LOG *log = wevt_openlog6(query_has_fts(lqs) ? WEVT_QUERY_FTS : WEVT_QUERY_NORMAL);
1012 if(!log) {
1013 // release the files
1014 for(size_t f = 0; f < files_used ;f++)
1015 dictionary_acquired_item_release(wevt_sources, file_items[f]);
1016
1017 netdata_log_error("WINDOWS EVENTS: cannot open windows event log");
1018 return rrd_call_function_error(wb, "cannot open windows events log", HTTP_RESP_INTERNAL_SERVER_ERROR);
1019 }
1020
1021 // sampling_query_init(lqs, facets);
1022
1023 buffer_json_member_add_array(wb, "_channels");
1024 for(size_t f = 0; f < files_used ;f++) {
1025 const char *fullname = dictionary_acquired_item_name(file_items[f]);
1026 src = dictionary_acquired_item_value(file_items[f]);
1027
1028 if(!source_is_mine(src, lqs))
1029 continue;
1030
1031 started_ut = ended_ut;
1032
1033 // do not even try to do the query if we expect it to pass the timeout
1034 if(ended_ut + max_duration_ut * 3 >= *lqs->stop_monotonic_ut) {
1035 partial = true;
1036 status = WEVT_TIMED_OUT;
1037 break;
1038 }
1039
1040 lqs->c.file_working++;
1041
1042 size_t rows_useful = lqs->c.rows_useful;
1043 size_t rows_read = lqs->c.rows_read;
1044 size_t bytes_read = lqs->c.bytes_read;
1045 size_t matches_setup_ut = lqs->c.matches_setup_ut;
1046
1047 // sampling_file_init(lqs, src);
1048
1049 lqs->c.progress.entries.current_query_total = src->entries;
1050 WEVT_QUERY_STATUS tmp_status = wevt_query_one_channel(log, wb, facets, src, lqs);
1051
1052 rows_useful = lqs->c.rows_useful - rows_useful;
1053 rows_read = lqs->c.rows_read - rows_read;
1054 bytes_read = lqs->c.bytes_read - bytes_read;
1055 matches_setup_ut = lqs->c.matches_setup_ut - matches_setup_ut;
1056
1057 ended_ut = now_monotonic_usec();
1058 duration_ut = ended_ut - started_ut;
1059
1060 if(duration_ut > max_duration_ut)
1061 max_duration_ut = duration_ut;
1062
1063 buffer_json_add_array_item_object(wb); // channel source
1064 {
1065 // information about the file
1066 buffer_json_member_add_string(wb, "_name", fullname);
1067 buffer_json_member_add_uint64(wb, "_source_type", src->source_type);
1068 buffer_json_member_add_string(wb, "_source", string2str(src->source));
1069 buffer_json_member_add_uint64(wb, "_msg_first_ut", src->msg_first_ut);
1070 buffer_json_member_add_uint64(wb, "_msg_last_ut", src->msg_last_ut);
1071
1072 // information about the current use of the file
1073 buffer_json_member_add_uint64(wb, "duration_ut", ended_ut - started_ut);
1074 buffer_json_member_add_uint64(wb, "rows_read", rows_read);
1075 buffer_json_member_add_uint64(wb, "rows_useful", rows_useful);
1076 buffer_json_member_add_double(wb, "rows_per_second", (double) rows_read / (double) duration_ut * (double) USEC_PER_SEC);
1077 buffer_json_member_add_uint64(wb, "bytes_read", bytes_read);
1078 buffer_json_member_add_double(wb, "bytes_per_second", (double) bytes_read / (double) duration_ut * (double) USEC_PER_SEC);
1079 buffer_json_member_add_uint64(wb, "duration_matches_ut", matches_setup_ut);
1080
1081 // if(lqs->rq.sampling) {
1082 // buffer_json_member_add_object(wb, "_sampling");
1083 // {
1084 // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples_per_file.sampled);
1085 // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples_per_file.unsampled);
1086 // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples_per_file.estimated);
1087 // }
1088 // buffer_json_object_close(wb); // _sampling
1089 // }
1090 }
1091 buffer_json_object_close(wb); // channel source
1092
1093 bool stop = false;
1094 switch(tmp_status) {
1095 case WEVT_OK:
1096 case WEVT_NO_CHANNEL_MATCHED:
1097 status = (status == WEVT_OK) ? WEVT_OK : tmp_status;
1098 break;
1099
1100 case WEVT_FAILED_TO_OPEN:
1101 case WEVT_FAILED_TO_SEEK:
1102 partial = true;
1103 if(status == WEVT_NO_CHANNEL_MATCHED)
1104 status = tmp_status;
1105 break;
1106
1107 case WEVT_CANCELLED:
1108 case WEVT_TIMED_OUT:
1109 partial = true;
1110 stop = true;
1111 status = tmp_status;
1112 break;
1113
1114 case WEVT_NOT_MODIFIED:
1115 internal_fatal(true, "this should never be returned here");
1116 break;
1117 }
1118
1119 if(stop)
1120 break;
1121 }
1122 buffer_json_array_close(wb); // _channels
1123
1124 // release the files
1125 for(size_t f = 0; f < files_used ;f++)
1126 dictionary_acquired_item_release(wevt_sources, file_items[f]);
1127
1128 switch (status) {
1129 case WEVT_OK:
1130 if(lqs->rq.if_modified_since && !lqs->c.rows_useful)
1131 return rrd_call_function_error(wb, "no useful logs, not modified", HTTP_RESP_NOT_MODIFIED);
1132 break;
1133
1134 case WEVT_TIMED_OUT:
1135 case WEVT_NO_CHANNEL_MATCHED:
1136 break;
1137
1138 case WEVT_CANCELLED:
1139 return rrd_call_function_error(wb, "client closed connection", HTTP_RESP_CLIENT_CLOSED_REQUEST);
1140
1141 case WEVT_NOT_MODIFIED:
1142 return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
1143
1144 case WEVT_FAILED_TO_OPEN:
1145 return rrd_call_function_error(wb, "failed to open event log", HTTP_RESP_INTERNAL_SERVER_ERROR);
1146
1147 case WEVT_FAILED_TO_SEEK:
1148 return rrd_call_function_error(wb, "failed to execute event log query", HTTP_RESP_INTERNAL_SERVER_ERROR);
1149
1150 default:
1151 return rrd_call_function_error(wb, "unknown status", HTTP_RESP_INTERNAL_SERVER_ERROR);
1152 }
1153
1154 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
1155 buffer_json_member_add_boolean(wb, "partial", partial);
1156 buffer_json_member_add_string(wb, "type", "table");
1157
1158 // build a message for the query
1159 if(!lqs->rq.data_only) {
1160 CLEAN_BUFFER *msg = buffer_create(0, NULL);
1161 CLEAN_BUFFER *msg_description = buffer_create(0, NULL);
1162 ND_LOG_FIELD_PRIORITY msg_priority = NDLP_INFO;
1163
1164 // if(!journal_files_completed_once()) {
1165 // buffer_strcat(msg, "Journals are still being scanned. ");
1166 // buffer_strcat(msg_description
1167 // , "LIBRARY SCAN: The journal files are still being scanned, you are probably viewing incomplete data. ");
1168 // msg_priority = NDLP_WARNING;
1169 // }
1170
1171 if(partial) {
1172 buffer_strcat(msg, "Query timed-out, incomplete data. ");
1173 buffer_strcat(msg_description
1174 , "QUERY TIMEOUT: The query timed out and may not include all the data of the selected window. ");
1175 msg_priority = NDLP_WARNING;
1176 }
1177
1178 // if(lqs->c.samples.estimated || lqs->c.samples.unsampled) {
1179 // double percent = (double) (lqs->c.samples.sampled * 100.0 /
1180 // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1181 // buffer_sprintf(msg, "%.2f%% real data", percent);
1182 // buffer_sprintf(msg_description, "ACTUAL DATA: The filters counters reflect %0.2f%% of the data. ", percent);
1183 // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1184 // }
1185 //
1186 // if(lqs->c.samples.unsampled) {
1187 // double percent = (double) (lqs->c.samples.unsampled * 100.0 /
1188 // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1189 // buffer_sprintf(msg, ", %.2f%% unsampled", percent);
1190 // buffer_sprintf(msg_description
1191 // , "UNSAMPLED DATA: %0.2f%% of the events exist and have been counted, but their values have not been evaluated, so they are not included in the filters counters. "
1192 // , percent);
1193 // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1194 // }
1195 //
1196 // if(lqs->c.samples.estimated) {
1197 // double percent = (double) (lqs->c.samples.estimated * 100.0 /
1198 // (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
1199 // buffer_sprintf(msg, ", %.2f%% estimated", percent);
1200 // buffer_sprintf(msg_description
1201 // , "ESTIMATED DATA: The query selected a large amount of data, so to avoid delaying too much, the presented data are estimated by %0.2f%%. "
1202 // , percent);
1203 // msg_priority = MIN(msg_priority, NDLP_NOTICE);
1204 // }
1205
1206 buffer_json_member_add_object(wb, "message");
1207 if(buffer_tostring(msg)) {
1208 buffer_json_member_add_string(wb, "title", buffer_tostring(msg));
1209 buffer_json_member_add_string(wb, "description", buffer_tostring(msg_description));
1210 buffer_json_member_add_string(wb, "status", nd_log_id2priority(msg_priority));
1211 }
1212 // else send an empty object if there is nothing to tell
1213 buffer_json_object_close(wb); // message
1214 }
1215
1216 if(!lqs->rq.data_only) {
1217 buffer_json_member_add_time_t(wb, "update_every", 1);
1218 buffer_json_member_add_string(wb, "help", WEVT_FUNCTION_DESCRIPTION);
1219 }
1220
1221 if(!lqs->rq.data_only || lqs->rq.tail)
1222 buffer_json_member_add_uint64(wb, "last_modified", lqs->last_modified);
1223
1224 facets_sort_and_reorder_keys(facets);
1225 facets_report(facets, wb, used_hashes_registry);
1226
1227 wb->expires = now_realtime_sec() + (lqs->rq.data_only ? 3600 : 0);
1228 buffer_json_member_add_time_t(wb, "expires", wb->expires);
1229
1230 // if(lqs->rq.sampling) {
1231 // buffer_json_member_add_object(wb, "_sampling");
1232 // {
1233 // buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples.sampled);
1234 // buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples.unsampled);
1235 // buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples.estimated);
1236 // }
1237 // buffer_json_object_close(wb); // _sampling
1238 // }
1239
1240 wevt_closelog6(log);
1241
1242 wb->content_type = CT_APPLICATION_JSON;
1243 wb->response_code = HTTP_RESP_OK;
1244 return wb->response_code;
1245 }
1246
1247 void function_windows_events(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
1248 BUFFER *payload, HTTP_ACCESS access __maybe_unused,
1249 const char *source __maybe_unused, void *data __maybe_unused) {
1250 bool have_slice = LQS_DEFAULT_SLICE_MODE;
1251
1252 LOGS_QUERY_STATUS tmp_fqs = {
1253 .facets = lqs_facets_create(
1254 LQS_DEFAULT_ITEMS_PER_QUERY,
1255 FACETS_OPTION_ALL_KEYS_FTS | FACETS_OPTION_HASH_IDS,
1256 WEVT_ALWAYS_VISIBLE_KEYS,
1257 WEVT_KEYS_INCLUDED_IN_FACETS,
1258 WEVT_KEYS_EXCLUDED_FROM_FACETS,
1259 have_slice),
1260
1261 .rq = LOGS_QUERY_REQUEST_DEFAULTS(transaction, have_slice, FACETS_ANCHOR_DIRECTION_BACKWARD),
1262
1263 .cancelled = cancelled,
1264 .stop_monotonic_ut = stop_monotonic_ut,
1265 };
1266 LOGS_QUERY_STATUS *lqs = &tmp_fqs;
1267
1268 CLEAN_BUFFER *wb = lqs_create_output_buffer();
1269
1270 // ------------------------------------------------------------------------
1271 // parse the parameters
1272
1273 if(lqs_request_parse_and_validate(lqs, wb, function, payload, have_slice, WEVT_FIELD_LEVEL)) {
1274 wevt_register_fields(lqs);
1275
1276 // ------------------------------------------------------------------------
1277 // add versions to the response
1278
1279 buffer_json_wevt_versions(wb);
1280
1281 // ------------------------------------------------------------------------
1282 // run the request
1283
1284 if (lqs->rq.info)
1285 lqs_info_response(wb, lqs->facets);
1286 else {
1287 wevt_master_query(wb, lqs);
1288 if (wb->response_code == HTTP_RESP_OK)
1289 buffer_json_finalize(wb);
1290 }
1291 }
1292
1293 netdata_mutex_lock(&stdout_mutex);
1294 pluginsd_function_result_to_stdout(transaction, wb);
1295 netdata_mutex_unlock(&stdout_mutex);
1296
1297 lqs_cleanup(lqs);
1298 }
1299
1300 int main(int argc __maybe_unused, char **argv __maybe_unused) {
1301 nd_thread_tag_set("wevt.plugin");
1302 nd_log_initialize_for_external_plugins("windows-events.plugin");
1303 netdata_threads_init_for_external_plugins(0);
1304
1305 // ------------------------------------------------------------------------
1306 // initialization
1307
1308 wevt_sources_init();
1309 provider_cache_init();
1310 cached_sid_username_init();
1311 field_cache_init();
1312
1313 if(!EnableWindowsPrivilege(SE_SECURITY_NAME))
1314 nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to enable %s privilege", SE_SECURITY_NAME);
1315
1316 if(!EnableWindowsPrivilege(SE_BACKUP_NAME))
1317 nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to enable %s privilege", SE_BACKUP_NAME);
1318
1319 if(!EnableWindowsPrivilege(SE_AUDIT_NAME))
1320 nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to enable %s privilege", SE_AUDIT_NAME);
1321
1322 // ------------------------------------------------------------------------
1323 // debug
1324
1325 if(argc >= 2 && strcmp(argv[argc - 1], "debug") == 0) {
1326 wevt_sources_scan();
1327
1328 struct {
1329 const char *func;
1330 } array[] = {
1331 { "windows-events after:-8640000 before:0 last:200 source:All" },
1332 //{ "windows-events after:-86400 before:0 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,HytMJ9kj82B,JM3OPW3kHn6,H106l8MXSSr,HREiMN.4Ahu,ClaDGnYSQE7,ApYltST_icg,PtkRm91M0En data_only:false slice:true source:All" },
1333 //{ "windows-events after:1726055370 before:1726056270 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6,ClaDGnYSQE7,H106l8MXSSr,HREiMN.4Ahu data_only:false source:All HytMJ9kj82B:BlC24d5JBBV,PtVoyIuX.MU,HMj1B38kHTv KeCITtVD5AD:PY1JtCeWwSe,O9kz5J37nNl,JZoJURadhDb" },
1334 // { "windows-events after:1725636012 before:1726240812 direction:backward last:200 facets:HdUoSYab5wV,Cq2r7mRUv4a,LAnVlsIQfeD,BnPLNbA5VWT,KeCITtVD5AD,HytMJ9kj82B,JM3OPW3kHn6,H106l8MXSSr,HREiMN.4Ahu,ClaDGnYSQE7,ApYltST_icg,PtkRm91M0En data_only:false source:All PtkRm91M0En:LDzHbP5libb" },
1335 //{ "windows-events after:1725650386 before:1725736786 anchor:1725652420809461 direction:forward last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 if_modified_since:1725736649011085 data_only:true delta:true tail:true source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1336 //{ "windows-events info after:1725650420 before:1725736820" },
1337 //{ "windows-events after:1725650420 before:1725736820 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1338 //{ "windows-events after:1725650430 before:1725736830 last:200 facets:HWNGeY7tg6c,LAnVlsIQfeD,BnPLNbA5VWT,Cq2r7mRUv4a,KeCITtVD5AD,I_Amz_APBm3,HytMJ9kj82B,LT.Xp9I9tiP,No4kPTQbS.g,LQ2LQzfE8EG,PtkRm91M0En,JM3OPW3kHn6 source:all Cq2r7mRUv4a:PPc9fUy.q6o No4kPTQbS.g:Dwo9PhK27v3 HytMJ9kj82B:KbbznGjt_9r LAnVlsIQfeD:OfU1t5cpjgG JM3OPW3kHn6:CS_0g5AEpy2" },
1339 { NULL },
1340 };
1341
1342 for(int i = 0; array[i].func ;i++) {
1343 bool cancelled = false;
1344 usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
1345 //char buf[] = "windows-events after:-86400 before:0 direction:backward last:200 data_only:false slice:true source:all";
1346 function_windows_events("123", (char *)array[i].func, &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
1347 }
1348 printf("\n\nAll done!\n\n");
1349 fflush(stdout);
1350 exit(1);
1351 }
1352
1353 // ------------------------------------------------------------------------
1354 // the event loop for functions
1355
1356 struct functions_evloop_globals *wg =
1357 functions_evloop_init(WINDOWS_EVENTS_WORKER_THREADS, "WEVT", &stdout_mutex, &plugin_should_exit, NULL);
1358
1359 functions_evloop_add_function(wg,
1360 WEVT_FUNCTION_NAME,
1361 function_windows_events,
1362 WINDOWS_EVENTS_DEFAULT_TIMEOUT,
1363 NULL);
1364
1365 // ------------------------------------------------------------------------
1366 // register functions to netdata
1367
1368 netdata_mutex_lock(&stdout_mutex);
1369
1370 fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" "HTTP_ACCESS_FORMAT" %d\n",
1371 WEVT_FUNCTION_NAME, WINDOWS_EVENTS_DEFAULT_TIMEOUT, WEVT_FUNCTION_DESCRIPTION,
1372 (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
1373 RRDFUNCTIONS_PRIORITY_DEFAULT);
1374
1375 fflush(stdout);
1376 netdata_mutex_unlock(&stdout_mutex);
1377
1378 // ------------------------------------------------------------------------
1379
1380 usec_t send_newline_ut = 0;
1381 usec_t since_last_scan_ut = WINDOWS_EVENTS_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
1382 usec_t since_last_providers_release_ut = 0;
1383 const bool tty = isatty(fileno(stdout)) == 1;
1384
1385 heartbeat_t hb;
1386 heartbeat_init(&hb, USEC_PER_SEC);
1387 while(!__atomic_load_n(&plugin_should_exit, __ATOMIC_ACQUIRE)) {
1388
1389 if(since_last_scan_ut > WINDOWS_EVENTS_SCAN_EVERY_USEC) {
1390 wevt_sources_scan();
1391 since_last_scan_ut = 0;
1392 }
1393
1394 if(since_last_providers_release_ut > WINDOWS_EVENTS_RELEASE_PROVIDERS_HANDLES_EVERY_UT) {
1395 providers_release_unused_handles();
1396 since_last_providers_release_ut = 0;
1397 }
1398
1399 usec_t dt_ut = heartbeat_next(&hb);
1400 since_last_providers_release_ut += dt_ut;
1401 since_last_scan_ut += dt_ut;
1402 send_newline_ut += dt_ut;
1403
1404 if(!tty && send_newline_ut > USEC_PER_SEC) {
1405 send_newline_and_flush(&stdout_mutex);
1406 send_newline_ut = 0;
1407 }
1408 }
1409
1410 exit(0);
1411 }