@cryptotaxi247 / netdata-1 / commits / a59097ca5

journals management improvements (#16475)

* different approach to distribute estimatons evenly * log estimations calculation to spot the issue * better log * more logs * more logs * more logs * more logs * commented logs * process journal files and directories in reverse order, so that the system will immediately be available after start * put the right filename in the dictionary * fix scan dir recursion * add missing fields when sending logs to systemd-journal-remote * the standard datetime format is now switched to rfc3339

Costa Tsaousis committed Nov 26, 2023 at 02:22 UTC a59097ca507f01782ad4b401236770acf1175005
14 files changed +396 -123
Makefile.am
+2
@@ -156,6 +156,8 @@ LIBNETDATA_FILES = \
156 libnetdata/completion/completion.h \
157 libnetdata/datetime/iso8601.c \
158 libnetdata/datetime/iso8601.h \
159 + libnetdata/datetime/rfc3339.c \
160 + libnetdata/datetime/rfc3339.h \
161 libnetdata/datetime/rfc7231.c \
162 libnetdata/datetime/rfc7231.h \
163 libnetdata/dictionary/dictionary.c \
collectors/systemd-journal.plugin/systemd-internals.h
+5 -7
@@ -13,14 +13,14 @@
13 #define SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
14 #define SYSTEMD_JOURNAL_FUNCTION_NAME "systemd-journal"
15 #define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 60
16 +#define SYSTEMD_JOURNAL_ENABLE_ESTIMATIONS_FILE_PERCENTAGE 0.01
17 +#define SYSTEMD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS 250
18 +#define SYSTEMD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
19
20 #define SYSTEMD_UNITS_FUNCTION_DESCRIPTION "View the status of systemd units"
21 #define SYSTEMD_UNITS_FUNCTION_NAME "systemd-list-units"
22 #define SYSTEMD_UNITS_DEFAULT_TIMEOUT 30
23
21 -#define EXECUTE_WATCHER_PENDING_EVERY_MS 500
22 -#define FULL_JOURNAL_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
23 -
24 extern __thread size_t fstat_thread_calls;
25 extern __thread size_t fstat_thread_cached_responses;
26 void fstat_cache_enable_on_thread(void);
@@ -28,7 +28,6 @@ void fstat_cache_disable_on_thread(void);
28
29 extern netdata_mutex_t stdout_mutex;
30
31 -
31 typedef enum {
32 ND_SD_JOURNAL_NO_FILE_MATCHED,
33 ND_SD_JOURNAL_FAILED_TO_OPEN,
@@ -97,7 +96,8 @@ int journal_file_dict_items_forward_compar(const void *a, const void *b);
96 void buffer_json_journal_versions(BUFFER *wb);
97 void available_journal_file_sources_to_json_array(BUFFER *wb);
98 bool journal_files_completed_once(void);
100 -void journal_files_updater_all_headers_sorted(void);
99 +void journal_files_registry_update(void);
100 +void journal_directory_scan_recursively(DICTIONARY *files, DICTIONARY *dirs, const char *dirname, int depth);
101
102 FACET_ROW_SEVERITY syslog_priority_to_facet_severity(FACETS *facets, FACET_ROW *row, void *data);
103
@@ -116,14 +116,12 @@ usec_t journal_file_update_annotation_boot_id(sd_journal *j, struct journal_file
116 #define MAX_JOURNAL_DIRECTORIES 100
117 struct journal_directory {
118 char *path;
119 - bool logged_failure;
119 };
120 extern struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES];
121
122 void journal_init_files_and_directories(void);
123 void journal_init_query_status(void);
124 void function_systemd_journal(const char *transaction, char *function, int timeout, bool *cancelled);
126 -void journal_files_registry_update(void);
125 void journal_file_update_header(const char *filename, struct journal_file *jf);
126
127 void netdata_systemd_journal_message_ids_init(void);
collectors/systemd-journal.plugin/systemd-journal-annotations.c
+4 -4
@@ -427,8 +427,8 @@ void netdata_systemd_journal_transform_boot_id(FACETS *facets __maybe_unused, BU
427 ut = *p_ut;
428
429 if(ut && ut != UINT64_MAX) {
430 - char buffer[ISO8601_MAX_LENGTH];
431 - iso8601_datetime_ut(buffer, sizeof(buffer), ut, ISO8601_UTC);
430 + char buffer[RFC3339_MAX_LENGTH];
431 + rfc3339_datetime_ut(buffer, sizeof(buffer), ut, 0, true);
432
433 switch(scope) {
434 default:
@@ -506,8 +506,8 @@ void netdata_systemd_journal_transform_timestamp_usec(FACETS *facets __maybe_unu
506 if(*v && isdigit(*v)) {
507 uint64_t ut = str2ull(buffer_tostring(wb), NULL);
508 if(ut) {
509 - char buffer[ISO8601_MAX_LENGTH];
510 - iso8601_datetime_ut(buffer, sizeof(buffer), ut, ISO8601_UTC | ISO8601_MICROSECONDS);
509 + char buffer[RFC3339_MAX_LENGTH];
510 + rfc3339_datetime_ut(buffer, sizeof(buffer), ut, 6, true);
511 buffer_sprintf(wb, " (%s)", buffer);
512 }
513 }
collectors/systemd-journal.plugin/systemd-journal-files.c
+94 -57
@@ -602,17 +602,16 @@ static void files_registry_delete_cb(const DICTIONARY_ITEM *item, void *value, v
602 string_freez(jf->source);
603 }
604
605 -void journal_directory_scan(const char *dirname, int depth, usec_t last_scan_monotonic_ut) {
605 +void journal_directory_scan_recursively(DICTIONARY *files, DICTIONARY *dirs, const char *dirname, int depth) {
606 static const char *ext = ".journal";
607 - static const size_t ext_len = sizeof(".journal") - 1;
607 + static const ssize_t ext_len = sizeof(".journal") - 1;
608
609 if (depth > VAR_LOG_JOURNAL_MAX_DEPTH)
610 return;
611
612 DIR *dir;
613 struct dirent *entry;
614 - struct stat info;
615 - char absolute_path[FILENAME_MAX];
614 + char full_path[FILENAME_MAX];
615
616 // Open the directory.
617 if ((dir = opendir(dirname)) == NULL) {
@@ -621,33 +620,43 @@ void journal_directory_scan(const char *dirname, int depth, usec_t last_scan_mon
620 return;
621 }
622
623 + bool existing = false;
624 + bool *found = dictionary_set(dirs, dirname, &existing, sizeof(existing));
625 + if(*found) return;
626 + *found = true;
627 +
628 // Read each entry in the directory.
629 while ((entry = readdir(dir)) != NULL) {
626 - snprintfz(absolute_path, sizeof(absolute_path), "%s/%s", dirname, entry->d_name);
627 - if (stat(absolute_path, &info) != 0) {
628 - netdata_log_error("Failed to stat() '%s", absolute_path);
630 + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
631 continue;
630 - }
632
632 - if (S_ISDIR(info.st_mode)) {
633 - // If entry is a directory, call traverse recursively.
634 - if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0)
635 - journal_directory_scan(absolute_path, depth + 1, last_scan_monotonic_ut);
633 + ssize_t len = snprintfz(full_path, sizeof(full_path), "%s/%s", dirname, entry->d_name);
634 +
635 + if (entry->d_type == DT_DIR) {
636 + journal_directory_scan_recursively(files, dirs, full_path, depth++);
637 + }
638 + else if (entry->d_type == DT_REG && len > ext_len && strcmp(full_path + len - ext_len, ext) == 0) {
639 + if(files)
640 + dictionary_set(files, full_path, NULL, 0);
641
642 + send_newline_and_flush();
643 }
638 - else if (S_ISREG(info.st_mode)) {
639 - // If entry is a regular file, check if it ends with .journal.
640 - char *filename = entry->d_name;
641 - size_t len = strlen(filename);
642 -
643 - if (len > ext_len && strcmp(filename + len - ext_len, ext) == 0) {
644 - struct journal_file t = {
645 - .file_last_modified_ut = info.st_mtim.tv_sec * USEC_PER_SEC + info.st_mtim.tv_nsec / NSEC_PER_USEC,
646 - .last_scan_monotonic_ut = last_scan_monotonic_ut,
647 - .size = info.st_size,
648 - .max_journal_vs_realtime_delta_ut = JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT,
649 - };
650 - dictionary_set(journal_files_registry, absolute_path, &t, sizeof(t));
644 + else if (entry->d_type == DT_LNK) {
645 + struct stat info;
646 + if (stat(full_path, &info) == -1)
647 + continue;
648 +
649 + if (S_ISDIR(info.st_mode)) {
650 + // The symbolic link points to a directory
651 + char resolved_path[FILENAME_MAX + 1];
652 + if (realpath(full_path, resolved_path) != NULL) {
653 + journal_directory_scan_recursively(files, dirs, resolved_path, depth++);
654 + }
655 + }
656 + else if(S_ISREG(info.st_mode) && len > ext_len && strcmp(full_path + len - ext_len, ext) == 0) {
657 + if(files)
658 + dictionary_set(files, full_path, NULL, 0);
659 +
660 send_newline_and_flush();
661 }
662 }
@@ -661,41 +670,38 @@ bool journal_files_completed_once(void) {
670 return journal_files_scans > 0;
671 }
672
664 -static int journal_file_dict_items_last_modified_compar(const void *a, const void *b) {
665 - const DICTIONARY_ITEM **ad = (const DICTIONARY_ITEM **)a, **bd = (const DICTIONARY_ITEM **)b;
666 - struct journal_file *jfa = dictionary_acquired_item_value(*ad);
667 - struct journal_file *jfb = dictionary_acquired_item_value(*bd);
673 +int filenames_compar(const void *a, const void *b) {
674 + const char *p1 = *(const char **)a;
675 + const char *p2 = *(const char **)b;
676
669 - if(jfa->file_last_modified_ut > jfb->file_last_modified_ut)
677 + const char *at1 = strchr(p1, '@');
678 + const char *at2 = strchr(p2, '@');
679 +
680 + if(!at1 && at2)
681 return -1;
671 - else if(jfa->file_last_modified_ut < jfb->file_last_modified_ut)
682 +
683 + if(at1 && !at2)
684 return 1;
685
674 - return 0;
675 -}
686 + if(!at1 && !at2)
687 + return strcmp(p1, p2);
688
677 -void journal_files_updater_all_headers_sorted(void) {
678 - const DICTIONARY_ITEM *file_items[dictionary_entries(journal_files_registry)];
679 - size_t files_used = 0;
689 + const char *dash1 = strrchr(at1, '-');
690 + const char *dash2 = strrchr(at2, '-');
691
681 - struct journal_file *jf;
682 - dfe_start_write(journal_files_registry, jf){
683 - if(jf->last_scan_header_vs_last_modified_ut < jf->file_last_modified_ut)
684 - file_items[files_used++] = dictionary_acquired_item_dup(journal_files_registry, jf_dfe.item);
685 - }
686 - dfe_done(jf);
692 + if(!dash1 || !dash2)
693 + return strcmp(p1, p2);
694
688 - // sort them in reverse order (newer first)
689 - qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *),
690 - journal_file_dict_items_last_modified_compar);
695 + uint64_t ts1 = strtoul(dash1 + 1, NULL, 16);
696 + uint64_t ts2 = strtoul(dash2 + 1, NULL, 16);
697
692 - // update the header (first and last message ut, sequence numbers, etc)
693 - for(size_t i = 0; i < files_used ; i++) {
694 - jf = dictionary_acquired_item_value(file_items[i]);
695 - journal_file_update_header(jf->filename, jf);
696 - dictionary_acquired_item_release(journal_files_registry, file_items[i]);
697 - send_newline_and_flush();
698 - }
698 + if(ts1 > ts2)
699 + return -1;
700 +
701 + if(ts1 < ts2)
702 + return 1;
703 +
704 + return -strcmp(p1, p2);
705 }
706
707 void journal_files_registry_update(void) {
@@ -704,12 +710,45 @@ void journal_files_registry_update(void) {
710 if(spinlock_trylock(&spinlock)) {
711 usec_t scan_monotonic_ut = now_monotonic_usec();
712
713 + DICTIONARY *files = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
714 + DICTIONARY *dirs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
715 +
716 for(unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) {
708 - if(!journal_directories[i].path)
709 - break;
717 + if(!journal_directories[i].path) break;
718 + journal_directory_scan_recursively(files, dirs, journal_directories[i].path, 0);
719 + }
720 +
721 + const char **array = mallocz(sizeof(const char *) * dictionary_entries(files));
722 + size_t used = 0;
723 +
724 + void *x;
725 + dfe_start_read(files, x) {
726 + if(used >= dictionary_entries(files)) continue;
727 + array[used++] = x_dfe.name;
728 + }
729 + dfe_done(x);
730
711 - journal_directory_scan(journal_directories[i].path, 0, scan_monotonic_ut);
731 + qsort(array, used, sizeof(const char *), filenames_compar);
732 +
733 + for(size_t i = 0; i < used ;i++) {
734 + const char *full_path = array[i];
735 +
736 + struct stat info;
737 + if (stat(full_path, &info) == -1)
738 + continue;
739 +
740 + struct journal_file t = {
741 + .file_last_modified_ut = info.st_mtim.tv_sec * USEC_PER_SEC + info.st_mtim.tv_nsec / NSEC_PER_USEC,
742 + .last_scan_monotonic_ut = scan_monotonic_ut,
743 + .size = info.st_size,
744 + .max_journal_vs_realtime_delta_ut = JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT,
745 + };
746 + struct journal_file *jf = dictionary_set(journal_files_registry, full_path, &t, sizeof(t));
747 + journal_file_update_header(jf->filename, jf);
748 }
749 + freez(array);
750 + dictionary_destroy(files);
751 + dictionary_destroy(dirs);
752
753 struct journal_file *jf;
754 dfe_start_write(journal_files_registry, jf){
@@ -718,8 +757,6 @@ void journal_files_registry_update(void) {
757 }
758 dfe_done(jf);
759
721 - journal_files_updater_all_headers_sorted();
722 -
760 journal_files_scans++;
761 spinlock_unlock(&spinlock);
762
collectors/systemd-journal.plugin/systemd-journal-watcher.c
+23 -21
@@ -136,29 +136,31 @@ static char* get_path_from_wd(Watcher *watcher, int wd) {
136 return NULL;
137 }
138
139 -static void watch_directory_recursively(Watcher *watcher, int inotifyFd, const char *basePath) {
140 - // First, add a watch for the top-level directory itself
141 - add_watch(watcher, inotifyFd, basePath);
142 -
143 - char path[PATH_MAX];
144 - struct dirent *dp;
145 - DIR *dir = opendir(basePath);
139 +static bool is_directory_watched(Watcher *watcher, const char *path) {
140 + for (int i = 0; i < watcher->watchCount; ++i) {
141 + if (watcher->watchList[i].wd != -1 && strcmp(watcher->watchList[i].path, path) == 0) {
142 + return true;
143 + }
144 + }
145 + return false;
146 +}
147
147 - if (!dir)
148 - return;
148 +static void watch_directory_and_subdirectories(Watcher *watcher, int inotifyFd, const char *basePath) {
149 + DICTIONARY *dirs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
150
150 - while ((dp = readdir(dir)) != NULL) {
151 - if (strcmp(dp->d_name, ".") != 0 && strcmp(dp->d_name, "..") != 0) {
152 - snprintfz(path, sizeof(path), "%s/%s", basePath, dp->d_name);
151 + journal_directory_scan_recursively(NULL, dirs, basePath, 0);
152
154 - if (dp->d_type == DT_DIR) {
155 - // Recursively watch this directory
156 - watch_directory_recursively(watcher, inotifyFd, path);
157 - }
153 + void *x;
154 + dfe_start_read(dirs, x) {
155 + const char *dirname = x_dfe.name;
156 + // Check if this directory is already being watched
157 + if (!is_directory_watched(watcher, dirname)) {
158 + add_watch(watcher, inotifyFd, dirname);
159 }
160 }
161 + dfe_done(x);
162
161 - closedir(dir);
163 + dictionary_destroy(dirs);
164 }
165
166 static bool is_subpath(const char *path, const char *subpath) {
@@ -236,7 +238,7 @@ void process_event(Watcher *watcher, int inotifyFd, struct inotify_event *event)
238 fullPath);
239
240 // Start watching the new directory - recursive watch
239 - watch_directory_recursively(watcher, inotifyFd, fullPath);
241 + watch_directory_and_subdirectories(watcher, inotifyFd, fullPath);
242 }
243 else
244 nd_log(NDLS_COLLECTORS, NDLP_WARNING,
@@ -310,14 +312,14 @@ void *journal_watcher_main(void *arg __maybe_unused) {
312
313 for (unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) {
314 if (!journal_directories[i].path) break;
313 - watch_directory_recursively(&watcher, inotifyFd, journal_directories[i].path);
315 + watch_directory_and_subdirectories(&watcher, inotifyFd, journal_directories[i].path);
316 }
317
318 usec_t last_headers_update_ut = now_monotonic_usec();
319 struct buffered_reader reader;
320 while (1) {
321 buffered_reader_ret_t rc = buffered_reader_read_timeout(
320 - &reader, inotifyFd, EXECUTE_WATCHER_PENDING_EVERY_MS, false);
322 + &reader, inotifyFd, SYSTEMD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS, false);
323
324 if (rc != BUFFERED_READER_READ_OK && rc != BUFFERED_READER_READ_POLL_TIMEOUT) {
325 nd_log(NDLS_COLLECTORS, NDLP_CRIT,
@@ -353,7 +355,7 @@ void *journal_watcher_main(void *arg __maybe_unused) {
355
356 usec_t ut = now_monotonic_usec();
357 if (dictionary_entries(watcher.pending) && (rc == BUFFERED_READER_READ_POLL_TIMEOUT ||
356 - last_headers_update_ut + (EXECUTE_WATCHER_PENDING_EVERY_MS * USEC_PER_MS) <= ut)) {
358 + last_headers_update_ut + (SYSTEMD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS * USEC_PER_MS) <= ut)) {
359 process_pending(&watcher);
360 last_headers_update_ut = ut;
361 }
collectors/systemd-journal.plugin/systemd-journal.c
+36 -4
@@ -474,9 +474,9 @@ static size_t sampling_running_file_query_estimate_remaining_lines_by_time(FUNCT
474 size_t scanned_lines = sampling_file_lines_scanned_so_far(fqs);
475
476 // Calculate the proportion of time covered
477 - usec_t total_time_ut;
477 + usec_t total_time_ut, remaining_start_ut, remaining_end_ut;
478 usec_t remaining_time_ut = sampling_running_file_query_remaining_time(fqs, jf, direction, msg_ut, &total_time_ut,
479 - NULL, NULL);
479 + &remaining_start_ut, &remaining_end_ut);
480 if (total_time_ut == 0) total_time_ut = 1;
481
482 double proportion_by_time = (double) (total_time_ut - remaining_time_ut) / (double) total_time_ut;
@@ -494,10 +494,30 @@ static size_t sampling_running_file_query_estimate_remaining_lines_by_time(FUNCT
494 size_t remaining_logs_by_time = expected_matching_logs_by_time - scanned_lines;
495 if (remaining_logs_by_time < 1) remaining_logs_by_time = 1;
496
497 +// nd_log(NDLS_COLLECTORS, NDLP_INFO,
498 +// "JOURNAL ESTIMATION: '%s' "
499 +// "scanned_lines=%zu [sampled=%zu, unsampled=%zu, estimated=%zu], "
500 +// "file [%"PRIu64" - %"PRIu64", duration %"PRId64", known lines in file %zu], "
501 +// "query [%"PRIu64" - %"PRIu64", duration %"PRId64"], "
502 +// "first message read from the file at %"PRIu64", current message at %"PRIu64", "
503 +// "proportion of time %.2f %%, "
504 +// "expected total lines in file %zu, "
505 +// "remaining lines %zu, "
506 +// "remaining time %"PRIu64" [%"PRIu64" - %"PRIu64", duration %"PRId64"]"
507 +// , jf->filename
508 +// , scanned_lines, fqs->samples_per_file.sampled, fqs->samples_per_file.unsampled, fqs->samples_per_file.estimated
509 +// , jf->msg_first_ut, jf->msg_last_ut, jf->msg_last_ut - jf->msg_first_ut, jf->messages_in_file
510 +// , fqs->query_file.start_ut, fqs->query_file.stop_ut, fqs->query_file.stop_ut - fqs->query_file.start_ut
511 +// , fqs->query_file.first_msg_ut, msg_ut
512 +// , proportion_by_time * 100.0
513 +// , expected_matching_logs_by_time
514 +// , remaining_logs_by_time
515 +// , remaining_time_ut, remaining_start_ut, remaining_end_ut, remaining_end_ut - remaining_start_ut
516 +// );
517 +
518 return remaining_logs_by_time;
519 }
520
500 -
521 static size_t sampling_running_file_query_estimate_remaining_lines(sd_journal *j, FUNCTION_QUERY_STATUS *fqs, struct journal_file *jf, FACETS_ANCHOR_DIRECTION direction, usec_t msg_ut) {
522 size_t expected_matching_logs_by_seqnum = 0;
523 double proportion_by_seqnum = 0.0;
@@ -613,7 +633,7 @@ static inline sampling_t is_row_in_sample(sd_journal *j, FUNCTION_QUERY_STATUS *
633 if(fqs->samples_per_file.unsampled > fqs->samples_per_file.sampled) {
634 double progress_by_time = sampling_running_file_query_progress_by_time(fqs, jf, direction, msg_ut);
635
616 - if(progress_by_time > 0.05)
636 + if(progress_by_time > SYSTEMD_JOURNAL_ENABLE_ESTIMATIONS_FILE_PERCENTAGE)
637 return SAMPLING_STOP_AND_ESTIMATE;
638 }
639
@@ -1166,6 +1186,18 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1186
1187 ND_SD_JOURNAL_STATUS tmp_status = netdata_systemd_journal_query_one_file(filename, wb, facets, jf, fqs);
1188
1189 +// nd_log(NDLS_COLLECTORS, NDLP_INFO,
1190 +// "JOURNAL ESTIMATION FINAL: '%s' "
1191 +// "total lines %zu [sampled=%zu, unsampled=%zu, estimated=%zu], "
1192 +// "file [%"PRIu64" - %"PRIu64", duration %"PRId64", known lines in file %zu], "
1193 +// "query [%"PRIu64" - %"PRIu64", duration %"PRId64"], "
1194 +// , jf->filename
1195 +// , fqs->samples_per_file.sampled + fqs->samples_per_file.unsampled + fqs->samples_per_file.estimated
1196 +// , fqs->samples_per_file.sampled, fqs->samples_per_file.unsampled, fqs->samples_per_file.estimated
1197 +// , jf->msg_first_ut, jf->msg_last_ut, jf->msg_last_ut - jf->msg_first_ut, jf->messages_in_file
1198 +// , fqs->query_file.start_ut, fqs->query_file.stop_ut, fqs->query_file.stop_ut - fqs->query_file.start_ut
1199 +// );
1200 +
1201 rows_useful = fqs->rows_useful - rows_useful;
1202 rows_read = fqs->rows_read - rows_read;
1203 bytes_read = fqs->bytes_read - bytes_read;
collectors/systemd-journal.plugin/systemd-main.c
+2 -2
@@ -86,14 +86,14 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
86
87 usec_t step_ut = 100 * USEC_PER_MS;
88 usec_t send_newline_ut = 0;
89 - usec_t since_last_scan_ut = FULL_JOURNAL_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
89 + usec_t since_last_scan_ut = SYSTEMD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
90 bool tty = isatty(fileno(stderr)) == 1;
91
92 heartbeat_t hb;
93 heartbeat_init(&hb);
94 while(!plugin_should_exit) {
95
96 - if(since_last_scan_ut > FULL_JOURNAL_SCAN_EVERY_USEC) {
96 + if(since_last_scan_ut > SYSTEMD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC) {
97 journal_files_registry_update();
98 since_last_scan_ut = 0;
99 }
libnetdata/datetime/rfc3339.c new
+135
@@ -0,0 +1,135 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "../libnetdata.h"
4 +
5 +#include "rfc3339.h"
6 +
7 +size_t rfc3339_datetime_ut(char *buffer, size_t len, usec_t now_ut, size_t fractional_digits, bool utc) {
8 + if (!buffer || len == 0)
9 + return 0;
10 +
11 + time_t t = (time_t)(now_ut / USEC_PER_SEC);
12 + struct tm *tmp, tmbuf;
13 +
14 + if (utc)
15 + tmp = gmtime_r(&t, &tmbuf);
16 + else
17 + tmp = localtime_r(&t, &tmbuf);
18 +
19 + if (!tmp) {
20 + buffer[0] = '\0';
21 + return 0;
22 + }
23 +
24 + size_t used_length = strftime(buffer, len, "%Y-%m-%dT%H:%M:%S", tmp);
25 + if (used_length == 0) {
26 + buffer[0] = '\0';
27 + return 0;
28 + }
29 +
30 + if (fractional_digits >= 0 && fractional_digits <= 9) {
31 + int fractional_part = (int)(now_ut % USEC_PER_SEC);
32 + if (fractional_part && len - used_length > fractional_digits + 1) {
33 + char format[] = ".%01d";
34 + format[3] = (char)('0' + fractional_digits);
35 +
36 + // Adjust fractional part
37 + fractional_part /= (int)pow(10, 6 - fractional_digits);
38 +
39 + used_length += snprintf(buffer + used_length, len - used_length,
40 + format, fractional_part);
41 + }
42 + }
43 +
44 + if (utc) {
45 + if (used_length + 1 < len) {
46 + buffer[used_length++] = 'Z';
47 + buffer[used_length] = '\0';
48 + }
49 + }
50 + else {
51 + long offset = tmbuf.tm_gmtoff;
52 + int hours = (int)(offset / 3600);
53 + int minutes = abs((int)((offset % 3600) / 60));
54 +
55 + if (used_length + 7 < len) { // Space for "+HH:MM\0"
56 + used_length += snprintf(buffer + used_length, len - used_length, "%+03d:%02d", hours, minutes);
57 + }
58 + }
59 +
60 + return used_length;
61 +}
62 +
63 +usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr) {
64 + struct tm tm = { 0 };
65 + int tz_hours = 0, tz_mins = 0;
66 + char *s;
67 + usec_t timestamp, usec = 0;
68 +
69 + // Use strptime to parse up to seconds
70 + s = strptime(rfc3339, "%Y-%m-%dT%H:%M:%S", &tm);
71 + if (!s)
72 + return 0; // Parsing error
73 +
74 + // Parse fractional seconds if present
75 + if (*s == '.') {
76 + char *next;
77 + usec = strtoul(s + 1, &next, 10);
78 + int digits_parsed = (int)(next - (s + 1));
79 +
80 + if (digits_parsed < 1 || digits_parsed > 9)
81 + return 0; // parsing error
82 +
83 + static const usec_t fix_usec[] = {
84 + 1000000, // 0 digits (not used)
85 + 100000, // 1 digit
86 + 10000, // 2 digits
87 + 1000, // 3 digits
88 + 100, // 4 digits
89 + 10, // 5 digits
90 + 1, // 6 digits
91 + 10, // 7 digits
92 + 100, // 8 digits
93 + 1000, // 9 digits
94 + };
95 + usec = digits_parsed <= 6 ? usec * fix_usec[digits_parsed] : usec / fix_usec[digits_parsed];
96 +
97 + s = next;
98 + }
99 +
100 + // Check and parse timezone if present
101 + int tz_offset = 0;
102 + if (*s == '+' || *s == '-') {
103 + // Parse the hours:mins part of the timezone
104 +
105 + if (!isdigit(s[1]) || !isdigit(s[2]) || s[3] != ':' ||
106 + !isdigit(s[4]) || !isdigit(s[5]))
107 + return 0; // Parsing error
108 +
109 + char tz_sign = *s;
110 + tz_hours = (s[1] - '0') * 10 + (s[2] - '0');
111 + tz_mins = (s[4] - '0') * 10 + (s[5] - '0');
112 +
113 + tz_offset = tz_hours * 3600 + tz_mins * 60;
114 + tz_offset *= (tz_sign == '+' ? 1 : -1);
115 +
116 + s += 6; // Move past the timezone part
117 + }
118 + else if (*s == 'Z')
119 + s++;
120 + else
121 + return 0; // Invalid RFC 3339 format
122 +
123 + // Convert to time_t (assuming local time, then adjusting for timezone later)
124 + time_t epoch_s = mktime(&tm);
125 + if (epoch_s == -1)
126 + return 0; // Error in time conversion
127 +
128 + timestamp = (usec_t)epoch_s * USEC_PER_SEC + usec;
129 + timestamp -= tz_offset * USEC_PER_SEC;
130 +
131 + if(endptr)
132 + *endptr = s;
133 +
134 + return timestamp;
135 +}
libnetdata/datetime/rfc3339.h new
+12
@@ -0,0 +1,12 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "../libnetdata.h"
4 +
5 +#ifndef NETDATA_RFC3339_H
6 +#define NETDATA_RFC3339_H
7 +
8 +#define RFC3339_MAX_LENGTH 36
9 +size_t rfc3339_datetime_ut(char *buffer, size_t len, usec_t now_ut, size_t fractional_digits, bool utc);
10 +usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr);
11 +
12 +#endif //NETDATA_RFC3339_H
libnetdata/facets/facets.c
+25 -10
@@ -931,7 +931,23 @@ static inline void facets_histogram_update_value(FACETS *facets, usec_t usec) {
931 facets_histogram_update_value_slot(facets, usec, v);
932 }
933
934 +static usec_t overlap_duration_ut(usec_t start1, usec_t end1, usec_t start2, usec_t end2) {
935 + usec_t overlap_start = MAX(start1, start2);
936 + usec_t overlap_end = MIN(end1, end2);
937 +
938 + if (overlap_start < overlap_end)
939 + return overlap_end - overlap_start;
940 + else
941 + return 0; // No overlap
942 +}
943 +
944 void facets_update_estimations(FACETS *facets, usec_t from_ut, usec_t to_ut, size_t entries) {
945 + if(unlikely(!facets->histogram.enabled))
946 + return;
947 +
948 + if(unlikely(!overlap_duration_ut(facets->histogram.after_ut, facets->histogram.before_ut, from_ut, to_ut)))
949 + return;
950 +
951 facets->operations.rows.evaluated += entries;
952 facets->operations.rows.matched += entries;
953 facets->operations.rows.estimated += entries;
@@ -952,29 +968,28 @@ void facets_update_estimations(FACETS *facets, usec_t from_ut, usec_t to_ut, siz
968
969 FACET_VALUE *v = facets->histogram.key->estimated_value.v;
970
955 - size_t from_slot = facets_histogram_slot_at_time_ut(facets, from_ut, v);
956 - size_t to_slot = facets_histogram_slot_at_time_ut(facets, to_ut, v);
971 + size_t slots = 0;
972 size_t total_ut = to_ut - from_ut;
973 ssize_t remaining_entries = (ssize_t)entries;
974 + size_t slot = facets_histogram_slot_at_time_ut(facets, from_ut, v);
975 + for(; slot < facets->histogram.slots ;slot++) {
976 + usec_t slot_start_ut = facets->histogram.after_ut + slot * facets->histogram.slot_width_ut;
977 + usec_t slot_end_ut = slot_start_ut + facets->histogram.slot_width_ut;
978
960 - for (size_t slot = from_slot; slot <= to_slot; slot++) {
961 - if (unlikely(slot >= facets->histogram.slots))
979 + if(slot_start_ut > to_ut)
980 break;
981
964 - usec_t slot_start_ut = facets->histogram.after_ut + slot * facets->histogram.slot_width_ut;
965 - usec_t slot_end_ut = slot_start_ut + facets->histogram.slot_width_ut;
966 - usec_t overlap_start_ut = (from_ut > slot_start_ut) ? from_ut : slot_start_ut;
967 - usec_t overlap_end_ut = (to_ut < slot_end_ut) ? to_ut : slot_end_ut;
968 - usec_t overlap_ut = (overlap_end_ut > overlap_start_ut) ? (overlap_end_ut - overlap_start_ut) : 0;
982 + usec_t overlap_ut = overlap_duration_ut(from_ut, to_ut, slot_start_ut, slot_end_ut);
983
984 size_t slot_entries = (overlap_ut * entries) / total_ut;
985 v->histogram[slot] += slot_entries;
986 remaining_entries -= (ssize_t)slot_entries;
987 + slots++;
988 }
989
990 // Check if all entries are assigned
991 // This should always be true if the distribution is correct
977 - internal_fatal(remaining_entries < 0 || remaining_entries > (ssize_t)(to_slot - from_slot),
992 + internal_fatal(remaining_entries < 0 || remaining_entries >= (ssize_t)(slots),
993 "distribution of estimations is not accurate - there are %zd remaining entries",
994 remaining_entries);
995 }
libnetdata/libnetdata.h
+1
@@ -723,6 +723,7 @@ extern char *netdata_configured_host_prefix;
723 #include "line_splitter/line_splitter.h"
724 #include "clocks/clocks.h"
725 #include "datetime/iso8601.h"
726 +#include "datetime/rfc3339.h"
727 #include "datetime/rfc7231.h"
728 #include "completion/completion.h"
729 #include "popen/popen.h"
libnetdata/log/log.c
+2 -2
@@ -1498,8 +1498,8 @@ static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_fie
1498 if(!ut)
1499 return;
1500
1501 - char datetime[ISO8601_MAX_LENGTH];
1502 - iso8601_datetime_ut(datetime, sizeof(datetime), ut, ISO8601_LOCAL_TIMEZONE | ISO8601_MILLISECONDS);
1501 + char datetime[RFC3339_MAX_LENGTH];
1502 + rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false);
1503
1504 if(buffer_strlen(wb))
1505 buffer_fast_strcat(wb, " ", 1);
libnetdata/log/systemd-cat-native.c
+53 -14
@@ -135,7 +135,12 @@ static inline void buffer_memcat_replacing_newlines(BUFFER *wb, const char *src,
135
136 char global_hostname[HOST_NAME_MAX] = "";
137 char global_boot_id[UUID_COMPACT_STR_LEN] = "";
138 +char global_machine_id[UUID_COMPACT_STR_LEN] = "";
139 +char global_stream_id[UUID_COMPACT_STR_LEN] = "";
140 +char global_namespace[1024] = "";
141 +char global_systemd_invocation_id[1024] = "";
142 #define BOOT_ID_PATH "/proc/sys/kernel/random/boot_id"
143 +#define MACHINE_ID_PATH "/etc/machine-id"
144
145 #define DEFAULT_PRIVATE_KEY "/etc/ssl/private/journal-upload.pem"
146 #define DEFAULT_PUBLIC_KEY "/etc/ssl/certs/journal-upload.pem"
@@ -197,16 +202,25 @@ static void journal_remote_complete_event(BUFFER *msg, usec_t *monotonic_ut) {
202 *monotonic_ut = ut;
203
204 buffer_sprintf(msg,
200 - ""
201 - "__REALTIME_TIMESTAMP=%llu\n"
202 - "__MONOTONIC_TIMESTAMP=%llu\n"
203 - "_BOOT_ID=%s\n"
204 - "_HOSTNAME=%s\n"
205 - "\n"
205 + ""
206 + "__REALTIME_TIMESTAMP=%llu\n"
207 + "__MONOTONIC_TIMESTAMP=%llu\n"
208 + "_MACHINE_ID=%s\n"
209 + "_BOOT_ID=%s\n"
210 + "_HOSTNAME=%s\n"
211 + "_TRANSPORT=stdout\n"
212 + "_LINE_BREAK=nul\n"
213 + "_STREAM_ID=%s\n"
214 + "_RUNTIME_SCOPE=system\n"
215 + "%s%s\n"
216 , now_realtime_usec()
217 , ut
218 + , global_machine_id
219 , global_boot_id
220 , global_hostname
221 + , global_stream_id
222 + , global_namespace
223 + , global_systemd_invocation_id
224 );
225 }
226
@@ -245,10 +259,10 @@ static log_to_journal_remote_ret_t log_input_to_journal_remote(const char *url,
259 timeout_ms = 10;
260
261 global_boot_id[0] = '\0';
248 - char boot_id[1024];
249 - if(read_file(BOOT_ID_PATH, boot_id, sizeof(boot_id)) == 0) {
262 + char buffer[1024];
263 + if(read_file(BOOT_ID_PATH, buffer, sizeof(buffer)) == 0) {
264 uuid_t uuid;
251 - if(uuid_parse_flexi(boot_id, uuid) == 0)
265 + if(uuid_parse_flexi(buffer, uuid) == 0)
266 uuid_unparse_lower_compact(uuid, global_boot_id);
267 else
268 fprintf(stderr, "WARNING: cannot parse the UUID found in '%s'.\n", BOOT_ID_PATH);
@@ -261,6 +275,27 @@ static log_to_journal_remote_ret_t log_input_to_journal_remote(const char *url,
275 uuid_unparse_lower_compact(uuid, global_boot_id);
276 }
277
278 + if(read_file(MACHINE_ID_PATH, buffer, sizeof(buffer)) == 0) {
279 + uuid_t uuid;
280 + if(uuid_parse_flexi(buffer, uuid) == 0)
281 + uuid_unparse_lower_compact(uuid, global_machine_id);
282 + else
283 + fprintf(stderr, "WARNING: cannot parse the UUID found in '%s'.\n", MACHINE_ID_PATH);
284 + }
285 +
286 + if(global_machine_id[0] == '\0') {
287 + fprintf(stderr, "WARNING: cannot read '%s'. Will generate a random _MACHINE_ID.\n", MACHINE_ID_PATH);
288 + uuid_t uuid;
289 + uuid_generate_random(uuid);
290 + uuid_unparse_lower_compact(uuid, global_boot_id);
291 + }
292 +
293 + if(global_stream_id[0] == '\0') {
294 + uuid_t uuid;
295 + uuid_generate_random(uuid);
296 + uuid_unparse_lower_compact(uuid, global_stream_id);
297 + }
298 +
299 if(global_hostname[0] == '\0') {
300 if(gethostname(global_hostname, sizeof(global_hostname)) != 0) {
301 fprintf(stderr, "WARNING: cannot get system's hostname. Will use internal default.\n");
@@ -268,6 +303,9 @@ static log_to_journal_remote_ret_t log_input_to_journal_remote(const char *url,
303 }
304 }
305
306 + if(global_systemd_invocation_id[0] == '\0' && getenv("INVOCATION_ID"))
307 + snprintfz(global_systemd_invocation_id, sizeof(global_systemd_invocation_id), "_SYSTEMD_INVOCATION_ID=%s\n", getenv("INVOCATION_ID"));
308 +
309 if(!key)
310 key = DEFAULT_PRIVATE_KEY;
311
@@ -483,6 +521,9 @@ static int help(void) {
521 " The default is: " DEFAULT_CA_CERT "\n"
522 " The keyword 'all' can be used to trust all CAs.\n"
523 "\n"
524 + " --namespace=NAMESPACE\n"
525 + " Set the namespace of the messages sent.\n"
526 + "\n"
527 " --keep-trying\n"
528 " Keep trying to send the message, if the remote journal is not there.\n"
529 #endif
@@ -752,11 +793,6 @@ int main(int argc, char *argv[]) {
793 return 1;
794 }
795
755 - if(url && namespace) {
756 - fprintf(stderr, "Cannot log to a systemd-journal-remote URL using a namespace. "
757 - "Please either give --url or --namespace, not both.\n");
758 - return 1;
759 - }
796 #endif
797
798 if(log_as_netdata && namespace) {
@@ -770,6 +806,9 @@ int main(int argc, char *argv[]) {
806
807 #ifdef HAVE_CURL
808 if(url) {
809 + if(url && namespace && *namespace)
810 + snprintfz(global_namespace, sizeof(global_namespace), "_NAMESPACE=%s\n", namespace);
811 +
812 log_to_journal_remote_ret_t rc;
813 do {
814 rc = log_input_to_journal_remote(url, key, cert, trust, newline, timeout_ms);
streaming/sender.c
+2 -2
@@ -557,8 +557,8 @@ static inline bool rrdpush_sender_validate_response(RRDHOST *host, struct sender
557 };
558 ND_LOG_STACK_PUSH(lgs);
559
560 - char buf[ISO8601_MAX_LENGTH];
561 - iso8601_datetime_ut(buf, sizeof(buf), host->destination->postpone_reconnection_until * USEC_PER_SEC, 0);
560 + char buf[RFC3339_MAX_LENGTH];
561 + rfc3339_datetime_ut(buf, sizeof(buf), host->destination->postpone_reconnection_until * USEC_PER_SEC, 0, false);
562
563 nd_log(NDLS_DAEMON, priority,
564 "STREAM %s [send to %s]: %s - will retry in %d secs, at %s",