Improve journal file access error logging protect retention recalculation (#20098)
* Improve journal file access error logging and handle corrupted journal files during retention calculation * Limit log frequency in PROTECTED_ACCESS_SETUP
Stelios Fragkakis committed
Apr 9, 2025 at 18:46 UTC
76d067971e69a30ed2fd1d21007752c6de7879a7
3 files changed
+47
-36
src/daemon/protected-access.h
+8
-7
@@ -89,13 +89,14 @@ static inline void protected_access_end(volatile int *ptr __maybe_unused) {
89
90
#define PROTECTED_ACCESS_END() protected_access_end(NULL);
91
92
-#define PROTECTED_ACCESS_SETUP(start, size, resource, op) \
93
- PROTECTED_ACCESS_AUTO_CLEANUP(); \
94
- bool no_signal_received = PROTECTED_ACCESS_START(start, size, resource, op); \
95
- if (!no_signal_received) { \
96
- char __pa_error_buf[1024]; \
97
- protected_access_format_error(__pa_error_buf, sizeof(__pa_error_buf)); \
98
- nd_log(NDLS_DAEMON, NDLP_ERR, "%s", __pa_error_buf); \
92
+#define PROTECTED_ACCESS_SETUP(start, size, resource, op) \
93
+ PROTECTED_ACCESS_AUTO_CLEANUP(); \
94
+ bool no_signal_received = PROTECTED_ACCESS_START(start, size, resource, op); \
95
+ if (!no_signal_received) { \
96
+ char __pa_error_buf[1024]; \
97
+ protected_access_format_error(__pa_error_buf, sizeof(__pa_error_buf)); \
98
+ nd_log_limit_static_thread_var(_erl, 10, 0); \
99
+ nd_log_limit(&_erl, NDLS_DAEMON, NDLP_ERR, "%s", __pa_error_buf); \
100
}
101
102
void signal_protected_access_check(int sig, siginfo_t *si, void *context);
src/database/engine/pagecache.c
+3
-3
@@ -624,9 +624,9 @@ static NOT_INLINE_HOT size_t get_page_list_from_journal_v2(struct rrdengine_inst
624
}
625
}
626
else {
627
- nd_log(NDLS_DAEMON, NDLP_ERR,
628
- "DBENGINE: failed to journal file %u of tier %u (SIGBUS)",
629
- datafile->fileno, datafile->tier);
627
+ nd_log_limit_static_thread_var(erl, 10, 0);
628
+ nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR, "DBENGINE: failed to access journal file %u of tier %d (SIGBUS)",
629
+ datafile->fileno, datafile->ctx->config.tier);
630
}
631
632
journalfile_v2_data_release(datafile->journalfile);
src/database/engine/rrdengine.c
+36
-26
@@ -989,37 +989,47 @@ time_t find_uuid_first_time(
989
struct uuid_first_time_s *uuid_original_entry;
990
991
size_t journal_metric_count = j2_header->metric_count;
992
+ char file_path[RRDENG_PATH_MAX];
993
+ journalfile_v2_generate_path(datafile, file_path, sizeof(file_path));
994
+ PROTECTED_ACCESS_SETUP(datafile->journalfile->mmap.data, datafile->journalfile->mmap.size, file_path, "read");
995
+ if(no_signal_received) {
996
+ for (size_t index = 0; index < count; ++index) {
997
+ uuid_original_entry = &uuid_first_entry_list[index];
998
+
999
+ // Check here if we should skip this
1000
+ if (uuid_original_entry->df_matched > 3 || uuid_original_entry->pages_found > 5)
1001
+ continue;
1002
+
1003
+ struct journal_metric_list *live_entry = bsearch(
1004
+ uuid_original_entry->uuid,
1005
+ uuid_list,
1006
+ journal_metric_count,
1007
+ sizeof(*uuid_list),
1008
+ journal_metric_uuid_compare);
1009
+
1010
+ if (!live_entry) {
1011
+ // Not found in this journal
1012
+ not_matching_bsearches++;
1013
+ continue;
1014
+ }
1015
993
- for (size_t index = 0; index < count; ++index) {
994
- uuid_original_entry = &uuid_first_entry_list[index];
995
-
996
- // Check here if we should skip this
997
- if (uuid_original_entry->df_matched > 3 || uuid_original_entry->pages_found > 5)
998
- continue;
999
-
1000
- struct journal_metric_list *live_entry =
1001
- bsearch(uuid_original_entry->uuid,uuid_list,journal_metric_count,
1002
- sizeof(*uuid_list), journal_metric_uuid_compare);
1003
-
1004
- if (!live_entry) {
1005
- // Not found in this journal
1006
- not_matching_bsearches++;
1007
- continue;
1008
- }
1009
-
1010
- uuid_original_entry->pages_found += live_entry->entries;
1011
- uuid_original_entry->df_matched++;
1016
+ uuid_original_entry->pages_found += live_entry->entries;
1017
+ uuid_original_entry->df_matched++;
1018
1013
- time_t old_first_time_s = uuid_original_entry->first_time_s;
1019
+ time_t old_first_time_s = uuid_original_entry->first_time_s;
1020
1015
- // Calculate first / last for this match
1016
- time_t first_time_s = live_entry->delta_start_s + journal_start_time_s;
1017
- uuid_original_entry->first_time_s = MIN(uuid_original_entry->first_time_s, first_time_s);
1021
+ // Calculate first / last for this match
1022
+ time_t first_time_s = live_entry->delta_start_s + journal_start_time_s;
1023
+ uuid_original_entry->first_time_s = MIN(uuid_original_entry->first_time_s, first_time_s);
1024
1019
- if (uuid_original_entry->first_time_s != old_first_time_s)
1020
- uuid_original_entry->df_index_oldest = uuid_original_entry->df_matched;
1025
+ if (uuid_original_entry->first_time_s != old_first_time_s)
1026
+ uuid_original_entry->df_index_oldest = uuid_original_entry->df_matched;
1027
1022
- binary_match++;
1028
+ binary_match++;
1029
+ }
1030
+ } else {
1031
+ nd_log_daemon(
1032
+ NDLP_ERR, "DBENGINE: journalfile \"%s\" is corrupted, skipping it", file_path);
1033
}
1034
1035
journalfile_count++;