better logging of invalid pages detected on dbengine files (#14420)
* better logging for pages that cannot be loaded * log uuid too
Costa Tsaousis committed
Feb 3, 2023 at 11:11 UTC
a136acf0f63a75691705512a2f59e6aa3b0a9f16
1 file changed
+73
-19
database/engine/pdc.c
+73
-19
@@ -822,6 +822,66 @@ static inline struct page_details *epdl_get_pd_load_link_list_from_metric_start_
822
return pd_list;
823
}
824
825
+static void epdl_extent_loading_error_log(struct rrdengine_instance *ctx, EPDL *epdl, struct rrdeng_extent_page_descr *descr, const char *msg) {
826
+ char uuid[UUID_STR_LEN] = "";
827
+ time_t start_time_s = 0;
828
+ time_t end_time_s = 0;
829
+ bool used_epdl = false;
830
+ bool used_descr = false;
831
+
832
+ if (descr) {
833
+ start_time_s = (time_t)(descr->start_time_ut / USEC_PER_SEC);
834
+ end_time_s = (time_t)(descr->end_time_ut / USEC_PER_SEC);
835
+ uuid_unparse_lower(descr->uuid, uuid);
836
+ used_descr = true;
837
+ }
838
+ else if (epdl) {
839
+ struct page_details *pd = NULL;
840
+
841
+ Word_t start = 0;
842
+ Pvoid_t *pd_by_start_time_s_judyL = PDCJudyLFirst(epdl->page_details_by_metric_id_JudyL, &start, PJE0);
843
+ if(pd_by_start_time_s_judyL) {
844
+ start = 0;
845
+ Pvoid_t *pd_pptr = PDCJudyLFirst(*pd_by_start_time_s_judyL, &start, PJE0);
846
+ if(pd_pptr) {
847
+ pd = *pd_pptr;
848
+ start_time_s = pd->first_time_s;
849
+ end_time_s = pd->last_time_s;
850
+ METRIC *metric = (METRIC *)pd->metric_id;
851
+ uuid_t *u = mrg_metric_uuid(main_mrg, metric);
852
+ uuid_unparse_lower(*u, uuid);
853
+ used_epdl = true;
854
+ }
855
+ }
856
+ }
857
+
858
+ if(!used_epdl && !used_descr && epdl && epdl->pdc) {
859
+ start_time_s = epdl->pdc->start_time_s;
860
+ end_time_s = epdl->pdc->end_time_s;
861
+ }
862
+
863
+ char start_time_str[LOG_DATE_LENGTH + 1] = "";
864
+ if(start_time_s)
865
+ log_date(start_time_str, LOG_DATE_LENGTH, start_time_s);
866
+
867
+ char end_time_str[LOG_DATE_LENGTH + 1] = "";
868
+ if(end_time_s)
869
+ log_date(end_time_str, LOG_DATE_LENGTH, end_time_s);
870
+
871
+ error_limit_static_global_var(erl, 1, 0);
872
+ error_limit(&erl,
873
+ "DBENGINE: error while reading extent from datafile %u of tier %d, at offset %" PRIu64 " (%u bytes) "
874
+ "%s from %ld (%s) to %ld (%s) %s%s: "
875
+ "%s",
876
+ epdl->datafile->fileno, ctx->config.tier,
877
+ epdl->extent_offset, epdl->extent_size,
878
+ used_epdl ? "to extract page (PD)" : used_descr ? "expected page (DESCR)" : "part of a query (PDC)",
879
+ start_time_s, start_time_str, end_time_s, end_time_str,
880
+ used_epdl || used_descr ? " of metric " : "",
881
+ used_epdl || used_descr ? uuid : "",
882
+ msg);
883
+}
884
+
885
static bool epdl_populate_pages_from_extent_data(
886
struct rrdengine_instance *ctx,
887
void *data,
@@ -870,11 +930,7 @@ static bool epdl_populate_pages_from_extent_data(
930
(payload_length != trailer_offset - payload_offset) ||
931
(data_length != payload_offset + payload_length + sizeof(*trailer))
932
) {
873
-
874
- error_limit_static_global_var(erl, 1, 0);
875
- error_limit(&erl, "%s: Extent at offset %"PRIu64" (%u bytes) was read from datafile %u, but header is INVALID", __func__,
876
- epdl->extent_offset, epdl->extent_size, epdl->datafile->fileno);
877
-
933
+ epdl_extent_loading_error_log(ctx, epdl, NULL, "header is INVALID");
934
return false;
935
}
936
@@ -884,10 +940,7 @@ static bool epdl_populate_pages_from_extent_data(
940
if (unlikely(ret)) {
941
ctx_io_error(ctx);
942
have_read_error = true;
887
-
888
- error_limit_static_global_var(erl, 1, 0);
889
- error_limit(&erl, "%s: Extent at offset %"PRIu64" (%u bytes) was read from datafile %u, but CRC32 check FAILED", __func__,
890
- epdl->extent_offset, epdl->extent_size, epdl->datafile->fileno);
943
+ epdl_extent_loading_error_log(ctx, epdl, NULL, "CRC32 checksum FAILED");
944
}
945
946
if(worker)
@@ -938,18 +991,18 @@ static bool epdl_populate_pages_from_extent_data(
991
time_t start_time_s = (time_t) (header->descr[i].start_time_ut / USEC_PER_SEC);
992
993
if(!page_length || !start_time_s) {
941
- error_limit_static_global_var(erl, 1, 0);
942
- error_limit(&erl, "%s: Extent at offset %"PRIu64" (%u bytes) was read from datafile %u, having page %u (out of %u) EMPTY",
943
- __func__, epdl->extent_offset, epdl->extent_size, epdl->datafile->fileno, i, count);
994
+ char log[200 + 1];
995
+ snprintfz(log, 200, "page %u (out of %u) is EMPTY", i, count);
996
+ epdl_extent_loading_error_log(ctx, epdl, &header->descr[i], log);
997
continue;
998
}
999
1000
METRIC *metric = mrg_metric_get_and_acquire(main_mrg, &header->descr[i].uuid, (Word_t)ctx);
1001
Word_t metric_id = (Word_t)metric;
1002
if(!metric) {
950
- error_limit_static_global_var(erl, 1, 0);
951
- error_limit(&erl, "%s: Extent at offset %"PRIu64" (%u bytes) was read from datafile %u, having page %u (out of %u) for unknown UUID",
952
- __func__, epdl->extent_offset, epdl->extent_size, epdl->datafile->fileno, i, count);
1003
+ char log[200 + 1];
1004
+ snprintfz(log, 200, "page %u (out of %u) has unknown UUID", i, count);
1005
+ epdl_extent_loading_error_log(ctx, epdl, &header->descr[i], log);
1006
continue;
1007
}
1008
mrg_metric_release(main_mrg, metric);
@@ -980,10 +1033,11 @@ static bool epdl_populate_pages_from_extent_data(
1033
}
1034
else {
1035
if (unlikely(page_offset + vd.page_length > uncompressed_payload_length)) {
983
- error_limit_static_global_var(erl, 10, 0);
984
- error_limit(&erl,
985
- "DBENGINE: page %u offset %u + page length %zu exceeds the uncompressed buffer size %u",
986
- i, page_offset, vd.page_length, uncompressed_payload_length);
1036
+ char log[200 + 1];
1037
+ snprintfz(log, 200, "page %u (out of %u) offset %u + page length %zu, "
1038
+ "exceeds the uncompressed buffer size %u",
1039
+ i, count, page_offset, vd.page_length, uncompressed_payload_length);
1040
+ epdl_extent_loading_error_log(ctx, epdl, &header->descr[i], log);
1041
1042
page_data = DBENGINE_EMPTY_PAGE;
1043
stats_load_invalid_page++;