Handle orphan journal files by deleting unmatched entries (#20462)
* Handle orphan journal files by deleting unmatched entries * If failed to store datafile, skip validation
Stelios Fragkakis committed
Jun 11, 2025 at 21:01 UTC
ed125a1a8fbbdddfa5b1e2751a6a49187f4c3a93
1 file changed
+84
-3
src/database/engine/datafile.c
+84
-3
@@ -385,7 +385,7 @@ static int scan_data_files_cmp(const void *a, const void *b)
385
static int scan_data_files(struct rrdengine_instance *ctx)
386
{
387
int ret, matched_files, failed_to_load, i;
388
- unsigned tier, no;
388
+ unsigned tier, fileno;
389
uv_fs_t req;
390
uv_dirent_t dent;
391
struct rrdengine_datafile **datafiles, *datafile;
@@ -401,13 +401,44 @@ static int scan_data_files(struct rrdengine_instance *ctx)
401
}
402
netdata_log_info("DBENGINE: found %d files in path %s", ret, ctx->config.dbfiles_path);
403
404
+ Pvoid_t datafiles_JudyL = NULL;
405
+ Pvoid_t journafile_JudyL = NULL;
406
datafiles = callocz(MIN(ret, MAX_DATAFILES), sizeof(*datafiles));
407
+ bool validate_files = true;
408
for (matched_files = 0 ; UV_EOF != uv_fs_scandir_next(&req, &dent) && matched_files < MAX_DATAFILES ; ) {
406
- ret = sscanf(dent.name, DATAFILE_PREFIX RRDENG_FILE_NUMBER_SCAN_TMPL DATAFILE_EXTENSION, &tier, &no);
409
+ ret = sscanf(dent.name, DATAFILE_PREFIX RRDENG_FILE_NUMBER_SCAN_TMPL DATAFILE_EXTENSION, &tier, &fileno);
410
+
411
+ // This is a datafile
412
if (2 == ret) {
408
- datafile = datafile_alloc_and_init(ctx, tier, no);
413
+ datafile = datafile_alloc_and_init(ctx, tier, fileno);
414
datafiles[matched_files++] = datafile;
415
+ Pvoid_t *Pvalue = JudyLIns(&datafiles_JudyL, (Word_t)fileno, PJE0);
416
+ if (!Pvalue || Pvalue == PJERR)
417
+ validate_files = false;
418
+ continue;
419
+ }
420
+
421
+ // Check for journal v1 or v2
422
+ char expected_name[RRDENG_PATH_MAX];
423
+ ret = sscanf(dent.name, WALFILE_PREFIX RRDENG_FILE_NUMBER_SCAN_TMPL WALFILE_EXTENSION, &tier, &fileno);
424
+ bool unknown_file = true;
425
+ if (2 == ret) {
426
+ (void) snprintfz(expected_name, sizeof(expected_name), WALFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL WALFILE_EXTENSION,
427
+ 1, fileno);
428
+
429
+ unknown_file = (strcmp(dent.name, expected_name) != 0);
430
+ if (unknown_file) {
431
+ (void) snprintfz(expected_name, sizeof(expected_name), WALFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL WALFILE_EXTENSION_V2,
432
+ 1, fileno);
433
+ unknown_file = (strcmp(dent.name, expected_name) != 0);
434
+ }
435
+
436
+ if (!unknown_file)
437
+ (void) JudyLIns(&journafile_JudyL, (Word_t)fileno, PJE0);
438
}
439
+
440
+ if (unknown_file)
441
+ nd_log_daemon(NDLP_WARNING, "Unknown file detected : \"%s/%s\"", ctx->config.dbfiles_path, dent.name);
442
}
443
uv_fs_req_cleanup(&req);
444
@@ -423,6 +454,56 @@ static int scan_data_files(struct rrdengine_instance *ctx)
454
455
ctx->atomic.last_fileno = datafiles[matched_files - 1]->fileno;
456
457
+ // Remove journal files that do not have a matching data file
458
+ // by scanning the judy array of the journal files
459
+ if (validate_files) {
460
+ bool first_then_next = true;
461
+ Word_t idx = 0;
462
+ Pvoid_t *PValue;
463
+ size_t deleted_journals = 0;
464
+ while ((PValue = JudyLFirstThenNext(journafile_JudyL, &idx, &first_then_next))) {
465
+ char path[RRDENG_PATH_MAX];
466
+ if (unlikely(!JudyLGet(datafiles_JudyL, (Word_t)idx, PJE0))) {
467
+ (void)snprintfz(
468
+ path,
469
+ sizeof(path),
470
+ "%s/" WALFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL WALFILE_EXTENSION,
471
+ datafile_ctx(datafile)->config.dbfiles_path,
472
+ 1,
473
+ (unsigned)idx);
474
+
475
+ UNLINK_FILE(ctx, path, ret);
476
+ if (ret == 0) {
477
+ netdata_log_info("DBENGINE: deleting journal file without matching data file: %s", path);
478
+ __atomic_add_fetch(&ctx->stats.journalfile_deletions, 1, __ATOMIC_RELAXED);
479
+ deleted_journals++;
480
+ }
481
+
482
+ (void)snprintfz(
483
+ path,
484
+ sizeof(path),
485
+ "%s/" WALFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL WALFILE_EXTENSION_V2,
486
+ datafile_ctx(datafile)->config.dbfiles_path,
487
+ 1,
488
+ (unsigned)idx);
489
+
490
+ UNLINK_FILE(ctx, path, ret);
491
+ if (ret == 0) {
492
+ netdata_log_info("DBENGINE: deleting journal file without matching data file: %s", path);
493
+ __atomic_add_fetch(&ctx->stats.journalfile_deletions, 1, __ATOMIC_RELAXED);
494
+ deleted_journals++;
495
+ }
496
+ }
497
+ }
498
+
499
+ if (deleted_journals)
500
+ netdata_log_info("DBENGINE: deleted %zu journal files without matching data files", deleted_journals);
501
+ }
502
+
503
+ (void) JudyLFreeArray(&journafile_JudyL, NULL);
504
+ (void) JudyLFreeArray(&datafiles_JudyL, NULL);
505
+
506
+
507
netdata_log_info("DBENGINE: loading %d data/journal of tier %d...", matched_files, ctx->config.tier);
508
for (failed_to_load = 0, i = 0 ; i < matched_files ; ++i) {
509
uint8_t must_delete_pair = 0;