Fix empty dbengine files (#9820)
* Stop aborting when loading empty metadata log files. * Stop aborting when loading empty dbengine data or journal files. * Improve consistency of metadata log file buffers and reduce likelyhood of zero length files on disk.
Markos Fountoulakis committed
Aug 27, 2020 at 14:27 UTC
ea8b0e976bdd39849e0d91ae7d209c82bae79bcf
7 files changed
+108
-17
database/engine/datafile.c
+46
-11
@@ -75,6 +75,27 @@ int close_data_file(struct rrdengine_datafile *datafile)
75
return ret;
76
}
77
78
+int unlink_data_file(struct rrdengine_datafile *datafile)
79
+{
80
+ struct rrdengine_instance *ctx = datafile->ctx;
81
+ uv_fs_t req;
82
+ int ret;
83
+ char path[RRDENG_PATH_MAX];
84
+
85
+ generate_datafilepath(datafile, path, sizeof(path));
86
+
87
+ ret = uv_fs_unlink(NULL, &req, path, NULL);
88
+ if (ret < 0) {
89
+ error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
90
+ ++ctx->stats.fs_errors;
91
+ rrd_stat_atomic_add(&global_fs_errors, 1);
92
+ }
93
+ uv_fs_req_cleanup(&req);
94
+
95
+ ++ctx->stats.datafile_deletions;
96
+
97
+ return ret;
98
+}
99
100
int destroy_data_file(struct rrdengine_datafile *datafile)
101
{
@@ -305,33 +326,47 @@ static int scan_data_files(struct rrdengine_instance *ctx)
326
ctx->last_fileno = datafiles[matched_files - 1]->fileno;
327
328
for (failed_to_load = 0, i = 0 ; i < matched_files ; ++i) {
329
+ uint8_t must_delete_pair = 0;
330
+
331
datafile = datafiles[i];
332
ret = load_data_file(datafile);
333
if (0 != ret) {
311
- freez(datafile);
312
- ++failed_to_load;
313
- break;
334
+ must_delete_pair = 1;
335
}
336
journalfile = mallocz(sizeof(*journalfile));
337
datafile->journalfile = journalfile;
338
journalfile_init(journalfile, datafile);
339
ret = load_journal_file(ctx, journalfile, datafile);
340
if (0 != ret) {
320
- close_data_file(datafile);
321
- freez(datafile);
341
+ if (!must_delete_pair) /* If datafile is still open close it */
342
+ close_data_file(datafile);
343
+ must_delete_pair = 1;
344
+ }
345
+ if (must_delete_pair) {
346
+ char path[RRDENG_PATH_MAX];
347
+
348
+ error("Deleting invalid data and journal file pair.");
349
+ ret = unlink_journal_file(journalfile);
350
+ if (!ret) {
351
+ generate_journalfilepath(datafile, path, sizeof(path));
352
+ info("Deleted journal file \"%s\".", path);
353
+ }
354
+ ret = unlink_data_file(datafile);
355
+ if (!ret) {
356
+ generate_datafilepath(datafile, path, sizeof(path));
357
+ info("Deleted data file \"%s\".", path);
358
+ }
359
freez(journalfile);
360
+ freez(datafile);
361
++failed_to_load;
324
- break;
362
+ continue;
363
}
364
+
365
datafile_list_insert(ctx, datafile);
366
ctx->disk_space += datafile->pos + journalfile->pos;
367
}
368
+ matched_files -= failed_to_load;
369
freez(datafiles);
330
- if (failed_to_load) {
331
- error("%u datafiles failed to load.", failed_to_load);
332
- finalize_data_files(ctx);
333
- return UV_EIO;
334
- }
370
371
return matched_files;
372
}
database/engine/datafile.h
+1
@@ -57,6 +57,7 @@ extern void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengin
57
extern void datafile_list_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile);
58
extern void generate_datafilepath(struct rrdengine_datafile *datafile, char *str, size_t maxlen);
59
extern int close_data_file(struct rrdengine_datafile *datafile);
60
+extern int unlink_data_file(struct rrdengine_datafile *datafile);
61
extern int destroy_data_file(struct rrdengine_datafile *datafile);
62
extern int create_data_file(struct rrdengine_datafile *datafile);
63
extern int create_new_datafile_pair(struct rrdengine_instance *ctx, unsigned tier, unsigned fileno);
database/engine/journalfile.c
+23
@@ -125,6 +125,29 @@ int close_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengi
125
return ret;
126
}
127
128
+int unlink_journal_file(struct rrdengine_journalfile *journalfile)
129
+{
130
+ struct rrdengine_datafile *datafile = journalfile->datafile;
131
+ struct rrdengine_instance *ctx = datafile->ctx;
132
+ uv_fs_t req;
133
+ int ret;
134
+ char path[RRDENG_PATH_MAX];
135
+
136
+ generate_journalfilepath(datafile, path, sizeof(path));
137
+
138
+ ret = uv_fs_unlink(NULL, &req, path, NULL);
139
+ if (ret < 0) {
140
+ error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
141
+ ++ctx->stats.fs_errors;
142
+ rrd_stat_atomic_add(&global_fs_errors, 1);
143
+ }
144
+ uv_fs_req_cleanup(&req);
145
+
146
+ ++ctx->stats.journalfile_deletions;
147
+
148
+ return ret;
149
+}
150
+
151
int destroy_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile)
152
{
153
struct rrdengine_instance *ctx = datafile->ctx;
database/engine/journalfile.h
+1
@@ -38,6 +38,7 @@ extern void journalfile_init(struct rrdengine_journalfile *journalfile, struct r
38
extern void *wal_get_transaction_buffer(struct rrdengine_worker_config* wc, unsigned size);
39
extern void wal_flush_transaction_buffer(struct rrdengine_worker_config* wc);
40
extern int close_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);
41
+extern int unlink_journal_file(struct rrdengine_journalfile *journalfile);
42
extern int destroy_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);
43
extern int create_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);
44
extern int load_journal_file(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile,
database/engine/metadata_log/logfile.c
+34
-6
@@ -248,6 +248,26 @@ int close_metadata_logfile(struct metadata_logfile *metalogfile)
248
return ret;
249
}
250
251
+int fsync_metadata_logfile(struct metadata_logfile *metalogfile)
252
+{
253
+ struct metalog_instance *ctx = metalogfile->ctx;
254
+ uv_fs_t req;
255
+ int ret;
256
+ char path[RRDENG_PATH_MAX];
257
+
258
+ generate_metadata_logfile_path(metalogfile, path, sizeof(path));
259
+
260
+ ret = uv_fs_fsync(NULL, &req, metalogfile->file, NULL);
261
+ if (ret < 0) {
262
+ error("uv_fs_close(%s): %s", path, uv_strerror(ret));
263
+ ++ctx->stats.fs_errors;
264
+ rrd_stat_atomic_add(&global_fs_errors, 1);
265
+ }
266
+ uv_fs_req_cleanup(&req);
267
+
268
+ return ret;
269
+}
270
+
271
int unlink_metadata_logfile(struct metadata_logfile *metalogfile)
272
{
273
struct metalog_instance *ctx = metalogfile->ctx;
@@ -344,6 +364,15 @@ int create_metadata_logfile(struct metadata_logfile *metalogfile)
364
rrd_stat_atomic_add(&global_io_errors, 1);
365
}
366
uv_fs_req_cleanup(&req);
367
+
368
+ ret = uv_fs_fsync(NULL, &req, metalogfile->file, NULL);
369
+ if (ret < 0) {
370
+ error("uv_fs_close(%s): %s", path, uv_strerror(ret));
371
+ ++ctx->stats.fs_errors;
372
+ rrd_stat_atomic_add(&global_fs_errors, 1);
373
+ }
374
+ uv_fs_req_cleanup(&req);
375
+
376
free(superblock);
377
if (ret < 0) {
378
destroy_metadata_logfile(metalogfile);
@@ -695,13 +724,17 @@ static int scan_metalog_files(struct metalog_instance *ctx)
724
metalogfile = metalogfiles[i];
725
ret = load_metadata_logfile(ctx, metalogfile);
726
if (0 != ret) {
727
+ error("Deleting invalid metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
728
+ METALOG_EXTENSION"\"", dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
729
+ unlink_metadata_logfile(metalogfile);
730
freez(metalogfile);
731
++failed_to_load;
700
- break;
732
+ continue;
733
}
734
metadata_logfile_list_insert(&ctx->metadata_logfiles, metalogfile);
735
rrd_atomic_fetch_add(&ctx->disk_space, metalogfile->pos);
736
}
737
+ matched_files -= failed_to_load;
738
debug(D_METADATALOG, "PARSER ended");
739
740
parser_destroy(parser);
@@ -712,11 +745,6 @@ static int scan_metalog_files(struct metalog_instance *ctx)
745
after_failed_to_parse:
746
747
freez(metalogfiles);
715
- if (failed_to_load) {
716
- error("%u metadata log files failed to load.", failed_to_load);
717
- finalize_metalog_files(ctx);
718
- return UV_EIO;
719
- }
748
749
return matched_files;
750
}
database/engine/metadata_log/logfile.h
+1
@@ -83,6 +83,7 @@ extern void metadata_logfile_init(struct metadata_logfile *metadatalog, struct m
83
extern int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno,
84
unsigned new_fileno);
85
extern int close_metadata_logfile(struct metadata_logfile *metadatalog);
86
+extern int fsync_metadata_logfile(struct metadata_logfile *metalogfile);
87
extern int unlink_metadata_logfile(struct metadata_logfile *metalogfile);
88
extern int destroy_metadata_logfile(struct metadata_logfile *metalogfile);
89
extern int create_metadata_logfile(struct metadata_logfile *metalogfile);
database/engine/metadata_log/metadatalog.c
+2
@@ -134,6 +134,7 @@ void metalog_try_link_new_metadata_logfile(struct metalog_worker_config *wc)
134
if (metalogfile->records.first) { /* it has records */
135
/* Finalize metadata log file and create a new one */
136
mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
137
+ fsync_metadata_logfile(ctx->metadata_logfiles.last);
138
ret = add_new_metadata_logfile(ctx, &ctx->metadata_logfiles, 0, ctx->last_fileno + 1);
139
if (likely(!ret)) {
140
++ctx->last_fileno;
@@ -364,6 +365,7 @@ void metalog_worker(void* arg)
365
case METALOG_COMPACTION_FLUSH:
366
mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log,
367
&ctx->compaction_state.new_metadata_logfiles);
368
+ fsync_metadata_logfile(ctx->compaction_state.new_metadata_logfiles.last);
369
complete(cmd.record_io_descr.completion);
370
break;
371
default: