@cryptotaxi247 / netdata-1 / commits / 525556aad

Improve error handling and logging for journal and data files (#20112)

Stelios Fragkakis committed Apr 17, 2025 at 20:31 UTC 525556aad6a46889a9426e043275b04d42b31014

5 files changed +67 -61

src/database/engine/datafile.c

+15 -7

@@ -259,7 +259,6 @@ int create_data_file(struct rrdengine_datafile *datafile)

259 return fd;

260 }

261 datafile->file = file;

262 - __atomic_add_fetch(&ctx->stats.datafile_creations, 1, __ATOMIC_RELAXED);

262

263 (void)posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

264 memset(superblock, 0, sizeof(*superblock));

@@ -269,19 +268,28 @@ int create_data_file(struct rrdengine_datafile *datafile)

268

269 iov = uv_buf_init((void *)superblock, sizeof(*superblock));

270

272 - ret = uv_fs_write(NULL, &req, file, &iov, 1, 0, NULL);

273 - if (ret < 0) {

274 - fatal_assert(req.result < 0);

275 - netdata_log_error("DBENGINE: uv_fs_write: %s", uv_strerror(ret));

276 - ctx_io_error(ctx);

271 + int retries = 10;

272 + ret = -1;

273 + while (ret < 0 && --retries) {

274 + ret = uv_fs_write(NULL, &req, file, &iov, 1, 0, NULL);

275 + uv_fs_req_cleanup(&req);

276 + if (ret < 0) {

277 + if (ret == -ENOSPC || ret == -EBADF || ret == -EACCES || ret == -EROFS || ret == -EINVAL)

278 + break;

279 + sleep_usec(300 * USEC_PER_MS);

280 + }

281 }

278 - uv_fs_req_cleanup(&req);

282 +

283 posix_memalign_freez(superblock);

284 if (ret < 0) {

285 destroy_data_file_unsafe(datafile);

286 + ctx_io_error(ctx);

287 + nd_log_limit_static_global_var(dbengine_erl, 10, 0);

288 + nd_log_limit(&dbengine_erl, NDLS_DAEMON, NDLP_ERR, "DBENGINE: Failed to create datafile %s", path);

289 return ret;

290 }

291

292 + __atomic_add_fetch(&ctx->stats.datafile_creations, 1, __ATOMIC_RELAXED);

293 datafile->pos = sizeof(*superblock);

294 ctx_io_write_op_bytes(ctx, sizeof(*superblock));

295

src/database/engine/journalfile.c

+26 -25

@@ -6,7 +6,7 @@

6 time_t dbengine_journal_v2_unmount_time = 120;

8 /* Careful to always call this before creating a new journal file */

9 -void journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, WAL *wal)

9 +int journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, WAL *wal)

10 {

11 uv_fs_t request;

12 struct rrdengine_journalfile *journalfile = datafile->journalfile;

@@ -27,35 +27,28 @@ void journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengin

28 int retries = 10;

29 int ret = -1;

30 - while (ret == -1 && --retries) {

30 + while (ret < 0 && --retries) {

31 ret = uv_fs_write(NULL, &request, journalfile->file, &iov, 1, (int64_t)journalfile_position, NULL);

32 - if (ret == -1) {

32 + uv_fs_req_cleanup(&request);

33 + if (ret < 0) {

34 + if (ret == -ENOSPC || ret == -EBADF || ret == -EACCES || ret == -EROFS || ret == -EINVAL)

35 + break;

36 sleep_usec(300 * USEC_PER_MS);

34 - uv_fs_req_cleanup(&request);

37 }

38 }

38 - bool jf_write_error = (ret == -1 || request.result < 0);

39 -

40 - if (unlikely(jf_write_error)) {

40 + if (unlikely(ret < 0)) {

41 ctx_io_error(ctx);

42 - if (ret == -1)

43 - netdata_log_error(

44 - "DBENGINE: %s: uv_fs_write: failed to store metadata in journalfile %u, offset %"PRIu64,

45 - __func__,

46 - datafile->fileno,

47 - journalfile_position);

48 - else

49 - netdata_log_error("DBENGINE: %s: uv_fs_write: %s", __func__, uv_strerror((int)request.result));

42 + goto done;

43 }

52 - uv_fs_req_cleanup(&request);

45 ctx_current_disk_space_increase(ctx, wal->buf_size);

46 ctx_io_write_op_bytes(ctx, wal->buf_size);

48 +done:

49 wal_release(wal);

57 - __atomic_sub_fetch(&ctx->atomic.extents_currently_being_flushed, 1, __ATOMIC_RELAXED);

50 worker_is_idle();

51 + return ret;

52 }

54 void journalfile_v2_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen)

@@ -599,7 +592,6 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi

592 return fd;

593 }

594 journalfile->file = file;

602 - __atomic_add_fetch(&ctx->stats.journalfile_creations, 1, __ATOMIC_RELAXED);

595

596 (void)posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

597 memset(superblock, 0, sizeof(*superblock));

@@ -608,21 +600,30 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi

600

601 iov = uv_buf_init((void *)superblock, sizeof(*superblock));

602

611 - ret = uv_fs_write(NULL, &req, file, &iov, 1, 0, NULL);

612 - if (ret < 0) {

613 - fatal_assert(req.result < 0);

614 - netdata_log_error("DBENGINE: uv_fs_write: %s", uv_strerror(ret));

615 - ctx_io_error(ctx);

603 + int retries = 10;

604 + ret = -1;

605 + while (ret < 0 && --retries) {

606 + ret = uv_fs_write(NULL, &req, file, &iov, 1, 0, NULL);

607 + uv_fs_req_cleanup(&req);

608 + if (ret < 0) {

609 + if (ret == -ENOSPC || ret == -EBADF || ret == -EACCES || ret == -EROFS || ret == -EINVAL)

610 + break;

611 + sleep_usec(300 * USEC_PER_MS);

612 + }

613 }

617 - uv_fs_req_cleanup(&req);

614 +

615 posix_memalign_freez(superblock);

616 +

617 if (ret < 0) {

618 journalfile_destroy_unsafe(journalfile, datafile);

619 + ctx_io_error(ctx);

620 + nd_log_limit_static_global_var(dbengine_erl, 10, 0);

621 + nd_log_limit(&dbengine_erl, NDLS_DAEMON, NDLP_ERR, "DBENGINE: Failed to create journlfile %s", path);

622 return ret;

623 }

624

625 + __atomic_add_fetch(&ctx->stats.journalfile_creations, 1, __ATOMIC_RELAXED);

626 journalfile->unsafe.pos = sizeof(*superblock);

625 -

627 ctx_io_write_op_bytes(ctx, sizeof(*superblock));

628

629 return 0;

src/database/engine/journalfile.h

+1 -1

@@ -255,7 +255,7 @@ struct wal;

255 void journalfile_v1_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen);

256 void journalfile_v2_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen);

257 struct rrdengine_journalfile *journalfile_alloc_and_init(struct rrdengine_datafile *datafile);

258 -void journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, struct wal *wal);

258 +int journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, struct wal *wal);

259 int journalfile_close(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);

260 int journalfile_unlink(struct rrdengine_journalfile *journalfile);

261 int journalfile_destroy_unsafe(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);

src/database/engine/rrdengine.c

+22 -26

@@ -730,7 +730,8 @@ static struct rrdengine_datafile *get_datafile_to_write_extent(struct rrdengine_

730 static struct extent_io_descriptor *

731 datafile_extent_build(struct rrdengine_instance *ctx, struct page_descr_with_data *base, uv_buf_t *iov)

732 {

733 - unsigned i, count, size_bytes, pos, real_io_size;

733 + unsigned i;

734 + uint32_t real_io_size, size_bytes, count, pos;

735 uint32_t uncompressed_payload_length, max_compressed_size, payload_offset;

736 struct page_descr_with_data *descr, *eligible_pages[MAX_PAGES_PER_EXTENT];

737 struct extent_io_descriptor *xt_io_descr;

@@ -848,15 +849,15 @@ datafile_extent_build(struct rrdengine_instance *ctx, struct page_descr_with_dat

849 journalfile_extent_build(ctx, xt_io_descr);

850

851 ctx_last_flush_fileno_set(ctx, datafile->fileno);

851 - ctx_current_disk_space_increase(ctx, real_io_size);

852 - ctx_io_write_op_bytes(ctx, real_io_size);

852 + xt_io_descr->real_io_size = real_io_size;

853

854 return xt_io_descr;

855 }

856

857 static void after_extent_write(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t* uv_work_req __maybe_unused, int status __maybe_unused)

858 {

859 - ;

859 + if(completion)

860 + completion_mark_complete(completion);

861 }

862

863 static void *extent_write_tp_worker(

@@ -878,32 +879,27 @@ static void *extent_write_tp_worker(

879

880 int retries = 10;

881 int ret = -1;

881 - while (ret == -1 && --retries) {

882 + while (ret < 0 && --retries) {

883 ret = uv_fs_write(NULL, &request, datafile->file, &iov, 1, (int64_t)xt_io_descr->pos, NULL);

883 - if (ret == -1) {

884 + uv_fs_req_cleanup(&request);

885 + if (ret < 0) {

886 + if (ret == -ENOSPC || ret == -EBADF || ret == -EACCES || ret == -EROFS || ret == -EINVAL)

887 + break;

888 sleep_usec(300 * USEC_PER_MS);

885 - uv_fs_req_cleanup(&request);

889 }

890 }

891

889 - bool df_write_error = (ret == -1 || request.result < 0);

890 -

891 - if (unlikely(df_write_error)) {

892 + if (unlikely(ret < 0))

893 ctx_io_error(ctx);

893 - if (ret == -1)

894 - netdata_log_error(

895 - "DBENGINE: %s: uv_fs_write: failed to store metrics in datafile %u, offset %ld",

896 - __func__,

897 - datafile->fileno,

898 - (int64_t)xt_io_descr->pos);

899 - else

900 - netdata_log_error(

901 - "DBENGINE: %s: uv_fs_write: %s", __func__, uv_strerror((int)request.result));

894 + else {

895 + ctx_current_disk_space_increase(ctx, xt_io_descr->real_io_size);

896 + ctx_io_write_op_bytes(ctx, xt_io_descr->real_io_size);

897 + ret = journalfile_v1_extent_write(ctx, datafile, xt_io_descr->wal);

898 }

903 - uv_fs_req_cleanup(&request);

899

905 - if (likely(!df_write_error)) {

906 - journalfile_v1_extent_write(ctx, datafile, xt_io_descr->wal);

900 + if (ret < 0) {

901 + nd_log_limit_static_global_var(dbengine_erl, 10, 0);

902 + nd_log_limit(&dbengine_erl, NDLS_DAEMON, NDLP_ERR, "DBENGINE: Tier %d, %s", ctx->config.tier, uv_strerror(ret));

903 }

904

905 spinlock_lock(&datafile->writers.spinlock);

@@ -911,14 +907,12 @@ static void *extent_write_tp_worker(

907 datafile->writers.flushed_to_open_running++;

908 spinlock_unlock(&datafile->writers.spinlock);

909

914 - extent_flush_to_open(ctx, xt_io_descr, df_write_error);

910 + extent_flush_to_open(ctx, xt_io_descr, ret < 0);

911

912 if(ctx_is_available_for_queries(ctx) && rrdeng_ctx_tier_cap_exceeded(ctx))

913 rrdeng_enq_cmd(ctx, RRDENG_OPCODE_DATABASE_ROTATE, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);

914 done:

919 - if(completion)

920 - completion_mark_complete(completion);

921 -

915 + __atomic_sub_fetch(&ctx->atomic.extents_currently_being_flushed, 1, __ATOMIC_RELAXED);

916 worker_is_idle();

917 return NULL;

918 }

@@ -1273,6 +1267,7 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *

1267 deleted_bytes = journalfile_v2_data_size_get(journal_file);

1268

1269 netdata_log_info("DBENGINE: deleting data and journal files to maintain disk quota");

1270 + // This will delete journalfile_v2 and journalfile_v1

1271 ret = journalfile_destroy_unsafe(journal_file, datafile);

1272 if (!ret) {

1273 journalfile_v1_generate_path(datafile, path, sizeof(path));

@@ -1281,6 +1276,7 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *

1276 netdata_log_info("DBENGINE: deleted journal file \"%s\".", path);

1277 deleted_bytes += journal_file_bytes;

1278 }

1279 + // This will delete the datafile

1280 ret = destroy_data_file_unsafe(datafile);

1281 if (!ret) {

1282 generate_datafilepath(datafile, path, sizeof(path));

src/database/engine/rrdengine.h

+3 -2

@@ -288,8 +288,9 @@ struct extent_io_descriptor {

288 struct rrdengine_instance *ctx;

289 void *buf;

290 uint64_t pos;

291 - unsigned descr_count;

292 - unsigned bytes;

291 + uint32_t descr_count;

292 + uint32_t bytes;

293 + uint32_t real_io_size;

294 struct wal *wal;

295 uv_file file;

296 struct page_descr_with_data *descr_array[MAX_PAGES_PER_EXTENT];