@cryptotaxi247 / netdata-1 / commits / 4ee083053

unified out of memory handling (#19717)

* unified out of memory handling * fix windows cleanup and exit

Costa Tsaousis committed Feb 26, 2025 at 13:58 UTC 4ee0830534942520ff6dada6274795dce6ae64b9

8 files changed +66 -67

src/daemon/daemon-status-file.c

-1

@@ -56,7 +56,6 @@ static XXH64_hash_t daemon_status_file_hash(DAEMON_STATUS_FILE *ds, const char *

56 buffer_json_member_add_string_or_empty(wb, "fatal_filename", ds->fatal.filename);

57 buffer_json_member_add_string_or_empty(wb, "fatal_errno", ds->fatal.errno_str);

58 buffer_json_member_add_string_or_empty(wb, "fatal_function", ds->fatal.function);

59 - buffer_json_member_add_string_or_empty(wb, "fatal_message", ds->fatal.message);

59 buffer_json_member_add_string_or_empty(wb, "fatal_stack_trace", ds->fatal.stack_trace);

60 buffer_json_member_add_string(wb, "message", msg);

61 buffer_json_member_add_string(wb, "cause", cause);

src/daemon/winsvc.cc

@@ -2,6 +2,7 @@ extern "C" {

3 #include "daemon.h"

4 #include "libnetdata/libnetdata.h"

5 +#include "daemon/daemon-shutdown.h"

7 int netdata_main(int argc, char *argv[]);

8 void nd_process_signals(void);

src/database/engine/datafile.c

+2 -8

@@ -260,10 +260,7 @@ int create_data_file(struct rrdengine_datafile *datafile)

260 datafile->file = file;

261 __atomic_add_fetch(&ctx->stats.datafile_creations, 1, __ATOMIC_RELAXED);

262

263 - ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

264 - if (unlikely(ret)) {

265 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));

266 - }

263 + (void)posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

264 memset(superblock, 0, sizeof(*superblock));

265 (void) strncpy(superblock->magic_number, RRDENG_DF_MAGIC, RRDENG_MAGIC_SZ);

266 (void) strncpy(superblock->version, RRDENG_DF_VER, RRDENG_VER_SZ);

@@ -297,10 +294,7 @@ static int check_data_file_superblock(uv_file file)

294 uv_buf_t iov;

295 uv_fs_t req;

296

300 - ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

301 - if (unlikely(ret)) {

302 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));

303 - }

297 + (void)posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

298 iov = uv_buf_init((void *)superblock, sizeof(*superblock));

299

300 ret = uv_fs_read(NULL, &req, file, &iov, 1, 0, NULL);

src/database/engine/journalfile.c

+3 -11

@@ -580,10 +580,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi

580 journalfile->file = file;

581 __atomic_add_fetch(&ctx->stats.journalfile_creations, 1, __ATOMIC_RELAXED);

582

583 - ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

584 - if (unlikely(ret)) {

585 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));

586 - }

583 + (void)posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

584 memset(superblock, 0, sizeof(*superblock));

585 (void) strncpy(superblock->magic_number, RRDENG_JF_MAGIC, RRDENG_MAGIC_SZ);

586 (void) strncpy(superblock->version, RRDENG_JF_VER, RRDENG_VER_SZ);

@@ -617,10 +614,7 @@ static int journalfile_check_superblock(uv_file file)

614 uv_buf_t iov;

615 uv_fs_t req;

616

620 - ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

621 - if (unlikely(ret)) {

622 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));

623 - }

617 + (void)posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

618 iov = uv_buf_init((void *)superblock, sizeof(*superblock));

619

620 ret = uv_fs_read(NULL, &req, file, &iov, 1, 0, NULL);

@@ -813,9 +807,7 @@ static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx,

807 file_size = journalfile->unsafe.pos;

808

809 max_id = 1;

816 - ret = posix_memalignz((void *)&buf, RRDFILE_ALIGNMENT, READAHEAD_BYTES);

817 - if (unlikely(ret))

818 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));

810 + (void)posix_memalignz((void *)&buf, RRDFILE_ALIGNMENT, READAHEAD_BYTES);

811

812 for (pos = sizeof(struct rrdeng_jf_sb); pos < file_size; pos += READAHEAD_BYTES) {

813 size_bytes = MIN(READAHEAD_BYTES, file_size - pos);

src/database/engine/pdc.c

+2 -4

@@ -1216,12 +1216,10 @@ static inline void *datafile_extent_read(struct rrdengine_instance *ctx, uv_file

1216 uv_fs_t request;

1217

1218 unsigned real_io_size = ALIGN_BYTES_CEILING(size_bytes);

1219 - int ret = posix_memalignz(&buffer, RRDFILE_ALIGNMENT, real_io_size);

1220 - if (unlikely(ret))

1221 - fatal("DBENGINE: posix_memalign(): %s", strerror(ret));

1219 + (void)posix_memalignz(&buffer, RRDFILE_ALIGNMENT, real_io_size);

1220

1221 uv_buf_t iov = uv_buf_init(buffer, real_io_size);

1224 - ret = uv_fs_read(NULL, &request, file, &iov, 1, (int64_t)pos, NULL);

1222 + int ret = uv_fs_read(NULL, &request, file, &iov, 1, (int64_t)pos, NULL);

1223 if (unlikely(-1 == ret)) {

1224 ctx_io_error(ctx);

1225 posix_memalign_freez(buffer);

src/database/engine/rrdengine.c

+2 -9

@@ -393,9 +393,7 @@ WAL *wal_get(struct rrdengine_instance *ctx, unsigned size) {

393 if(unlikely(!wal)) {

394 wal = mallocz(sizeof(WAL));

395 wal->buf_size = RRDENG_BLOCK_SIZE;

396 - int ret = posix_memalignz((void *)&wal->buf, RRDFILE_ALIGNMENT, wal->buf_size);

397 - if (unlikely(ret))

398 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));

396 + (void)posix_memalignz((void *)&wal->buf, RRDFILE_ALIGNMENT, wal->buf_size);

397 __atomic_add_fetch(&wal_globals.atomics.allocated, 1, __ATOMIC_RELAXED);

398 }

399

@@ -732,7 +730,6 @@ static struct rrdengine_datafile *get_datafile_to_write_extent(struct rrdengine_

730 static struct extent_io_descriptor *

731 datafile_extent_build(struct rrdengine_instance *ctx, struct page_descr_with_data *base, uv_buf_t *iov)

732 {

735 - int ret;

733 unsigned i, count, size_bytes, pos, real_io_size;

734 uint32_t uncompressed_payload_length, max_compressed_size, payload_offset;

735 struct page_descr_with_data *descr, *eligible_pages[MAX_PAGES_PER_EXTENT];

@@ -763,11 +760,7 @@ datafile_extent_build(struct rrdengine_instance *ctx, struct page_descr_with_dat

760 payload_offset = sizeof(*header) + count * sizeof(header->descr[0]);

761 max_compressed_size = dbengine_max_compressed_size(uncompressed_payload_length, compression_algorithm);

762 size_bytes = payload_offset + MAX(uncompressed_payload_length, max_compressed_size) + sizeof(*trailer);

766 - ret = posix_memalignz((void *)&xt_io_descr->buf, RRDFILE_ALIGNMENT, ALIGN_BYTES_CEILING(size_bytes));

767 - if (unlikely(ret)) {

768 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));

769 - /* freez(xt_io_descr);*/

770 - }

763 + (void)posix_memalignz((void *)&xt_io_descr->buf, RRDFILE_ALIGNMENT, ALIGN_BYTES_CEILING(size_bytes));

764 memset(xt_io_descr->buf, 0, ALIGN_BYTES_CEILING(size_bytes));

765 (void) memcpy(xt_io_descr->descr_array, eligible_pages, sizeof(struct page_descr_with_data *) * count);

766 xt_io_descr->descr_count = count;

src/libnetdata/memory/nd-mallocz.c

+51 -29

@@ -386,72 +386,94 @@ void freez_int(void *ptr, const char *file, const char *function, size_t line) {

386 }

387 #else

388

389 -ALWAYS_INLINE char *strdupz(const char *s) {

389 +ALWAYS_INLINE NORETURN

390 +static void out_of_memory(const char *call, size_t size) {

391 + struct rusage usage = { 0 };

392 + if(getrusage(RUSAGE_SELF, &usage) != 0)

393 + usage.ru_maxrss = 0;

394 +

395 + OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

396 + fatal("Out of memory on %s(%zu bytes)!\n"

397 + "System memory available: %lu, while our max RSS usage is: %ld\n"

398 + "O/S mmap limit: %llu, while our mmap count is: %zu",

399 + call, size,

400 + sm.ram_available_bytes, usage.ru_maxrss,

401 + os_mmap_limit(), __atomic_load_n(&nd_mmap_count, __ATOMIC_RELAXED));

402 +}

403 +

404 +ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED

405 +char *strdupz(const char *s) {

406 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_STRDUP);

407

408 char *t = strdup(s);

393 - if (unlikely(!t)) {

394 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

395 - fatal("Cannot strdup() string '%s' (system memory available bytes: %lu)", s, sm.ram_available_bytes);

396 - }

409 + if (unlikely(!t))

410 + out_of_memory(__FUNCTION__ , strlen(s) + 1);

411 +

412 return t;

413 }

414

400 -ALWAYS_INLINE char *strndupz(const char *s, size_t len) {

415 +ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED

416 +char *strndupz(const char *s, size_t len) {

417 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_STRNDUP);

418

419 char *t = strndup(s, len);

404 - if (unlikely(!t)) {

405 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

406 - fatal("Cannot strndup() string '%s' of len %zu (system memory available bytes: %lu)", s, len, sm.ram_available_bytes);

407 - }

420 + if (unlikely(!t))

421 + out_of_memory(__FUNCTION__ , len + 1);

422 +

423 return t;

424 }

425

426 // If ptr is NULL, no operation is performed.

412 -ALWAYS_INLINE void freez(void *ptr) {

427 +ALWAYS_INLINE

428 +void freez(void *ptr) {

429 if(likely(ptr)) {

430 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_FREE);

431 free(ptr);

432 }

433 }

434

419 -ALWAYS_INLINE void *mallocz(size_t size) {

435 +ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED

436 +void *mallocz(size_t size) {

437 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_MALLOC);

438 void *p = malloc(size);

422 - if (unlikely(!p)) {

423 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

424 - fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", size, sm.ram_available_bytes);

425 - }

439 + if (unlikely(!p))

440 + out_of_memory(__FUNCTION__, size);

441 +

442 return p;

443 }

444

429 -ALWAYS_INLINE void *callocz(size_t nmemb, size_t size) {

445 +ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED

446 +void *callocz(size_t nmemb, size_t size) {

447 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_CALLOC);

448 void *p = calloc(nmemb, size);

432 - if (unlikely(!p)) {

433 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

434 - fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", nmemb * size, sm.ram_available_bytes);

435 - }

449 + if (unlikely(!p))

450 + out_of_memory(__FUNCTION__, nmemb * size);

451 +

452 return p;

453 }

454

439 -ALWAYS_INLINE void *reallocz(void *ptr, size_t size) {

455 +ALWAYS_INLINE MALLOCLIKE NEVERNULL WARNUNUSED

456 +void *reallocz(void *ptr, size_t size) {

457 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_REALLOC);

458 void *p = realloc(ptr, size);

442 - if (unlikely(!p)) {

443 - OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

444 - fatal("Cannot re-allocate memory to %zu bytes. (system memory available bytes: %lu)", size, sm.ram_available_bytes);

445 - }

459 + if (unlikely(!p))

460 + out_of_memory(__FUNCTION__, size);

461 +

462 return p;

463 }

464

449 -ALWAYS_INLINE int posix_memalignz(void **memptr, size_t alignment, size_t size) {

465 +ALWAYS_INLINE

466 +int posix_memalignz(void **memptr, size_t alignment, size_t size) {

467 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN);

451 - return posix_memalign(memptr, alignment, size);

468 + int rc = posix_memalign(memptr, alignment, size);

469 + if(unlikely(rc))

470 + out_of_memory(__FUNCTION__, size);

471 +

472 + return rc;

473 }

474

454 -ALWAYS_INLINE void posix_memalign_freez(void *ptr) {

475 +ALWAYS_INLINE

476 +void posix_memalign_freez(void *ptr) {

477 workers_memory_call(WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN_FREE);

478 free(ptr);

479 }

src/libnetdata/memory/nd-mallocz.h

+5 -5

@@ -51,11 +51,11 @@ void freez_int(void *ptr, const char *file, const char *function, size_t line);

51 size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line);

53 #else // NETDATA_TRACE_ALLOCATIONS

54 -char *strdupz(const char *s) MALLOCLIKE NEVERNULL;

55 -char *strndupz(const char *s, size_t len) MALLOCLIKE NEVERNULL;

56 -void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;

57 -void *mallocz(size_t size) MALLOCLIKE NEVERNULL;

58 -void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL;

54 +char *strdupz(const char *s) MALLOCLIKE NEVERNULL WARNUNUSED;

55 +char *strndupz(const char *s, size_t len) MALLOCLIKE NEVERNULL WARNUNUSED;

56 +void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL WARNUNUSED;

57 +void *mallocz(size_t size) MALLOCLIKE NEVERNULL WARNUNUSED;

58 +void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL WARNUNUSED;

59 void freez(void *ptr);

60 #endif // NETDATA_TRACE_ALLOCATIONS