@cryptotaxi247 / netdata-1 / commits / 6da3a1650

status file improvements (dedup and signal handler use) (#19751)

* remove the address from the stack trace to make it unique * log backtrace failure to get symbols * improve status file handling via the signal handler * add already-running exit reason

Costa Tsaousis committed Mar 3, 2025 at 11:59 UTC 6da3a165090ab00f5966cd3ce431b3f315c52b25

8 files changed +243 -152

src/aclk/aclk_proxy.c

+2 -1

@@ -112,7 +112,8 @@ static inline int check_http_environment(const char **proxy)

112

113 char buf[1024];

114 snprintfz(buf, sizeof(buf),

115 - "Environment var '%s' defined but of unknown format. Supported syntax: 'http://[user:pass@]host:port'.",

115 + "Environment var '%s' defined but of unknown format '%s'. "

116 + "Supported syntax: 'http://[user:pass@]host:port'.",

117 var, tmp);

118 safe_log_proxy_error(buf, tmp);

119

src/daemon/daemon-status-file.c

+196 -132

@@ -37,14 +37,37 @@ ENUM_STR_MAP_DEFINE(DAEMON_OS_TYPE) = {

37 };

38 ENUM_STR_DEFINE_FUNCTIONS(DAEMON_OS_TYPE, DAEMON_OS_TYPE_UNKNOWN, "unknown");

40 -static DAEMON_STATUS_FILE last_session_status = { 0 };

41 -static DAEMON_STATUS_FILE session_status = { 0 };

42 -static SPINLOCK dsf_spinlock = SPINLOCK_INITIALIZER;

40 +static DAEMON_STATUS_FILE last_session_status = {

41 + .spinlock = SPINLOCK_INITIALIZER,

42 + .fatal = {

43 + .spinlock = SPINLOCK_INITIALIZER,

44 + },

45 + .dedup = {

46 + .spinlock = SPINLOCK_INITIALIZER,

47 + },

48 +};

49 +

50 +static DAEMON_STATUS_FILE session_status = {

51 + .spinlock = SPINLOCK_INITIALIZER,

52 + .fatal = {

53 + .spinlock = SPINLOCK_INITIALIZER,

54 + },

55 + .dedup = {

56 + .spinlock = SPINLOCK_INITIALIZER,

57 + },

58 +};

59 +

60 +static void daemon_status_file_out_of_memory(void);

61 +

62 +// these are used instead of locks when locks cannot be used (signal handler, out of memory, etc)

63 +#define dsf_acquire(ds) __atomic_load_n(&(ds).v, __ATOMIC_ACQUIRE)

64 +#define dsf_release(ds) __atomic_store_n(&(ds).v, STATUS_FILE_VERSION, __ATOMIC_RELEASE)

66 // --------------------------------------------------------------------------------------------------------------------

67 // json generation

69 static XXH64_hash_t daemon_status_file_hash(DAEMON_STATUS_FILE *ds, const char *msg, const char *cause) {

70 + dsf_acquire(*ds);

71 CLEAN_BUFFER *wb = buffer_create(0, NULL);

72 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);

73 buffer_json_member_add_uint64(wb, "version", STATUS_FILE_VERSION);

@@ -62,12 +85,15 @@ static XXH64_hash_t daemon_status_file_hash(DAEMON_STATUS_FILE *ds, const char *

85 buffer_json_member_add_string(wb, "status", DAEMON_STATUS_2str(ds->status));

86 EXIT_REASON_2json(wb, "exit_reason", ds->exit_reason);

87 ND_PROFILE_2json(wb, "profile", ds->profile);

88 + dsf_release(*ds);

89 buffer_json_finalize(wb);

90 XXH64_hash_t hash = XXH3_64bits((const void *)buffer_tostring(wb), buffer_strlen(wb));

91 return hash;

92 }

94 static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

95 + dsf_acquire(*ds);

96 +

97 buffer_json_member_add_datetime_rfc3339(wb, "@timestamp", ds->timestamp_ut, true); // ECS

98 buffer_json_member_add_uint64(wb, "version", STATUS_FILE_VERSION); // custom

@@ -159,19 +185,21 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

185

186 buffer_json_member_add_array(wb, "dedup"); // custom

187 {

162 - for(size_t i = 0; i < _countof(ds->dedup); i++) {

163 - if (ds->dedup[i].timestamp_ut == 0)

188 + for(size_t i = 0; i < _countof(ds->dedup.slot); i++) {

189 + if (ds->dedup.slot[i].timestamp_ut == 0)

190 continue;

191

192 buffer_json_add_array_item_object(wb); // custom

193 {

168 - buffer_json_member_add_datetime_rfc3339(wb, "@timestamp", ds->dedup[i].timestamp_ut, true); // custom

169 - buffer_json_member_add_uint64(wb, "hash", ds->dedup[i].hash); // custom

194 + buffer_json_member_add_datetime_rfc3339(wb, "@timestamp", ds->dedup.slot[i].timestamp_ut, true); // custom

195 + buffer_json_member_add_uint64(wb, "hash", ds->dedup.slot[i].hash); // custom

196 }

197 buffer_json_object_close(wb);

198 }

199 }

200 buffer_json_array_close(wb);

201 +

202 + dsf_release(*ds);

203 }

204

205 // --------------------------------------------------------------------------------------------------------------------

@@ -278,9 +306,9 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

306 datetime[0] = '\0';

307 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required_v3);

308 if (datetime[0])

281 - ds->dedup[0].timestamp_ut = rfc3339_parse_ut(datetime, NULL);

309 + ds->dedup.slot[0].timestamp_ut = rfc3339_parse_ut(datetime, NULL);

310

283 - JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "hash", ds->dedup[0].hash, error, required_v3);

311 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "hash", ds->dedup.slot[0].hash, error, required_v3);

312 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "restarts", ds->restarts, error, required_v3);

313 });

314 }

@@ -288,14 +316,14 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

316 JSONC_PARSE_ARRAY(jobj, path, "dedup", error, required_v4, {

317 size_t i = 0;

318 JSONC_PARSE_ARRAY_ITEM_OBJECT(jobj, path, i, required_v4, {

291 - if(i >= _countof(ds->dedup))

319 + if(i >= _countof(ds->dedup.slot))

320 break;

321

322 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required_v4);

323 if (datetime[0])

296 - ds->dedup[i].timestamp_ut = rfc3339_parse_ut(datetime, NULL);

324 + ds->dedup.slot[i].timestamp_ut = rfc3339_parse_ut(datetime, NULL);

325

298 - JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "hash", ds->dedup[i].hash, error, required_v4);

326 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "hash", ds->dedup.slot[i].hash, error, required_v4);

327 });

328 });

329 }

@@ -307,10 +335,11 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

335 // get the current status

336

337 static void daemon_status_file_refresh(DAEMON_STATUS status) {

310 - spinlock_lock(&dsf_spinlock);

311 -

338 usec_t now_ut = now_realtime_usec();

339

340 + dsf_acquire(session_status);

341 + spinlock_lock(&session_status.spinlock);

342 +

343 #if defined(OS_LINUX)

344 session_status.os_type = DAEMON_OS_TYPE_LINUX;

345 #elif defined(OS_FREEBSD)

@@ -321,11 +350,17 @@ static void daemon_status_file_refresh(DAEMON_STATUS status) {

350 session_status.os_type = DAEMON_OS_TYPE_WINDOWS;

351 #endif

352

353 + if(session_status.status != DAEMON_STATUS_INITIALIZING && status == DAEMON_STATUS_INITIALIZING)

354 + session_status.timings.init_started_ut = now_ut;

355 +

356 + if(session_status.status != DAEMON_STATUS_EXITING && status == DAEMON_STATUS_EXITING)

357 + session_status.timings.exit_started_ut = now_ut;

358 +

359 if(session_status.status == DAEMON_STATUS_INITIALIZING && status == DAEMON_STATUS_RUNNING)

325 - session_status.timings.init = (time_t)((now_ut - session_status.timestamp_ut + USEC_PER_SEC/2) / USEC_PER_SEC);

360 + session_status.timings.init = (time_t)((now_ut - session_status.timings.init_started_ut + USEC_PER_SEC/2) / USEC_PER_SEC);

361

362 if(session_status.status == DAEMON_STATUS_EXITING && status == DAEMON_STATUS_EXITED)

328 - session_status.timings.exit = (time_t)((now_ut - session_status.timestamp_ut + USEC_PER_SEC/2) / USEC_PER_SEC);

363 + session_status.timings.exit = (time_t)((now_ut - session_status.timings.exit_started_ut + USEC_PER_SEC/2) / USEC_PER_SEC);

364

365 strncpyz(session_status.version, NETDATA_VERSION, sizeof(session_status.version) - 1);

366

@@ -384,9 +419,9 @@ static void daemon_status_file_refresh(DAEMON_STATUS status) {

419 session_status.os_id_like = strdupz(last_session_status.os_id_like);

420 if(!session_status.restarts)

421 session_status.restarts = last_session_status.restarts + 1;

387 - if(!session_status.dedup[0].timestamp_ut || !session_status.dedup[0].hash) {

388 - for (size_t i = 0; i < _countof(session_status.dedup); i++)

389 - session_status.dedup[i] = last_session_status.dedup[i];

422 + if(!session_status.dedup.slot[0].timestamp_ut || !session_status.dedup.slot[0].hash) {

423 + for (size_t i = 0; i < _countof(session_status.dedup.slot); i++)

424 + session_status.dedup.slot[i] = last_session_status.dedup.slot[i];

425 }

426

427 if(!session_status.install_type) {

@@ -408,7 +443,8 @@ static void daemon_status_file_refresh(DAEMON_STATUS status) {

443 session_status.memory = os_system_memory(true);

444 session_status.var_cache = os_disk_space(netdata_configured_cache_dir);

445

411 - spinlock_unlock(&dsf_spinlock);

446 + spinlock_unlock(&session_status.spinlock);

447 + dsf_release(session_status);

448 }

449

450 // --------------------------------------------------------------------------------------------------------------------

@@ -608,136 +644,97 @@ static void static_save_buffer_init(void) {

644 buffer_flush(static_save_buffer);

645 }

646

611 -static void daemon_status_file_save(DAEMON_STATUS_FILE *ds, bool have_lock) {

612 - if(!have_lock)

613 - spinlock_lock(&dsf_spinlock);

614 -

615 - static_save_buffer_init();

616 -

617 - buffer_json_initialize(static_save_buffer, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);

618 - daemon_status_file_to_json(static_save_buffer, ds);

619 - buffer_json_finalize(static_save_buffer);

647 +static void daemon_status_file_save(BUFFER *wb, DAEMON_STATUS_FILE *ds, bool log) {

648 + buffer_flush(wb);

649 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);

650 + daemon_status_file_to_json(wb, ds);

651 + buffer_json_finalize(wb);

652

621 - const char *content = buffer_tostring(static_save_buffer);

622 - size_t content_size = buffer_strlen(static_save_buffer);

653 + const char *content = buffer_tostring(wb);

654 + size_t content_size = buffer_strlen(wb);

655

656 // Try primary directory first

657 bool saved = false;

658 if (save_status_file(netdata_configured_cache_dir, content, content_size))

659 saved = true;

660 else {

629 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Failed to save status file in primary directory %s",

630 - netdata_configured_cache_dir);

661 + if(log)

662 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Failed to save status file in primary directory %s",

663 + netdata_configured_cache_dir);

664

665 // Try each fallback directory until successful

666 for(size_t i = 0; i < _countof(status_file_fallbacks); i++) {

667 if (save_status_file(status_file_fallbacks[i], content, content_size)) {

635 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Saved status file in fallback %s", status_file_fallbacks[i]);

668 + if(log)

669 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Saved status file in fallback %s", status_file_fallbacks[i]);

670 saved = true;

671 break;

672 }

673 }

674 }

675

642 - if (!saved)

676 + if (!saved && log)

677 nd_log(NDLS_DAEMON, NDLP_ERR, "Failed to save status file in any location");

644 -

645 - if(!have_lock)

646 - spinlock_unlock(&dsf_spinlock);

647 -}

648 -

649 -void daemon_status_file_update_status(DAEMON_STATUS status) {

650 - daemon_status_file_refresh(status);

651 - daemon_status_file_save(&session_status, false);

652 -}

653 -

654 -void daemon_status_file_exit_reason_save(EXIT_REASON reason) {

655 - exit_initiated_add(reason);

656 - spinlock_lock(&dsf_spinlock);

657 - session_status.exit_reason = exit_initiated;

658 - spinlock_unlock(&dsf_spinlock);

659 - daemon_status_file_save(&session_status, false);

660 -}

661 -

662 -static void daemon_status_file_out_of_memory(void) {

663 - daemon_status_file_exit_reason_save(EXIT_REASON_OUT_OF_MEMORY);

664 -}

665 -

666 -void daemon_status_file_deadly_signal_received(EXIT_REASON reason) {

667 - // DO NOT LOCK IN THIS FUNCTION - WE CRASHED ALREADY AND WE ARE INSIDE THE SIGNAL HANDLER!

668 -

669 - session_status.exit_reason |= reason;

670 -

671 - if(!session_status.fatal.thread[0])

672 - strncpyz(session_status.fatal.thread, nd_thread_tag(), sizeof(session_status.fatal.thread) - 1);

673 -

674 - if(!session_status.fatal.stack_trace[0]) {

675 - static_save_buffer_init();

676 - capture_stack_trace(static_save_buffer);

677 - strncpyz(session_status.fatal.stack_trace, buffer_tostring(static_save_buffer), sizeof(session_status.fatal.stack_trace) - 1);

678 - }

679 -

680 - daemon_status_file_save(&session_status, true);

678 }

679

680 // --------------------------------------------------------------------------------------------------------------------

681 // deduplication hashes management

682

686 -static bool dedup_already_posted(XXH64_hash_t hash) {

687 - spinlock_lock(&dsf_spinlock);

683 +static bool dedup_already_posted(DAEMON_STATUS_FILE *ds, XXH64_hash_t hash) {

684 + spinlock_lock(&ds->dedup.spinlock);

685

686 usec_t now_ut = now_realtime_usec();

687

691 - for(size_t i = 0; i < _countof(session_status.dedup); i++) {

692 - if(session_status.dedup[i].timestamp_ut == 0)

688 + for(size_t i = 0; i < _countof(ds->dedup.slot); i++) {

689 + if(ds->dedup.slot[i].timestamp_ut == 0)

690 continue;

691

695 - if(hash == session_status.dedup[i].hash &&

696 - now_ut - session_status.dedup[i].timestamp_ut < 86400 * USEC_PER_SEC) {

692 + if(hash == ds->dedup.slot[i].hash &&

693 + now_ut - ds->dedup.slot[i].timestamp_ut < 86400 * USEC_PER_SEC) {

694 // we have already posted this crash

698 - spinlock_unlock(&dsf_spinlock);

695 + spinlock_unlock(&ds->dedup.spinlock);

696 return true;

697 }

698 }

699

703 - spinlock_unlock(&dsf_spinlock);

700 + spinlock_unlock(&ds->dedup.spinlock);

701 return false;

702 }

703

704 static void dedup_keep_hash(DAEMON_STATUS_FILE *ds, XXH64_hash_t hash) {

708 - spinlock_lock(&dsf_spinlock);

705 + spinlock_lock(&ds->dedup.spinlock);

706

707 // find the same hash

711 - for(size_t i = 0; i < _countof(ds->dedup); i++) {

712 - if(ds->dedup[i].hash == hash) {

713 - ds->dedup[i].hash = hash;

714 - ds->dedup[i].timestamp_ut = now_realtime_usec();

715 - spinlock_unlock(&dsf_spinlock);

708 + for(size_t i = 0; i < _countof(ds->dedup.slot); i++) {

709 + if(ds->dedup.slot[i].hash == hash) {

710 + ds->dedup.slot[i].hash = hash;

711 + ds->dedup.slot[i].timestamp_ut = now_realtime_usec();

712 + spinlock_unlock(&ds->dedup.spinlock);

713 return;

714 }

715 }

716

717 // find an empty slot

721 - for(size_t i = 0; i < _countof(ds->dedup); i++) {

722 - if(!ds->dedup[i].hash) {

723 - ds->dedup[i].hash = hash;

724 - ds->dedup[i].timestamp_ut = now_realtime_usec();

725 - spinlock_unlock(&dsf_spinlock);

718 + for(size_t i = 0; i < _countof(ds->dedup.slot); i++) {

719 + if(!ds->dedup.slot[i].hash) {

720 + ds->dedup.slot[i].hash = hash;

721 + ds->dedup.slot[i].timestamp_ut = now_realtime_usec();

722 + spinlock_unlock(&ds->dedup.spinlock);

723 return;

724 }

725 }

726

727 // find the oldest slot

728 size_t store_at_slot = 0;

732 - for(size_t i = 1; i < _countof(ds->dedup); i++) {

733 - if(ds->dedup[i].timestamp_ut < ds->dedup[store_at_slot].timestamp_ut)

729 + for(size_t i = 1; i < _countof(ds->dedup.slot); i++) {

730 + if(ds->dedup.slot[i].timestamp_ut < ds->dedup.slot[store_at_slot].timestamp_ut)

731 store_at_slot = i;

732 }

733

737 - ds->dedup[store_at_slot].hash = hash;

738 - ds->dedup[store_at_slot].timestamp_ut = now_realtime_usec();

734 + ds->dedup.slot[store_at_slot].hash = hash;

735 + ds->dedup.slot[store_at_slot].timestamp_ut = now_realtime_usec();

736

740 - spinlock_unlock(&dsf_spinlock);

737 + spinlock_unlock(&ds->dedup.spinlock);

738 }

739

740 // --------------------------------------------------------------------------------------------------------------------

@@ -776,7 +773,7 @@ void post_status_file(struct post_status_file_thread_data *d) {

773 if(rc == CURLE_OK) {

774 XXH64_hash_t hash = daemon_status_file_hash(d->status, d->msg, d->cause);

775 dedup_keep_hash(&session_status, hash);

779 - daemon_status_file_save(&session_status, false);

776 + daemon_status_file_save(wb, &session_status, true);

777 }

778

779 curl_easy_cleanup(curl);

@@ -808,6 +805,8 @@ struct log_priority PRI_BAD_BUT_NO_REASON = { NDLP_ERR, NDLP_WARNING };

805 void daemon_status_file_check_crash(void) {

806 FUNCTION_RUN_ONCE();

807

808 + static_save_buffer_init();

809 +

810 mallocz_register_out_of_memory_cb(daemon_status_file_out_of_memory);

811

812 last_session_status = daemon_status_file_load();

@@ -875,6 +874,11 @@ void daemon_status_file_check_crash(void) {

874 msg = "Netdata was last crashed while starting, because it couldn't allocate memory";

875 pri = PRI_USER_SHOULD_FIX;

876 }

877 + else if (last_session_status.exit_reason & EXIT_REASON_ALREADY_RUNNING) {

878 + cause = "out of memory";

879 + msg = "Netdata couldn't start, because it was already running";

880 + pri = PRI_USER_SHOULD_FIX;

881 + }

882 else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&

883 last_session_status.var_cache.is_read_only) {

884 cause = "disk read-only";

@@ -979,7 +983,7 @@ void daemon_status_file_check_crash(void) {

983

984 // check if we have already posted this crash in the last 24 hours

985 XXH64_hash_t hash = daemon_status_file_hash(&last_session_status, msg, cause);

982 - if(dedup_already_posted(hash))

986 + if(dedup_already_posted(&session_status, hash))

987 disable_crash_report = true;

988

989 if(!disable_crash_report && (analytics_check_enabled() || post_crash_report)) {

@@ -994,6 +998,97 @@ void daemon_status_file_check_crash(void) {

998 }

999 }

1000

1001 +// --------------------------------------------------------------------------------------------------------------------

1002 +// ng_log() hook for receiving fatal message information

1003 +

1004 +void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line) {

1005 + FUNCTION_RUN_ONCE();

1006 +

1007 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

1008 +

1009 + dsf_acquire(session_status);

1010 + spinlock_lock(&session_status.fatal.spinlock);

1011 +

1012 + // do not check the function, because it may have a startup step in it

1013 + if(session_status.fatal.filename || session_status.fatal.message || session_status.fatal.errno_str || session_status.fatal.thread[0]) {

1014 + spinlock_unlock(&session_status.fatal.spinlock);

1015 + freez((void *)filename);

1016 + freez((void *)function);

1017 + freez((void *)message);

1018 + freez((void *)errno_str);

1019 + freez((void *)stack_trace);

1020 + return;

1021 + }

1022 +

1023 + exit_initiated_add(EXIT_REASON_FATAL);

1024 + strncpyz(session_status.fatal.thread, nd_thread_tag(), sizeof(session_status.fatal.thread) - 1);

1025 +

1026 + session_status.fatal.filename = filename;

1027 + freez((char *)session_status.fatal.function); // it may have a startup step

1028 + session_status.fatal.function = function;

1029 + session_status.fatal.message = message;

1030 + session_status.fatal.errno_str = errno_str;

1031 + strncpyz(session_status.fatal.stack_trace, stack_trace, sizeof(session_status.fatal.stack_trace) - 1);

1032 + freez((char *)stack_trace);

1033 + session_status.fatal.line = line;

1034 +

1035 + spinlock_unlock(&session_status.fatal.spinlock);

1036 + dsf_release(session_status);

1037 +

1038 + daemon_status_file_save(wb, &session_status, false);

1039 +}

1040 +

1041 +// --------------------------------------------------------------------------------------------------------------------

1042 +

1043 +void daemon_status_file_update_status(DAEMON_STATUS status) {

1044 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

1045 + daemon_status_file_refresh(status);

1046 + daemon_status_file_save(wb, &session_status, true);

1047 +}

1048 +

1049 +static void daemon_status_file_out_of_memory(void) {

1050 + FUNCTION_RUN_ONCE();

1051 +

1052 + // DO NOT LOCK OR ALLOCATE IN THIS FUNCTION - WE DON'T HAVE ANY MEMORY AVAILABLE - IT HAPPENED ALREADY!

1053 +

1054 + exit_initiated_add(EXIT_REASON_OUT_OF_MEMORY);

1055 +

1056 + // the buffer should already be allocated, so this should normally do nothing

1057 + static_save_buffer_init();

1058 +

1059 + dsf_acquire(session_status);

1060 + session_status.exit_reason = exit_initiated;

1061 + dsf_release(session_status);

1062 +

1063 + daemon_status_file_save(static_save_buffer, &session_status, false);

1064 +}

1065 +

1066 +void daemon_status_file_deadly_signal_received(EXIT_REASON reason) {

1067 + FUNCTION_RUN_ONCE();

1068 +

1069 + // DO NOT LOCK OR ALLOCATE IN THIS FUNCTION - WE CRASHED ALREADY AND WE ARE INSIDE THE SIGNAL HANDLER!

1070 +

1071 + dsf_acquire(session_status);

1072 +

1073 + session_status.exit_reason |= reason;

1074 + if(!session_status.fatal.thread[0])

1075 + strncpyz(session_status.fatal.thread, nd_thread_tag(), sizeof(session_status.fatal.thread) - 1);

1076 +

1077 + dsf_release(session_status);

1078 +

1079 + // the buffer should already be allocated, so this should normally do nothing

1080 + static_save_buffer_init();

1081 +

1082 + // save what we know already

1083 + daemon_status_file_save(static_save_buffer, &session_status, false);

1084 +

1085 + if(!session_status.fatal.stack_trace[0]) {

1086 + capture_stack_trace(static_save_buffer);

1087 + strncpyz(session_status.fatal.stack_trace, buffer_tostring(static_save_buffer), sizeof(session_status.fatal.stack_trace) - 1);

1088 + daemon_status_file_save(static_save_buffer, &session_status, false);

1089 + }

1090 +}

1091 +

1092 bool daemon_status_file_has_last_crashed(void) {

1093 return last_session_status.status != DAEMON_STATUS_EXITED || !is_exit_reason_normal(last_session_status.exit_reason);

1094 }

@@ -1002,6 +1097,9 @@ bool daemon_status_file_was_incomplete_shutdown(void) {

1097 return last_session_status.status == DAEMON_STATUS_EXITING;

1098 }

1099

1100 +// --------------------------------------------------------------------------------------------------------------------

1101 +// startup and shutdown steps

1102 +

1103 void daemon_status_file_startup_step(const char *step) {

1104 if(session_status.fatal.filename)

1105 // we have a fatal logged

@@ -1030,37 +1128,3 @@ void daemon_status_file_shutdown_step(const char *step) {

1128

1129 daemon_status_file_update_status(DAEMON_STATUS_NONE);

1130 }

1033 -

1034 -// --------------------------------------------------------------------------------------------------------------------

1035 -// ng_log() hook for receiving fatal message information

1036 -

1037 -void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line) {

1038 - spinlock_lock(&dsf_spinlock);

1039 -

1040 - // do not check the function, because it may have a startup step in it

1041 - if(session_status.fatal.filename || session_status.fatal.message || session_status.fatal.errno_str || session_status.fatal.thread[0]) {

1042 - spinlock_unlock(&dsf_spinlock);

1043 - freez((void *)filename);

1044 - freez((void *)function);

1045 - freez((void *)message);

1046 - freez((void *)errno_str);

1047 - freez((void *)stack_trace);

1048 - return;

1049 - }

1050 -

1051 - exit_initiated_add(EXIT_REASON_FATAL);

1052 - strncpyz(session_status.fatal.thread, nd_thread_tag(), sizeof(session_status.fatal.thread) - 1);

1053 -

1054 - session_status.fatal.filename = filename;

1055 - freez((char *)session_status.fatal.function); // it may have a startup step

1056 - session_status.fatal.function = function;

1057 - session_status.fatal.message = message;

1058 - session_status.fatal.errno_str = errno_str;

1059 - strncpyz(session_status.fatal.stack_trace, stack_trace, sizeof(session_status.fatal.stack_trace) - 1);

1060 - freez((char *)stack_trace);

1061 - session_status.fatal.line = line;

1062 -

1063 - spinlock_unlock(&dsf_spinlock);

1064 -

1065 - daemon_status_file_save(&session_status, false);

1066 -}

src/daemon/daemon-status-file.h

+12 -4

@@ -25,6 +25,9 @@ typedef enum {

25 ENUM_STR_DEFINE_FUNCTIONS_EXTERN(DAEMON_OS_TYPE);

27 typedef struct daemon_status_file {

28 + SPINLOCK spinlock;

29 + uint32_t v; // the version of the status file

30 +

31 char version[32]; // the netdata version

32 DAEMON_STATUS status; // the daemon status

33 EXIT_REASON exit_reason; // the exit reason (maybe empty)

@@ -43,7 +46,9 @@ typedef struct daemon_status_file {

46 ND_UUID claim_id; // the Netdata Cloud claim id of the agent

48 struct {

49 + usec_t init_started_ut;

50 time_t init;

51 + usec_t exit_started_ut;

52 time_t exit;

53 } timings;

@@ -62,6 +67,7 @@ typedef struct daemon_status_file {

67 bool read_system_info;

69 struct {

70 + SPINLOCK spinlock;

71 long line;

72 const char *filename;

73 const char *function;

@@ -72,9 +78,12 @@ typedef struct daemon_status_file {

78 } fatal;

80 struct {

75 - XXH64_hash_t hash;

76 - usec_t timestamp_ut;

77 - } dedup[20];

81 + SPINLOCK spinlock;

82 + struct {

83 + XXH64_hash_t hash;

84 + usec_t timestamp_ut;

85 + } slot[20];

86 + } dedup;

87 } DAEMON_STATUS_FILE;

89 // loads the last status saved

@@ -82,7 +91,6 @@ DAEMON_STATUS_FILE daemon_status_file_load(void);

92 // saves the current status

93 void daemon_status_file_update_status(DAEMON_STATUS status);

85 -void daemon_status_file_exit_reason_save(EXIT_REASON reason);

94 void daemon_status_file_deadly_signal_received(EXIT_REASON reason);

96 // check for a crash

src/daemon/main.c

@@ -902,6 +902,8 @@ int netdata_main(int argc, char **argv) {

902 delta_startup_time("web server sockets");

903 if(web_server_mode != WEB_SERVER_MODE_NONE) {

904 if (!api_listen_sockets_setup()) {

905 + exit_initiated_add(EXIT_REASON_ALREADY_RUNNING);

906 + daemon_status_file_update_status(DAEMON_STATUS_NONE);

907 fatal("Cannot setup listen port(s). Is Netdata already running?");

908 exit(1);

909 }

src/libnetdata/common.h

+13 -2

@@ -426,8 +426,19 @@ typedef uint32_t uid_t;

426

427 // --------------------------------------------------------------------------------------------------------------------

428

429 -#define FUNCTION_RUN_ONCE() { static bool __run_once = false; if(__run_once) return; __run_once = true; }

430 -#define FUNCTION_RUN_ONCE_RET(ret) { static bool __run_once = false; if(__run_once) return (ret); __run_once = true; }

429 +#define FUNCTION_RUN_ONCE() { \

430 + static bool __run_once = false; \

431 + if (!__sync_bool_compare_and_swap(&__run_once, false, true)) { \

432 + return; \

433 + } \

434 +}

435 +

436 +#define FUNCTION_RUN_ONCE_RET(ret) { \

437 + static bool __run_once = false; \

438 + if (!__sync_bool_compare_and_swap(&__run_once, false, true)) { \

439 + return (ret); \

440 + } \

441 +}

442

443 // --------------------------------------------------------------------------------------------------------------------

444

src/libnetdata/exit/exit_initiated.c

@@ -10,6 +10,7 @@ ENUM_STR_MAP_DEFINE(EXIT_REASON) = {

10 { EXIT_REASON_SIGFPE, "signal-floating-point-exception"},

11 { EXIT_REASON_SIGILL, "signal-illegal-instruction"},

12 { EXIT_REASON_OUT_OF_MEMORY, "out-of-memory"},

13 + { EXIT_REASON_ALREADY_RUNNING, "already-running"},

15 { EXIT_REASON_FATAL, "fatal"},

src/libnetdata/exit/exit_initiated.h

+10 -9

@@ -15,28 +15,29 @@ typedef enum {

15 EXIT_REASON_SIGFPE = (1 << 2),

16 EXIT_REASON_SIGILL = (1 << 3),

17 EXIT_REASON_OUT_OF_MEMORY = (1 << 4),

18 + EXIT_REASON_ALREADY_RUNNING = (1 << 5),

20 // abnormal termination via a fatal message

20 - EXIT_REASON_FATAL = (1 << 5), // a fatal message

21 + EXIT_REASON_FATAL = (1 << 6), // a fatal message

23 // normal termination via APIs

23 - EXIT_REASON_API_QUIT = (1 << 6), // developer only

24 - EXIT_REASON_CMD_EXIT = (1 << 7), // netdatacli

24 + EXIT_REASON_API_QUIT = (1 << 7), // developer only

25 + EXIT_REASON_CMD_EXIT = (1 << 8), // netdatacli

27 // signals - normal termination

27 - EXIT_REASON_SIGQUIT = (1 << 8), // rare, but graceful

28 - EXIT_REASON_SIGTERM = (1 << 9), // received on Linux, FreeBSD, MacOS

29 - EXIT_REASON_SIGINT = (1 << 10), // received on Windows on normal termination

28 + EXIT_REASON_SIGQUIT = (1 << 9), // rare, but graceful

29 + EXIT_REASON_SIGTERM = (1 << 10), // received on Linux, FreeBSD, MacOS

30 + EXIT_REASON_SIGINT = (1 << 11), // received on Windows on normal termination

32 // windows specific, service stop

32 - EXIT_REASON_SERVICE_STOP = (1 << 11),

33 + EXIT_REASON_SERVICE_STOP = (1 << 12),

35 // automatically detect when exit_initiated_set() is called

36 // supports Linux, FreeBSD, MacOS, Windows

36 - EXIT_REASON_SYSTEM_SHUTDOWN = (1 << 12),

37 + EXIT_REASON_SYSTEM_SHUTDOWN = (1 << 13),

39 // netdata update

39 - EXIT_REASON_UPDATE = (1 << 13),

40 + EXIT_REASON_UPDATE = (1 << 14),

41 } EXIT_REASON;

src/libnetdata/log/nd_log-stacktrace.c

+7 -4

@@ -54,13 +54,16 @@ void capture_stack_trace(BUFFER *wb) {

54 messages = backtrace_symbols(array, size);

56 if (messages == NULL) {

57 - // Failed to get backtrace symbols

57 + buffer_strcat(wb, "backtrace failed to get symbols");

58 return;

59 }

61 - // Format the stack trace

62 - for (i = 0; i < size ; i++)

63 - buffer_sprintf(wb, "#%d %s\n", i, messages[i]);

61 + // Format the stack trace (removing the address part)

62 + for (i = 0; i < size; i++) {

63 + char *p = strstr(messages[i], " [");

64 + size_t len = p ? (size_t)(p - messages[i]) : strlen(messages[i]);

65 + buffer_sprintf(wb, "#%d %.*s\n", i, (int)len, messages[i]);

66 + }

68 free(messages);

69 }