@cryptotaxi247 / netdata-1 / commits / 84a207f55

Avoid post initialization errors repeateadly (#19709)

do not post initialization errors repeateadly; detect almost full disks; increase status file version

Costa Tsaousis committed Feb 25, 2025 at 21:03 UTC 84a207f559e1656ccda87941acbd87a416c77fd5

5 files changed +77 -25

src/daemon/daemon-shutdown.c

+2 -2

@@ -173,7 +173,7 @@ void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char

173 exit(ret);

174 }

175 run = true;

176 - daemon_status_file_save(DAEMON_STATUS_EXITING);

176 + daemon_status_file_update_status(DAEMON_STATUS_EXITING);

177

178 nd_log_limits_unlimited();

179 netdata_log_exit_reason();

@@ -322,7 +322,7 @@ void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char

322 watcher_thread_stop();

323 curl_global_cleanup();

324

325 - daemon_status_file_save(DAEMON_STATUS_EXITED);

325 + daemon_status_file_update_status(DAEMON_STATUS_EXITED);

326

327 #ifdef OS_WINDOWS

328 return;

src/daemon/daemon-status-file.c

+65 -19

@@ -37,13 +37,14 @@ ENUM_STR_DEFINE_FUNCTIONS(DAEMON_OS_TYPE, DAEMON_OS_TYPE_UNKNOWN, "unknown");

38 static DAEMON_STATUS_FILE last_session_status = { 0 };

39 static DAEMON_STATUS_FILE session_status = { 0 };

40 +static SPINLOCK dsf_spinlock = SPINLOCK_INITIALIZER;

42 // --------------------------------------------------------------------------------------------------------------------

43 // json generation

45 static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

46 buffer_json_member_add_datetime_rfc3339(wb, "@timestamp", ds->timestamp_ut, true); // ECS

46 - buffer_json_member_add_uint64(wb, "version", 1); // custom

47 + buffer_json_member_add_uint64(wb, "version", 2); // custom

49 buffer_json_member_add_object(wb, "agent"); // ECS

50 {

@@ -116,7 +117,7 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

117 }

118 buffer_json_object_close(wb);

119

119 - buffer_json_member_add_object(wb, "fatal");

120 + buffer_json_member_add_object(wb, "fatal"); // custom

121 {

122 buffer_json_member_add_uint64(wb, "line", ds->fatal.line);

123 buffer_json_member_add_string_or_empty(wb, "filename", ds->fatal.filename);

@@ -125,6 +126,14 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

126 buffer_json_member_add_string_or_empty(wb, "stack_trace", ds->fatal.stack_trace);

127 }

128 buffer_json_object_close(wb);

129 +

130 + buffer_json_member_add_object(wb, "dedup"); // custom

131 + {

132 + buffer_json_member_add_time_t(wb, "timestamp", ds->dedup.timestamp); // custom

133 + buffer_json_member_add_string(wb, "status", DAEMON_STATUS_2str(ds->dedup.status)); // custom

134 + EXIT_REASON_2json(wb, "exit_reason", ds->dedup.exit_reason); // custom

135 + }

136 + buffer_json_object_close(wb);

137 }

138

139 // --------------------------------------------------------------------------------------------------------------------

@@ -141,6 +150,7 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

150 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "version", version, error, true);

151

152 bool required = false; // allow missing fields and values

153 + bool required_v2 = false; // allow missing fields and values for version 2

154

155 // Parse timestamp

156 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required);

@@ -215,13 +225,22 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

225 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "line", ds->fatal.line, error, required);

226 });

227

228 + // Parse the last posted object

229 + JSONC_PARSE_SUBOBJECT(jobj, path, "dedup", error, required_v2, {

230 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "timestamp", ds->dedup.timestamp, error, required_v2);

231 + JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "status", DAEMON_STATUS_2id, ds->dedup.status, error, required_v2);

232 + JSONC_PARSE_ARRAY_OF_TXT2BITMAP_OR_ERROR_AND_RETURN(jobj, path, "exit_reason", EXIT_REASON_2id_one, ds->dedup.exit_reason, error, required_v2);

233 + });

234 +

235 return true;

236 }

237

238 // --------------------------------------------------------------------------------------------------------------------

239 // get the current status

240

224 -static DAEMON_STATUS_FILE daemon_status_file_get(DAEMON_STATUS status) {

241 +static void daemon_status_file_refresh(DAEMON_STATUS status) {

242 + spinlock_lock(&dsf_spinlock);

243 +

244 usec_t now_ut = now_realtime_usec();

245

246 #if defined(OS_LINUX)

@@ -295,6 +314,11 @@ static DAEMON_STATUS_FILE daemon_status_file_get(DAEMON_STATUS status) {

314 session_status.os_id = strdupz(last_session_status.os_id);

315 if(!session_status.os_id_like && last_session_status.os_id_like)

316 session_status.os_id_like = strdupz(last_session_status.os_id_like);

317 + if(!session_status.dedup.timestamp) {

318 + session_status.dedup.timestamp = last_session_status.dedup.timestamp;

319 + session_status.dedup.status = last_session_status.dedup.status;

320 + session_status.dedup.exit_reason = last_session_status.dedup.exit_reason;

321 + }

322

323 get_daemon_status_fields_from_system_info(&session_status);

324

@@ -307,7 +331,7 @@ static DAEMON_STATUS_FILE daemon_status_file_get(DAEMON_STATUS status) {

331 session_status.memory = os_system_memory(true);

332 session_status.var_cache = os_disk_space(netdata_configured_cache_dir);

333

310 - return session_status;

334 + spinlock_unlock(&dsf_spinlock);

335 }

336

337 // --------------------------------------------------------------------------------------------------------------------

@@ -439,17 +463,13 @@ static bool save_status_file(const char *directory, const char *content, size_t

463 return true;

464 }

465

442 -void daemon_status_file_save(DAEMON_STATUS status) {

443 - static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

444 - spinlock_lock(&spinlock);

445 -

446 - // Get current status

447 - DAEMON_STATUS_FILE ds = daemon_status_file_get(status);

466 +static void daemon_status_file_save(DAEMON_STATUS_FILE *ds) {

467 + spinlock_lock(&dsf_spinlock);

468

469 // Prepare JSON content

470 CLEAN_BUFFER *wb = buffer_create(0, NULL);

471 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);

452 - daemon_status_file_to_json(wb, &ds);

472 + daemon_status_file_to_json(wb, ds);

473 buffer_json_finalize(wb);

474

475 const char *content = buffer_tostring(wb);

@@ -476,7 +496,12 @@ void daemon_status_file_save(DAEMON_STATUS status) {

496 if (!saved)

497 nd_log(NDLS_DAEMON, NDLP_ERR, "Failed to save status file in any location");

498

479 - spinlock_unlock(&spinlock);

499 + spinlock_unlock(&dsf_spinlock);

500 +}

501 +

502 +void daemon_status_file_update_status(DAEMON_STATUS status) {

503 + daemon_status_file_refresh(status);

504 + daemon_status_file_save(&session_status);

505 }

506

507 // --------------------------------------------------------------------------------------------------------------------

@@ -512,7 +537,14 @@ void post_status_file(struct post_status_file_thread_data *d) {

537 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

538

539 CURLcode rc = curl_easy_perform(curl);

515 - (void)rc;

540 + if(rc == CURLE_OK) {

541 + spinlock_lock(&dsf_spinlock);

542 + session_status.dedup.timestamp = now_realtime_sec();

543 + session_status.dedup.status = d->status.status;

544 + session_status.dedup.exit_reason = d->status.exit_reason;

545 + spinlock_unlock(&dsf_spinlock);

546 + daemon_status_file_save(&session_status);

547 + }

548

549 curl_easy_cleanup(curl);

550 curl_slist_free_all(headers);

@@ -532,7 +564,7 @@ void *post_status_file_thread(void *ptr) {

564

565 void daemon_status_file_check_crash(void) {

566 last_session_status = daemon_status_file_load();

535 - daemon_status_file_save(DAEMON_STATUS_INITIALIZING);

567 + daemon_status_file_update_status(DAEMON_STATUS_INITIALIZING);

568 ND_LOG_FIELD_PRIORITY pri = NDLP_NOTICE;

569

570 bool new_version = strcmp(last_session_status.version, session_status.version) != 0;

@@ -592,12 +624,24 @@ void daemon_status_file_check_crash(void) {

624 cause = "disk full";

625 msg = "Netdata couldn't start because the disk is full";

626 }

627 + else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&

628 + last_session_status.var_cache.free_bytes < 1 * 1024 * 1024) {

629 + cause = "disk almost full";

630 + msg = "Netdata couldn't start while the disk is almost full";

631 + }

632 else {

633 cause = "crashed on start";

634 msg = "Netdata was last killed/crashed while starting";

635 }

636 pri = NDLP_ERR;

637 post_crash_report = true;

638 +

639 + if(session_status.dedup.status == DAEMON_STATUS_INITIALIZING &&

640 + now_realtime_sec() - last_session_status.dedup.timestamp < 86400) {

641 + // we have already posted this crash

642 + disable_crash_report = true;

643 + }

644 +

645 break;

646

647 case DAEMON_STATUS_EXITING:

@@ -678,19 +722,18 @@ void daemon_status_file_startup_step(const char *step) {

722 freez((char *)session_status.fatal.function);

723 session_status.fatal.function = step ? strdupz(step) : NULL;

724 if(step != NULL)

681 - daemon_status_file_save(DAEMON_STATUS_NONE);

725 + daemon_status_file_update_status(DAEMON_STATUS_NONE);

726 }

727

728 // --------------------------------------------------------------------------------------------------------------------

729 // ng_log() hook for receiving fatal message information

730

731 void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *stack_trace, long line) {

688 - static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

689 - spinlock_lock(&spinlock);

732 + spinlock_lock(&dsf_spinlock);

733

734 // do not check the function, because it may have a startup step in it

735 if(session_status.fatal.filename || session_status.fatal.message || session_status.fatal.stack_trace) {

693 - spinlock_unlock(&spinlock);

736 + spinlock_unlock(&dsf_spinlock);

737 freez((void *)filename);

738 freez((void *)function);

739 freez((void *)message);

@@ -705,5 +748,8 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio

748 session_status.fatal.stack_trace = stack_trace;

749 session_status.fatal.line = line;

750

708 - spinlock_unlock(&spinlock);

751 + spinlock_unlock(&dsf_spinlock);

752 +

753 + exit_initiated |= EXIT_REASON_FATAL;

754 + daemon_status_file_save(&session_status);

755 }

src/daemon/daemon-status-file.h

+7 -1

@@ -66,13 +66,19 @@ typedef struct daemon_status_file {

66 const char *stack_trace;

67 const char *message;

68 } fatal;

69 +

70 + struct {

71 + time_t timestamp;

72 + DAEMON_STATUS status;

73 + EXIT_REASON exit_reason;

74 + } dedup;

75 } DAEMON_STATUS_FILE;

77 // loads the last status saved

78 DAEMON_STATUS_FILE daemon_status_file_load(void);

80 // saves the current status

75 -void daemon_status_file_save(DAEMON_STATUS status);

81 +void daemon_status_file_update_status(DAEMON_STATUS status);

83 // check for a crash

84 void daemon_status_file_check_crash(void);

src/daemon/main.c

+1 -1

@@ -1058,7 +1058,7 @@ int netdata_main(int argc, char **argv) {

1058 webrtc_initialize();

1059

1060 daemon_status_file_startup_step(NULL);

1061 - daemon_status_file_save(DAEMON_STATUS_RUNNING);

1061 + daemon_status_file_update_status(DAEMON_STATUS_RUNNING);

1062 return 10;

1063 }

1064

src/daemon/signals.c

+2 -2

@@ -113,7 +113,7 @@ void nd_process_signals(void) {

113 // is delivered that either terminates the process or causes the invocation

114 // of a signal-catching function.

115 if(pause() == -1 && errno == EINTR) {

116 - daemon_status_file_save(DAEMON_STATUS_NONE);

116 + daemon_status_file_update_status(DAEMON_STATUS_NONE);

117 errno_clear();

118

119 // loop once, but keep looping while signals are coming in,

@@ -157,7 +157,7 @@ void nd_process_signals(void) {

157 case NETDATA_SIGNAL_FATAL:

158 nd_log_limits_unlimited();

159 exit_initiated_set(signals_waiting[i].reason);

160 - daemon_status_file_save(DAEMON_STATUS_NONE);

160 + daemon_status_file_update_status(DAEMON_STATUS_NONE);

161 fatal("SIGNAL: Received %s. netdata now exits.", name);

162 break;

163