more status file annotations (#19721)
more status annotations
Costa Tsaousis committed
Feb 26, 2025 at 20:51 UTC
9c6d274d39b3e57f2274e11e6bcd965a046cb2ce
1 file changed
+53
-9
src/daemon/daemon-status-file.c
+53
-9
@@ -607,6 +607,16 @@ void *post_status_file_thread(void *ptr) {
607
// --------------------------------------------------------------------------------------------------------------------
608
// check last status on startup and post-crash report
609
610
+struct log_priority {
611
+ ND_LOG_FIELD_PRIORITY user;
612
+ ND_LOG_FIELD_PRIORITY post;
613
+};
614
+
615
+struct log_priority PRI_ALL_NORMAL = { NDLP_NOTICE, NDLP_DEBUG };
616
+struct log_priority PRI_USER_SHOULD_FIX = { NDLP_WARNING, NDLP_INFO };
617
+struct log_priority PRI_NETDATA_BUG = { NDLP_CRIT, NDLP_ERR };
618
+struct log_priority PRI_BAD_BUT_NO_REASON = { NDLP_ERR, NDLP_WARNING };
619
+
620
void daemon_status_file_check_crash(void) {
621
FUNCTION_RUN_ONCE();
622
@@ -614,7 +624,7 @@ void daemon_status_file_check_crash(void) {
624
625
last_session_status = daemon_status_file_load();
626
daemon_status_file_update_status(DAEMON_STATUS_INITIALIZING);
617
- ND_LOG_FIELD_PRIORITY pri = NDLP_NOTICE;
627
+ struct log_priority pri = PRI_ALL_NORMAL;
628
629
bool new_version = strcmp(last_session_status.version, session_status.version) != 0;
630
bool post_crash_report = false;
@@ -640,7 +650,7 @@ void daemon_status_file_check_crash(void) {
650
else if(!is_exit_reason_normal(last_session_status.exit_reason)) {
651
cause = "exit on fatal";
652
msg = "Netdata was last stopped gracefully (encountered an error)";
643
- pri = NDLP_ERR;
653
+ pri = PRI_NETDATA_BUG;
654
post_crash_report = true;
655
}
656
else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {
@@ -667,22 +677,43 @@ void daemon_status_file_check_crash(void) {
677
last_session_status.var_cache.is_read_only) {
678
cause = "disk read-only";
679
msg = "Netdata couldn't start because the disk is readonly";
680
+ pri = PRI_USER_SHOULD_FIX;
681
}
682
else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&
683
last_session_status.var_cache.free_bytes == 0) {
684
cause = "disk full";
685
msg = "Netdata couldn't start because the disk is full";
686
+ pri = PRI_USER_SHOULD_FIX;
687
}
688
else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&
689
last_session_status.var_cache.free_bytes < 1 * 1024 * 1024) {
690
cause = "disk almost full";
691
msg = "Netdata couldn't start while the disk is almost full";
692
+ pri = PRI_USER_SHOULD_FIX;
693
+ }
694
+ else if (last_session_status.exit_reason == EXIT_REASON_NONE &&
695
+ !UUIDiszero(session_status.boot_id) &&
696
+ !UUIDiszero(last_session_status.boot_id) &&
697
+ !UUIDeq(session_status.boot_id, last_session_status.boot_id)) {
698
+ cause = "abnormal power off";
699
+ msg = "The system was abnormally powered off while Netdata was starting";
700
+ pri = PRI_USER_SHOULD_FIX;
701
+ }
702
+ else if (last_session_status.exit_reason &= (EXIT_REASON_SIGBUS|EXIT_REASON_SIGFPE|EXIT_REASON_SIGILL|EXIT_REASON_SIGSEGV)) {
703
+ cause = "killed signal";
704
+ msg = "Netdata was last crashed while starting, with a signal indicating a bug";
705
+ pri = PRI_NETDATA_BUG;
706
+ }
707
+ else if (last_session_status.exit_reason &= EXIT_REASON_OUT_OF_MEMORY) {
708
+ cause = "out of memory";
709
+ msg = "Netdata was last crashed while starting, because it couldn't allocate memory";
710
+ pri = PRI_USER_SHOULD_FIX;
711
}
712
else {
713
cause = "crashed on start";
714
msg = "Netdata was last killed/crashed while starting";
715
+ pri = PRI_BAD_BUT_NO_REASON;
716
}
685
- pri = NDLP_ERR;
717
post_crash_report = true;
718
719
break;
@@ -704,20 +735,33 @@ void daemon_status_file_check_crash(void) {
735
cause = "crashed on exit";
736
msg = "Netdata was last killed/crashed while exiting (instructed to do so)";
737
}
707
- pri = NDLP_ERR;
738
+ pri = PRI_NETDATA_BUG;
739
post_crash_report = true;
740
break;
741
742
case DAEMON_STATUS_RUNNING: {
712
- if (!UUIDeq(session_status.boot_id, last_session_status.boot_id)) {
743
+ if (last_session_status.exit_reason == EXIT_REASON_NONE &&
744
+ !UUIDiszero(session_status.boot_id) &&
745
+ !UUIDiszero(last_session_status.boot_id) &&
746
+ !UUIDeq(session_status.boot_id, last_session_status.boot_id)) {
747
cause = "abnormal power off";
748
msg = "The system was abnormally powered off while Netdata was running";
715
- pri = NDLP_CRIT;
749
+ pri = PRI_USER_SHOULD_FIX;
750
+ }
751
+ else if (last_session_status.exit_reason &= (EXIT_REASON_SIGBUS|EXIT_REASON_SIGFPE|EXIT_REASON_SIGILL|EXIT_REASON_SIGSEGV)) {
752
+ cause = "killed signal";
753
+ msg = "Netdata was last crashed with a signal indicating a bug";
754
+ pri = PRI_NETDATA_BUG;
755
+ }
756
+ else if (last_session_status.exit_reason &= EXIT_REASON_OUT_OF_MEMORY) {
757
+ cause = "out of memory";
758
+ msg = "Netdata was last crashed because it couldn't allocate memory";
759
+ pri = PRI_USER_SHOULD_FIX;
760
}
761
else {
762
cause = "killed hard";
763
msg = "Netdata was last killed/crashed while operating normally";
720
- pri = NDLP_CRIT;
764
+ pri = PRI_BAD_BUT_NO_REASON;
765
post_crash_report = true;
766
}
767
break;
@@ -736,7 +780,7 @@ void daemon_status_file_check_crash(void) {
780
};
781
ND_LOG_STACK_PUSH(lgs);
782
739
- nd_log(NDLS_DAEMON, pri,
783
+ nd_log(NDLS_DAEMON, pri.user,
784
"Netdata Agent version '%s' is starting...\n"
785
"Last exit status: %s (%s):\n\n%s",
786
NETDATA_VERSION, msg, cause, buffer_tostring(wb));
@@ -757,7 +801,7 @@ void daemon_status_file_check_crash(void) {
801
d->cause = strdupz(cause);
802
d->msg = strdupz(msg);
803
d->status = last_session_status;
760
- d->priority = pri;
804
+ d->priority = pri.post;
805
nd_thread_create("post_status_file", NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_DEFAULT, post_status_file_thread, d);
806
}
807
}