User configurable crash reporting (#19789)
* use a counter for temp status files * provide user configuration to reporting crashes * docs for crash reports * updated documentation * revert condition
Costa Tsaousis committed
Mar 6, 2025 at 19:39 UTC
35c6875bbbbd14667f81b35e525224378f92f1cf
2 files changed
+87
-65
src/daemon/config/README.md
+12
-11
@@ -35,17 +35,18 @@ After `netdata.conf` has been modified, Netdata needs to be [restarted](/docs/ne
35
36
### `global` section options
37
38
-| setting | default | info |
39
-|:----------------------------------:|:-------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
40
-| process scheduling policy | `keep` | See [Netdata process scheduling policy](/src/daemon/README.md#process-scheduling-policy-unix-only) |
41
-| OOM score | `0` | |
42
-| glibc malloc arena max for plugins | `1` | |
43
-| glibc malloc arena max for Netdata | `1` | |
44
-| hostname | auto-detected | The hostname of the computer running Netdata. |
45
-| host access prefix | empty | This is used in Docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). |
46
-| timezone | auto-detected | The timezone retrieved from the environment variable |
47
-| run as user | `netdata` | The user Netdata will run as. |
48
-| pthread stack size | auto-detected | |
38
+| setting | default | info |
39
+|:----------------------------------:|:--------------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
40
+| process scheduling policy | `keep` | See [Netdata process scheduling policy](/src/daemon/README.md#process-scheduling-policy-unix-only) |
41
+| OOM score | `0` | |
42
+| glibc malloc arena max for plugins | `1` | |
43
+| glibc malloc arena max for Netdata | `1` | |
44
+| hostname | auto-detected | The hostname of the computer running Netdata. |
45
+| host access prefix | empty | This is used in Docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). |
46
+| timezone | auto-detected | The timezone retrieved from the environment variable |
47
+| run as user | `netdata` | The user Netdata will run as. |
48
+| pthread stack size | auto-detected | |
49
+| crash reports | `all` or `off` | It is `off` when anonymous telemetry is disabled, otherwise `all`. When it is `all` Netdata reports agent restarts and crashes. It can also be `crashes` to report only crashes. Each kind of event is deduplicated and reported at most once per day. |
50
51
### `db` section options
52
src/daemon/daemon-status-file.c
+75
-54
@@ -568,45 +568,34 @@ static bool save_status_file(const char *directory, const char *content, size_t
568
if(!directory || !*directory)
569
return false;
570
571
+ static uint64_t tmp_attempt_counter = 0;
572
+
573
char filename[FILENAME_MAX];
574
char temp_filename[FILENAME_MAX];
575
+ char tid_str[UINT64_MAX_LENGTH];
576
574
- /* Construct filenames using async-safe string operations */
575
- /* Using simple string concatenation instead of snprintf */
577
+ print_uint64(tid_str, __atomic_add_fetch(&tmp_attempt_counter, 1, __ATOMIC_RELAXED));
578
size_t dir_len = strlen(directory);
577
- if (dir_len + 1 + strlen(STATUS_FILENAME) >= FILENAME_MAX)
578
- return false; /* Path too long */
579
-
580
- memcpy(filename, directory, dir_len);
581
- filename[dir_len] = '/';
582
- memcpy(filename + dir_len + 1, STATUS_FILENAME, strlen(STATUS_FILENAME) + 1);
583
-
584
- /* Create a unique temp filename using thread id */
585
- unsigned int tid = (unsigned int)gettid_cached();
586
- char tid_str[16];
587
- char *tid_ptr = tid_str + sizeof(tid_str) - 1;
588
- *tid_ptr = '\0';
589
-
590
- unsigned int tid_copy = tid;
591
- do {
592
- tid_ptr--;
593
- *tid_ptr = "0123456789abcdef"[tid_copy & 0xf];
594
- tid_copy >>= 4;
595
- } while (tid_copy && tid_ptr > tid_str);
596
-
597
- size_t temp_name_len = dir_len + 1 + strlen(STATUS_FILENAME) + 1 + (sizeof(tid_str) - (tid_ptr - tid_str));
598
- if (temp_name_len >= FILENAME_MAX)
599
- return false; /* Path too long */
600
-
601
- memcpy(temp_filename, directory, dir_len);
602
- temp_filename[dir_len] = '/';
603
- char *ptr = temp_filename + dir_len + 1;
604
- memcpy(ptr, STATUS_FILENAME, strlen(STATUS_FILENAME));
605
- ptr += strlen(STATUS_FILENAME);
606
- *ptr++ = '-';
607
- memcpy(ptr, tid_ptr, strlen(tid_ptr) + 1);
608
-
609
- /* Open file with O_WRONLY, O_CREAT, and O_TRUNC flags */
579
+ size_t fil_len = strlen(STATUS_FILENAME);
580
+ size_t tid_len = strlen(tid_str);
581
+
582
+ if (dir_len + 1 + fil_len + 1 + tid_len + 1 >= sizeof(filename))
583
+ return false; // cannot fit the filename
584
+
585
+ // create the filename
586
+ size_t pos = 0;
587
+ memcpy(&filename[pos], directory, dir_len); pos += dir_len;
588
+ filename[pos] = '/'; pos++;
589
+ memcpy(&filename[pos], STATUS_FILENAME, fil_len); pos += fil_len;
590
+ filename[pos] = '\0';
591
+
592
+ // create the temp filename
593
+ memcpy(temp_filename, filename, pos);
594
+ temp_filename[pos] = '-'; pos++;
595
+ memcpy(&temp_filename[pos], tid_str, tid_len); pos += tid_len;
596
+ temp_filename[pos] = '\0';
597
+
598
+ // Open file with O_WRONLY, O_CREAT, and O_TRUNC flags
599
int fd = open(temp_filename, O_WRONLY | O_CREAT | O_TRUNC, 0664);
600
if (fd == -1)
601
return false;
@@ -620,7 +609,7 @@ static bool save_status_file(const char *directory, const char *content, size_t
609
610
if (bytes_written == -1) {
611
if (errno == EINTR)
623
- continue; /* Retry if interrupted by signal */
612
+ continue; /* Retry if interrupted by signal */
613
614
close(fd);
615
unlink(temp_filename); /* Remove the temp file */
@@ -822,6 +811,30 @@ static bool is_ci(void) {
811
return ci && *ci && strcasecmp(ci, "true") == 0;
812
}
813
814
+enum crash_report_t {
815
+ DSF_REPORT_DISABLED = 0,
816
+ DSF_REPORT_ALL,
817
+ DSF_REPORT_CRASHES,
818
+};
819
+
820
+static enum crash_report_t check_crash_reports_config(void) {
821
+ bool analytics = analytics_check_enabled();
822
+
823
+ const char *t = inicfg_get(&netdata_config, CONFIG_SECTION_GLOBAL, "crash reports", analytics ? "all" : "off");
824
+
825
+ enum crash_report_t rc;
826
+ if(!t || !*t)
827
+ rc = analytics ? DSF_REPORT_ALL : DSF_REPORT_DISABLED;
828
+ else if(strcmp(t, "all") == 0)
829
+ rc = DSF_REPORT_ALL;
830
+ else if(strcmp(t, "crashes") == 0)
831
+ rc = DSF_REPORT_CRASHES;
832
+ else
833
+ rc = DSF_REPORT_DISABLED;
834
+
835
+ return rc;
836
+}
837
+
838
void daemon_status_file_check_crash(void) {
839
FUNCTION_RUN_ONCE();
840
@@ -834,8 +847,8 @@ void daemon_status_file_check_crash(void) {
847
struct log_priority pri = PRI_ALL_NORMAL;
848
849
bool new_version = strcmp(last_session_status.version, session_status.version) != 0;
837
- bool post_crash_report = false;
838
- bool disable_crash_report = false;
850
+ bool this_is_a_crash = false;
851
+ bool crash_report_ignore = false;
852
bool dump_json = true;
853
const char *msg = "", *cause = "";
854
switch(last_session_status.status) {
@@ -844,7 +857,7 @@ void daemon_status_file_check_crash(void) {
857
// probably a previous version of netdata was running
858
cause = "no last status";
859
msg = "No status found for the previous Netdata session";
847
- disable_crash_report = true;
860
+ crash_report_ignore = true;
861
break;
862
863
case DAEMON_STATUS_EXITED:
@@ -858,14 +871,14 @@ void daemon_status_file_check_crash(void) {
871
cause = "deadly signal and exit";
872
msg = "Netdata was last stopped gracefully after receiving a deadly signal";
873
pri = PRI_NETDATA_BUG;
861
- post_crash_report = true;
874
+ this_is_a_crash = true;
875
}
876
else if(last_session_status.exit_reason != EXIT_REASON_NONE &&
877
!is_exit_reason_normal(last_session_status.exit_reason)) {
878
cause = "fatal and exit";
879
msg = "Netdata was last stopped gracefully after it encountered a fatal error";
880
pri = PRI_NETDATA_BUG;
868
- post_crash_report = true;
881
+ this_is_a_crash = true;
882
}
883
else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {
884
cause = "exit on system shutdown";
@@ -899,7 +912,7 @@ void daemon_status_file_check_crash(void) {
912
cause = "deadly signal on start";
913
msg = "Netdata was last crashed while starting after receiving a deadly signal";
914
pri = PRI_NETDATA_BUG;
902
- post_crash_report = true;
915
+ this_is_a_crash = true;
916
}
917
else if (last_session_status.exit_reason & EXIT_REASON_OUT_OF_MEMORY) {
918
cause = "out of memory";
@@ -940,8 +953,7 @@ void daemon_status_file_check_crash(void) {
953
msg = "Netdata was last killed/crashed while starting";
954
pri = PRI_BAD_BUT_NO_REASON;
955
}
943
- post_crash_report = true;
944
-
956
+ this_is_a_crash = true;
957
break;
958
959
case DAEMON_STATUS_EXITING:
@@ -949,7 +961,7 @@ void daemon_status_file_check_crash(void) {
961
cause = "deadly signal on exit";
962
msg = "Netdata was last crashed while exiting after receiving a deadly signal";
963
pri = PRI_NETDATA_BUG;
952
- post_crash_report = true;
964
+ this_is_a_crash = true;
965
}
966
else if(last_session_status.exit_reason != EXIT_REASON_NONE &&
967
!is_exit_reason_normal(last_session_status.exit_reason)) {
@@ -969,7 +981,7 @@ void daemon_status_file_check_crash(void) {
981
msg = "Netdata was last killed/crashed while it was instructed to exit";
982
}
983
pri = PRI_NETDATA_BUG;
972
- post_crash_report = true;
984
+ this_is_a_crash = true;
985
break;
986
987
case DAEMON_STATUS_RUNNING: {
@@ -990,7 +1002,7 @@ void daemon_status_file_check_crash(void) {
1002
cause = "deadly signal";
1003
msg = "Netdata was last crashed after receiving a deadly signal";
1004
pri = PRI_NETDATA_BUG;
993
- post_crash_report = true;
1005
+ this_is_a_crash = true;
1006
}
1007
else if (last_session_status.exit_reason != EXIT_REASON_NONE &&
1008
!is_exit_reason_normal(last_session_status.exit_reason)) {
@@ -1002,7 +1014,7 @@ void daemon_status_file_check_crash(void) {
1014
cause = "killed hard";
1015
msg = "Netdata was last killed/crashed while operating normally";
1016
pri = PRI_BAD_BUT_NO_REASON;
1005
- post_crash_report = true;
1017
+ this_is_a_crash = true;
1018
}
1019
break;
1020
}
@@ -1025,22 +1037,31 @@ void daemon_status_file_check_crash(void) {
1037
"Last exit status: %s (%s):\n\n%s",
1038
NETDATA_VERSION, msg, cause, buffer_tostring(wb));
1039
1028
- // check if we have already posted this crash in the last 24 hours
1029
- XXH64_hash_t hash = daemon_status_file_hash(&last_session_status, msg, cause);
1030
- if(dedup_already_posted(&session_status, hash) || (last_session_status.restarts < 10 && is_ci()))
1031
- disable_crash_report = true;
1040
+ enum crash_report_t r = check_crash_reports_config();
1041
+ if( // must be first for netdata.conf option to be used
1042
+ (r == DSF_REPORT_ALL || (this_is_a_crash && r == DSF_REPORT_CRASHES)) &&
1043
1033
- if(!disable_crash_report && (analytics_check_enabled() || post_crash_report)) {
1034
- netdata_conf_ssl();
1044
+ // not a useful report (no previous status file)
1045
+ !crash_report_ignore &&
1046
+
1047
+ // we are not running in CI
1048
+ (last_session_status.restarts >= 10 || !is_ci()) &&
1049
1050
+ // we have not already reported this
1051
+ !dedup_already_posted(&session_status, daemon_status_file_hash(&last_session_status, msg, cause))
1052
+
1053
+ ) {
1054
daemon_status_file_startup_step("startup(post status file)");
1055
1056
+ netdata_conf_ssl();
1057
+
1058
struct post_status_file_thread_data d = {
1059
.cause = cause,
1060
.msg = msg,
1061
.status = &last_session_status,
1062
.priority = pri.post,
1063
};
1064
+
1065
post_status_file(&d);
1066
1067
// MacOS crashes when starting under launchctl, when we create a thread to post the status file,