detect ECC memory correctable and uncorrectable errors; fixes #1508
Costa Tsaousis (ktsaou) committed
Jan 11, 2017 at 23:24 UTC
2ecf423c40b6a774781711d2e6843c44aad05f1c
6 files changed
+228
-2
CMakeLists.txt
+1
-1
@@ -101,7 +101,7 @@ set(NETDATA_SOURCE_FILES
101
src/registry_person.c
102
src/registry_person.h
103
src/registry_machine.c
104
- src/registry_machine.h src/registry_internals.c src/registry_init.c src/registry_db.c src/registry_log.c src/proc_uptime.c)
104
+ src/registry_machine.h src/registry_internals.c src/registry_init.c src/registry_db.c src/registry_log.c src/proc_uptime.c src/sys_devices_system_edac_mc.c)
105
106
set(APPS_PLUGIN_SOURCE_FILES
107
src/appconfig.c
conf.d/health.d/memory.conf
new
+30
@@ -0,0 +1,30 @@
1
+
2
+ alarm: 1hour_ecc_memory_correctable
3
+ on: mem.ecc_ce
4
+ lookup: sum -10m unaligned
5
+ units: errors
6
+ every: 1m
7
+ warn: $this > 0
8
+ delay: down 1h multiplier 1.5 max 1h
9
+ info: number of ECC correctable errors during the last hour
10
+ to: sysadmin
11
+
12
+ alarm: 1hour_ecc_memory_uncorrectable
13
+ on: mem.ecc_ue
14
+ lookup: sum -10m unaligned
15
+ units: errors
16
+ every: 1m
17
+ crit: $this > 0
18
+ delay: down 1h multiplier 1.5 max 1h
19
+ info: number of ECC uncorrectable errors during the last hour
20
+ to: sysadmin
21
+
22
+ alarm: 1hour_memory_hw_corrupted
23
+ on: mem.hwcorrupt
24
+ calc: $HardwareCorrupted
25
+ units: MB
26
+ every: 10s
27
+ warn: $this > 0
28
+ delay: down 1h multiplier 1.5 max 1h
29
+ info: amount of memory corrupted due to a hardware failure
30
+ to: sysadmin
src/Makefile.am
+1
@@ -47,6 +47,7 @@ netdata_SOURCES = \
47
popen.c popen.h \
48
socket.c socket.h \
49
sys_fs_cgroup.c \
50
+ sys_devices_system_edac_mc.c \
51
procfile.c procfile.h \
52
proc_self_mountinfo.c proc_self_mountinfo.h \
53
registry.c registry.h \
src/plugin_proc.c
+11
-1
@@ -30,7 +30,8 @@ void *proc_main(void *ptr)
30
int vdo_proc_vmstat = !config_get_boolean("plugin:proc", "/proc/vmstat", 1);
31
int vdo_proc_net_rpc_nfs = !config_get_boolean("plugin:proc", "/proc/net/rpc/nfs", 1);
32
int vdo_proc_net_rpc_nfsd = !config_get_boolean("plugin:proc", "/proc/net/rpc/nfsd", 1);
33
- int vdo_proc_sys_kernel_random_entropy_avail = !config_get_boolean("plugin:proc", "/proc/sys/kernel/random/entropy_avail", 1);
33
+ int vdo_proc_sys_kernel_random_entropy_avail = !config_get_boolean("plugin:proc", "/proc/sys/kernel/random/entropy_avail", 1);
34
+ int vdo_proc_sys_devices_system_edac_mc = !config_get_boolean("plugin:proc", "/sys/devices/system/edac/mc", 1);
35
int vdo_proc_interrupts = !config_get_boolean("plugin:proc", "/proc/interrupts", 1);
36
int vdo_proc_softirqs = !config_get_boolean("plugin:proc", "/proc/softirqs", 1);
37
int vdo_proc_net_softnet_stat = !config_get_boolean("plugin:proc", "/proc/net/softnet_stat", 1);
@@ -55,6 +56,7 @@ void *proc_main(void *ptr)
56
usec_t sutime_proc_net_rpc_nfs = 0ULL;
57
usec_t sutime_proc_net_rpc_nfsd = 0ULL;
58
usec_t sutime_proc_sys_kernel_random_entropy_avail = 0ULL;
59
+ usec_t sutime_proc_sys_devices_system_edac_mc = 0ULL;
60
usec_t sutime_proc_interrupts = 0ULL;
61
usec_t sutime_proc_softirqs = 0ULL;
62
usec_t sutime_proc_net_softnet_stat = 0ULL;
@@ -142,6 +144,14 @@ void *proc_main(void *ptr)
144
}
145
if(unlikely(netdata_exit)) break;
146
147
+ if(!vdo_proc_sys_devices_system_edac_mc) {
148
+ debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_sys_devices_system_edac_mc().");
149
+ now = now_realtime_usec();
150
+ vdo_proc_sys_devices_system_edac_mc = do_proc_sys_devices_system_edac_mc(rrd_update_every, (sutime_proc_sys_devices_system_edac_mc > 0)?now - sutime_proc_sys_devices_system_edac_mc:0ULL);
151
+ sutime_proc_sys_devices_system_edac_mc = now;
152
+ }
153
+ if(unlikely(netdata_exit)) break;
154
+
155
if(!vdo_proc_net_dev) {
156
debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_dev().");
157
now = now_realtime_usec();
src/plugin_proc.h
+1
@@ -23,5 +23,6 @@ extern int do_proc_loadavg(int update_every, usec_t dt);
23
extern int do_proc_net_stat_synproxy(int update_every, usec_t dt);
24
extern int do_proc_net_softnet_stat(int update_every, usec_t dt);
25
extern int do_proc_uptime(int update_every, usec_t dt);
26
+extern int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt);
27
28
#endif /* NETDATA_PLUGIN_PROC_H */
src/sys_devices_system_edac_mc.c
new
+184
@@ -0,0 +1,184 @@
1
+#include "common.h"
2
+
3
+struct mc {
4
+ char *name;
5
+ char ce_updated;
6
+ char ue_updated;
7
+
8
+ char *ce_count_filename;
9
+ char *ue_count_filename;
10
+
11
+ procfile *ce_ff;
12
+ procfile *ue_ff;
13
+
14
+ collected_number ce_count;
15
+ collected_number ue_count;
16
+
17
+ RRDDIM *ce_rd;
18
+ RRDDIM *ue_rd;
19
+
20
+ struct mc *next;
21
+};
22
+static struct mc *mc_root = NULL;
23
+
24
+static void find_all_mc() {
25
+ char name[FILENAME_MAX + 1];
26
+ snprintfz(name, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/devices/system/edac/mc");
27
+ char *dirname = config_get("plugin:proc:/sys/devices/system/edac/mc", "directory to monitor", name);
28
+
29
+ DIR *dir = opendir(dirname);
30
+ if(!dir) {
31
+ error("Cannot read ECC memory errors directory '%s'", dirname);
32
+ return;
33
+ }
34
+
35
+ struct dirent *de = NULL;
36
+ while((de = readdir(dir))) {
37
+ if(de->d_type == DT_DIR && de->d_name[0] == 'm' && de->d_name[1] == 'c' && isdigit(de->d_name[2])) {
38
+ struct mc *m = callocz(1, sizeof(struct mc));
39
+ m->name = strdupz(de->d_name);
40
+
41
+ struct stat st;
42
+
43
+ snprintfz(name, FILENAME_MAX, "%s/%s/ce_count", dirname, de->d_name);
44
+ if(stat(name, &st) != -1)
45
+ m->ce_count_filename = strdupz(name);
46
+
47
+ snprintfz(name, FILENAME_MAX, "%s/%s/ue_count", dirname, de->d_name);
48
+ if(stat(name, &st) != -1)
49
+ m->ue_count_filename = strdupz(name);
50
+
51
+ if(!m->ce_count_filename && !m->ue_count_filename) {
52
+ freez(m->name);
53
+ freez(m);
54
+ }
55
+ else {
56
+ m->next = mc_root;
57
+ mc_root = m;
58
+ }
59
+ }
60
+ }
61
+
62
+ closedir(dir);
63
+}
64
+
65
+int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) {
66
+ (void)dt;
67
+
68
+ if(unlikely(mc_root == NULL)) {
69
+ find_all_mc();
70
+ if(unlikely(mc_root == NULL))
71
+ return 1;
72
+ }
73
+
74
+ static int do_ce = -1, do_ue = -1;
75
+ calculated_number ce_sum = 0, ue_sum = 0;
76
+ struct mc *m;
77
+
78
+ if(unlikely(do_ce == -1)) {
79
+ do_ce = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory correctable errors", CONFIG_ONDEMAND_ONDEMAND);
80
+ do_ue = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory uncorrectable errors", CONFIG_ONDEMAND_ONDEMAND);
81
+ }
82
+
83
+ if(do_ce != CONFIG_ONDEMAND_NO) {
84
+ for(m = mc_root; m; m = m->next) {
85
+ if(m->ce_count_filename) {
86
+ m->ce_updated = 0;
87
+
88
+ if(unlikely(!m->ce_ff)) {
89
+ m->ce_ff = procfile_open(m->ce_count_filename, " \t", PROCFILE_FLAG_DEFAULT);
90
+ if(unlikely(!m->ce_ff))
91
+ continue;
92
+ }
93
+
94
+ m->ce_ff = procfile_readall(m->ce_ff);
95
+ if(unlikely(!m->ce_ff || procfile_lines(m->ce_ff) < 1 || procfile_linewords(m->ce_ff, 0) < 1))
96
+ continue;
97
+
98
+ m->ce_count = strtoull(procfile_lineword(m->ce_ff, 0, 0), NULL, 0);
99
+ ce_sum += m->ce_count;
100
+ m->ce_updated = 1;
101
+ }
102
+ }
103
+ }
104
+
105
+ if(do_ue != CONFIG_ONDEMAND_NO) {
106
+ for(m = mc_root; m; m = m->next) {
107
+ if(m->ue_count_filename) {
108
+ m->ue_updated = 0;
109
+
110
+ if(unlikely(!m->ue_ff)) {
111
+ m->ue_ff = procfile_open(m->ue_count_filename, " \t", PROCFILE_FLAG_DEFAULT);
112
+ if(unlikely(!m->ue_ff))
113
+ continue;
114
+ }
115
+
116
+ m->ue_ff = procfile_readall(m->ue_ff);
117
+ if(unlikely(!m->ue_ff || procfile_lines(m->ue_ff) < 1 || procfile_linewords(m->ue_ff, 0) < 1))
118
+ continue;
119
+
120
+ m->ue_count = strtoull(procfile_lineword(m->ue_ff, 0, 0), NULL, 0);
121
+ ue_sum += m->ue_count;
122
+ m->ue_updated = 1;
123
+ }
124
+ }
125
+ }
126
+
127
+ // --------------------------------------------------------------------
128
+
129
+ if(do_ce == CONFIG_ONDEMAND_YES || (do_ce == CONFIG_ONDEMAND_ONDEMAND && ce_sum > 0)) {
130
+ do_ce = CONFIG_ONDEMAND_YES;
131
+
132
+ static RRDSET *ce_st = NULL;
133
+
134
+ if(unlikely(!ce_st))
135
+ ce_st = rrdset_find("mem.ecc_ce");
136
+
137
+ if(unlikely(!ce_st)) {
138
+ ce_st = rrdset_create("mem", "ecc_ce", NULL, "ecc", NULL, "ECC Memory Correctable Errors", "errors", 6600
139
+ , update_every, RRDSET_TYPE_LINE);
140
+
141
+ for(m = mc_root; m; m = m->next)
142
+ if(m->ce_count_filename)
143
+ m->ce_rd = rrddim_add(ce_st, m->name, NULL, 1, 1, RRDDIM_INCREMENTAL);
144
+ }
145
+ else
146
+ rrdset_next(ce_st);
147
+
148
+ for(m = mc_root; m; m = m->next)
149
+ if(m->ce_count_filename && m->ce_updated)
150
+ rrddim_set_by_pointer(ce_st, m->ce_rd, m->ce_count);
151
+
152
+ rrdset_done(ce_st);
153
+ }
154
+
155
+ // --------------------------------------------------------------------
156
+
157
+ if(do_ue == CONFIG_ONDEMAND_YES || (do_ue == CONFIG_ONDEMAND_ONDEMAND && ue_sum > 0)) {
158
+ do_ue = CONFIG_ONDEMAND_YES;
159
+
160
+ static RRDSET *ue_st = NULL;
161
+
162
+ if(unlikely(!ue_st))
163
+ ue_st = rrdset_find("mem.ecc_ue");
164
+
165
+ if(unlikely(!ue_st)) {
166
+ ue_st = rrdset_create("mem", "ecc_ue", NULL, "ecc", NULL, "ECC Memory Uncorrectable Errors", "errors", 6610
167
+ , update_every, RRDSET_TYPE_LINE);
168
+
169
+ for(m = mc_root; m; m = m->next)
170
+ if(m->ue_count_filename)
171
+ m->ue_rd = rrddim_add(ue_st, m->name, NULL, 1, 1, RRDDIM_INCREMENTAL);
172
+ }
173
+ else
174
+ rrdset_next(ue_st);
175
+
176
+ for(m = mc_root; m; m = m->next)
177
+ if(m->ue_count_filename && m->ue_updated)
178
+ rrddim_set_by_pointer(ue_st, m->ue_rd, m->ue_count);
179
+
180
+ rrdset_done(ue_st);
181
+ }
182
+
183
+ return 0;
184
+}