Memory Controller (MC) and DIMM Error Detection And Correction (EDAC) (#15473)
Co-authored-by: ilyam8 <ilya@netdata.cloud>
Costa Tsaousis committed
Jul 21, 2023 at 16:48 UTC
173a3f9bedeb2b8d367e908fdf99a46af19b6f1a
3 files changed
+321
-156
collectors/proc.plugin/multi_metadata.yaml
+60
-21
@@ -1307,7 +1307,7 @@ modules:
1307
plugin_name: proc.plugin
1308
module_name: /sys/devices/system/edac/mc
1309
monitored_instance:
1310
- name: System Memory Errors
1310
+ name: Memory modules (DIMMs)
1311
link: ""
1312
categories:
1313
- data-collection.linux-systems.memory-metrics
@@ -1319,9 +1319,10 @@ modules:
1319
description: ""
1320
keywords:
1321
- edac
1322
- - error detection and correction memory controllers
1322
- ecc
1323
+ - dimm
1324
- ram
1325
+ - hardware
1326
most_popular: false
1327
overview:
1328
data_collection:
@@ -1362,16 +1363,22 @@ modules:
1363
problems:
1364
list: []
1365
alerts:
1365
- - name: 1hour_ecc_memory_correctable
1366
+ - name: ecc_memory_mc_noinfo_correctable
1367
+ metric: mem.edac_mc
1368
+ info: memory controller ${label:controller} ECC correctable errors (unknown DIMM slot) in the last 10 minutes
1369
link: https://github.com/netdata/netdata/blob/master/health/health.d/memory.conf
1367
- metric: mem.ecc_ce
1368
- info: number of ECC correctable errors in the last 10 minutes
1369
- os: "linux"
1370
- - name: 1hour_ecc_memory_uncorrectable
1370
+ - name: ecc_memory_mc_noinfo_uncorrectable
1371
+ metric: mem.edac_mc
1372
+ info: memory controller ${label:controller} ECC uncorrectable errors (unknown DIMM slot) in the last 10 minutes
1373
+ link: https://github.com/netdata/netdata/blob/master/health/health.d/memory.conf
1374
+ - name: ecc_memory_dimm_correctable
1375
+ metric: mem.edac_mc_dimm
1376
+ info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC correctable errors in the last 10 minutes
1377
+ link: https://github.com/netdata/netdata/blob/master/health/health.d/memory.conf
1378
+ - name: ecc_memory_dimm_uncorrectable
1379
+ metric: mem.edac_mc_dimm
1380
+ info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC uncorrectable errors in the last 10 minutes
1381
link: https://github.com/netdata/netdata/blob/master/health/health.d/memory.conf
1372
- metric: mem.ecc_ue
1373
- info: number of ECC uncorrectable errors in the last 10 minutes
1374
- os: "linux"
1382
metrics:
1383
folding:
1384
title: Metrics
@@ -1379,22 +1386,54 @@ modules:
1386
description: ""
1387
availability: []
1388
scopes:
1382
- - name: global
1383
- description: ""
1384
- labels: []
1389
+ - name: memory controller
1390
+ description: These metrics refer to the memory controller.
1391
+ labels:
1392
+ - name: controller
1393
+ description: "[mcX](https://www.kernel.org/doc/html/v5.0/admin-guide/ras.html#mcx-directories) directory name of this memory controller."
1394
+ - name: mc_name
1395
+ description: Memory controller type.
1396
+ - name: size_mb
1397
+ description: The amount of memory in megabytes that this memory controller manages.
1398
+ - name: max_location
1399
+ description: Last available memory slot in this memory controller.
1400
metrics:
1386
- - name: mem.ecc_ce
1387
- description: ECC Memory Correctable Errors
1388
- unit: "errors"
1401
+ - name: mem.edac_mc
1402
+ description: Memory Controller (MC) Error Detection And Correction (EDAC) Errors
1403
+ unit: errors/s
1404
chart_type: line
1405
dimensions:
1391
- - name: a dimension per mem controller
1392
- - name: mem.ecc_ue
1393
- description: ECC Memory Uncorrectable Errors
1394
- unit: "errors"
1406
+ - name: correctable
1407
+ - name: uncorrectable
1408
+ - name: correctable_noinfo
1409
+ - name: uncorrectable_noinfo
1410
+ - name: memory module
1411
+ description: These metrics refer to the memory module (or rank, [depends on the memory controller](https://www.kernel.org/doc/html/v5.0/admin-guide/ras.html#f5)).
1412
+ labels:
1413
+ - name: controller
1414
+ description: "[mcX](https://www.kernel.org/doc/html/v5.0/admin-guide/ras.html#mcx-directories) directory name of this memory controller."
1415
+ - name: dimm
1416
+ description: "[dimmX or rankX](https://www.kernel.org/doc/html/v5.0/admin-guide/ras.html#dimmx-or-rankx-directories) directory name of this memory module."
1417
+ - name: dimm_dev_type
1418
+ description: Type of DRAM device used in this memory module. For example, x1, x2, x4, x8.
1419
+ - name: dimm_edac_mode
1420
+ description: Used type of error detection and correction. For example, S4ECD4ED would mean a Chipkill with x4 DRAM.
1421
+ - name: dimm_label
1422
+ description: Label assigned to this memory module.
1423
+ - name: dimm_location
1424
+ description: Location of the memory module.
1425
+ - name: dimm_mem_type
1426
+ description: Type of the memory module. Usually either buffered or unbuffered memory.
1427
+ - name: size
1428
+ description: The amount of memory in megabytes that this memory module manages.
1429
+ metrics:
1430
+ - name: mem.edac_mc
1431
+ description: DIMM Error Detection And Correction (EDAC) Errors
1432
+ unit: errors/s
1433
chart_type: line
1434
dimensions:
1397
- - name: a dimension per mem controller
1435
+ - name: correctable
1436
+ - name: uncorrectable
1437
- meta:
1438
plugin_name: proc.plugin
1439
module_name: /sys/devices/system/node
collectors/proc.plugin/sys_devices_system_edac_mc.c
+213
-120
@@ -2,35 +2,51 @@
2
3
#include "plugin_proc.h"
4
5
+struct edac_count {
6
+ bool updated;
7
+ char *filename;
8
+ procfile *ff;
9
+ kernel_uint_t count;
10
+ RRDDIM *rd;
11
+};
12
+
13
+struct edac_dimm {
14
+ char *name;
15
+
16
+ struct edac_count ce;
17
+ struct edac_count ue;
18
+
19
+ RRDSET *st;
20
+
21
+ struct edac_dimm *prev, *next;
22
+};
23
+
24
struct mc {
25
char *name;
7
- char ce_updated;
8
- char ue_updated;
26
10
- char *ce_count_filename;
11
- char *ue_count_filename;
27
+ struct edac_count ce;
28
+ struct edac_count ue;
29
+ struct edac_count ce_noinfo;
30
+ struct edac_count ue_noinfo;
31
13
- procfile *ce_ff;
14
- procfile *ue_ff;
32
+ RRDSET *st;
33
16
- collected_number ce_count;
17
- collected_number ue_count;
34
+ struct edac_dimm *dimms;
35
19
- RRDDIM *ce_rd;
20
- RRDDIM *ue_rd;
21
-
22
- struct mc *next;
36
+ struct mc *prev, *next;
37
};
38
+
39
static struct mc *mc_root = NULL;
40
+static char *mc_dirname = NULL;
41
42
static void find_all_mc() {
43
char name[FILENAME_MAX + 1];
44
snprintfz(name, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/edac/mc");
29
- char *dirname = config_get("plugin:proc:/sys/devices/system/edac/mc", "directory to monitor", name);
45
+ mc_dirname = config_get("plugin:proc:/sys/devices/system/edac/mc", "directory to monitor", name);
46
31
- DIR *dir = opendir(dirname);
47
+ DIR *dir = opendir(mc_dirname);
48
if(unlikely(!dir)) {
33
- collector_error("Cannot read ECC memory errors directory '%s'", dirname);
49
+ collector_error("Cannot read EDAC memory errors directory '%s'", mc_dirname);
50
return;
51
}
52
@@ -42,162 +58,239 @@ static void find_all_mc() {
58
59
struct stat st;
60
45
- snprintfz(name, FILENAME_MAX, "%s/%s/ce_count", dirname, de->d_name);
61
+ snprintfz(name, FILENAME_MAX, "%s/%s/ce_count", mc_dirname, de->d_name);
62
if(stat(name, &st) != -1)
47
- m->ce_count_filename = strdupz(name);
63
+ m->ce.filename = strdupz(name);
64
49
- snprintfz(name, FILENAME_MAX, "%s/%s/ue_count", dirname, de->d_name);
65
+ snprintfz(name, FILENAME_MAX, "%s/%s/ue_count", mc_dirname, de->d_name);
66
if(stat(name, &st) != -1)
51
- m->ue_count_filename = strdupz(name);
67
+ m->ue.filename = strdupz(name);
68
53
- if(!m->ce_count_filename && !m->ue_count_filename) {
69
+ snprintfz(name, FILENAME_MAX, "%s/%s/ce_noinfo_count", mc_dirname, de->d_name);
70
+ if(stat(name, &st) != -1)
71
+ m->ce_noinfo.filename = strdupz(name);
72
+
73
+ snprintfz(name, FILENAME_MAX, "%s/%s/ue_noinfo_count", mc_dirname, de->d_name);
74
+ if(stat(name, &st) != -1)
75
+ m->ue_noinfo.filename = strdupz(name);
76
+
77
+ if(!m->ce.filename && !m->ue.filename && !m->ce_noinfo.filename && !m->ue_noinfo.filename) {
78
freez(m->name);
79
freez(m);
80
}
57
- else {
58
- m->next = mc_root;
59
- mc_root = m;
60
- }
81
+ else
82
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(mc_root, m, prev, next);
83
}
84
}
63
-
85
closedir(dir);
65
-}
86
67
-int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) {
68
- (void)dt;
87
+ for(struct mc *m = mc_root; m ;m = m->next) {
88
+ snprintfz(name, FILENAME_MAX, "%s/%s", mc_dirname, m->name);
89
+ dir = opendir(name);
90
+ if(!dir) {
91
+ collector_error("Cannot read EDAC memory errors directory '%s'", name);
92
+ continue;
93
+ }
94
70
- if(unlikely(mc_root == NULL)) {
71
- find_all_mc();
72
- if(unlikely(mc_root == NULL))
73
- return 1;
74
- }
95
+ while((de = readdir(dir))) {
96
+ // it can be dimmX or rankX directory
97
+ // https://www.kernel.org/doc/html/v5.0/admin-guide/ras.html#f5
98
76
- static int do_ce = -1, do_ue = -1;
77
- NETDATA_DOUBLE ce_sum = 0, ue_sum = 0;
78
- struct mc *m;
99
+ if (de->d_type == DT_DIR &&
100
+ ((strncmp(de->d_name, "rank", 4) == 0 || strncmp(de->d_name, "dimm", 4) == 0)) &&
101
+ isdigit(de->d_name[4])) {
102
80
- if(unlikely(do_ce == -1)) {
81
- do_ce = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory correctable errors", CONFIG_BOOLEAN_YES);
82
- do_ue = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory uncorrectable errors", CONFIG_BOOLEAN_YES);
83
- }
103
+ struct edac_dimm *d = callocz(1, sizeof(struct edac_dimm));
104
+ d->name = strdupz(de->d_name);
105
85
- if(do_ce != CONFIG_BOOLEAN_NO) {
86
- for(m = mc_root; m; m = m->next) {
87
- if(m->ce_count_filename) {
88
- m->ce_updated = 0;
106
+ struct stat st;
107
90
- if(unlikely(!m->ce_ff)) {
91
- m->ce_ff = procfile_open(m->ce_count_filename, " \t", PROCFILE_FLAG_DEFAULT);
92
- if(unlikely(!m->ce_ff))
93
- continue;
94
- }
108
+ snprintfz(name, FILENAME_MAX, "%s/%s/%s/dimm_ce_count", mc_dirname, m->name, de->d_name);
109
+ if(stat(name, &st) != -1)
110
+ d->ce.filename = strdupz(name);
111
96
- m->ce_ff = procfile_readall(m->ce_ff);
97
- if(unlikely(!m->ce_ff || procfile_lines(m->ce_ff) < 1 || procfile_linewords(m->ce_ff, 0) < 1))
98
- continue;
112
+ snprintfz(name, FILENAME_MAX, "%s/%s/%s/dimm_ue_count", mc_dirname, m->name, de->d_name);
113
+ if(stat(name, &st) != -1)
114
+ d->ue.filename = strdupz(name);
115
100
- m->ce_count = str2ull(procfile_lineword(m->ce_ff, 0, 0), NULL);
101
- ce_sum += m->ce_count;
102
- m->ce_updated = 1;
116
+ if(!d->ce.filename && !d->ue.filename) {
117
+ freez(d->name);
118
+ freez(d);
119
+ }
120
+ else
121
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(m->dimms, d, prev, next);
122
}
123
}
124
}
125
+}
126
107
- if(do_ue != CONFIG_BOOLEAN_NO) {
108
- for(m = mc_root; m; m = m->next) {
109
- if(m->ue_count_filename) {
110
- m->ue_updated = 0;
127
+static kernel_uint_t read_edac_count(struct edac_count *t) {
128
+ t->updated = false;
129
+ t->count = 0;
130
112
- if(unlikely(!m->ue_ff)) {
113
- m->ue_ff = procfile_open(m->ue_count_filename, " \t", PROCFILE_FLAG_DEFAULT);
114
- if(unlikely(!m->ue_ff))
115
- continue;
116
- }
131
+ if(t->filename) {
132
+ if(unlikely(!t->ff)) {
133
+ t->ff = procfile_open(t->filename, " \t", PROCFILE_FLAG_DEFAULT);
134
+ if(unlikely(!t->ff))
135
+ return 0;
136
+ }
137
118
- m->ue_ff = procfile_readall(m->ue_ff);
119
- if(unlikely(!m->ue_ff || procfile_lines(m->ue_ff) < 1 || procfile_linewords(m->ue_ff, 0) < 1))
120
- continue;
138
+ t->ff = procfile_readall(t->ff);
139
+ if(unlikely(!t->ff || procfile_lines(t->ff) < 1 || procfile_linewords(t->ff, 0) < 1))
140
+ return 0;
141
122
- m->ue_count = str2ull(procfile_lineword(m->ue_ff, 0, 0), NULL);
123
- ue_sum += m->ue_count;
124
- m->ue_updated = 1;
125
- }
142
+ t->count = str2ull(procfile_lineword(t->ff, 0, 0), NULL);
143
+ t->updated = true;
144
+ }
145
+
146
+ return t->count;
147
+}
148
+
149
+static bool read_edac_mc_file(const char *mc, const char *filename, char *out, size_t out_size) {
150
+ char f[FILENAME_MAX + 1];
151
+ snprintfz(f, FILENAME_MAX, "%s/%s/%s", mc_dirname, mc, filename);
152
+ if(read_file(f, out, out_size) != 0) {
153
+ collector_error("EDAC: cannot read file '%s'", f);
154
+ return false;
155
+ }
156
+ return true;
157
+}
158
+
159
+static bool read_edac_mc_rank_file(const char *mc, const char *rank, const char *filename, char *out, size_t out_size) {
160
+ char f[FILENAME_MAX + 1];
161
+ snprintfz(f, FILENAME_MAX, "%s/%s/%s/%s", mc_dirname, mc, rank, filename);
162
+ if(read_file(f, out, out_size) != 0) {
163
+ collector_error("EDAC: cannot read file '%s'", f);
164
+ return false;
165
+ }
166
+ return true;
167
+}
168
+
169
+int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt __maybe_unused) {
170
+ if(unlikely(!mc_root)) {
171
+ find_all_mc();
172
+
173
+ if(!mc_root)
174
+ // don't call this again
175
+ return 1;
176
+ }
177
+
178
+ for(struct mc *m = mc_root; m; m = m->next) {
179
+ read_edac_count(&m->ce);
180
+ read_edac_count(&m->ce_noinfo);
181
+ read_edac_count(&m->ue);
182
+ read_edac_count(&m->ue_noinfo);
183
+
184
+ for(struct edac_dimm *d = m->dimms; d ;d = d->next) {
185
+ read_edac_count(&d->ce);
186
+ read_edac_count(&d->ue);
187
}
188
}
189
190
// --------------------------------------------------------------------
191
131
- if(do_ce == CONFIG_BOOLEAN_YES || (do_ce == CONFIG_BOOLEAN_AUTO &&
132
- (ce_sum > 0 || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
133
- do_ce = CONFIG_BOOLEAN_YES;
192
+ for(struct mc *m = mc_root; m ; m = m->next) {
193
+ if(unlikely(!m->ce.updated && !m->ue.updated && !m->ce_noinfo.updated && !m->ue_noinfo.updated))
194
+ continue;
195
135
- static RRDSET *ce_st = NULL;
136
-
137
- if(unlikely(!ce_st)) {
138
- ce_st = rrdset_create_localhost(
196
+ if(unlikely(!m->st)) {
197
+ char id[RRD_ID_LENGTH_MAX + 1];
198
+ snprintfz(id, RRD_ID_LENGTH_MAX, "edac_%s", m->name);
199
+ m->st = rrdset_create_localhost(
200
"mem"
140
- , "ecc_ce"
141
- , NULL
142
- , "ecc"
201
+ , id
202
, NULL
144
- , "ECC Memory Correctable Errors"
145
- , "errors"
203
+ , "edac"
204
+ , "mem.edac_mc"
205
+ , "Memory Controller (MC) Error Detection And Correction (EDAC) Errors"
206
+ , "errors/s"
207
, PLUGIN_PROC_NAME
208
, "/sys/devices/system/edac/mc"
209
, NETDATA_CHART_PRIO_MEM_HW_ECC_CE
210
, update_every
211
, RRDSET_TYPE_LINE
212
);
152
- }
213
154
- for(m = mc_root; m; m = m->next) {
155
- if (m->ce_count_filename && m->ce_updated) {
156
- if(unlikely(!m->ce_rd))
157
- m->ce_rd = rrddim_add(ce_st, m->name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
214
+ rrdlabels_add(m->st->rrdlabels, "controller", m->name, RRDLABEL_SRC_AUTO);
215
159
- rrddim_set_by_pointer(ce_st, m->ce_rd, m->ce_count);
160
- }
216
+ char buffer[1024 + 1];
217
+
218
+ if(read_edac_mc_file(m->name, "mc_name", buffer, 1024))
219
+ rrdlabels_add(m->st->rrdlabels, "mc_name", buffer, RRDLABEL_SRC_AUTO);
220
+
221
+ if(read_edac_mc_file(m->name, "size_mb", buffer, 1024))
222
+ rrdlabels_add(m->st->rrdlabels, "size_mb", buffer, RRDLABEL_SRC_AUTO);
223
+
224
+ if(read_edac_mc_file(m->name, "max_location", buffer, 1024))
225
+ rrdlabels_add(m->st->rrdlabels, "max_location", buffer, RRDLABEL_SRC_AUTO);
226
+
227
+ m->ce.rd = rrddim_add(m->st, "correctable", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
228
+ m->ue.rd = rrddim_add(m->st, "uncorrectable", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
229
+ m->ce_noinfo.rd = rrddim_add(m->st, "correctable_noinfo", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
230
+ m->ue_noinfo.rd = rrddim_add(m->st, "uncorrectable_noinfo", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
231
}
232
163
- rrdset_done(ce_st);
164
- }
233
+ rrddim_set_by_pointer(m->st, m->ce.rd, (collected_number)m->ce.count);
234
+ rrddim_set_by_pointer(m->st, m->ue.rd, (collected_number)m->ue.count);
235
+ rrddim_set_by_pointer(m->st, m->ce_noinfo.rd, (collected_number)m->ce_noinfo.count);
236
+ rrddim_set_by_pointer(m->st, m->ue_noinfo.rd, (collected_number)m->ue_noinfo.count);
237
166
- // --------------------------------------------------------------------
238
+ rrdset_done(m->st);
239
168
- if(do_ue == CONFIG_BOOLEAN_YES || (do_ue == CONFIG_BOOLEAN_AUTO &&
169
- (ue_sum > 0 || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
170
- do_ue = CONFIG_BOOLEAN_YES;
240
+ for(struct edac_dimm *d = m->dimms; d ;d = d->next) {
241
+ if(unlikely(!d->ce.updated && !d->ue.updated))
242
+ continue;
243
172
- static RRDSET *ue_st = NULL;
244
+ if(unlikely(!d->st)) {
245
+ char id[RRD_ID_LENGTH_MAX + 1];
246
+ snprintfz(id, RRD_ID_LENGTH_MAX, "edac_%s_%s", m->name, d->name);
247
+ d->st = rrdset_create_localhost(
248
+ "mem"
249
+ , id
250
+ , NULL
251
+ , "edac"
252
+ , "mem.edac_mc_dimm"
253
+ , "DIMM Error Detection And Correction (EDAC) Errors"
254
+ , "errors/s"
255
+ , PLUGIN_PROC_NAME
256
+ , "/sys/devices/system/edac/mc"
257
+ , NETDATA_CHART_PRIO_MEM_HW_ECC_CE + 1
258
+ , update_every
259
+ , RRDSET_TYPE_LINE
260
+ );
261
174
- if(unlikely(!ue_st)) {
175
- ue_st = rrdset_create_localhost(
176
- "mem"
177
- , "ecc_ue"
178
- , NULL
179
- , "ecc"
180
- , NULL
181
- , "ECC Memory Uncorrectable Errors"
182
- , "errors"
183
- , PLUGIN_PROC_NAME
184
- , "/sys/devices/system/edac/mc"
185
- , NETDATA_CHART_PRIO_MEM_HW_ECC_UE
186
- , update_every
187
- , RRDSET_TYPE_LINE
188
- );
189
- }
262
+ rrdlabels_add(d->st->rrdlabels, "controller", m->name, RRDLABEL_SRC_AUTO);
263
+ rrdlabels_add(d->st->rrdlabels, "dimm", d->name, RRDLABEL_SRC_AUTO);
264
+
265
+ char buffer[1024 + 1];
266
191
- for(m = mc_root; m; m = m->next) {
192
- if (m->ue_count_filename && m->ue_updated) {
193
- if(unlikely(!m->ue_rd))
194
- m->ue_rd = rrddim_add(ue_st, m->name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
267
+ if(read_edac_mc_rank_file(m->name, d->name, "dimm_dev_type", buffer, 1024))
268
+ rrdlabels_add(d->st->rrdlabels, "dimm_dev_type", buffer, RRDLABEL_SRC_AUTO);
269
196
- rrddim_set_by_pointer(ue_st, m->ue_rd, m->ue_count);
270
+ if(read_edac_mc_rank_file(m->name, d->name, "dimm_edac_mode", buffer, 1024))
271
+ rrdlabels_add(d->st->rrdlabels, "dimm_edac_mode", buffer, RRDLABEL_SRC_AUTO);
272
+
273
+ if(read_edac_mc_rank_file(m->name, d->name, "dimm_label", buffer, 1024))
274
+ rrdlabels_add(d->st->rrdlabels, "dimm_label", buffer, RRDLABEL_SRC_AUTO);
275
+
276
+ if(read_edac_mc_rank_file(m->name, d->name, "dimm_location", buffer, 1024))
277
+ rrdlabels_add(d->st->rrdlabels, "dimm_location", buffer, RRDLABEL_SRC_AUTO);
278
+
279
+ if(read_edac_mc_rank_file(m->name, d->name, "dimm_mem_type", buffer, 1024))
280
+ rrdlabels_add(d->st->rrdlabels, "dimm_mem_type", buffer, RRDLABEL_SRC_AUTO);
281
+
282
+ if(read_edac_mc_rank_file(m->name, d->name, "size", buffer, 1024))
283
+ rrdlabels_add(d->st->rrdlabels, "size", buffer, RRDLABEL_SRC_AUTO);
284
+
285
+ d->ce.rd = rrddim_add(d->st, "correctable", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
286
+ d->ue.rd = rrddim_add(d->st, "uncorrectable", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
287
}
198
- }
288
200
- rrdset_done(ue_st);
289
+ rrddim_set_by_pointer(d->st, d->ce.rd, (collected_number)d->ce.count);
290
+ rrddim_set_by_pointer(d->st, d->ue.rd, (collected_number)d->ue.count);
291
+
292
+ rrdset_done(d->st);
293
+ }
294
}
295
296
return 0;
health/health.d/memory.conf
+48
-15
@@ -1,47 +1,80 @@
1
-
1
# you can disable an alarm notification by setting the 'to' line to: silent
2
4
- alarm: 1hour_ecc_memory_correctable
5
- on: mem.ecc_ce
3
+ alarm: 1hour_memory_hw_corrupted
4
+ on: mem.hwcorrupt
5
class: Errors
6
type: System
7
component: Memory
8
os: linux
9
hosts: *
11
- lookup: sum -10m unaligned
10
+ calc: $HardwareCorrupted
11
+ units: MB
12
+ every: 10s
13
+ warn: $this > 0
14
+ delay: down 1h multiplier 1.5 max 1h
15
+ info: amount of memory corrupted due to a hardware failure
16
+ to: sysadmin
17
+
18
+## ECC Controller
19
+
20
+ template: ecc_memory_mc_correctable
21
+ on: mem.edac_mc
22
+ class: Errors
23
+ type: System
24
+component: Memory
25
+ os: linux
26
+ hosts: *
27
+ lookup: sum -10m unaligned of correctable, correctable_noinfo
28
units: errors
29
every: 1m
30
warn: $this > 0
31
delay: down 1h multiplier 1.5 max 1h
16
- info: number of ECC correctable errors in the last 10 minutes
32
+ info: memory controller ${label:controller} ECC correctable errors in the last 10 minutes
33
to: sysadmin
34
19
- alarm: 1hour_ecc_memory_uncorrectable
20
- on: mem.ecc_ue
35
+ template: ecc_memory_mc_uncorrectable
36
+ on: mem.edac_mc
37
class: Errors
38
type: System
39
component: Memory
40
os: linux
41
hosts: *
26
- lookup: sum -10m unaligned
42
+ lookup: sum -10m unaligned of uncorrectable,uncorrectable_noinfo
43
units: errors
44
every: 1m
45
crit: $this > 0
46
delay: down 1h multiplier 1.5 max 1h
31
- info: number of ECC uncorrectable errors in the last 10 minutes
47
+ info: memory controller ${label:controller} ECC uncorrectable errors in the last 10 minutes
48
to: sysadmin
49
34
- alarm: 1hour_memory_hw_corrupted
35
- on: mem.hwcorrupt
50
+## ECC DIMM
51
+
52
+ template: ecc_memory_dimm_correctable
53
+ on: mem.edac_mc_dimm
54
class: Errors
55
type: System
56
component: Memory
57
os: linux
58
hosts: *
41
- calc: $HardwareCorrupted
42
- units: MB
43
- every: 10s
59
+ lookup: sum -10m unaligned of correctable
60
+ units: errors
61
+ every: 1m
62
warn: $this > 0
63
delay: down 1h multiplier 1.5 max 1h
46
- info: amount of memory corrupted due to a hardware failure
64
+ info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC correctable errors in the last 10 minutes
65
+ to: sysadmin
66
+
67
+ template: ecc_memory_dimm_uncorrectable
68
+ on: mem.edac_mc_dimm
69
+ class: Errors
70
+ type: System
71
+component: Memory
72
+ os: linux
73
+ hosts: *
74
+ lookup: sum -10m unaligned of uncorrectable
75
+ units: errors
76
+ every: 1m
77
+ crit: $this > 0
78
+ delay: down 1h multiplier 1.5 max 1h
79
+ info: DIMM ${label:dimm} controller ${label:controller} (location ${label:dimm_location}) ECC uncorrectable errors in the last 10 minutes
80
to: sysadmin