@cryptotaxi247 / netdata-1 / commits / 9bb40ae57

PCI Advanced Error Reporting (AER) (#15488)

Costa Tsaousis committed Jul 26, 2023 at 01:32 UTC 9bb40ae57c1a62fef0411f034ef4b26d017359f8
5 files changed +341 -6
Makefile.am
+1
@@ -409,6 +409,7 @@ PROC_PLUGIN_FILES = \
409 collectors/proc.plugin/sys_kernel_mm_ksm.c \
410 collectors/proc.plugin/sys_block_zram.c \
411 collectors/proc.plugin/sys_devices_system_edac_mc.c \
412 + collectors/proc.plugin/sys_devices_pci_aer.c \
413 collectors/proc.plugin/sys_devices_system_node.c \
414 collectors/proc.plugin/sys_fs_btrfs.c \
415 collectors/proc.plugin/sys_class_power_supply.c \
collectors/all.h
+2 -5
@@ -390,12 +390,9 @@
390
391 #define NETDATA_CHART_PRIO_STATSD_PRIVATE 90000 // many charts
392
393 -// INTERNAL NETDATA INFO
393 +// PCI
394
395 -#define NETDATA_CHART_PRIO_CHECKS 99999
396 -
397 -#define NETDATA_CHART_PRIO_NETDATA_TIMEX 132030
398 -#define NETDATA_CHART_PRIO_NETDATA_TC_TIME 1000100
395 +#define NETDATA_CHART_PRIO_PCI_AER 100000
396
397 // NETDATA ML CHARTS
398
collectors/proc.plugin/plugin_proc.c
+2 -1
@@ -33,7 +33,8 @@ static struct proc_module {
33 {.name = "/proc/meminfo", .dim = "meminfo", .func = do_proc_meminfo},
34 {.name = "/sys/kernel/mm/ksm", .dim = "ksm", .func = do_sys_kernel_mm_ksm},
35 {.name = "/sys/block/zram", .dim = "zram", .func = do_sys_block_zram},
36 - {.name = "/sys/devices/system/edac/mc", .dim = "ecc", .func = do_proc_sys_devices_system_edac_mc},
36 + {.name = "/sys/devices/system/edac/mc", .dim = "edac", .func = do_proc_sys_devices_system_edac_mc},
37 + {.name = "/sys/devices/pci/aer", .dim = "pci_aer", .func = do_proc_sys_devices_pci_aer},
38 {.name = "/sys/devices/system/node", .dim = "numa", .func = do_proc_sys_devices_system_node},
39 {.name = "/proc/pagetypeinfo", .dim = "pagetypeinfo", .func = do_proc_pagetypeinfo},
40
collectors/proc.plugin/plugin_proc.h
+1
@@ -34,6 +34,7 @@ int do_proc_net_stat_synproxy(int update_every, usec_t dt);
34 int do_proc_net_softnet_stat(int update_every, usec_t dt);
35 int do_proc_uptime(int update_every, usec_t dt);
36 int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt);
37 +int do_proc_sys_devices_pci_aer(int update_every, usec_t dt);
38 int do_proc_sys_devices_system_node(int update_every, usec_t dt);
39 int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt);
40 int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt);
collectors/proc.plugin/sys_devices_pci_aer.c new
+335
@@ -0,0 +1,335 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "plugin_proc.h"
4 +
5 +static char *pci_aer_dirname = NULL;
6 +
7 +typedef enum __attribute__((packed)) {
8 + AER_DEV_NONFATAL = (1 << 0),
9 + AER_DEV_CORRECTABLE = (1 << 1),
10 + AER_DEV_FATAL = (1 << 2),
11 + AER_ROOTPORT_TOTAL_ERR_COR = (1 << 3),
12 + AER_ROOTPORT_TOTAL_ERR_FATAL = (1 << 4),
13 +} AER_TYPE;
14 +
15 +struct aer_value {
16 + kernel_uint_t count;
17 + RRDDIM *rd;
18 +};
19 +
20 +struct aer_entry {
21 + bool updated;
22 +
23 + STRING *name;
24 + AER_TYPE type;
25 +
26 + procfile *ff;
27 + DICTIONARY *values;
28 +
29 + RRDSET *st;
30 +};
31 +
32 +DICTIONARY *aer_root = NULL;
33 +
34 +static bool aer_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
35 + struct aer_value *v = old_value;
36 + struct aer_value *nv = new_value;
37 +
38 + v->count = nv->count;
39 +
40 + return false;
41 +}
42 +
43 +static void aer_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
44 + struct aer_entry *a = value;
45 + a->values = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
46 + dictionary_register_conflict_callback(a->values, aer_value_conflict_callback, NULL);
47 +}
48 +
49 +static void add_pci_aer(const char *base_dir, const char *d_name, AER_TYPE type) {
50 + char buffer[FILENAME_MAX + 1];
51 + snprintfz(buffer, FILENAME_MAX, "%s/%s", base_dir, d_name);
52 + struct aer_entry *a = dictionary_set(aer_root, buffer, NULL, sizeof(struct aer_entry));
53 +
54 + if(!a->name)
55 + a->name = string_strdupz(d_name);
56 +
57 + a->type = type;
58 +}
59 +
60 +static bool recursively_find_pci_aer(AER_TYPE types, const char *base_dir, const char *d_name, int depth) {
61 + if(depth > 100)
62 + return false;
63 +
64 + char buffer[FILENAME_MAX + 1];
65 + snprintfz(buffer, FILENAME_MAX, "%s/%s", base_dir, d_name);
66 + DIR *dir = opendir(buffer);
67 + if(unlikely(!dir)) {
68 + collector_error("Cannot read PCI_AER directory '%s'", buffer);
69 + return true;
70 + }
71 +
72 + struct dirent *de = NULL;
73 + while((de = readdir(dir))) {
74 + if(de->d_type == DT_DIR) {
75 + if(de->d_name[0] == '.')
76 + continue;
77 +
78 + recursively_find_pci_aer(types, buffer, de->d_name, depth + 1);
79 + }
80 + else if(de->d_type == DT_REG) {
81 + if((types & AER_DEV_NONFATAL) && strcmp(de->d_name, "aer_dev_nonfatal") == 0) {
82 + add_pci_aer(buffer, de->d_name, AER_DEV_NONFATAL);
83 + }
84 + else if((types & AER_DEV_CORRECTABLE) && strcmp(de->d_name, "aer_dev_correctable") == 0) {
85 + add_pci_aer(buffer, de->d_name, AER_DEV_CORRECTABLE);
86 + }
87 + else if((types & AER_DEV_FATAL) && strcmp(de->d_name, "aer_dev_fatal") == 0) {
88 + add_pci_aer(buffer, de->d_name, AER_DEV_FATAL);
89 + }
90 + else if((types & AER_ROOTPORT_TOTAL_ERR_COR) && strcmp(de->d_name, "aer_rootport_total_err_cor") == 0) {
91 + add_pci_aer(buffer, de->d_name, AER_ROOTPORT_TOTAL_ERR_COR);
92 + }
93 + else if((types & AER_ROOTPORT_TOTAL_ERR_FATAL) && strcmp(de->d_name, "aer_rootport_total_err_fatal") == 0) {
94 + add_pci_aer(buffer, de->d_name, AER_ROOTPORT_TOTAL_ERR_FATAL);
95 + }
96 + }
97 + }
98 + closedir(dir);
99 + return true;
100 +}
101 +
102 +static void find_all_pci_aer(AER_TYPE types) {
103 + char name[FILENAME_MAX + 1];
104 + snprintfz(name, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices");
105 + pci_aer_dirname = config_get("plugin:proc:/sys/devices/pci/aer", "directory to monitor", name);
106 +
107 + DIR *dir = opendir(pci_aer_dirname);
108 + if(unlikely(!dir)) {
109 + collector_error("Cannot read PCI_AER directory '%s'", pci_aer_dirname);
110 + return;
111 + }
112 +
113 + struct dirent *de = NULL;
114 + while((de = readdir(dir))) {
115 + if(de->d_type == DT_DIR && de->d_name[0] == 'p' && de->d_name[1] == 'c' && de->d_name[2] == 'i' && isdigit(de->d_name[3]))
116 + recursively_find_pci_aer(types, pci_aer_dirname, de->d_name, 1);
117 + }
118 + closedir(dir);
119 +}
120 +
121 +static void read_pci_aer_values(const char *filename, struct aer_entry *t) {
122 + t->updated = false;
123 +
124 + if(unlikely(!t->ff)) {
125 + t->ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
126 + if(unlikely(!t->ff))
127 + return;
128 + }
129 +
130 + t->ff = procfile_readall(t->ff);
131 + if(unlikely(!t->ff || procfile_lines(t->ff) < 1 || procfile_linewords(t->ff, 0) < 1))
132 + return;
133 +
134 + size_t lines = procfile_lines(t->ff);
135 + for(size_t l = 0; l < lines ; l++) {
136 + if(procfile_linewords(t->ff, l) != 2)
137 + continue;
138 +
139 + struct aer_value v = {
140 + .count = str2ull(procfile_lineword(t->ff, l, 1), NULL)
141 + };
142 +
143 + char *key = procfile_lineword(t->ff, l, 0);
144 + if(!key || !*key || (key[0] == 'T' && key[1] == 'O' && key[2] == 'T' && key[3] == 'A' && key[4] == 'L' && key[5] == '_'))
145 + continue;
146 +
147 + dictionary_set(t->values, key, &v, sizeof(v));
148 + }
149 +
150 + t->updated = true;
151 +}
152 +
153 +static void read_pci_aer_count(const char *filename, struct aer_entry *t) {
154 + t->updated = false;
155 +
156 + if(unlikely(!t->ff)) {
157 + t->ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
158 + if(unlikely(!t->ff))
159 + return;
160 + }
161 +
162 + t->ff = procfile_readall(t->ff);
163 + if(unlikely(!t->ff || procfile_lines(t->ff) < 1 || procfile_linewords(t->ff, 0) < 1))
164 + return;
165 +
166 + struct aer_value v = {
167 + .count = str2ull(procfile_lineword(t->ff, 0, 0), NULL)
168 + };
169 + dictionary_set(t->values, "count", &v, sizeof(v));
170 + t->updated = true;
171 +}
172 +
173 +static void add_label_from_link(struct aer_entry *a, const char *path, const char *link) {
174 + char name[FILENAME_MAX + 1];
175 + strncpyz(name, path, FILENAME_MAX);
176 + char *slash = strrchr(name, '/');
177 + if(slash)
178 + *slash = '\0';
179 +
180 + char name2[FILENAME_MAX + 1];
181 + snprintfz(name2, FILENAME_MAX, "%s/%s", name, link);
182 +
183 + ssize_t len = readlink(name2, name, FILENAME_MAX);
184 + if(len != -1) {
185 + name[len] = '\0'; // Null-terminate the string
186 + slash = strrchr(name, '/');
187 + if(slash) slash++;
188 + else slash = name;
189 + rrdlabels_add(a->st->rrdlabels, link, slash, RRDLABEL_SRC_AUTO);
190 + }
191 +}
192 +
193 +int do_proc_sys_devices_pci_aer(int update_every, usec_t dt __maybe_unused) {
194 + if(unlikely(!aer_root)) {
195 + int do_root_ports = CONFIG_BOOLEAN_AUTO;
196 + int do_pci_slots = CONFIG_BOOLEAN_NO;
197 +
198 + char buffer[100 + 1] = "";
199 + rrdlabels_get_value_strcpyz(localhost->rrdlabels, buffer, 100, "_virtualization");
200 + if(strcmp(buffer, "none") != 0) {
201 + // no need to run on virtualized environments
202 + do_root_ports = CONFIG_BOOLEAN_NO;
203 + do_pci_slots = CONFIG_BOOLEAN_NO;
204 + }
205 +
206 + do_root_ports = config_get_boolean("plugin:proc:/sys/class/pci/aer", "enable root ports", do_root_ports);
207 + do_pci_slots = config_get_boolean("plugin:proc:/sys/class/pci/aer", "enable pci slots", do_pci_slots);
208 +
209 + if(!do_root_ports && !do_pci_slots)
210 + return 1;
211 +
212 + aer_root = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
213 + dictionary_register_insert_callback(aer_root, aer_insert_callback, NULL);
214 +
215 + AER_TYPE types = ((do_root_ports) ? (AER_ROOTPORT_TOTAL_ERR_COR|AER_ROOTPORT_TOTAL_ERR_FATAL) : 0) |
216 + ((do_pci_slots) ? (AER_DEV_FATAL|AER_DEV_NONFATAL|AER_DEV_CORRECTABLE) : 0);
217 +
218 + find_all_pci_aer(types);
219 +
220 + if(!dictionary_entries(aer_root))
221 + return 1;
222 + }
223 +
224 + struct aer_entry *a;
225 + dfe_start_read(aer_root, a) {
226 + switch(a->type) {
227 + case AER_DEV_NONFATAL:
228 + case AER_DEV_FATAL:
229 + case AER_DEV_CORRECTABLE:
230 + read_pci_aer_values(a_dfe.name, a);
231 + break;
232 +
233 + case AER_ROOTPORT_TOTAL_ERR_COR:
234 + case AER_ROOTPORT_TOTAL_ERR_FATAL:
235 + read_pci_aer_count(a_dfe.name, a);
236 + break;
237 + }
238 +
239 + if(!a->updated)
240 + continue;
241 +
242 + if(!a->st) {
243 + const char *title;
244 + const char *context;
245 +
246 + switch(a->type) {
247 + case AER_DEV_NONFATAL:
248 + title = "PCI Advanced Error Reporting (AER) Non-Fatal Errors";
249 + context = "pci.aer_nonfatal";
250 + break;
251 +
252 + case AER_DEV_FATAL:
253 + title = "PCI Advanced Error Reporting (AER) Fatal Errors";
254 + context = "pci.aer_fatal";
255 + break;
256 +
257 + case AER_DEV_CORRECTABLE:
258 + title = "PCI Advanced Error Reporting (AER) Correctable Errors";
259 + context = "pci.aer_correctable";
260 + break;
261 +
262 + case AER_ROOTPORT_TOTAL_ERR_COR:
263 + title = "PCI Root-Port Advanced Error Reporting (AER) Correctable Errors";
264 + context = "pci.rootport_aer_correctable";
265 + break;
266 +
267 + case AER_ROOTPORT_TOTAL_ERR_FATAL:
268 + title = "PCI Root-Port Advanced Error Reporting (AER) Fatal Errors";
269 + context = "pci.rootport_aer_fatal";
270 + break;
271 + }
272 +
273 + char id[RRD_ID_LENGTH_MAX + 1];
274 + char nm[RRD_ID_LENGTH_MAX + 1];
275 + size_t len = strlen(pci_aer_dirname);
276 +
277 + const char *fname = a_dfe.name;
278 + if(strncmp(a_dfe.name, pci_aer_dirname, len) == 0)
279 + fname = &a_dfe.name[len];
280 +
281 + if(*fname == '/')
282 + fname++;
283 +
284 + snprintfz(id, RRD_ID_LENGTH_MAX, "%s_%s", &context[4], fname);
285 + char *slash = strrchr(id, '/');
286 + if(slash)
287 + *slash = '\0';
288 +
289 + netdata_fix_chart_id(id);
290 +
291 + snprintfz(nm, RRD_ID_LENGTH_MAX, "%s", fname);
292 + slash = strrchr(nm, '/');
293 + if(slash)
294 + *slash = '\0';
295 +
296 + a->st = rrdset_create_localhost(
297 + "pci"
298 + , id
299 + , NULL
300 + , "aer"
301 + , context
302 + , title
303 + , "errors/s"
304 + , PLUGIN_PROC_NAME
305 + , "/sys/devices/pci/aer"
306 + , NETDATA_CHART_PRIO_PCI_AER
307 + , update_every
308 + , RRDSET_TYPE_LINE
309 + );
310 +
311 + rrdlabels_add(a->st->rrdlabels, "device", nm, RRDLABEL_SRC_AUTO);
312 + add_label_from_link(a, a_dfe.name, "driver");
313 +
314 + struct aer_value *v;
315 + dfe_start_read(a->values, v) {
316 + v->rd = rrddim_add(a->st, v_dfe.name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
317 + }
318 + dfe_done(v);
319 + }
320 +
321 + struct aer_value *v;
322 + dfe_start_read(a->values, v) {
323 + if(unlikely(!v->rd))
324 + v->rd = rrddim_add(a->st, v_dfe.name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
325 +
326 + rrddim_set_by_pointer(a->st, v->rd, (collected_number)v->count);
327 + }
328 + dfe_done(v);
329 +
330 + rrdset_done(a->st);
331 + }
332 + dfe_done(a);
333 +
334 + return 0;
335 +}