@cryptotaxi247 / netdata-1 / commits / 42b1de4da

Add infiniband monitoring to collector proc.plugin (#9091)

Adrien Mahieux committed Jul 2, 2020 at 10:19 UTC 42b1de4dad8afbcd36a34019002889fc4257b9b0
7 files changed +751
CMakeLists.txt
+1
@@ -476,6 +476,7 @@ set(PROC_PLUGIN_FILES
476 collectors/proc.plugin/sys_block_zram.c
477 collectors/proc.plugin/sys_devices_system_edac_mc.c
478 collectors/proc.plugin/sys_devices_system_node.c
479 + collectors/proc.plugin/sys_class_infiniband.c
480 collectors/proc.plugin/sys_fs_btrfs.c
481 collectors/proc.plugin/sys_class_power_supply.c
482 )
Makefile.am
+1
@@ -319,6 +319,7 @@ PROC_PLUGIN_FILES = \
319 collectors/proc.plugin/sys_devices_system_node.c \
320 collectors/proc.plugin/sys_fs_btrfs.c \
321 collectors/proc.plugin/sys_class_power_supply.c \
322 + collectors/proc.plugin/sys_class_infiniband.c \
323 $(NULL)
324
325 TC_PLUGIN_FILES = \
collectors/all.h
+2
@@ -282,6 +282,8 @@
282 #define NETDATA_CHART_PRIO_TC_QOS_TOCKENS 7030
283 #define NETDATA_CHART_PRIO_TC_QOS_CTOCKENS 7040
284
285 +// Infiniband
286 +#define NETDATA_CHART_PRIO_INFINIBAND 7100
287
288 // Netfilter
289
collectors/proc.plugin/README.md
+43
@@ -28,6 +28,7 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/proc.
28 - `/proc/pressure/{cpu,memory,io}` (pressure stall information)
29 - `/proc/sys/kernel/random/entropy_avail` (random numbers pool availability - used in cryptography)
30 - `/sys/class/power_supply` (power supply properties)
31 +- `/sys/class/infiniband` (infiniband interconnect)
32 - `ipc` (IPC semaphores and message queues)
33 - `ksm` Kernel Same-Page Merging performance (several files under `/sys/kernel/mm/ksm`).
34 - `netdata` (internal Netdata resources utilization)
@@ -462,6 +463,48 @@ and metrics:
463 the corresponding `min` or `empty`, which will then always read as zero.
464 This way, alerts which match on these will still work.
465
466 +## Infiniband interconnect
467 +
468 +This module monitors every active Infiniband port. It provides generic counters statistics, and per-vendor hw-counters (if vendor is supported).
469 +
470 +### Monitored interface metrics
471 +
472 +Each port will have its counters metrics monitored, grouped in the following charts:
473 +
474 +- **Bandwidth usage**
475 + Sent/Received data, in KB/s
476 +
477 +- **Packets Statistics**
478 + Sent/Received packets, in 3 categories: total, unicast and multicast.
479 +
480 +- **Errors Statistics**
481 + Many errors counters are provided, presenting statistics for:
482 + - Packets: malformated, sent/received discarded by card/switch, missing ressource
483 + - Link: downed, recovered, integrity error, minor error
484 + - Other events: Tick Wait to send, buffer overrun
485 +
486 +If your vendor is supported, you'll also get HW-Counters statistics. These being vendor specific, please refer to their documentation.
487 +
488 +- Mellanox: [see statistics documentation](https://community.mellanox.com/s/article/understanding-mlx5-linux-counters-and-status-parameters)
489 +
490 +### configuration
491 +
492 +Default configuration will monitor only enabled infiniband ports, and refresh newly activated or created ports every 30 seconds
493 +
494 +```
495 +[plugin:proc:/sys/class/infiniband]
496 + # dirname to monitor = /sys/class/infiniband
497 + # bandwidth counters = yes
498 + # packets counters = yes
499 + # errors counters = yes
500 + # hardware packets counters = auto
501 + # hardware errors counters = auto
502 + # monitor only ports being active = auto
503 + # disable by default interfaces matching =
504 + # refresh ports state every seconds = 30
505 +```
506 +
507 +
508 ## IPC
509
510 ### Monitored IPC metrics
collectors/proc.plugin/plugin_proc.c
+1
@@ -47,6 +47,7 @@ static struct proc_module {
47 { .name = "/proc/net/sctp/snmp", .dim = "sctp", .func = do_proc_net_sctp_snmp },
48 { .name = "/proc/net/softnet_stat", .dim = "softnet", .func = do_proc_net_softnet_stat },
49 { .name = "/proc/net/ip_vs/stats", .dim = "ipvs", .func = do_proc_net_ip_vs_stats },
50 + { .name = "/sys/class/infiniband", .dim = "infiniband", .func = do_sys_class_infiniband },
51
52 // firewall metrics
53 { .name = "/proc/net/stat/conntrack", .dim = "conntrack", .func = do_proc_net_stat_conntrack },
collectors/proc.plugin/plugin_proc.h
+1
@@ -57,6 +57,7 @@ extern int do_proc_net_sctp_snmp(int update_every, usec_t dt);
57 extern int do_ipc(int update_every, usec_t dt);
58 extern int do_sys_class_power_supply(int update_every, usec_t dt);
59 extern int do_proc_pagetypeinfo(int update_every, usec_t dt);
60 +extern int do_sys_class_infiniband(int update_every, usec_t dt);
61 extern int get_numa_node_count(void);
62
63 // metrics that need to be shared among data collectors
collectors/proc.plugin/sys_class_infiniband.c new
+702
@@ -0,0 +1,702 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +// Heavily inspired from proc_net_dev.c
4 +#include "plugin_proc.h"
5 +
6 +#define PLUGIN_PROC_MODULE_INFINIBAND_NAME "/sys/class/infiniband"
7 +#define CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND \
8 + "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_INFINIBAND_NAME
9 +
10 +// ib_device::name[IB_DEVICE_NAME_MAX(64)] + "-" + ib_device::phys_port_cnt[u8 = 3 chars]
11 +#define IBNAME_MAX 68
12 +
13 +// ----------------------------------------------------------------------------
14 +// infiniband & omnipath standard counters
15 +
16 +// I use macro as there's no single file acting as summary, but a lot of different files, so can't use helpers like
17 +// procfile(). Also, omnipath generates other counters, that are not provided by infiniband
18 +#define FOREACH_COUNTER(GEN, ...) \
19 + FOREACH_COUNTER_BYTES(GEN, __VA_ARGS__) \
20 + FOREACH_COUNTER_PACKETS(GEN, __VA_ARGS__) \
21 + FOREACH_COUNTER_ERRORS(GEN, __VA_ARGS__)
22 +
23 +#define FOREACH_COUNTER_BYTES(GEN, ...) \
24 + GEN(port_rcv_data, bytes, "Received", 1, __VA_ARGS__) \
25 + GEN(port_xmit_data, bytes, "Sent", -1, __VA_ARGS__)
26 +
27 +#define FOREACH_COUNTER_PACKETS(GEN, ...) \
28 + GEN(port_rcv_packets, packets, "Received", 1, __VA_ARGS__) \
29 + GEN(port_xmit_packets, packets, "Sent", -1, __VA_ARGS__) \
30 + GEN(multicast_rcv_packets, packets, "Mcast rcvd", 1, __VA_ARGS__) \
31 + GEN(multicast_xmit_packets, packets, "Mcast sent", -1, __VA_ARGS__) \
32 + GEN(unicast_rcv_packets, packets, "Ucast rcvd", 1, __VA_ARGS__) \
33 + GEN(unicast_xmit_packets, packets, "Ucast sent", -1, __VA_ARGS__)
34 +
35 +#define FOREACH_COUNTER_ERRORS(GEN, ...) \
36 + GEN(port_rcv_errors, errors, "Pkts malformated", 1, __VA_ARGS__) \
37 + GEN(port_rcv_constraint_errors, errors, "Pkts rcvd discarded ", 1, __VA_ARGS__) \
38 + GEN(port_xmit_discards, errors, "Pkts sent discarded", 1, __VA_ARGS__) \
39 + GEN(port_xmit_wait, errors, "Tick Wait to send", 1, __VA_ARGS__) \
40 + GEN(VL15_dropped, errors, "Pkts missed ressource", 1, __VA_ARGS__) \
41 + GEN(excessive_buffer_overrun_errors, errors, "Buffer overrun", 1, __VA_ARGS__) \
42 + GEN(link_downed, errors, "Link Downed", 1, __VA_ARGS__) \
43 + GEN(link_error_recovery, errors, "Link recovered", 1, __VA_ARGS__) \
44 + GEN(local_link_integrity_errors, errors, "Link integrity err", 1, __VA_ARGS__) \
45 + GEN(symbol_error, errors, "Link minor errors", 1, __VA_ARGS__) \
46 + GEN(port_rcv_remote_physical_errors, errors, "Pkts rcvd with EBP", 1, __VA_ARGS__) \
47 + GEN(port_rcv_switch_relay_errors, errors, "Pkts rcvd discarded by switch", 1, __VA_ARGS__) \
48 + GEN(port_xmit_constraint_errors, errors, "Pkts sent discarded by switch", 1, __VA_ARGS__)
49 +
50 +//
51 +// Hardware Counters
52 +//
53 +
54 +// IMPORTANT: These are vendor-specific fields.
55 +// If you want to add a new vendor, search this for for 'VENDORS:' keyword and
56 +// add your definition as 'VENDOR-<key>:' where <key> if the string part that
57 +// is shown in /sys/class/infiniband/<key>X_Y
58 +// EG: for Mellanox, shown as mlx0_1, it's 'mlx'
59 +// for Intel, shown as hfi1_1, it's 'hfi'
60 +
61 +// VENDORS: List of implemented hardware vendors
62 +#define FOREACH_HWCOUNTER_NAME(GEN, ...) GEN(mlx, __VA_ARGS__)
63 +
64 +// VENDOR-MLX: HW Counters for Mellanox ConnectX Devices
65 +#define FOREACH_HWCOUNTER_MLX(GEN, ...) \
66 + FOREACH_HWCOUNTER_MLX_PACKETS(GEN, __VA_ARGS__) \
67 + FOREACH_HWCOUNTER_MLX_ERRORS(GEN, __VA_ARGS__)
68 +
69 +#define FOREACH_HWCOUNTER_MLX_PACKETS(GEN, ...) \
70 + GEN(np_cnp_sent, hwpackets, "RoCEv2 Congestion sent", 1, __VA_ARGS__) \
71 + GEN(np_ecn_marked_roce_packets, hwpackets, "RoCEv2 Congestion rcvd", -1, __VA_ARGS__) \
72 + GEN(rp_cnp_handled, hwpackets, "IB Congestion handled", 1, __VA_ARGS__) \
73 + GEN(rx_atomic_requests, hwpackets, "ATOMIC req. rcvd", 1, __VA_ARGS__) \
74 + GEN(rx_dct_connect, hwpackets, "Connection req. rcvd", 1, __VA_ARGS__) \
75 + GEN(rx_read_requests, hwpackets, "Read req. rcvd", 1, __VA_ARGS__) \
76 + GEN(rx_write_requests, hwpackets, "Write req. rcvd", 1, __VA_ARGS__) \
77 + GEN(roce_adp_retrans, hwpackets, "RoCE retrans adaptive", 1, __VA_ARGS__) \
78 + GEN(roce_adp_retrans_to, hwpackets, "RoCE retrans timeout", 1, __VA_ARGS__) \
79 + GEN(roce_slow_restart, hwpackets, "RoCE slow restart", 1, __VA_ARGS__) \
80 + GEN(roce_slow_restart_cnps, hwpackets, "RoCE slow restart congestion", 1, __VA_ARGS__) \
81 + GEN(roce_slow_restart_trans, hwpackets, "RoCE slow restart count", 1, __VA_ARGS__)
82 +
83 +#define FOREACH_HWCOUNTER_MLX_ERRORS(GEN, ...) \
84 + GEN(duplicate_request, hwerrors, "Duplicated packets", -1, __VA_ARGS__) \
85 + GEN(implied_nak_seq_err, hwerrors, "Pkt Seq Num gap", 1, __VA_ARGS__) \
86 + GEN(local_ack_timeout_err, hwerrors, "Ack timer expired", 1, __VA_ARGS__) \
87 + GEN(out_of_buffer, hwerrors, "Drop missing buffer", 1, __VA_ARGS__) \
88 + GEN(out_of_sequence, hwerrors, "Drop out of sequence", 1, __VA_ARGS__) \
89 + GEN(packet_seq_err, hwerrors, "NAK sequence rcvd", 1, __VA_ARGS__) \
90 + GEN(req_cqe_error, hwerrors, "CQE err Req", 1, __VA_ARGS__) \
91 + GEN(resp_cqe_error, hwerrors, "CQE err Resp", 1, __VA_ARGS__) \
92 + GEN(req_cqe_flush_error, hwerrors, "CQE Flushed err Req", 1, __VA_ARGS__) \
93 + GEN(resp_cqe_flush_error, hwerrors, "CQE Flushed err Resp", 1, __VA_ARGS__) \
94 + GEN(req_remote_access_errors, hwerrors, "Remote access err Req", 1, __VA_ARGS__) \
95 + GEN(resp_remote_access_errors, hwerrors, "Remote access err Resp", 1, __VA_ARGS__) \
96 + GEN(req_remote_invalid_request, hwerrors, "Remote invalid req", 1, __VA_ARGS__) \
97 + GEN(resp_local_length_error, hwerrors, "Local length err Resp", 1, __VA_ARGS__) \
98 + GEN(rnr_nak_retry_err, hwerrors, "RNR NAK Packets", 1, __VA_ARGS__) \
99 + GEN(rp_cnp_ignored, hwerrors, "CNP Pkts ignored", 1, __VA_ARGS__) \
100 + GEN(rx_icrc_encapsulated, hwerrors, "RoCE ICRC Errors", 1, __VA_ARGS__)
101 +
102 +// Common definitions used more than once
103 +#define GEN_RRD_DIM_ADD(NAME, GRP, DESC, DIR, PORT) \
104 + GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, 1, 1, RRD_ALGORITHM_INCREMENTAL)
105 +
106 +#define GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, MULT, DIV, ALGO) \
107 + PORT->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR * MULT, DIV, ALGO);
108 +
109 +#define GEN_RRD_DIM_ADD_HW(NAME, GRP, DESC, DIR, PORT, HW) \
110 + HW->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR, 1, RRD_ALGORITHM_INCREMENTAL);
111 +
112 +#define GEN_RRD_DIM_SETP(NAME, GRP, DESC, DIR, PORT) \
113 + rrddim_set_by_pointer(PORT->st_##GRP, PORT->rd_##NAME, (collected_number)PORT->NAME);
114 +
115 +#define GEN_RRD_DIM_SETP_HW(NAME, GRP, DESC, DIR, PORT, HW) \
116 + rrddim_set_by_pointer(PORT->st_##GRP, HW->rd_##NAME, (collected_number)HW->NAME);
117 +
118 +// https://community.mellanox.com/s/article/understanding-mlx5-linux-counters-and-status-parameters
119 +// https://community.mellanox.com/s/article/infiniband-port-counters
120 +static struct ibport {
121 + char *name;
122 + char *counters_path;
123 + char *hwcounters_path;
124 + int len;
125 +
126 + // flags
127 + int configured;
128 + int enabled;
129 + int discovered;
130 +
131 + int do_bytes;
132 + int do_packets;
133 + int do_errors;
134 + int do_hwpackets;
135 + int do_hwerrors;
136 +
137 + const char *chart_type_bytes;
138 + const char *chart_type_packets;
139 + const char *chart_type_errors;
140 + const char *chart_type_hwpackets;
141 + const char *chart_type_hwerrors;
142 +
143 + const char *chart_id_bytes;
144 + const char *chart_id_packets;
145 + const char *chart_id_errors;
146 + const char *chart_id_hwpackets;
147 + const char *chart_id_hwerrors;
148 +
149 + const char *chart_family;
150 +
151 + unsigned long priority;
152 +
153 + // Port details using drivers/infiniband/core/sysfs.c :: rate_show()
154 + RRDDIM *rd_speed;
155 + uint64_t speed;
156 + uint64_t width;
157 +
158 +// Stats from /$device/ports/$portid/counters
159 +// as drivers/infiniband/hw/qib/qib_verbs.h
160 +// All uint64 except vl15_dropped, local_link_integrity_errors, excessive_buffer_overrun_errors uint32
161 +// Will generate 2 elements for each counter:
162 +// - uint64_t to store the value
163 +// - char* to store the filename path
164 +// - RRDDIM* to store the RRD Dimension
165 +#define GEN_DEF_COUNTER(NAME, ...) \
166 + uint64_t NAME; \
167 + char *file_##NAME; \
168 + RRDDIM *rd_##NAME;
169 + FOREACH_COUNTER(GEN_DEF_COUNTER)
170 +
171 +// Vendor specific hwcounters from /$device/ports/$portid/hw_counters
172 +// We will generate one struct pointer per vendor to avoid future casting
173 +#define GEN_DEF_HWCOUNTER_PTR(VENDOR, ...) struct ibporthw_##VENDOR *hwcounters_##VENDOR;
174 + FOREACH_HWCOUNTER_NAME(GEN_DEF_HWCOUNTER_PTR)
175 +
176 + // Function pointer to the "infiniband_hwcounters_parse_<vendor>" function
177 + void (*hwcounters_parse)(struct ibport *);
178 + void (*hwcounters_dorrd)(struct ibport *);
179 +
180 + // charts and dim
181 + RRDSET *st_bytes;
182 + RRDSET *st_packets;
183 + RRDSET *st_errors;
184 + RRDSET *st_hwpackets;
185 + RRDSET *st_hwerrors;
186 +
187 + RRDSETVAR *stv_speed;
188 +
189 + usec_t speed_last_collected_usec;
190 +
191 + struct ibport *next;
192 +} *ibport_root = NULL, *ibport_last_used = NULL;
193 +
194 +// VENDORS: reading / calculation functions
195 +#define GEN_DEF_HWCOUNTER(NAME, ...) \
196 + uint64_t NAME; \
197 + char *file_##NAME; \
198 + RRDDIM *rd_##NAME;
199 +
200 +#define GEN_DO_HWCOUNTER_READ(NAME, GRP, DESC, DIR, PORT, HW, ...) \
201 + if (HW->file_##NAME) { \
202 + if (read_single_number_file(HW->file_##NAME, (unsigned long long *)&HW->NAME)) { \
203 + error("cannot read iface '%s' hwcounter '" #HW "'", PORT->name); \
204 + HW->file_##NAME = NULL; \
205 + } \
206 + }
207 +
208 +// VENDOR-MLX: Mellanox
209 +struct ibporthw_mlx {
210 + FOREACH_HWCOUNTER_MLX(GEN_DEF_HWCOUNTER)
211 +};
212 +void infiniband_hwcounters_parse_mlx(struct ibport *port)
213 +{
214 + if (port->do_hwerrors != CONFIG_BOOLEAN_NO)
215 + FOREACH_HWCOUNTER_MLX_ERRORS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
216 + if (port->do_hwpackets != CONFIG_BOOLEAN_NO)
217 + FOREACH_HWCOUNTER_MLX_PACKETS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
218 +}
219 +void infiniband_hwcounters_dorrd_mlx(struct ibport *port)
220 +{
221 + if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
222 + FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
223 + rrdset_done(port->st_hwerrors);
224 + }
225 + if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
226 + FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
227 + rrdset_done(port->st_hwpackets);
228 + }
229 +}
230 +
231 +// ----------------------------------------------------------------------------
232 +
233 +static struct ibport *get_ibport(const char *dev, const char *port)
234 +{
235 + struct ibport *p;
236 +
237 + char name[IBNAME_MAX + 1];
238 + snprintfz(name, IBNAME_MAX, "%s-%s", dev, port);
239 +
240 + // search it, resuming from the last position in sequence
241 + for (p = ibport_last_used; p; p = p->next) {
242 + if (unlikely(!strcmp(name, p->name))) {
243 + ibport_last_used = p->next;
244 + return p;
245 + }
246 + }
247 +
248 + // new round, from the beginning to the last position used this time
249 + for (p = ibport_root; p != ibport_last_used; p = p->next) {
250 + if (unlikely(!strcmp(name, p->name))) {
251 + ibport_last_used = p->next;
252 + return p;
253 + }
254 + }
255 +
256 + // create a new one
257 + p = callocz(1, sizeof(struct ibport));
258 + p->name = strdupz(name);
259 + p->len = strlen(p->name);
260 +
261 + p->chart_type_bytes = strdupz("infiniband_cnt_bytes");
262 + p->chart_type_packets = strdupz("infiniband_cnt_packets");
263 + p->chart_type_errors = strdupz("infiniband_cnt_errors");
264 + p->chart_type_hwpackets = strdupz("infiniband_hwc_packets");
265 + p->chart_type_hwerrors = strdupz("infiniband_hwc_errors");
266 +
267 + char buffer[RRD_ID_LENGTH_MAX + 1];
268 + snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntbytes_%s", p->name);
269 + p->chart_id_bytes = strdupz(buffer);
270 +
271 + snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntpackets_%s", p->name);
272 + p->chart_id_packets = strdupz(buffer);
273 +
274 + snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cnterrors_%s", p->name);
275 + p->chart_id_errors = strdupz(buffer);
276 +
277 + snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcntpackets_%s", p->name);
278 + p->chart_id_hwpackets = strdupz(buffer);
279 +
280 + snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcnterrors_%s", p->name);
281 + p->chart_id_hwerrors = strdupz(buffer);
282 +
283 + p->chart_family = strdupz(p->name);
284 + p->priority = NETDATA_CHART_PRIO_INFINIBAND;
285 +
286 + // Link current ibport to last one in the list
287 + if (ibport_root) {
288 + struct ibport *t;
289 + for (t = ibport_root; t->next; t = t->next)
290 + ;
291 + t->next = p;
292 + } else
293 + ibport_root = p;
294 +
295 + return p;
296 +}
297 +
298 +int do_sys_class_infiniband(int update_every, usec_t dt)
299 +{
300 + (void)dt;
301 + static SIMPLE_PATTERN *disabled_list = NULL;
302 + static int initialized = 0;
303 + static int enable_new_ports = -1, enable_only_active = CONFIG_BOOLEAN_YES;
304 + static int do_bytes = -1, do_packets = -1, do_errors = -1, do_hwpackets = -1, do_hwerrors = -1;
305 + static char *sys_class_infiniband_dirname = NULL;
306 +
307 + static long long int dt_to_refresh_ports = 0, last_refresh_ports_usec = 0;
308 +
309 + if (unlikely(enable_new_ports == -1)) {
310 + char dirname[FILENAME_MAX + 1];
311 +
312 + snprintfz(dirname, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/infiniband");
313 + sys_class_infiniband_dirname =
314 + config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "dirname to monitor", dirname);
315 +
316 + do_bytes = config_get_boolean_ondemand(
317 + CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "bandwidth counters", CONFIG_BOOLEAN_YES);
318 + do_packets = config_get_boolean_ondemand(
319 + CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "packets counters", CONFIG_BOOLEAN_YES);
320 + do_errors = config_get_boolean_ondemand(
321 + CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "errors counters", CONFIG_BOOLEAN_YES);
322 + do_hwpackets = config_get_boolean_ondemand(
323 + CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware packets counters", CONFIG_BOOLEAN_AUTO);
324 + do_hwerrors = config_get_boolean_ondemand(
325 + CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware errors counters", CONFIG_BOOLEAN_AUTO);
326 +
327 + enable_only_active = config_get_boolean_ondemand(
328 + CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "monitor only active ports", CONFIG_BOOLEAN_AUTO);
329 + disabled_list = simple_pattern_create(
330 + config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""), NULL,
331 + SIMPLE_PATTERN_EXACT);
332 +
333 + dt_to_refresh_ports =
334 + config_get_number(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "refresh ports state every seconds", 30) *
335 + USEC_PER_SEC;
336 + if (dt_to_refresh_ports < 0)
337 + dt_to_refresh_ports = 0;
338 + }
339 +
340 + // init listing of /sys/class/infiniband/ (or rediscovery)
341 + if (unlikely(!initialized) || unlikely(last_refresh_ports_usec >= dt_to_refresh_ports)) {
342 + // If folder does not exists, return 1 to disable
343 + DIR *devices_dir = opendir(sys_class_infiniband_dirname);
344 + if (unlikely(!devices_dir))
345 + return 1;
346 +
347 + // Work on all device available
348 + struct dirent *dev_dent;
349 + while ((dev_dent = readdir(devices_dir))) {
350 + // Skip special folders
351 + if (!strcmp(dev_dent->d_name, "..") || !strcmp(dev_dent->d_name, "."))
352 + continue;
353 +
354 + // /sys/class/infiniband/<dev>/ports
355 + char ports_dirname[FILENAME_MAX + 1];
356 + snprintfz(ports_dirname, FILENAME_MAX, "%s/%s/%s", sys_class_infiniband_dirname, dev_dent->d_name, "ports");
357 +
358 + DIR *ports_dir = opendir(ports_dirname);
359 + if (unlikely(!ports_dir))
360 + continue;
361 +
362 + struct dirent *port_dent;
363 + while ((port_dent = readdir(ports_dir))) {
364 + // Skip special folders
365 + if (!strcmp(port_dent->d_name, "..") || !strcmp(port_dent->d_name, "."))
366 + continue;
367 +
368 + char buffer[FILENAME_MAX + 1];
369 +
370 + // Check if counters are availablea (mandatory)
371 + // /sys/class/infiniband/<device>/ports/<port>/counters
372 + char counters_dirname[FILENAME_MAX + 1];
373 + snprintfz(counters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "counters");
374 + DIR *counters_dir = opendir(counters_dirname);
375 + // Standard counters are mandatory
376 + if (!counters_dir)
377 + continue;
378 +
379 + // Nearly same with hardware counters
380 + char hwcounters_dirname[FILENAME_MAX + 1];
381 + snprintfz(
382 + hwcounters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "hw_counters");
383 + DIR *hwcounters_dir = opendir(hwcounters_dirname);
384 +
385 + // Get new or existing ibport
386 + struct ibport *p = get_ibport(dev_dent->d_name, port_dent->d_name);
387 + if (!p)
388 + continue;
389 +
390 + // Prepare configuration
391 + if (!p->configured) {
392 + p->configured = 1;
393 +
394 + // Enable by default, will be filtered out later
395 + p->enabled = 1;
396 +
397 + p->counters_path = strdupz(counters_dirname);
398 + p->hwcounters_path = strdupz(hwcounters_dirname);
399 +
400 + snprintfz(buffer, FILENAME_MAX, "plugin:proc:/sys/class/infiniband:%s", p->name);
401 +
402 + // Standard counters
403 + p->do_bytes = config_get_boolean_ondemand(buffer, "bytes", do_bytes);
404 + p->do_packets = config_get_boolean_ondemand(buffer, "packets", do_packets);
405 + p->do_errors = config_get_boolean_ondemand(buffer, "errors", do_errors);
406 +
407 +// Gen filename allocation and concatenation
408 +#define GEN_DO_COUNTER_NAME(NAME, GRP, DESC, DIR, PORT, ...) \
409 + PORT->file_##NAME = callocz(1, strlen(PORT->counters_path) + sizeof(#NAME) + 3); \
410 + strcat(PORT->file_##NAME, PORT->counters_path); \
411 + strcat(PORT->file_##NAME, "/" #NAME);
412 + FOREACH_COUNTER(GEN_DO_COUNTER_NAME, p)
413 +
414 + // Check HW Counters vendor dependent
415 + if (hwcounters_dir) {
416 + // By default set standard
417 + p->do_hwpackets = config_get_boolean_ondemand(buffer, "hwpackets", do_hwpackets);
418 + p->do_hwerrors = config_get_boolean_ondemand(buffer, "hwerrors", do_hwerrors);
419 +
420 +// VENDORS: Set your own
421 +
422 +// Allocate the chars to the filenames
423 +#define GEN_DO_HWCOUNTER_NAME(NAME, GRP, DESC, DIR, PORT, HW, ...) \
424 + HW->file_##NAME = callocz(1, strlen(PORT->hwcounters_path) + sizeof(#NAME) + 3); \
425 + strcat(HW->file_##NAME, PORT->hwcounters_path); \
426 + strcat(HW->file_##NAME, "/" #NAME);
427 +
428 + // VENDOR-MLX: Mellanox
429 + if (strncmp(dev_dent->d_name, "mlx", 3) == 0) {
430 + // Allocate the vendor specific struct
431 + p->hwcounters_mlx = callocz(1, sizeof(struct ibporthw_mlx));
432 +
433 + FOREACH_HWCOUNTER_MLX(GEN_DO_HWCOUNTER_NAME, p, p->hwcounters_mlx)
434 +
435 + // Set the function pointer for hwcounter parsing
436 + p->hwcounters_parse = &infiniband_hwcounters_parse_mlx;
437 + p->hwcounters_dorrd = &infiniband_hwcounters_dorrd_mlx;
438 + }
439 +
440 + // VENDOR: Unknown
441 + else {
442 + p->do_hwpackets = CONFIG_BOOLEAN_NO;
443 + p->do_hwerrors = CONFIG_BOOLEAN_NO;
444 + }
445 + }
446 + }
447 +
448 + // Check port state to keep activation
449 + if (enable_only_active) {
450 + snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "state");
451 + unsigned long long active;
452 + // File is "1: DOWN" or "4: ACTIVE", but str2ull will stop on first non-decimal char
453 + read_single_number_file(buffer, &active);
454 +
455 + // Want "IB_PORT_ACTIVE" == "4", as defined by drivers/infiniband/core/sysfs.c::state_show()
456 + if (active == 4)
457 + p->enabled = 1;
458 + else
459 + p->enabled = 0;
460 + }
461 +
462 + if (p->enabled)
463 + p->enabled = !simple_pattern_matches(disabled_list, p->name);
464 +
465 + // Check / Update the link speed & width frm "rate" file
466 + // Sample output: "100 Gb/sec (4X EDR)"
467 + snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "rate");
468 + char buffer_rate[65];
469 + if (read_file(buffer, buffer_rate, 64)) {
470 + error("Unable to read '%s'", buffer);
471 + p->width = 1;
472 + } else {
473 + char *buffer_width = strstr(buffer_rate, "(");
474 + buffer_width++;
475 + // str2ull will stop on first non-decimal value
476 + p->speed = str2ull(buffer_rate);
477 + p->width = str2ull(buffer_width);
478 + }
479 +
480 + if (!p->discovered)
481 + info(
482 + "Infiniband card %s port %s at speed %lu width %lu", dev_dent->d_name, port_dent->d_name,
483 + p->speed, p->width);
484 +
485 + p->discovered = 1;
486 + }
487 + closedir(ports_dir);
488 + }
489 + closedir(devices_dir);
490 +
491 + initialized = 1;
492 + last_refresh_ports_usec = 0;
493 + }
494 + last_refresh_ports_usec += dt;
495 +
496 + // Update all ports values
497 + struct ibport *port;
498 + for (port = ibport_root; port; port = port->next) {
499 + if (!port->enabled)
500 + continue;
501 + //
502 + // Read values from system to struct
503 + //
504 +
505 +// counter from file and place it in ibport struct
506 +#define GEN_DO_COUNTER_READ(NAME, GRP, DESC, DIR, PORT, ...) \
507 + if (PORT->file_##NAME) { \
508 + if (read_single_number_file(PORT->file_##NAME, (unsigned long long *)&PORT->NAME)) { \
509 + error("cannot read iface '%s' counter '" #NAME "'", PORT->name); \
510 + PORT->file_##NAME = NULL; \
511 + } \
512 + }
513 +
514 + // Update charts
515 + if (port->do_bytes != CONFIG_BOOLEAN_NO) {
516 + // Read values from sysfs
517 + FOREACH_COUNTER_BYTES(GEN_DO_COUNTER_READ, port)
518 +
519 + // First creation of RRD Set (charts)
520 + if (unlikely(!port->st_bytes)) {
521 + port->st_bytes = rrdset_create_localhost(
522 + "Infiniband",
523 + port->chart_id_bytes,
524 + NULL,
525 + port->chart_family,
526 + "ib.bytes",
527 + "Bandwidth usage",
528 + "kilobits/s",
529 + PLUGIN_PROC_NAME,
530 + PLUGIN_PROC_MODULE_INFINIBAND_NAME,
531 + port->priority + 1,
532 + update_every,
533 + RRDSET_TYPE_AREA);
534 + // Create Dimensions
535 + rrdset_flag_set(port->st_bytes, RRDSET_FLAG_DETAIL);
536 + // On this chart, we want to have a KB/s so the dashboard will autoscale it
537 + // The reported values are also per-lane, so we must multiply it by the width
538 + FOREACH_COUNTER_BYTES(GEN_RRD_DIM_ADD_CUSTOM, port, 8 * port->width, 1024, RRD_ALGORITHM_INCREMENTAL)
539 +
540 + port->stv_speed = rrdsetvar_custom_chart_variable_create(port->st_bytes, "link_speed");
541 + } else
542 + rrdset_next(port->st_bytes);
543 +
544 + // Link read values to dimensions
545 + FOREACH_COUNTER_BYTES(GEN_RRD_DIM_SETP, port)
546 +
547 + // For link speed set only variable
548 + rrdsetvar_custom_chart_variable_set(port->stv_speed, port->speed);
549 +
550 + rrdset_done(port->st_bytes);
551 + }
552 +
553 + if (port->do_packets != CONFIG_BOOLEAN_NO) {
554 + // Read values from sysfs
555 + FOREACH_COUNTER_PACKETS(GEN_DO_COUNTER_READ, port)
556 +
557 + // First creation of RRD Set (charts)
558 + if (unlikely(!port->st_packets)) {
559 + port->st_packets = rrdset_create_localhost(
560 + "Infiniband",
561 + port->chart_id_packets,
562 + NULL,
563 + port->chart_family,
564 + "ib.packets",
565 + "Packets Statistics",
566 + "packets/s",
567 + PLUGIN_PROC_NAME,
568 + PLUGIN_PROC_MODULE_INFINIBAND_NAME,
569 + port->priority + 2,
570 + update_every,
571 + RRDSET_TYPE_AREA);
572 + // Create Dimensions
573 + rrdset_flag_set(port->st_packets, RRDSET_FLAG_DETAIL);
574 + FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_ADD, port)
575 + } else
576 + rrdset_next(port->st_packets);
577 +
578 + // Link read values to dimensions
579 + FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_SETP, port)
580 + rrdset_done(port->st_packets);
581 + }
582 +
583 + if (port->do_errors != CONFIG_BOOLEAN_NO) {
584 + // Read values from sysfs
585 + FOREACH_COUNTER_ERRORS(GEN_DO_COUNTER_READ, port)
586 +
587 + // First creation of RRD Set (charts)
588 + if (unlikely(!port->st_errors)) {
589 + port->st_errors = rrdset_create_localhost(
590 + "Infiniband",
591 + port->chart_id_errors,
592 + NULL,
593 + port->chart_family,
594 + "ib.errors",
595 + "Error Counters",
596 + "errors/s",
597 + PLUGIN_PROC_NAME,
598 + PLUGIN_PROC_MODULE_INFINIBAND_NAME,
599 + port->priority + 3,
600 + update_every,
601 + RRDSET_TYPE_LINE);
602 + // Create Dimensions
603 + rrdset_flag_set(port->st_errors, RRDSET_FLAG_DETAIL);
604 + FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_ADD, port)
605 + } else
606 + rrdset_next(port->st_errors);
607 +
608 + // Link read values to dimensions
609 + FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_SETP, port)
610 + rrdset_done(port->st_errors);
611 + }
612 +
613 + //
614 + // HW Counters
615 + //
616 +
617 + // Call the function for parsing and setting hwcounters
618 + if (port->hwcounters_parse && port->hwcounters_dorrd) {
619 + // Read all values (done by vendor-specific function)
620 + (*port->hwcounters_parse)(port);
621 +
622 + if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
623 + // First creation of RRD Set (charts)
624 + if (unlikely(!port->st_hwerrors)) {
625 + port->st_hwerrors = rrdset_create_localhost(
626 + "Infiniband",
627 + port->chart_id_hwerrors,
628 + NULL,
629 + port->chart_family,
630 + "ib.hwerrors",
631 + "Hardware Errors",
632 + "errors/s",
633 + PLUGIN_PROC_NAME,
634 + PLUGIN_PROC_MODULE_INFINIBAND_NAME,
635 + port->priority + 4,
636 + update_every,
637 + RRDSET_TYPE_LINE);
638 +
639 + rrdset_flag_set(port->st_hwerrors, RRDSET_FLAG_DETAIL);
640 +
641 + // VENDORS: Set your selection
642 +
643 + // VENDOR: Mellanox
644 + if (strncmp(port->name, "mlx", 3) == 0) {
645 + FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
646 + }
647 +
648 + // Unknown vendor, should not happen
649 + else {
650 + error(
651 + "Unmanaged vendor for '%s', do_hwerrors should have been set to no. Please report this bug",
652 + port->name);
653 + port->do_hwerrors = CONFIG_BOOLEAN_NO;
654 + }
655 + } else
656 + rrdset_next(port->st_hwerrors);
657 + }
658 +
659 + if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
660 + // First creation of RRD Set (charts)
661 + if (unlikely(!port->st_hwpackets)) {
662 + port->st_hwpackets = rrdset_create_localhost(
663 + "Infiniband",
664 + port->chart_id_hwpackets,
665 + NULL,
666 + port->chart_family,
667 + "ib.hwpackets",
668 + "Hardware Packets Statistics",
669 + "packets/s",
670 + PLUGIN_PROC_NAME,
671 + PLUGIN_PROC_MODULE_INFINIBAND_NAME,
672 + port->priority + 5,
673 + update_every,
674 + RRDSET_TYPE_LINE);
675 +
676 + rrdset_flag_set(port->st_hwpackets, RRDSET_FLAG_DETAIL);
677 +
678 + // VENDORS: Set your selection
679 +
680 + // VENDOR: Mellanox
681 + if (strncmp(port->name, "mlx", 3) == 0) {
682 + FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
683 + }
684 +
685 + // Unknown vendor, should not happen
686 + else {
687 + error(
688 + "Unmanaged vendor for '%s', do_hwpackets should have been set to no. Please report this bug",
689 + port->name);
690 + port->do_hwpackets = CONFIG_BOOLEAN_NO;
691 + }
692 + } else
693 + rrdset_next(port->st_hwpackets);
694 + }
695 +
696 + // Update values to rrd (done by vendor-specific function)
697 + (*port->hwcounters_dorrd)(port);
698 + }
699 + }
700 +
701 + return 0;
702 +}