added /proc/net/softnet_stat monitoring which provides information about common network device misconfigurations and issues
Costa Tsaousis (ktsaou) committed
Sep 16, 2016 at 00:04 UTC
079134376fc83d1d7b1ae96423de31512f5c3e38
6 files changed
+166
conf.d/Makefile.am
+1
@@ -57,6 +57,7 @@ dist_healthconfig_DATA = \
57
health.d/ram.conf \
58
health.d/redis.conf \
59
health.d/retroshare.conf \
60
+ health.d/softnet.conf \
61
health.d/swap.conf \
62
health.d/squid.conf \
63
$(NULL)
conf.d/health.d/softnet.conf
new
+31
@@ -0,0 +1,31 @@
1
+# check for common /proc/net/softnet_stat errors
2
+
3
+ alarm: 1hour_netdev_backlog_exceeded
4
+ on: system.softnet_stat
5
+ lookup: sum -1h unaligned absolute of dropped
6
+ units: packets
7
+ every: 5m
8
+ warn: $this > 0
9
+ delay: down 30m multiplier 1.5 max 1h
10
+ info: number of packets dropped because sysctl net.core.netdev_max_backlog was exceeded
11
+ to: sysadmin
12
+
13
+ alarm: 1hour_netdev_budget_ran_outs
14
+ on: system.softnet_stat
15
+ lookup: sum -1h unaligned absolute of squeezed
16
+ units: events
17
+ every: 5m
18
+ warn: $this > 0
19
+ delay: down 30m multiplier 1.5 max 1h
20
+ info: number of times ksoftirq ran out of sysctl net.core.netdev_budget or time slice, with work remaining
21
+ to: sysadmin
22
+
23
+ alarm: 1hour_cpu_lock_collisions
24
+ on: system.softnet_stat
25
+ lookup: sum -1h unaligned absolute of collisions
26
+ units: collisions
27
+ every: 5m
28
+ warn: $this > 0
29
+ delay: down 30m multiplier 1.5 max 1h
30
+ info: number of times two cpus collided trying to get a network device queue lock
31
+ to: sysadmin
src/Makefile.am
+1
@@ -55,6 +55,7 @@ netdata_SOURCES = \
55
proc_net_rpc_nfsd.c \
56
proc_net_snmp.c \
57
proc_net_snmp6.c \
58
+ proc_net_softnet_stat.c \
59
proc_net_stat_conntrack.c \
60
proc_net_stat_synproxy.c \
61
proc_stat.c \
src/plugin_proc.c
+10
@@ -32,6 +32,7 @@ void *proc_main(void *ptr)
32
int vdo_proc_sys_kernel_random_entropy_avail = !config_get_boolean("plugin:proc", "/proc/sys/kernel/random/entropy_avail", 1);
33
int vdo_proc_interrupts = !config_get_boolean("plugin:proc", "/proc/interrupts", 1);
34
int vdo_proc_softirqs = !config_get_boolean("plugin:proc", "/proc/softirqs", 1);
35
+ int vdo_proc_net_softnet_stat = !config_get_boolean("plugin:proc", "/proc/net/softnet_stat", 1);
36
int vdo_proc_loadavg = !config_get_boolean("plugin:proc", "/proc/loadavg", 1);
37
int vdo_sys_kernel_mm_ksm = !config_get_boolean("plugin:proc", "/sys/kernel/mm/ksm", 1);
38
int vdo_cpu_netdata = !config_get_boolean("plugin:proc", "netdata server resources", 1);
@@ -52,6 +53,7 @@ void *proc_main(void *ptr)
53
unsigned long long sutime_proc_sys_kernel_random_entropy_avail = 0ULL;
54
unsigned long long sutime_proc_interrupts = 0ULL;
55
unsigned long long sutime_proc_softirqs = 0ULL;
56
+ unsigned long long sutime_proc_net_softnet_stat = 0ULL;
57
unsigned long long sutime_proc_loadavg = 0ULL;
58
unsigned long long sutime_sys_kernel_mm_ksm = 0ULL;
59
@@ -107,6 +109,14 @@ void *proc_main(void *ptr)
109
}
110
if(unlikely(netdata_exit)) break;
111
112
+ if(!vdo_proc_net_softnet_stat) {
113
+ debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_softnet_stat().");
114
+ sunow = time_usec();
115
+ vdo_proc_net_softnet_stat = do_proc_net_softnet_stat(rrd_update_every, (sutime_proc_net_softnet_stat > 0)?sunow - sutime_proc_net_softnet_stat:0ULL);
116
+ sutime_proc_net_softnet_stat = sunow;
117
+ }
118
+ if(unlikely(netdata_exit)) break;
119
+
120
if(!vdo_proc_sys_kernel_random_entropy_avail) {
121
debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_sys_kernel_random_entropy_avail().");
122
sunow = time_usec();
src/plugin_proc.h
+1
@@ -20,5 +20,6 @@ extern int do_proc_softirqs(int update_every, unsigned long long dt);
20
extern int do_sys_kernel_mm_ksm(int update_every, unsigned long long dt);
21
extern int do_proc_loadavg(int update_every, unsigned long long dt);
22
extern int do_proc_net_stat_synproxy(int update_every, unsigned long long dt);
23
+extern int do_proc_net_softnet_stat(int update_every, unsigned long long dt);
24
25
#endif /* NETDATA_PLUGIN_PROC_H */
src/proc_net_softnet_stat.c
new
+122
@@ -0,0 +1,122 @@
1
+#include "common.h"
2
+
3
+static inline char *softnet_column_name(uint32_t column) {
4
+ static char buf[4] = "c00";
5
+ char *s;
6
+
7
+ switch(column) {
8
+ case 0: s = "total"; break;
9
+ case 1: s = "dropped"; break;
10
+ case 2: s = "squeezed"; break;
11
+ case 8: s = "collisions"; break;
12
+ default: {
13
+ uint32_t c = column + 1;
14
+ buf[1] = '0' + ( c / 10); c = c % 10;
15
+ buf[2] = '0' + c;
16
+ s = buf;
17
+ break;
18
+ }
19
+ }
20
+
21
+ return s;
22
+}
23
+
24
+int do_proc_net_softnet_stat(int update_every, unsigned long long dt) {
25
+ (void)dt;
26
+
27
+ static procfile *ff = NULL;
28
+ static int do_per_core = -1;
29
+ static uint32_t allocated_lines = 0, allocated_columns = 0, *data = NULL;
30
+
31
+ if(do_per_core == -1) do_per_core = config_get_boolean("plugin:proc:/proc/net/softnet_stat", "softnet_stat per core", 1);
32
+
33
+ if(!ff) {
34
+ char filename[FILENAME_MAX + 1];
35
+ snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/softnet_stat");
36
+ ff = procfile_open(config_get("plugin:proc:/proc/net/softnet_stat", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT);
37
+ }
38
+ if(!ff) return 1;
39
+
40
+ ff = procfile_readall(ff);
41
+ if(!ff) return 0; // we return 0, so that we will retry to open it next time
42
+
43
+ uint32_t lines = procfile_lines(ff), l;
44
+ uint32_t words = procfile_linewords(ff, 0), w;
45
+
46
+ if(!lines || !words) {
47
+ error("Cannot read /proc/net/softnet_stat, %u lines and %u columns reported.", lines, words);
48
+ return 1;
49
+ }
50
+
51
+ if(lines > 200) lines = 200;
52
+ if(words > 50) words = 50;
53
+
54
+ if(unlikely(!data || lines > allocated_lines || words > allocated_columns)) {
55
+ freez(data);
56
+ data = mallocz((lines + 1) * words * sizeof(uint32_t));
57
+ allocated_lines = lines;
58
+ allocated_columns = words;
59
+ }
60
+
61
+ // initialize to zero
62
+ bzero(data, allocated_lines * allocated_columns * sizeof(uint32_t));
63
+
64
+ // parse the values
65
+ for(l = 0; l < lines ;l++) {
66
+ words = procfile_linewords(ff, l);
67
+ if(!words) continue;
68
+
69
+ if(words > allocated_columns) words = allocated_columns;
70
+
71
+ for(w = 0; w < words ; w++) {
72
+ uint32_t t = strtoul(procfile_lineword(ff, l, w), NULL, 16);
73
+ data[w] += t;
74
+ data[((l + 1) * allocated_columns) + w] = t;
75
+ }
76
+ }
77
+
78
+ if(data[(lines * allocated_columns)] == 0)
79
+ lines--;
80
+
81
+ RRDSET *st;
82
+
83
+ // --------------------------------------------------------------------
84
+
85
+ st = rrdset_find_bytype("system", "softnet_stat");
86
+ if(!st) {
87
+ st = rrdset_create("system", "softnet_stat", NULL, "softnet_stat", NULL, "System softnet_stat", "events/s", 955, update_every, RRDSET_TYPE_LINE);
88
+ for(w = 0; w < allocated_columns ;w++)
89
+ rrddim_add(st, softnet_column_name(w), NULL, 1, 1, RRDDIM_INCREMENTAL);
90
+ }
91
+ else rrdset_next(st);
92
+
93
+ for(w = 0; w < allocated_columns ;w++)
94
+ rrddim_set(st, softnet_column_name(w), data[w]);
95
+
96
+ rrdset_done(st);
97
+
98
+ if(do_per_core) {
99
+ for(l = 0; l < lines ;l++) {
100
+ char id[50+1];
101
+ snprintfz(id, 50, "cpu%d_softnet_stat", l);
102
+
103
+ st = rrdset_find_bytype("cpu", id);
104
+ if(!st) {
105
+ char title[100+1];
106
+ snprintfz(title, 100, "CPU%d softnet_stat", l);
107
+
108
+ st = rrdset_create("cpu", id, NULL, "softnet_stat", NULL, title, "events/s", 4101 + l, update_every, RRDSET_TYPE_LINE);
109
+ for(w = 0; w < allocated_columns ;w++)
110
+ rrddim_add(st, softnet_column_name(w), NULL, 1, 1, RRDDIM_INCREMENTAL);
111
+ }
112
+ else rrdset_next(st);
113
+
114
+ for(w = 0; w < allocated_columns ;w++)
115
+ rrddim_set(st, softnet_column_name(w), data[((l + 1) * allocated_columns) + w]);
116
+
117
+ rrdset_done(st);
118
+ }
119
+ }
120
+
121
+ return 0;
122
+}