add tcp ListenOverflows and ListenDrops chart and alarm; fixes #2855
Costa Tsaousis (ktsaou) committed
Oct 9, 2017 at 22:58 UTC
b5e49ea74c4f095279c178534deb0d5f9fb5bb17
3 files changed
+50
-3
conf.d/Makefile.am
+1
@@ -102,6 +102,7 @@ dist_healthconfig_DATA = \
102
health.d/softnet.conf \
103
health.d/squid.conf \
104
health.d/swap.conf \
105
+ health.d/tcp_listen.conf \
106
health.d/tcp_resets.conf \
107
health.d/udp_errors.conf \
108
health.d/varnish.conf \
conf.d/health.d/tcp_resets.conf
+2
-2
@@ -37,7 +37,7 @@
37
every: 10s
38
warn: $this > ((($1m_ipv4_tcp_resets_sent < 5)?(5):($1m_ipv4_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (20)))
39
delay: up 0 down 60m multiplier 1.2 max 2h
40
-options: no-clear-notification
40
+ options: no-clear-notification
41
info: average TCP RESETS this host is sending, over the last 10 seconds (this can be an indication that a port scan is made, or that a service running on this host has crashed; clear notification for this alarm will not be sent)
42
to: sysadmin
43
@@ -62,6 +62,6 @@ options: no-clear-notification
62
every: 10s
63
warn: $this > ((($1m_ipv4_tcp_resets_received < 5)?(5):($1m_ipv4_tcp_resets_received)) * (($status >= $WARNING) ? (1) : (10)))
64
delay: up 0 down 60m multiplier 1.2 max 2h
65
-options: no-clear-notification
65
+ options: no-clear-notification
66
info: average TCP RESETS this host is receiving, over the last 10 seconds (this can be an indication that a service this host needs, has crashed; clear notification for this alarm will not be sent)
67
to: sysadmin
src/proc_net_netstat.c
+47
-1
@@ -22,7 +22,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
22
(void)dt;
23
24
static int do_bandwidth = -1, do_inerrors = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, do_bcast_p = -1, do_ecn = -1, \
25
- do_tcpext_reorder = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, do_tcpext_memory = -1;
25
+ do_tcpext_reorder = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, do_tcpext_memory = -1,
26
+ do_tcpext_listen = -1;
27
+
28
static uint32_t hash_ipext = 0, hash_tcpext = 0;
29
static procfile *ff = NULL;
30
@@ -93,6 +95,10 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
95
static unsigned long long tcpext_TCPAbortOnLinger = 0; // connections aborted after user close in linger timeout
96
static unsigned long long tcpext_TCPAbortFailed = 0; // times unable to send RST due to no memory
97
98
+ // https://perfchron.com/2015/12/26/investigating-linux-network-issues-with-netstat-and-nstat/
99
+ static unsigned long long tcpext_ListenOverflows = 0; // times the listen queue of a socket overflowed
100
+ static unsigned long long tcpext_ListenDrops = 0; // SYNs to LISTEN sockets ignored
101
+
102
// IPv4 TCP memory pressures
103
static unsigned long long tcpext_TCPMemoryPressures = 0;
104
@@ -116,6 +122,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
122
do_tcpext_ofo = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP out-of-order queue", CONFIG_BOOLEAN_AUTO);
123
do_tcpext_connaborts = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP connection aborts", CONFIG_BOOLEAN_AUTO);
124
do_tcpext_memory = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP memory pressures", CONFIG_BOOLEAN_AUTO);
125
+ do_tcpext_listen = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP listen errors", CONFIG_BOOLEAN_AUTO);
126
127
arl_ipext = arl_create("netstat/ipext", NULL, 60);
128
arl_tcpext = arl_create("netstat/tcpext", NULL, 60);
@@ -197,6 +204,11 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
204
arl_expect(arl_tcpext, "TCPMemoryPressures", &tcpext_TCPMemoryPressures);
205
}
206
207
+ if(do_tcpext_listen != CONFIG_BOOLEAN_NO) {
208
+ arl_expect(arl_tcpext, "ListenOverflows", &tcpext_ListenOverflows);
209
+ arl_expect(arl_tcpext, "ListenDrops", &tcpext_ListenDrops);
210
+ }
211
+
212
// shared metrics
213
arl_expect(arl_tcpext, "TCPSynRetrans", &tcpext_TCPSynRetrans);
214
}
@@ -682,6 +694,40 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
694
rrdset_done(st_syncookies);
695
}
696
697
+ // --------------------------------------------------------------------
698
+
699
+ if(do_tcpext_listen == CONFIG_BOOLEAN_YES || (do_tcpext_listen == CONFIG_BOOLEAN_AUTO && (tcpext_ListenOverflows || tcpext_ListenDrops))) {
700
+ do_tcpext_listen = CONFIG_BOOLEAN_YES;
701
+
702
+ static RRDSET *st_listen = NULL;
703
+ static RRDDIM *rd_overflows = NULL, *rd_drops = NULL;
704
+
705
+ if(unlikely(!st_listen)) {
706
+
707
+ st_listen = rrdset_create_localhost(
708
+ "ipv4"
709
+ , "tcplistenissues"
710
+ , NULL
711
+ , "tcp"
712
+ , NULL
713
+ , "TCP Listen Socket Issues"
714
+ , "packets/s"
715
+ , 3015
716
+ , update_every
717
+ , RRDSET_TYPE_LINE
718
+ );
719
+
720
+ rd_overflows = rrddim_add(st_listen, "ListenOverflows", "overflows", 1, 1, RRD_ALGORITHM_INCREMENTAL);
721
+ rd_drops = rrddim_add(st_listen, "ListenDrops", "drops", 1, 1, RRD_ALGORITHM_INCREMENTAL);
722
+ }
723
+ else
724
+ rrdset_next(st_listen);
725
+
726
+ rrddim_set_by_pointer(st_listen, rd_overflows, tcpext_ListenOverflows);
727
+ rrddim_set_by_pointer(st_listen, rd_drops, tcpext_ListenDrops);
728
+
729
+ rrdset_done(st_listen);
730
+ }
731
}
732
}
733