added orphans percentage alarm; fixes #14
Costa Tsaousis (ktsaou) committed
Nov 7, 2017 at 02:38 UTC
5185f1dc53be5bee2ca55c66e78bab90da22f3a4
3 files changed
+55
-3
conf.d/Makefile.am
+1
@@ -103,6 +103,7 @@ dist_healthconfig_DATA = \
103
health.d/squid.conf \
104
health.d/swap.conf \
105
health.d/tcp_listen.conf \
106
+ health.d/tcp_orphans.conf \
107
health.d/tcp_resets.conf \
108
health.d/udp_errors.conf \
109
health.d/varnish.conf \
conf.d/health.d/tcp_orphans.conf
new
+21
@@ -0,0 +1,21 @@
1
+
2
+#
3
+# check
4
+# http://blog.tsunanet.net/2011/03/out-of-socket-memory.html
5
+#
6
+# The kernel may penalize orphans by 2x or even 4x
7
+# so we alarm warning at 25% and critical at 50%
8
+#
9
+
10
+ alarm: tcp_orphans_percentage
11
+ on: ipv4.sockstat_tcp_sockets
12
+ os: linux
13
+ hosts: *
14
+ calc: ${orphan} * 100 / ${tcp_max_orphans}
15
+ units: %
16
+ every: 10s
17
+ warn: $this > (($status >= $WARNING ) ? ( 20 ) : ( 25 ))
18
+ crit: $this > (($status >= $CRITICAL) ? ( 25 ) : ( 50 ))
19
+ delay: up 0 down 5m multiplier 1.5 max 1h
20
+ info: the percentage of orphan IPv4 TCP sockets over the max allowed (this may lead to out-of-socket-memory errors)
21
+ to: sysadmin
src/proc_net_sockstat.c
+33
-3
@@ -32,6 +32,29 @@ static inline void arl_callback_str2kernel_uint_t(const char *name, uint32_t has
32
// fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, (unsigned long long)*d);
33
}
34
35
+static kernel_uint_t read_tcp_max_orphans(void) {
36
+ static char *filename = NULL;
37
+ static RRDVAR *tcp_max_orphans_var = NULL;
38
+
39
+ if(unlikely(!filename)) {
40
+ char buffer[FILENAME_MAX + 1];
41
+ snprintfz(buffer, FILENAME_MAX, "%s/proc/sys/net/ipv4/tcp_max_orphans", netdata_configured_host_prefix);
42
+ filename = strdupz(buffer);
43
+ }
44
+
45
+ unsigned long long tcp_max_orphans = 0;
46
+ if(read_single_number_file(filename, &tcp_max_orphans) == 0) {
47
+
48
+ if(unlikely(!tcp_max_orphans_var))
49
+ tcp_max_orphans_var = rrdvar_custom_host_variable_create(localhost, "tcp_max_orphans");
50
+
51
+ rrdvar_custom_host_variable_set(localhost, tcp_max_orphans_var, tcp_max_orphans);
52
+ return tcp_max_orphans;
53
+ }
54
+
55
+ return 0;
56
+}
57
+
58
int do_proc_net_sockstat(int update_every, usec_t dt) {
59
(void)dt;
60
@@ -44,6 +67,7 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
67
hash_udp = 0,
68
hash_udplite = 0;
69
70
+ static long long update_tcp_max_orphans_every = 60, update_tcp_max_orphans_count = 0;
71
72
static ARL_BASE *arl_sockets = NULL;
73
static ARL_BASE *arl_tcp = NULL;
@@ -68,6 +92,8 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
92
do_raw_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 RAW sockets", CONFIG_BOOLEAN_AUTO);
93
do_frag_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG sockets", CONFIG_BOOLEAN_AUTO);
94
do_frag_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG memory", CONFIG_BOOLEAN_AUTO);
95
+ update_tcp_max_orphans_every = config_get_number("plugin:proc:/proc/net/sockstat", "update tcp_max_orphans every", update_tcp_max_orphans_every);
96
+ update_tcp_max_orphans_count = update_tcp_max_orphans_every;
97
98
arl_sockets = arl_create("sockstat/sockets", arl_callback_str2kernel_uint_t, 60);
99
arl_expect(arl_sockets, "used", &sockstat_root.sockets_used);
@@ -108,6 +134,10 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
134
keys[5] = "FRAG"; hashes[5] = hash_frag; bases[5] = arl_frag;
135
}
136
137
+ update_tcp_max_orphans_count += update_every;
138
+ if(unlikely(update_tcp_max_orphans_count > update_tcp_max_orphans_every))
139
+ read_tcp_max_orphans();
140
+
141
if(unlikely(!ff)) {
142
char filename[FILENAME_MAX + 1];
143
snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/sockstat");
@@ -240,7 +270,7 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
270
, "net/sockstat"
271
, 2406
272
, update_every
243
- , RRDSET_TYPE_LINE
273
+ , RRDSET_TYPE_AREA
274
);
275
276
rd_mem = rrddim_add(st, "mem", NULL, sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -304,7 +334,7 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
334
, "net/sockstat"
335
, 2411
336
, update_every
307
- , RRDSET_TYPE_LINE
337
+ , RRDSET_TYPE_AREA
338
);
339
340
rd_mem = rrddim_add(st, "mem", NULL, sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -432,7 +462,7 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
462
, "net/sockstat"
463
, 2441
464
, update_every
435
- , RRDSET_TYPE_LINE
465
+ , RRDSET_TYPE_AREA
466
);
467
468
rd_mem = rrddim_add(st, "mem", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);