provide tcp memory alarm
Costa Tsaousis (ktsaou) committed
Nov 8, 2017 at 00:06 UTC
b3fee04bf0a12bf3dcf1b7aed4e4ea9b3665f0cf
3 files changed
+71
-5
conf.d/Makefile.am
+1
@@ -104,6 +104,7 @@ dist_healthconfig_DATA = \
104
health.d/swap.conf \
105
health.d/tcp_conn.conf \
106
health.d/tcp_listen.conf \
107
+ health.d/tcp_mem.conf \
108
health.d/tcp_orphans.conf \
109
health.d/tcp_resets.conf \
110
health.d/udp_errors.conf \
conf.d/health.d/tcp_mem.conf
new
+20
@@ -0,0 +1,20 @@
1
+#
2
+# check
3
+# http://blog.tsunanet.net/2011/03/out-of-socket-memory.html
4
+#
5
+# We give a warning when TCP is under memory pressure
6
+# and a critical when TCP is 90% of its upper memory limit
7
+#
8
+
9
+ alarm: tcp_memory
10
+ on: ipv4.sockstat_tcp_mem
11
+ os: linux
12
+ hosts: *
13
+ calc: ${mem} * 100 / ${tcp_mem_high}
14
+ units: %
15
+ every: 10s
16
+ warn: ${mem} > (($status >= $WARNING ) ? ( ${tcp_mem_pressure} * 0.8 ) : ( ${tcp_mem_pressure} ))
17
+ crit: ${mem} > (($status >= $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 ))
18
+ delay: up 0 down 5m multiplier 1.5 max 1h
19
+ info: the amount of TCP memory as a percentage of its max memory limit
20
+ to: sysadmin
src/proc_net_sockstat.c
+50
-5
@@ -23,6 +23,48 @@ static struct proc_net_sockstat {
23
} sockstat_root = { 0 };
24
25
26
+static int read_tcp_mem(void) {
27
+ static char *filename = NULL;
28
+ static RRDVAR *tcp_mem_low_threshold = NULL,
29
+ *tcp_mem_pressure_threshold = NULL,
30
+ *tcp_mem_high_threshold = NULL;
31
+
32
+ if(unlikely(!tcp_mem_low_threshold)) {
33
+ tcp_mem_low_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_low");
34
+ tcp_mem_pressure_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_pressure");
35
+ tcp_mem_high_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_high");
36
+ }
37
+
38
+ if(unlikely(!filename)) {
39
+ char buffer[FILENAME_MAX + 1];
40
+ snprintfz(buffer, FILENAME_MAX, "%s/proc/sys/net/ipv4/tcp_mem", netdata_configured_host_prefix);
41
+ filename = strdupz(buffer);
42
+ }
43
+
44
+ char buffer[200 + 1], *start, *end;
45
+ if(read_file(filename, buffer, 200) != 0) return 1;
46
+ buffer[200] = '\0';
47
+
48
+ unsigned long long low = 0, pressure = 0, high = 0;
49
+
50
+ start = buffer;
51
+ low = strtoull(start, &end, 10);
52
+
53
+ start = end;
54
+ pressure = strtoull(start, &end, 10);
55
+
56
+ start = end;
57
+ high = strtoull(start, &end, 10);
58
+
59
+ // fprintf(stderr, "TCP MEM low = %llu, pressure = %llu, high = %llu\n", low, pressure, high);
60
+
61
+ rrdvar_custom_host_variable_set(localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE));
62
+ rrdvar_custom_host_variable_set(localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE));
63
+ rrdvar_custom_host_variable_set(localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE));
64
+
65
+ return 0;
66
+}
67
+
68
static kernel_uint_t read_tcp_max_orphans(void) {
69
static char *filename = NULL;
70
static RRDVAR *tcp_max_orphans_var = NULL;
@@ -58,7 +100,7 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
100
hash_udp = 0,
101
hash_udplite = 0;
102
61
- static long long update_tcp_max_orphans_every = 60, update_tcp_max_orphans_count = 0;
103
+ static long long update_constants_every = 60, update_constants_count = 0;
104
105
static ARL_BASE *arl_sockets = NULL;
106
static ARL_BASE *arl_tcp = NULL;
@@ -84,8 +126,8 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
126
do_frag_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG sockets", CONFIG_BOOLEAN_AUTO);
127
do_frag_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG memory", CONFIG_BOOLEAN_AUTO);
128
87
- update_tcp_max_orphans_every = config_get_number("plugin:proc:/proc/net/sockstat", "update tcp_max_orphans every", update_tcp_max_orphans_every);
88
- update_tcp_max_orphans_count = update_tcp_max_orphans_every;
129
+ update_constants_every = config_get_number("plugin:proc:/proc/net/sockstat", "update constants every", update_constants_every);
130
+ update_constants_count = update_constants_every;
131
132
arl_sockets = arl_create("sockstat/sockets", arl_callback_str2kernel_uint_t, 60);
133
arl_expect(arl_sockets, "used", &sockstat_root.sockets_used);
@@ -126,9 +168,12 @@ int do_proc_net_sockstat(int update_every, usec_t dt) {
168
keys[5] = "FRAG"; hashes[5] = hash_frag; bases[5] = arl_frag;
169
}
170
129
- update_tcp_max_orphans_count += update_every;
130
- if(unlikely(update_tcp_max_orphans_count > update_tcp_max_orphans_every))
171
+ update_constants_count += update_every;
172
+ if(unlikely(update_constants_count > update_constants_every)) {
173
read_tcp_max_orphans();
174
+ read_tcp_mem();
175
+ update_constants_count = 0;
176
+ }
177
178
if(unlikely(!ff)) {
179
char filename[FILENAME_MAX + 1];