configurable tcp idle timeout for statsd tcp clients; disabled request timeout for statsd tcp clients
Costa Tsaousis (ktsaou) committed
Feb 3, 2018 at 13:01 UTC
980a9c4a809f8152e28d65e0c95b8724fb1f25b1
4 files changed
+89
-38
src/socket.c
+8
-6
@@ -1338,6 +1338,8 @@ void poll_events(LISTEN_SOCKETS *sockets
1338
, int (*snd_callback)(POLLINFO *pi, short int *events)
1339
, SIMPLE_PATTERN *access_list
1340
, void *data
1341
+ , time_t tcp_request_timeout
1342
+ , time_t tcp_idle_timeout
1343
) {
1344
if(!sockets || !sockets->opened) {
1345
error("POLLFD: internal error: no listening sockets are opened");
@@ -1354,9 +1356,9 @@ void poll_events(LISTEN_SOCKETS *sockets
1356
.inf = NULL,
1357
.first_free = NULL,
1358
1357
- .complete_request_timeout = web_client_first_request_timeout,
1358
- .idle_timeout = web_client_timeout,
1359
- .checks_every = (web_client_timeout / 3) + 1,
1359
+ .complete_request_timeout = tcp_request_timeout,
1360
+ .idle_timeout = tcp_idle_timeout,
1361
+ .checks_every = (tcp_idle_timeout / 3) + 1,
1362
1363
.access_list = access_list,
1364
@@ -1412,7 +1414,7 @@ void poll_events(LISTEN_SOCKETS *sockets
1414
}
1415
}
1416
1415
- if(unlikely(now - last_check > p.checks_every)) {
1417
+ if(unlikely(p.checks_every > 0 && now - last_check > p.checks_every)) {
1418
last_check = now;
1419
1420
// security checks
@@ -1420,7 +1422,7 @@ void poll_events(LISTEN_SOCKETS *sockets
1422
POLLINFO *pi = &p.inf[i];
1423
1424
if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) {
1423
- if (unlikely(pi->send_count == 0 && (now - pi->connected_t) >= p.complete_request_timeout)) {
1425
+ if (unlikely(pi->send_count == 0 && p.complete_request_timeout > 0 && (now - pi->connected_t) >= p.complete_request_timeout)) {
1426
info("POLLFD: LISTENER: client slot %zu (fd %d) from '%s:%s' has not sent a complete request in %zu seconds - closing it. "
1427
, i
1428
, pi->fd
@@ -1430,7 +1432,7 @@ void poll_events(LISTEN_SOCKETS *sockets
1432
);
1433
poll_close_fd(pi);
1434
}
1433
- else if(unlikely(pi->recv_count && now - ((pi->last_received_t > pi->last_sent_t) ? pi->last_received_t : pi->last_sent_t) >= p.idle_timeout )) {
1435
+ else if(unlikely(pi->recv_count && p.idle_timeout > 0 && now - ((pi->last_received_t > pi->last_sent_t) ? pi->last_received_t : pi->last_sent_t) >= p.idle_timeout )) {
1436
info("POLLFD: LISTENER: client slot %zu (fd %d) from '%s:%s' is idle for more than %zu seconds - closing it. "
1437
, i
1438
, pi->fd
src/socket.h
+2
@@ -145,6 +145,8 @@ extern void poll_events(LISTEN_SOCKETS *sockets
145
, int (*snd_callback)(POLLINFO *pi, short int *events)
146
, SIMPLE_PATTERN *access_list
147
, void *data
148
+ , time_t tcp_request_timeout
149
+ , time_t tcp_idle_timeout
150
);
151
152
#endif //NETDATA_SOCKET_H
src/statsd.c
+77
-32
@@ -229,6 +229,9 @@ static struct statsd {
229
STATSD_INDEX sets;
230
size_t unknown_types;
231
size_t socket_errors;
232
+ size_t tcp_socket_connects;
233
+ size_t tcp_socket_disconnects;
234
+ size_t tcp_socket_connected;
235
size_t tcp_socket_reads;
236
size_t tcp_packets_received;
237
size_t tcp_bytes_read;
@@ -240,6 +243,7 @@ static struct statsd {
243
int update_every;
244
SIMPLE_PATTERN *charts_for;
245
246
+ size_t tcp_idle_timeout;
247
size_t decimal_detail;
248
size_t private_charts;
249
size_t max_private_charts;
@@ -320,6 +324,8 @@ static struct statsd {
324
STATSD_FIRST_PTR_MUTEX_INIT
325
},
326
327
+ .tcp_idle_timeout = 600,
328
+
329
.apps = NULL,
330
.histogram_percentile = 95.0,
331
.histogram_increase_step = 10,
@@ -734,6 +740,8 @@ static void *statsd_add_callback(POLLINFO *pi, short int *events, void *data) {
740
struct statsd_tcp *t = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1);
741
t->type = STATSD_SOCKET_DATA_TYPE_TCP;
742
t->size = STATSD_TCP_BUFFER_SIZE - 1;
743
+ statsd.tcp_socket_connects++;
744
+ statsd.tcp_socket_connected++;
745
746
return t;
747
}
@@ -749,6 +757,8 @@ static void statsd_del_callback(POLLINFO *pi) {
757
error("STATSD: client is probably sending unterminated metrics. Closed socket left with '%s'. Trying to process it.", t->buffer);
758
statsd_process(t->buffer, t->len, 0);
759
}
760
+ statsd.tcp_socket_disconnects++;
761
+ statsd.tcp_socket_connected--;
762
}
763
else
764
error("STATSD: internal error: received socket data type is %d, but expected %d", (int)t->type, (int)STATSD_SOCKET_DATA_TYPE_TCP);
@@ -953,6 +963,8 @@ void *statsd_collector_thread(void *ptr) {
963
, statsd_snd_callback
964
, NULL
965
, (void *)d
966
+ , 0 // tcp request timeout, 0 = disabled
967
+ , statsd.tcp_idle_timeout // tcp idle timeout, 0 = disabled
968
);
969
970
netdata_thread_cleanup_pop(1);
@@ -2071,6 +2083,7 @@ void *statsd_main(void *ptr) {
2083
statsd.private_charts_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_STATSD, "private charts memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
2084
statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
2085
statsd.decimal_detail = (size_t)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
2086
+ statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);
2087
2088
statsd.histogram_percentile = (double)config_get_float(CONFIG_SECTION_STATSD, "histograms and timers percentile (percentThreshold)", statsd.histogram_percentile);
2089
if(isless(statsd.histogram_percentile, 0) || isgreater(statsd.histogram_percentile, 100)) {
@@ -2247,6 +2260,39 @@ void *statsd_main(void *ptr) {
2260
RRDDIM *rd_packets_tcp = rrddim_add(st_packets, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
2261
RRDDIM *rd_packets_udp = rrddim_add(st_packets, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
2262
2263
+ RRDSET *st_tcp_connects = rrdset_create_localhost(
2264
+ "netdata"
2265
+ , "tcp_connects"
2266
+ , NULL
2267
+ , "statsd"
2268
+ , NULL
2269
+ , "statsd server TCP connects and disconnects"
2270
+ , "events"
2271
+ , "netdata"
2272
+ , "stats"
2273
+ , 132005
2274
+ , statsd.update_every
2275
+ , RRDSET_TYPE_LINE
2276
+ );
2277
+ RRDDIM *rd_tcp_connects = rrddim_add(st_tcp_connects, "connects", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
2278
+ RRDDIM *rd_tcp_disconnects = rrddim_add(st_tcp_connects, "disconnects", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
2279
+
2280
+ RRDSET *st_tcp_connected = rrdset_create_localhost(
2281
+ "netdata"
2282
+ , "tcp_connected"
2283
+ , NULL
2284
+ , "statsd"
2285
+ , NULL
2286
+ , "statsd server TCP connected sockets"
2287
+ , "connected"
2288
+ , "netdata"
2289
+ , "stats"
2290
+ , 132006
2291
+ , statsd.update_every
2292
+ , RRDSET_TYPE_LINE
2293
+ );
2294
+ RRDDIM *rd_tcp_connected = rrddim_add(st_tcp_connected, "connected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
2295
+
2296
RRDSET *st_pcharts = rrdset_create_localhost(
2297
"netdata"
2298
, "private_charts"
@@ -2263,19 +2309,15 @@ void *statsd_main(void *ptr) {
2309
);
2310
RRDDIM *rd_pcharts = rrddim_add(st_pcharts, "charts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
2311
2266
-
2267
- // ----------------------------------------------------------------------------------------------------------------
2312
+ // ----------------------------------------------------------------------------------------------------------------
2313
// statsd thread to turn metrics into charts
2314
2315
usec_t step = statsd.update_every * USEC_PER_SEC;
2316
heartbeat_t hb;
2317
heartbeat_init(&hb);
2273
- for(;;) {
2318
+ while(!netdata_exit) {
2319
usec_t hb_dt = heartbeat_next(&hb, step);
2320
2276
- if(unlikely(netdata_exit))
2277
- break;
2278
-
2321
statsd_flush_index_metrics(&statsd.gauges, statsd_flush_gauge);
2322
statsd_flush_index_metrics(&statsd.counters, statsd_flush_counter);
2323
statsd_flush_index_metrics(&statsd.meters, statsd_flush_meter);
@@ -2294,48 +2336,51 @@ void *statsd_main(void *ptr) {
2336
rrdset_next(st_reads);
2337
rrdset_next(st_bytes);
2338
rrdset_next(st_packets);
2339
+ rrdset_next(st_tcp_connects);
2340
+ rrdset_next(st_tcp_connected);
2341
rrdset_next(st_pcharts);
2342
}
2343
2300
- rrddim_set_by_pointer(st_metrics, rd_metrics_gauge, (collected_number)statsd.gauges.metrics);
2301
- rrddim_set_by_pointer(st_metrics, rd_metrics_counter, (collected_number)statsd.counters.metrics);
2302
- rrddim_set_by_pointer(st_metrics, rd_metrics_timer, (collected_number)statsd.timers.metrics);
2303
- rrddim_set_by_pointer(st_metrics, rd_metrics_meter, (collected_number)statsd.meters.metrics);
2304
- rrddim_set_by_pointer(st_metrics, rd_metrics_histogram, (collected_number)statsd.histograms.metrics);
2305
- rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics);
2344
+ rrddim_set_by_pointer(st_metrics, rd_metrics_gauge, (collected_number)statsd.gauges.metrics);
2345
+ rrddim_set_by_pointer(st_metrics, rd_metrics_counter, (collected_number)statsd.counters.metrics);
2346
+ rrddim_set_by_pointer(st_metrics, rd_metrics_timer, (collected_number)statsd.timers.metrics);
2347
+ rrddim_set_by_pointer(st_metrics, rd_metrics_meter, (collected_number)statsd.meters.metrics);
2348
+ rrddim_set_by_pointer(st_metrics, rd_metrics_histogram, (collected_number)statsd.histograms.metrics);
2349
+ rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics);
2350
2307
- rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events);
2308
- rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events);
2309
- rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events);
2310
- rrddim_set_by_pointer(st_events, rd_events_meter, (collected_number)statsd.meters.events);
2311
- rrddim_set_by_pointer(st_events, rd_events_histogram, (collected_number)statsd.histograms.events);
2312
- rrddim_set_by_pointer(st_events, rd_events_set, (collected_number)statsd.sets.events);
2313
- rrddim_set_by_pointer(st_events, rd_events_unknown, (collected_number)statsd.unknown_types);
2314
- rrddim_set_by_pointer(st_events, rd_events_errors, (collected_number)statsd.socket_errors);
2351
+ rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events);
2352
+ rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events);
2353
+ rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events);
2354
+ rrddim_set_by_pointer(st_events, rd_events_meter, (collected_number)statsd.meters.events);
2355
+ rrddim_set_by_pointer(st_events, rd_events_histogram, (collected_number)statsd.histograms.events);
2356
+ rrddim_set_by_pointer(st_events, rd_events_set, (collected_number)statsd.sets.events);
2357
+ rrddim_set_by_pointer(st_events, rd_events_unknown, (collected_number)statsd.unknown_types);
2358
+ rrddim_set_by_pointer(st_events, rd_events_errors, (collected_number)statsd.socket_errors);
2359
2316
- rrddim_set_by_pointer(st_reads, rd_reads_tcp, (collected_number)statsd.tcp_socket_reads);
2317
- rrddim_set_by_pointer(st_reads, rd_reads_udp, (collected_number)statsd.udp_socket_reads);
2360
+ rrddim_set_by_pointer(st_reads, rd_reads_tcp, (collected_number)statsd.tcp_socket_reads);
2361
+ rrddim_set_by_pointer(st_reads, rd_reads_udp, (collected_number)statsd.udp_socket_reads);
2362
2319
- rrddim_set_by_pointer(st_bytes, rd_bytes_tcp, (collected_number)statsd.tcp_bytes_read);
2320
- rrddim_set_by_pointer(st_bytes, rd_bytes_udp, (collected_number)statsd.udp_bytes_read);
2363
+ rrddim_set_by_pointer(st_bytes, rd_bytes_tcp, (collected_number)statsd.tcp_bytes_read);
2364
+ rrddim_set_by_pointer(st_bytes, rd_bytes_udp, (collected_number)statsd.udp_bytes_read);
2365
2322
- rrddim_set_by_pointer(st_packets, rd_packets_tcp, (collected_number)statsd.tcp_packets_received);
2323
- rrddim_set_by_pointer(st_packets, rd_packets_udp, (collected_number)statsd.udp_packets_received);
2366
+ rrddim_set_by_pointer(st_packets, rd_packets_tcp, (collected_number)statsd.tcp_packets_received);
2367
+ rrddim_set_by_pointer(st_packets, rd_packets_udp, (collected_number)statsd.udp_packets_received);
2368
2325
- rrddim_set_by_pointer(st_pcharts, rd_pcharts, (collected_number)statsd.private_charts);
2369
+ rrddim_set_by_pointer(st_tcp_connects, rd_tcp_connects, (collected_number)statsd.tcp_socket_connects);
2370
+ rrddim_set_by_pointer(st_tcp_connects, rd_tcp_disconnects, (collected_number)statsd.tcp_socket_disconnects);
2371
2327
- if(unlikely(netdata_exit))
2328
- break;
2372
+ rrddim_set_by_pointer(st_tcp_connected, rd_tcp_connected, (collected_number)statsd.tcp_socket_connected);
2373
+
2374
+ rrddim_set_by_pointer(st_pcharts, rd_pcharts, (collected_number)statsd.private_charts);
2375
2376
rrdset_done(st_metrics);
2377
rrdset_done(st_events);
2378
rrdset_done(st_reads);
2379
rrdset_done(st_bytes);
2380
rrdset_done(st_packets);
2381
+ rrdset_done(st_tcp_connects);
2382
+ rrdset_done(st_tcp_connected);
2383
rrdset_done(st_pcharts);
2336
-
2337
- if(unlikely(netdata_exit))
2338
- break;
2384
}
2385
2386
cleanup:
src/web_server.c
+2
@@ -1140,6 +1140,8 @@ void *socket_listen_main_static_threaded_worker(void *ptr) {
1140
, web_server_snd_callback
1141
, web_allow_connections_from
1142
, NULL
1143
+ , web_client_first_request_timeout
1144
+ , web_client_timeout
1145
);
1146
1147
netdata_thread_cleanup_pop(1);