@cryptotaxi247 / netdata-1 / commits / f53fae77f

netdata web server protection against slowloris; #3501

Costa Tsaousis (ktsaou) committed Mar 27, 2018 at 01:47 UTC f53fae77f7853ca6b662a9d98c98717c022f232f

9 files changed +119 -28

src/common.c

@@ -19,6 +19,7 @@ char *netdata_configured_home_dir = NULL;

19 char *netdata_configured_host_prefix = NULL;

20 char *netdata_configured_timezone = NULL;

22 +struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };

23 int enable_ksm = 1;

25 volatile sig_atomic_t netdata_exit = 0;

src/common.h

@@ -323,6 +323,8 @@ extern int memory_file_save(const char *filename, void *mem, size_t size);

323

324 extern int fd_is_valid(int fd);

325

326 +extern struct rlimit rlimit_nofile;

327 +

328 extern int enable_ksm;

329

330 extern int sleep_usec(usec_t usec);

src/main.c

@@ -1005,6 +1005,11 @@ int main(int argc, char **argv) {

1005 }

1006 #endif /* NETDATA_INTERNAL_CHECKS */

1007

1008 + // get the max file limit

1009 + if(getrlimit(RLIMIT_NOFILE, &rlimit_nofile) != 0)

1010 + error("getrlimit(RLIMIT_NOFILE) failed");

1011 + else

1012 + info("resources control: allowed file descriptors: soft = %zu, max = %zu", rlimit_nofile.rlim_cur, rlimit_nofile.rlim_max);

1013

1014 // fork, switch user, create pid file, set process priority

1015 if(become_daemon(dont_fork, user) == -1)

src/socket.c

+52 -24

@@ -978,6 +978,12 @@ inline POLLINFO *poll_add_fd(POLLJOB *p

978

979 if(unlikely(fd < 0)) return NULL;

980

981 + //if(p->limit && p->used >= p->limit) {

982 + // info("Max sockets limit reached (%zu sockets), dropping connection", p->used);

983 + // close(fd);

984 + // return NULL;

985 + //}

986 +

987 if(unlikely(!p->first_free)) {

988 size_t new_slots = p->slots + POLL_FDS_INCREASE_STEP;

989 debug(D_POLLFD, "POLLFD: ADD: increasing size (current = %zu, new = %zu, used = %zu, min = %zu, max = %zu)", p->slots, new_slots, p->used, p->min, p->max);

@@ -1199,7 +1205,28 @@ static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, sho

1205 pi->last_received_t = now;

1206 pi->recv_count++;

1207

1202 - if(likely(pi->flags & POLLINFO_FLAG_SERVER_SOCKET)) {

1208 + if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) {

1209 + // read data from client TCP socket

1210 + debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", i, fd);

1211 +

1212 + pf->events = 0;

1213 + if (pi->rcv_callback(pi, &pf->events) == -1) {

1214 + poll_close_fd(&p->inf[i]);

1215 + return;

1216 + }

1217 + pf = &p->fds[i];

1218 + pi = &p->inf[i];

1219 +

1220 +#ifdef NETDATA_INTERNAL_CHECKS

1221 + // this is common - it is used for web server file copies

1222 + if(unlikely(!(pf->events & (POLLIN|POLLOUT)))) {

1223 + error("POLLFD: LISTENER: after reading, client slot %zu (fd %d) from '%s:%s' was left without expecting input or output. ", i, fd, pi->client_ip?pi->client_ip:"<undefined-ip>", pi->client_port?pi->client_port:"<undefined-port>");

1224 + //poll_close_fd(pi);

1225 + //return;

1226 + }

1227 +#endif

1228 + }

1229 + else if(likely(pi->flags & POLLINFO_FLAG_SERVER_SOCKET)) {

1230 // new connection

1231 // debug(D_POLLFD, "POLLFD: LISTENER: accepting connections from slot %zu (fd %d)", i, fd);

1232

@@ -1220,7 +1247,11 @@ static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, sho

1247

1248 debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", i, fd);

1249

1223 - if(errno != EWOULDBLOCK && errno != EAGAIN)

1250 + if(unlikely(errno == EMFILE)) {

1251 + error("POLLFD: LISTENER: too many open files - sleeping for 1ms - used by this thread %zu, max for this thread %zu", p->used, p->limit);

1252 + usleep(1000); // 10ms

1253 + }

1254 + else if(unlikely(errno != EWOULDBLOCK && errno != EAGAIN))

1255 error("POLLFD: LISTENER: accept() failed.");

1256

1257 break;

@@ -1245,7 +1276,7 @@ static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, sho

1276 pf = &p->fds[i];

1277 pi = &p->inf[i];

1278 }

1248 - } while (nfd >= 0);

1279 + } while (nfd >= 0 && (!p->limit || p->used < p->limit));

1280 break;

1281 }

1282

@@ -1268,26 +1299,6 @@ static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, sho

1299 }

1300 }

1301 }

1271 -

1272 - if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) {

1273 - // read data from client TCP socket

1274 - debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", i, fd);

1275 -

1276 - pf->events = 0;

1277 - if (pi->rcv_callback(pi, &pf->events) == -1) {

1278 - poll_close_fd(pi);

1279 - return;

1280 - }

1281 -

1282 -#ifdef NETDATA_INTERNAL_CHECKS

1283 - // this is common - it is used for web server file copies

1284 - if(unlikely(!(pf->events & (POLLIN|POLLOUT)))) {

1285 - error("POLLFD: LISTENER: after reading, client slot %zu (fd %d) from '%s:%s' was left without expecting input or output. ", i, fd, pi->client_ip?pi->client_ip:"<undefined-ip>", pi->client_port?pi->client_port:"<undefined-port>");

1286 - //poll_close_fd(pi);

1287 - //return;

1288 - }

1289 -#endif

1290 - }

1302 }

1303

1304 if(unlikely(revents & POLLOUT)) {

@@ -1299,9 +1310,11 @@ static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, sho

1310

1311 pf->events = 0;

1312 if (pi->snd_callback(pi, &pf->events) == -1) {

1302 - poll_close_fd(pi);

1313 + poll_close_fd(&p->inf[i]);

1314 return;

1315 }

1316 + pf = &p->fds[i];

1317 + pi = &p->inf[i];

1318

1319 #ifdef NETDATA_INTERNAL_CHECKS

1320 // this is common - it is used for streaming

@@ -1347,6 +1360,7 @@ void poll_events(LISTEN_SOCKETS *sockets

1360 , time_t tcp_idle_timeout_seconds

1361 , time_t timer_milliseconds

1362 , void *timer_data

1363 + , size_t max_tcp_sockets

1364 ) {

1365 if(!sockets || !sockets->opened) {

1366 error("POLLFD: internal error: no listening sockets are opened");

@@ -1361,6 +1375,7 @@ void poll_events(LISTEN_SOCKETS *sockets

1375 .slots = 0,

1376 .used = 0,

1377 .max = 0,

1378 + .limit = max_tcp_sockets,

1379 .fds = NULL,

1380 .inf = NULL,

1381 .first_free = NULL,

@@ -1401,6 +1416,8 @@ void poll_events(LISTEN_SOCKETS *sockets

1416 info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN");

1417 }

1418

1419 + int listen_sockets_active = 1;

1420 +

1421 int timeout_ms = 1000; // in milliseconds

1422 time_t last_check = now_boottime_sec();

1423

@@ -1432,6 +1449,17 @@ void poll_events(LISTEN_SOCKETS *sockets

1449 timeout_ms = (int)(dt_usec / USEC_PER_MS);

1450 }

1451

1452 + // enable or disable the TCP listening sockets, based on the current number of sockets used and the limit set

1453 + if((listen_sockets_active && (p.limit && p.used >= p.limit)) || (!listen_sockets_active && (!p.limit || p.used < p.limit))) {

1454 + listen_sockets_active = !listen_sockets_active;

1455 + info("%s listening sockets (used TCP sockets %zu, max allowed for this worker %zu)", (listen_sockets_active)?"ENABLING":"DISABLING", p.used, p.limit);

1456 + for (i = 0; i <= p.max; i++) {

1457 + if(p.inf[i].flags & POLLINFO_FLAG_SERVER_SOCKET && p.inf[i].socktype == SOCK_STREAM) {

1458 + p.fds[i].events = (short int) ((listen_sockets_active) ? POLLIN : 0);

1459 + }

1460 + }

1461 + }

1462 +

1463 debug(D_POLLFD, "POLLFD: LISTENER: Waiting on %zu sockets for %zu ms...", p.max + 1, (size_t)timeout_ms);

1464 retval = poll(p.fds, p.max + 1, timeout_ms);

1465 time_t now = now_boottime_sec();

src/socket.h

@@ -101,6 +101,8 @@ struct poll {

101 size_t min;

102 size_t max;

103

104 + size_t limit;

105 +

106 time_t complete_request_timeout;

107 time_t idle_timeout;

108 time_t checks_every;

@@ -154,6 +156,7 @@ extern void poll_events(LISTEN_SOCKETS *sockets

156 , time_t tcp_idle_timeout_seconds

157 , time_t timer_milliseconds

158 , void *timer_data

159 + , size_t max_tcp_sockets

160 );

161

162 #endif //NETDATA_SOCKET_H

src/statsd.c

+8 -2

@@ -222,6 +222,8 @@ typedef struct statsd_app {

222

223 struct collection_thread_status {

224 int status;

225 + size_t max_sockets;

226 +

227 netdata_thread_t thread;

228 struct rusage rusage;

229 RRDSET *st_cpu;

@@ -359,7 +361,7 @@ static int statsd_metric_compare(void* a, void* b) {

361 else return strcmp(((STATSD_METRIC *)a)->name, ((STATSD_METRIC *)b)->name);

362 }

363

362 -static inline STATSD_METRIC *stasd_metric_index_find(STATSD_INDEX *index, const char *name, uint32_t hash) {

364 +static inline STATSD_METRIC *statsd_metric_index_find(STATSD_INDEX *index, const char *name, uint32_t hash) {

365 STATSD_METRIC tmp;

366 tmp.name = name;

367 tmp.hash = (hash)?hash:simple_hash(tmp.name);

@@ -372,7 +374,7 @@ static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, cons

374

375 uint32_t hash = simple_hash(name);

376

375 - STATSD_METRIC *m = stasd_metric_index_find(index, name, hash);

377 + STATSD_METRIC *m = statsd_metric_index_find(index, name, hash);

378 if(unlikely(!m)) {

379 debug(D_STATSD, "Creating new %s metric '%s'", index->name, name);

380

@@ -982,6 +984,7 @@ void *statsd_collector_thread(void *ptr) {

984 , statsd.tcp_idle_timeout // tcp idle timeout, 0 = disabled

985 , statsd.update_every * 1000

986 , ptr // timer_data

987 + , status->max_sockets

988 );

989

990 netdata_thread_cleanup_pop(1);

@@ -2169,6 +2172,8 @@ void *statsd_main(void *ptr) {

2172 if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on timers (deleteTimers)", 0))

2173 statsd.timers.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;

2174

2175 + size_t max_sockets = (size_t)config_get_number(CONFIG_SECTION_STATSD, "statsd server max TCP sockets", (long long int)(rlimit_nofile.rlim_cur / 4));

2176 +

2177 #ifdef STATSD_MULTITHREADED

2178 statsd.threads = (int)config_get_number(CONFIG_SECTION_STATSD, "threads", processors);

2179 if(statsd.threads < 1) {

@@ -2202,6 +2207,7 @@ void *statsd_main(void *ptr) {

2207

2208 int i;

2209 for(i = 0; i < statsd.threads ;i++) {

2210 + statsd.collection_threads_status[i].max_sockets = max_sockets / statsd.threads;

2211 char tag[NETDATA_THREAD_TAG_MAX + 1];

2212 snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STATSD_COLLECTOR[%d]", i + 1);

2213 netdata_thread_create(&statsd.collection_threads_status[i].thread, tag, NETDATA_THREAD_OPTION_DEFAULT, statsd_collector_thread, &statsd.collection_threads_status[i]);

src/web_client.c

+32 -1

@@ -159,6 +159,9 @@ void web_client_request_done(struct web_client *w) {

159 w->response.sent = 0;

160 w->response.code = 0;

161

162 + w->header_parse_tries = 0;

163 + w->header_parse_last_size = 0;

164 +

165 web_client_enable_wait_receive(w);

166 web_client_disable_wait_send(w);

167

@@ -809,7 +812,31 @@ typedef enum {

812 } HTTP_VALIDATION;

813

814 static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {

812 - char *s = w->response.data->buffer, *encoded_url = NULL;

815 + char *s = (char *)buffer_tostring(w->response.data), *encoded_url = NULL;

816 +

817 + size_t last_pos = w->header_parse_last_size;

818 + if(last_pos > 4) last_pos -= 4; // allow searching for \r\n\r\n

819 + else last_pos = 0;

820 +

821 + w->header_parse_tries++;

822 + w->header_parse_last_size = buffer_strlen(w->response.data);

823 +

824 + if(w->header_parse_tries > 1) {

825 + if(w->header_parse_last_size < last_pos)

826 + last_pos = 0;

827 +

828 + if(strstr(&s[last_pos], "\r\n\r\n") == NULL) {

829 + if(w->header_parse_tries > 10) {

830 + info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));

831 + w->header_parse_tries = 0;

832 + w->header_parse_last_size = 0;

833 + web_client_disable_wait_receive(w);

834 + return HTTP_VALIDATION_NOT_SUPPORTED;

835 + }

836 +

837 + return HTTP_VALIDATION_INCOMPLETE;

838 + }

839 + }

840

841 // is is a valid request?

842 if(!strncmp(s, "GET ", 4)) {

@@ -825,6 +852,8 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {

852 w->mode = WEB_CLIENT_MODE_STREAM;

853 }

854 else {

855 + w->header_parse_tries = 0;

856 + w->header_parse_last_size = 0;

857 web_client_disable_wait_receive(w);

858 return HTTP_VALIDATION_NOT_SUPPORTED;

859 }

@@ -872,6 +901,8 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {

901 // FIXME -- we should avoid it

902 strncpyz(w->last_url, w->decoded_url, NETDATA_WEB_REQUEST_URL_SIZE);

903

904 + w->header_parse_tries = 0;

905 + w->header_parse_last_size = 0;

906 web_client_disable_wait_receive(w);

907 return HTTP_VALIDATION_OK;

908 }

src/web_client.h

@@ -128,6 +128,9 @@ struct web_client {

128 WEB_CLIENT_MODE mode; // the operational mode of the client

129 WEB_CLIENT_ACL acl; // the access list of the client

130

131 + size_t header_parse_tries;

132 + size_t header_parse_last_size;

133 +

134 int tcp_cork; // 1 = we have a cork on the socket

135

136 int ifd;

src/web_server.c

+13 -1

@@ -902,6 +902,8 @@ struct web_server_static_threaded_worker {

902 int id;

903 int running;

904

905 + size_t max_sockets;

906 +

907 volatile size_t connected;

908 volatile size_t disconnected;

909 volatile size_t receptions;

@@ -1078,7 +1080,12 @@ static int web_server_rcv_callback(POLLINFO *pi, short int *events) {

1080 , (void *) w

1081 );

1082

1081 - w->pollinfo_filecopy_slot = fpi->slot;

1083 + if(fpi)

1084 + w->pollinfo_filecopy_slot = fpi->slot;

1085 + else {

1086 + error("Failed to add filecopy fd. Closing client.");

1087 + return -1;

1088 + }

1089 }

1090 }

1091 }

@@ -1192,6 +1199,7 @@ void *socket_listen_main_static_threaded_worker(void *ptr) {

1199 , web_client_timeout

1200 , default_rrd_update_every * 1000 // timer_milliseconds

1201 , ptr // timer_data

1202 + , worker_private->max_sockets

1203 );

1204

1205 netdata_thread_cleanup_pop(1);

@@ -1257,6 +1265,8 @@ void *socket_listen_main_static_threaded(void *ptr) {

1265 static_threaded_workers_count = config_get_number(CONFIG_SECTION_WEB, "web server threads", def_thread_count);

1266 if(static_threaded_workers_count < 1) static_threaded_workers_count = 1;

1267

1268 + size_t max_sockets = (size_t)config_get_number(CONFIG_SECTION_WEB, "web server max sockets", (long long int)(rlimit_nofile.rlim_cur / 2));

1269 +

1270 static_workers_private_data = callocz((size_t)static_threaded_workers_count, sizeof(struct web_server_static_threaded_worker));

1271

1272 web_server_is_multithreaded = (static_threaded_workers_count > 1);

@@ -1264,6 +1274,7 @@ void *socket_listen_main_static_threaded(void *ptr) {

1274 int i;

1275 for(i = 1; i < static_threaded_workers_count; i++) {

1276 static_workers_private_data[i].id = i;

1277 + static_workers_private_data[i].max_sockets = max_sockets / static_threaded_workers_count;

1278

1279 char tag[50 + 1];

1280 snprintfz(tag, 50, "WEB_SERVER[static%d]", i+1);

@@ -1273,6 +1284,7 @@ void *socket_listen_main_static_threaded(void *ptr) {

1284 }

1285

1286 // and the main one

1287 + static_workers_private_data[0].max_sockets = max_sockets / static_threaded_workers_count;

1288 socket_listen_main_static_threaded_worker((void *)&static_workers_private_data[0]);

1289

1290 netdata_thread_cleanup_pop(1);