faster single threaded web server implementation
Costa Tsaousis committed
May 29, 2016 at 15:32 UTC
89c0686f32476b38c14104bfa23a6248a4407939
2 files changed
+123
-135
src/main.c
+3
@@ -99,6 +99,9 @@ void web_server_threading_selection(void) {
99
if(static_threads[i].start_routine == socket_listen_main_single_threaded)
100
static_threads[i].enabled = threaded?0:1;
101
}
102
+
103
+ web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS);
104
+ web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip);
105
}
106
107
src/web_server.c
+120
-135
@@ -220,10 +220,7 @@ void *socket_listen_main_multi_threaded(void *ptr) {
220
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
221
error("Cannot set pthread cancel state to ENABLE.");
222
223
- web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS);
224
- web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip);
225
-
226
- if(listen_fd < 0) fatal("LISTENER: Listen socket is not ready.");
223
+ if(listen_fd < 0 || listen_fd >= FD_SETSIZE) fatal("LISTENER: Listen socket %d is not ready, or invalid.", listen_fd);
224
225
fd_set ifds;
226
FD_ZERO (&ifds);
@@ -299,15 +296,57 @@ void *socket_listen_main_multi_threaded(void *ptr) {
296
}
297
298
error("LISTENER: exit!");
299
+ close(listen_fd);
300
+ listen_fd = -1;
301
+ return NULL;
302
+}
303
303
- if(listen_fd >= 0) close(listen_fd);
304
- exit(2);
304
+struct web_client *single_threaded_clients[FD_SETSIZE];
305
306
- return NULL;
306
+static inline int single_threaded_link_client(struct web_client *w, fd_set *ifds, fd_set *ofds, fd_set *efds, int *max) {
307
+ if(unlikely(w->obsolete || w->dead || (!w->wait_receive && !w->wait_send)))
308
+ return 1;
309
+
310
+ if(unlikely(w->ifd < 0 || w->ifd >= FD_SETSIZE || w->ofd < 0 || w->ofd >= FD_SETSIZE)) {
311
+ error("%llu: invalid file descriptor, ifd = %d, ofd = %d (required 0 <= fd < FD_SETSIZE (%d)", w->id, w->ifd, w->ofd, FD_SETSIZE);
312
+ return 1;
313
+ }
314
+
315
+ FD_SET(w->ifd, efds);
316
+ if(unlikely(*max < w->ifd)) *max = w->ifd;
317
+
318
+ if(unlikely(w->ifd != w->ofd)) {
319
+ if(*max < w->ofd) *max = w->ofd;
320
+ FD_SET(w->ofd, efds);
321
+ }
322
+
323
+ if(w->wait_receive) FD_SET(w->ifd, ifds);
324
+ if(w->wait_send) FD_SET(w->ofd, ofds);
325
+
326
+ single_threaded_clients[w->ifd] = w;
327
+ single_threaded_clients[w->ofd] = w;
328
+
329
+ return 0;
330
+}
331
+
332
+static inline int single_threaded_unlink_client(struct web_client *w, fd_set *ifds, fd_set *ofds, fd_set *efds) {
333
+ FD_CLR(w->ifd, efds);
334
+ if(unlikely(w->ifd != w->ofd)) FD_CLR(w->ofd, efds);
335
+
336
+ if(w->wait_receive) FD_CLR(w->ifd, ifds);
337
+ if(w->wait_send) FD_CLR(w->ofd, ofds);
338
+
339
+ single_threaded_clients[w->ifd] = NULL;
340
+ single_threaded_clients[w->ofd] = NULL;
341
+
342
+ if(unlikely(w->obsolete || w->dead || (!w->wait_receive && !w->wait_send)))
343
+ return 1;
344
+
345
+ return 0;
346
}
347
348
void *socket_listen_main_single_threaded(void *ptr) {
310
- if(ptr) { ; }
349
+ (void)ptr;
350
351
web_server_mode = WEB_SERVER_MODE_SINGLE_THREADED;
352
@@ -324,179 +363,125 @@ void *socket_listen_main_single_threaded(void *ptr) {
363
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
364
error("Cannot set pthread cancel state to ENABLE.");
365
327
- web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS);
328
- web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip);
366
+ if(listen_fd < 0 || listen_fd >= FD_SETSIZE)
367
+ fatal("LISTENER: Listen socket %d is not ready, or invalid.", listen_fd);
368
330
- if(listen_fd < 0) fatal("LISTENER: Listen socket is not ready.");
369
+ int i;
370
+ for(i = 0; i < FD_SETSIZE ; i++)
371
+ single_threaded_clients[i] = NULL;
372
332
- fd_set ifds, ofds, efds;
373
+ fd_set ifds, ofds, efds, rifds, rofds, refds;
374
+ FD_ZERO (&ifds);
375
+ FD_ZERO (&ofds);
376
+ FD_ZERO (&efds);
377
+ FD_SET(listen_fd, &ifds);
378
+ FD_SET(listen_fd, &efds);
379
int fdmax = listen_fd;
380
381
for(;;) {
336
- int has_obsolete = 0;
337
- FD_ZERO (&ifds);
338
- FD_ZERO (&ofds);
339
- FD_ZERO (&efds);
340
-
341
- if(listen_fd >= 0) {
342
- // debug(D_WEB_CLIENT_ACCESS, "LISTENER: adding listen socket %d to ifds, efds", listen_fd);
343
- FD_SET(listen_fd, &ifds);
344
- FD_SET(listen_fd, &efds);
345
- }
346
-
347
- for(w = web_clients; w ; w = w->next) {
348
- if(unlikely(w->dead)) {
349
- error("%llu: client is dead.", w->id);
350
- w->obsolete = 1;
351
- }
352
- else if(unlikely(!w->wait_receive && !w->wait_send)) {
353
- error("%llu: client is not set for neither receiving nor sending data.");
354
- w->obsolete = 1;
355
- }
356
-
357
- if(w->ifd < 0 || w->ifd >= FD_SETSIZE || w->ofd < 0 || w->ofd >= FD_SETSIZE) {
358
- error("%llu: invalid file descriptor, ifd = %d, ofd = %d (required 0 <= fd < FD_SETSIZE (%d)", w->id, w->ifd, w->ofd, FD_SETSIZE);
359
- w->obsolete = 1;
360
- }
361
-
362
- if(unlikely(w->obsolete)) {
363
- has_obsolete++;
364
- continue;
365
- }
366
-
367
- // debug(D_WEB_CLIENT_ACCESS, "%llu: adding input socket %d to efds", w->id, w->ifd);
368
- FD_SET(w->ifd, &efds);
369
- if(w->ifd > fdmax) fdmax = w->ifd;
370
-
371
- if(w->ifd != w->ofd) {
372
- // debug(D_WEB_CLIENT_ACCESS, "%llu: adding output socket %d to efds", w->id, w->ofd);
373
- FD_SET(w->ofd, &efds);
374
- if(w->ofd > fdmax) fdmax = w->ofd;
375
- }
376
-
377
- if (w->wait_receive) {
378
- // debug(D_WEB_CLIENT_ACCESS, "%llu: adding input socket %d to ifds", w->id, w->ifd);
379
- FD_SET(w->ifd, &ifds);
380
- if(w->ifd > fdmax) fdmax = w->ifd;
381
- }
382
-
383
- if (w->wait_send) {
384
- // debug(D_WEB_CLIENT_ACCESS, "%llu: adding output socket %d to ofds", w->id, w->ofd);
385
- FD_SET(w->ofd, &ofds);
386
- if(w->ofd > fdmax) fdmax = w->ofd;
387
- }
388
- }
389
-
390
- // cleanup unused clients
391
- if(unlikely(has_obsolete)) {
392
- for (w = web_clients; w; ) {
393
- if (w->obsolete) {
394
- debug(D_WEB_CLIENT, "%llu: Removing client.", w->id);
395
- w = web_client_free(w);
396
-#ifdef NETDATA_INTERNAL_CHECKS
397
- log_allocations();
398
-#endif
399
- }
400
- else w = w->next;
401
- }
402
- }
382
+ debug(D_WEB_CLIENT_ACCESS, "LISTENER: single threaded web server waiting (listen fd = %d, fdmax = %d)...", listen_fd, fdmax);
383
404
- debug(D_WEB_CLIENT_ACCESS, "LISTENER: Waiting...");
384
struct timeval tv = { .tv_sec = 1, .tv_usec = 0 };
406
- errno = 0;
407
- retval = select(fdmax+1, &ifds, &ofds, &efds, &tv);
385
+ rifds = ifds;
386
+ rofds = ofds;
387
+ refds = efds;
388
+ retval = select(fdmax+1, &rifds, &rofds, &refds, &tv);
389
409
- if(retval == -1) {
390
+ if(unlikely(retval == -1)) {
391
error("LISTENER: select() failed.");
411
-
412
- if(errno != EAGAIN) {
413
- // debug(D_WEB_CLIENT_ACCESS, "LISTENER: select() failed.");
414
- error("REMOVING ALL %lu WEB CLIENTS !", global_statistics.connected_clients);
415
- while (web_clients) web_client_free(web_clients);
416
- }
417
-
392
failures++;
393
if(failures > 10) {
394
+ if(global_statistics.connected_clients) {
395
+ error("REMOVING ALL %lu WEB CLIENTS !", global_statistics.connected_clients);
396
+ while (web_clients) {
397
+ single_threaded_unlink_client(web_clients, &ifds, &ofds, &efds);
398
+ web_client_free(web_clients);
399
+ }
400
+ }
401
+
402
error("LISTENER: our listen port %d seems dead. Re-opening it.", listen_fd);
403
+
404
close(listen_fd);
405
listen_fd = -1;
406
sleep(5);
407
+
408
create_listen_socket();
425
- if(listen_fd < 0)
409
+ if(listen_fd < 0 || listen_fd >= FD_SETSIZE)
410
fatal("Cannot listen for web clients (connected clients %llu).", global_statistics.connected_clients);
411
412
+ FD_ZERO (&ifds);
413
+ FD_ZERO (&ofds);
414
+ FD_ZERO (&efds);
415
+ FD_SET(listen_fd, &ifds);
416
+ FD_SET(listen_fd, &efds);
417
failures = 0;
418
}
430
-
431
- continue;
419
}
433
- else if(retval) {
434
- for(w = web_clients; w ; w = w->next) {
435
- if (unlikely(w->obsolete)) continue;
420
+ else if(likely(retval)) {
421
+ failures = 0;
422
+ debug(D_WEB_CLIENT_ACCESS, "LISTENER: got something.");
423
437
- if (unlikely(FD_ISSET(w->ifd, &efds))) {
438
- debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on input socket.", w->id);
439
- web_client_reset(w);
440
- w->obsolete = 1;
424
+ if(FD_ISSET(listen_fd, &rifds)) {
425
+ debug(D_WEB_CLIENT_ACCESS, "LISTENER: new connection.");
426
+ w = web_client_create(listen_fd);
427
+ if(single_threaded_link_client(w, &ifds, &ofds, &ifds, &fdmax) != 0) {
428
+ web_client_free(w);
429
+ }
430
+ }
431
+
432
+ for(i = 0 ; i <= fdmax ; i++) {
433
+ if(likely(!FD_ISSET(i, &rifds) && !FD_ISSET(i, &rofds) && !FD_ISSET(i, &refds)))
434
+ continue;
435
+
436
+ w = single_threaded_clients[i];
437
+ if(unlikely(!w))
438
+ continue;
439
+
440
+ if(unlikely(single_threaded_unlink_client(w, &ifds, &ofds, &efds) != 0)) {
441
+ web_client_free(w);
442
continue;
443
}
444
444
- if (unlikely(FD_ISSET(w->ofd, &efds))) {
445
- debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on output socket.", w->id);
446
- web_client_reset(w);
447
- w->obsolete = 1;
445
+ if (unlikely(FD_ISSET(w->ifd, &refds) || FD_ISSET(w->ofd, &refds))) {
446
+ web_client_free(w);
447
continue;
448
}
449
451
- if (unlikely(w->wait_receive && FD_ISSET(w->ifd, &ifds))) {
452
- long bytes;
453
- if (unlikely((bytes = web_client_receive(w)) < 0)) {
454
- debug(D_WEB_CLIENT, "%llu: Cannot receive data from client. Closing client.", w->id);
455
- errno = 0;
456
- web_client_reset(w);
457
- w->obsolete = 1;
450
+ if (unlikely(w->wait_receive && FD_ISSET(w->ifd, &rifds))) {
451
+ if (unlikely(web_client_receive(w) < 0)) {
452
+ web_client_free(w);
453
continue;
454
}
455
461
- if (w->mode == WEB_CLIENT_MODE_NORMAL) {
462
- debug(D_WEB_CLIENT, "%llu: Processing received data (%ld bytes).", w->id, bytes);
463
- // info("%llu: Attempting to process received data (%ld bytes).", w->id, bytes);
456
+ if (w->mode != WEB_CLIENT_MODE_FILECOPY) {
457
+ debug(D_WEB_CLIENT, "%llu: Processing received data.", w->id);
458
web_client_process(w);
459
}
466
- else {
467
- debug(D_WEB_CLIENT, "%llu: NO Processing for received data (%ld bytes).", w->id, bytes);
468
- }
460
}
461
471
- if (unlikely(w->wait_send && FD_ISSET(w->ofd, &ofds))) {
472
- ssize_t bytes;
473
- if (unlikely((bytes = web_client_send(w)) < 0)) {
462
+ if (unlikely(w->wait_send && FD_ISSET(w->ofd, &rofds))) {
463
+ if (unlikely(web_client_send(w) < 0)) {
464
debug(D_WEB_CLIENT, "%llu: Cannot send data to client. Closing client.", w->id);
475
- errno = 0;
476
- web_client_reset(w);
477
- w->obsolete = 1;
465
+ web_client_free(w);
466
continue;
467
}
468
}
481
- }
469
483
- // check for new incoming connections
484
- if(FD_ISSET(listen_fd, &ifds)) {
485
- debug(D_WEB_CLIENT_ACCESS, "LISTENER: new connection.");
486
- web_client_create(listen_fd);
470
+ if(unlikely(single_threaded_link_client(w, &ifds, &ofds, &efds, &fdmax) != 0)) {
471
+ web_client_free(w);
472
+ }
473
}
474
}
475
else {
490
- debug(D_WEB_CLIENT_ACCESS, "LISTENER: timeout.");
476
+ debug(D_WEB_CLIENT_ACCESS, "LISTENER: single threaded web server timeout.");
477
+#ifdef NETDATA_INTERNAL_CHECKS
478
+ log_allocations();
479
+#endif
480
}
492
-
493
- failures = 0;
481
}
482
483
error("LISTENER: exit!");
497
-
498
- if(listen_fd >= 0) close(listen_fd);
499
- exit(2);
500
-
484
+ close(listen_fd);
485
+ listen_fd = -1;
486
return NULL;
487
}