@cryptotaxi247 / netdata-1 / commits / 256b329d0

use recvmmsg() to receive multiple packets for each read operation; completed pollfd internal API

Costa Tsaousis (ktsaou) committed Apr 29, 2017 at 01:43 UTC 256b329d0140803ddb137f7032102fcad9c138c0

4 files changed +282 -33

configure.ac

@@ -126,6 +126,7 @@ AC_CHECK_TYPES([struct timespec, clockid_t], [], [], [[#include <time.h>]])

126 AC_SEARCH_LIBS([clock_gettime], [rt posix4])

127 AC_CHECK_FUNCS([clock_gettime])

128 AC_CHECK_FUNCS([sched_setscheduler sched_get_priority_min sched_get_priority_max nice])

129 +AC_CHECK_FUNCS([recvmmsg])

130

131 AC_TYPE_INT8_T

132 AC_TYPE_INT16_T

src/socket.c

+62 -6

@@ -851,11 +851,62 @@ static inline void poll_close_fd(struct poll *p, struct pollinfo *pi) {

851 debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);

852 }

853

854 +static void *add_callback_default(int fd, short int *events) {

855 + (void)fd;

856 + (void)events;

857 +

858 + return NULL;

859 +}

860 +static void del_callback_default(int fd, void *data) {

861 + (void)fd;

862 + (void)data;

863 +

864 + if(data)

865 + error("POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak");

866 +}

867 +

868 +static int rcv_callback_default(int fd, int socktype, void *data, short int *events) {

869 + (void)socktype;

870 + (void)data;

871 + (void)events;

872 +

873 + char buffer[1024 + 1];

874 +

875 + ssize_t rc;

876 + do {

877 + rc = recv(fd, buffer, 1024, MSG_DONTWAIT);

878 + if (rc < 0) {

879 + // read failed

880 + if (errno != EWOULDBLOCK && errno != EAGAIN) {

881 + error("POLLFD: recv() failed.");

882 + return -1;

883 + }

884 + } else if (rc) {

885 + // data received

886 + info("POLLFD: internal error: discarding %zd bytes received on socket %d", rc, fd);

887 + }

888 + } while (rc != -1);

889 +

890 + return 0;

891 +}

892 +

893 +static int snd_callback_default(int fd, int socktype, void *data, short int *events) {

894 + (void)socktype;

895 + (void)data;

896 + (void)events;

897 +

898 + *events &= ~POLLOUT;

899 +

900 + info("POLLFD: internal error: nothing to send on socket %d", fd);

901 + return 0;

902 +}

903 +

904 void poll_events(LISTEN_SOCKETS *sockets

905 , void *(*add_callback)(int fd, short int *events)

906 , void (*del_callback)(int fd, void *data)

907 , int (*rcv_callback)(int fd, int socktype, void *data, short int *events)

908 , int (*snd_callback)(int fd, int socktype, void *data, short int *events)

909 + , void *data

910 ) {

911 int retval;

912

@@ -867,15 +918,16 @@ void poll_events(LISTEN_SOCKETS *sockets

918 .inf = NULL,

919 .first_free = NULL,

920

870 - .add_callback = add_callback,

871 - .del_callback = del_callback,

872 - .rcv_callback = rcv_callback,

873 - .snd_callback = snd_callback

921 + .add_callback = add_callback?add_callback:add_callback_default,

922 + .del_callback = del_callback?del_callback:del_callback_default,

923 + .rcv_callback = rcv_callback?rcv_callback:rcv_callback_default,

924 + .snd_callback = snd_callback?snd_callback:snd_callback_default

925 };

926

927 size_t i;

928 for(i = 0; i < sockets->opened ;i++) {

878 - poll_add_fd(&p, sockets->fds[i], sockets->fds_types[i], POLLIN, POLLINFO_FLAG_SERVER_SOCKET);

929 + struct pollinfo *pi = poll_add_fd(&p, sockets->fds[i], sockets->fds_types[i], POLLIN, POLLINFO_FLAG_SERVER_SOCKET);

930 + pi->data = data;

931 info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN");

932 }

933

@@ -1008,7 +1060,11 @@ void poll_events(LISTEN_SOCKETS *sockets

1060 }

1061 }

1062

1063 + for(i = 0 ; i <= p.max ; i++) {

1064 + struct pollinfo *pi = &p.inf[i];

1065 + poll_close_fd(&p, pi);

1066 + }

1067 +

1068 freez(p.fds);

1069 freez(p.inf);

1070 }

1014 -

src/socket.h

@@ -48,6 +48,7 @@ extern void poll_events(LISTEN_SOCKETS *sockets

48 , void (*del_callback)(int fd, void *data)

49 , int (*rcv_callback)(int fd, int socktype, void *data, short int *events)

50 , int (*snd_callback)(int fd, int socktype, void *data, short int *events)

51 + , void *data

52 );

54 #endif //NETDATA_SOCKET_H

src/statsd.c

+218 -27

@@ -5,6 +5,7 @@

5 // #define STATSD_MULTITHREADED 1

7 #ifdef STATSD_MULTITHREADED

8 +// DO NOT ENABLE MULTITHREADING - IT IS NOT WELL TESTED

9 #define STATSD_AVL_TREE avl_tree_lock

10 #define STATSD_AVL_INSERT avl_insert_lock

11 #define STATSD_AVL_SEARCH avl_search_lock

@@ -132,7 +133,14 @@ static struct statsd {

133 STATSD_INDEX histograms;

134 STATSD_INDEX meters;

135 STATSD_INDEX sets;

135 - size_t unknown;

136 + size_t unknown_types;

137 + size_t socket_errors;

138 + size_t tcp_socket_reads;

139 + size_t tcp_packets_received;

140 + size_t tcp_bytes_read;

141 + size_t udp_socket_reads;

142 + size_t udp_packets_received;

143 + size_t udp_bytes_read;

144

145 int enabled;

146 int update_every;

@@ -144,6 +152,7 @@ static struct statsd {

152 RRD_MEMORY_MODE private_charts_memory_mode;

153 int private_charts_history;

154

155 + size_t recvmmsg_size;

156 size_t histogram_increase_step;

157 double histogram_percentile;

158 char *histogram_percentile_str;

@@ -151,10 +160,9 @@ static struct statsd {

160 LISTEN_SOCKETS sockets;

161 } statsd = {

162 .enabled = 1,

154 - .private_charts = 0,

163 .max_private_charts = 200,

164 .max_private_charts_hard = 1000,

157 - .unknown = 0,

165 + .recvmmsg_size = 10,

166

167 .gauges = {

168 .name = "gauge",

@@ -450,7 +458,7 @@ static void statsd_process_metric(const char *name, const char *value, const cha

458 statsd_find_or_add_metric(&statsd.meters, name),

459 value, sampling);

460 else

453 - statsd.unknown++;

461 + statsd.unknown_types++;

462 break;

463

464 case 'h':

@@ -466,7 +474,7 @@ static void statsd_process_metric(const char *name, const char *value, const cha

474 break;

475

476 default:

469 - statsd.unknown++;

477 + statsd.unknown_types++;

478 break;

479 }

480 }

@@ -586,21 +594,42 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli

594 // --------------------------------------------------------------------------------------------------------------------

595 // statsd pollfd interface

596

589 -#define STATSD_TCP_BUFFER_SIZE 16384 // minimize reads

590 -#define STATSD_UDP_BUFFER_SIZE 1500 // this should be up to MTU

597 +#define STATSD_TCP_BUFFER_SIZE 65536 // minimize tcp reads

598 +#define STATSD_UDP_BUFFER_SIZE 9000 // this should be up to MTU

599 +

600 +typedef enum {

601 + STATSD_SOCKET_DATA_TYPE_TCP,

602 + STATSD_SOCKET_DATA_TYPE_UDP

603 +} STATSD_SOCKET_DATA_TYPE;

604

605 struct statsd_tcp {

606 + STATSD_SOCKET_DATA_TYPE type;

607 size_t size;

608 size_t len;

609 char buffer[];

610 };

611

612 +#ifdef HAVE_RECVMMSG

613 +struct statsd_udp {

614 + STATSD_SOCKET_DATA_TYPE type;

615 + size_t size;

616 + struct iovec *iovecs;

617 + struct mmsghdr *msgs;

618 +};

619 +#else

620 +struct statsd_udp {

621 + STATSD_SOCKET_DATA_TYPE type;

622 + char buffer[STATSD_UDP_BUFFER_SIZE];

623 +};

624 +#endif

625 +

626 // new TCP client connected

627 static void *statsd_add_callback(int fd, short int *events) {

628 (void)fd;

629 *events = POLLIN;

630

631 struct statsd_tcp *data = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1);

632 + data->type = STATSD_SOCKET_DATA_TYPE_TCP;

633 data->size = STATSD_TCP_BUFFER_SIZE - 1;

634

635 return data;

@@ -610,29 +639,49 @@ static void *statsd_add_callback(int fd, short int *events) {

639 static void statsd_del_callback(int fd, void *data) {

640 (void)fd;

641

613 - freez(data);

642 + if(data) {

643 + struct statsd_tcp *t = data;

644 + if(t->type != STATSD_SOCKET_DATA_TYPE_TCP)

645 + error("STATSD: received socket data type is %d, but expected %d", (int)t->type, (int)STATSD_SOCKET_DATA_TYPE_TCP);

646 +

647 + freez(data);

648 + }

649

650 return;

651 }

652

653 // Receive data

654 static int statsd_rcv_callback(int fd, int socktype, void *data, short int *events) {

655 + *events = POLLIN;

656 +

657 switch(socktype) {

658 case SOCK_STREAM: {

659 struct statsd_tcp *d = (struct statsd_tcp *)data;

660 if(unlikely(!d)) {

624 - error("STATSD: internal error - tcp receive buffer is null");

661 + error("STATSD: internal error: expected TCP data pointer is NULL");

662 + statsd.socket_errors++;

663 return -1;

664 }

665

666 +#ifdef NETDATA_INTERNAL_CHECKS

667 + if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) {

668 + error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_TCP);

669 + statsd.socket_errors++;

670 + return -1;

671 + }

672 +#endif

673 +

674 int ret = 0;

675 ssize_t rc;

676 do {

677 rc = recv(fd, &d->buffer[d->len], d->size - d->len, MSG_DONTWAIT);

678 if (rc < 0) {

679 // read failed

634 - if (errno != EWOULDBLOCK && errno != EAGAIN)

680 + if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {

681 + error("STATSD: recv() on TCP socket %d failed.", fd);

682 + statsd.socket_errors++;

683 ret = -1;

684 + }

685 }

686 else if (!rc) {

687 // connection closed

@@ -642,10 +691,14 @@ static int statsd_rcv_callback(int fd, int socktype, void *data, short int *even

691 else {

692 // data received

693 d->len += rc;

694 + statsd.tcp_socket_reads++;

695 + statsd.tcp_bytes_read += rc;

696 }

697

647 - if(likely(d->len > 0))

698 + if(likely(d->len > 0)) {

699 + statsd.tcp_packets_received++;

700 d->len = statsd_process(d->buffer, d->len, 1);

701 + }

702

703 if(unlikely(ret == -1))

704 return -1;

@@ -655,33 +708,77 @@ static int statsd_rcv_callback(int fd, int socktype, void *data, short int *even

708 }

709

710 case SOCK_DGRAM: {

658 - char buffer[STATSD_UDP_BUFFER_SIZE + 1];

711 + struct statsd_udp *d = (struct statsd_udp *)data;

712 + if(unlikely(!d)) {

713 + error("STATSD: internal error: expected UDP data pointer is NULL");

714 + statsd.socket_errors++;

715 + return -1;

716 + }

717

718 +#ifdef NETDATA_INTERNAL_CHECKS

719 + if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) {

720 + error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_UDP);

721 + statsd.socket_errors++;

722 + return -1;

723 + }

724 +#endif

725 +

726 +#ifdef HAVE_RECVMMSG

727 + ssize_t rc;

728 + do {

729 + rc = recvmmsg(fd, d->msgs, (unsigned int)d->size, MSG_DONTWAIT, NULL);

730 + if (rc < 0) {

731 + // read failed

732 + if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {

733 + error("STATSD: recvmmsg() on UDP socket %d failed.", fd);

734 + statsd.socket_errors++;

735 + return -1;

736 + }

737 + } else if (rc) {

738 + // data received

739 + statsd.udp_socket_reads++;

740 + statsd.udp_packets_received += rc;

741 +

742 + size_t i;

743 + for (i = 0; i < (size_t)rc; ++i) {

744 + size_t len = (size_t)d->msgs[i].msg_len;

745 + statsd.udp_bytes_read += len;

746 + statsd_process(d->msgs[i].msg_hdr.msg_iov->iov_base, len, 0);

747 + }

748 + }

749 + } while (rc != -1);

750 +

751 +#else // !HAVE_RECVMMSG

752 ssize_t rc;

753 do {

662 - // FIXME: collect sender information

663 - rc = recvfrom(fd, buffer, STATSD_UDP_BUFFER_SIZE, MSG_DONTWAIT, NULL, NULL);

754 + rc = recv(fd, d->buffer, STATSD_UDP_BUFFER_SIZE, MSG_DONTWAIT);

755 if (rc < 0) {

756 // read failed

666 - if (errno != EWOULDBLOCK && errno != EAGAIN) {

667 - error("STATSD: recvfrom() failed.");

757 + if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {

758 + error("STATSD: recv() on UDP socket %d failed.", fd);

759 + statsd.errors++;

760 return -1;

761 }

762 } else if (rc) {

763 // data received

672 - statsd_process(buffer, (size_t) rc, 0);

764 + statsd.udp_socket_reads++;

765 + statsd.udp_packets_received++;

766 + statsd.udp_bytes_read += rc;

767 + statsd_process(d->buffer, (size_t) rc, 0);

768 }

769 } while (rc != -1);

770 +#endif

771 +

772 break;

773 }

774

775 default: {

679 - error("STATSD: unknown socktype %d on socket %d", socktype, fd);

776 + error("STATSD: internal error: unknown socktype %d on socket %d", socktype, fd);

777 + statsd.socket_errors++;

778 return -1;

779 }

780 }

781

684 - *events = POLLIN;

782 return 0;

783 }

784

@@ -709,13 +806,41 @@ void *statsd_collector_thread(void *ptr) {

806 if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)

807 error("Cannot set pthread cancel state to ENABLE.");

808

809 + struct statsd_udp *d = callocz(sizeof(struct statsd_udp), 1);

810 +

811 +#ifdef HAVE_RECVMMSG

812 + d->type = STATSD_SOCKET_DATA_TYPE_UDP;

813 + d->size = statsd.recvmmsg_size;

814 + d->iovecs = callocz(sizeof(struct iovec), d->size);

815 + d->msgs = callocz(sizeof(struct mmsghdr), d->size);

816 +

817 + size_t i;

818 + for (i = 0; i < d->size; i++) {

819 + d->iovecs[i].iov_base = mallocz(STATSD_UDP_BUFFER_SIZE);

820 + d->iovecs[i].iov_len = STATSD_UDP_BUFFER_SIZE - 1;

821 + d->msgs[i].msg_hdr.msg_iov = &d->iovecs[i];

822 + d->msgs[i].msg_hdr.msg_iovlen = 1;

823 + }

824 +#endif

825 +

826 poll_events(&statsd.sockets

827 , statsd_add_callback

828 , statsd_del_callback

829 , statsd_rcv_callback

830 , statsd_snd_callback

831 + , (void *)d

832 );

833

834 +#ifdef HAVE_RECVMMSG

835 + for (i = 0; i < d->size; i++)

836 + freez(d->iovecs[i].iov_base);

837 +

838 + freez(d->iovecs);

839 + freez(d->msgs);

840 +#endif

841 +

842 + freez(d);

843 +

844 debug(D_WEB_CLIENT, "STATSD: exit!");

845 listen_sockets_close(&statsd.sockets);

846

@@ -1044,6 +1169,10 @@ void *statsd_main(void *ptr) {

1169 statsd.update_every = default_rrd_update_every;

1170 }

1171

1172 +#ifdef HAVE_RECVMMSG

1173 + statsd.recvmmsg_size = (size_t)config_get_number(CONFIG_SECTION_STATSD, "udp messages to process at once", (long long)statsd.recvmmsg_size);

1174 +#endif

1175 +

1176 statsd.charts_for = simple_pattern_create(config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), SIMPLE_PATTERN_EXACT);

1177 statsd.max_private_charts = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts allowed", (long long)statsd.max_private_charts);

1178 statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts * 5);

@@ -1154,6 +1283,52 @@ void *statsd_main(void *ptr) {

1283 RRDDIM *rd_events_histogram = rrddim_add(st_events, "histograms", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1284 RRDDIM *rd_events_set = rrddim_add(st_events, "sets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1285 RRDDIM *rd_events_unknown = rrddim_add(st_events, "unknown", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1286 + RRDDIM *rd_events_errors = rrddim_add(st_events, "errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1287 +

1288 + RRDSET *st_reads = rrdset_create_localhost(

1289 + "netdata"

1290 + , "statsd_reads"

1291 + , NULL

1292 + , "statsd"

1293 + , NULL

1294 + , "Read operations made by the netdata statsd server"

1295 + , "reads/s"

1296 + , 132002

1297 + , statsd.update_every

1298 + , RRDSET_TYPE_STACKED

1299 + );

1300 + RRDDIM *rd_reads_tcp = rrddim_add(st_reads, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1301 + RRDDIM *rd_reads_udp = rrddim_add(st_reads, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1302 +

1303 + RRDSET *st_bytes = rrdset_create_localhost(

1304 + "netdata"

1305 + , "statsd_bytes"

1306 + , NULL

1307 + , "statsd"

1308 + , NULL

1309 + , "Bytes read by the netdata statsd server"

1310 + , "kbps"

1311 + , 132003

1312 + , statsd.update_every

1313 + , RRDSET_TYPE_STACKED

1314 + );

1315 + RRDDIM *rd_bytes_tcp = rrddim_add(st_bytes, "tcp", NULL, 8, 1024, RRD_ALGORITHM_INCREMENTAL);

1316 + RRDDIM *rd_bytes_udp = rrddim_add(st_bytes, "udp", NULL, 8, 1024, RRD_ALGORITHM_INCREMENTAL);

1317 +

1318 + RRDSET *st_packets = rrdset_create_localhost(

1319 + "netdata"

1320 + , "statsd_packets"

1321 + , NULL

1322 + , "statsd"

1323 + , NULL

1324 + , "Network packets processed by the netdata statsd server"

1325 + , "packets/s"

1326 + , 132004

1327 + , statsd.update_every

1328 + , RRDSET_TYPE_STACKED

1329 + );

1330 + RRDDIM *rd_packets_tcp = rrddim_add(st_packets, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1331 + RRDDIM *rd_packets_udp = rrddim_add(st_packets, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

1332

1333 RRDSET *st_pcharts = rrdset_create_localhost(

1334 "netdata"

@@ -1163,7 +1338,7 @@ void *statsd_main(void *ptr) {

1338 , NULL

1339 , "Private metric charts created by the netdata statsd server"

1340 , "charts"

1166 - , 132002

1341 + , 132010

1342 , statsd.update_every

1343 , RRDSET_TYPE_AREA

1344 );

@@ -1191,6 +1366,9 @@ void *statsd_main(void *ptr) {

1366 if(hb_dt) {

1367 rrdset_next(st_metrics);

1368 rrdset_next(st_events);

1369 + rrdset_next(st_reads);

1370 + rrdset_next(st_bytes);

1371 + rrdset_next(st_packets);

1372 rrdset_next(st_pcharts);

1373 }

1374

@@ -1201,13 +1379,23 @@ void *statsd_main(void *ptr) {

1379 rrddim_set_by_pointer(st_metrics, rd_metrics_histogram, (collected_number)statsd.histograms.metrics);

1380 rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics);

1381

1204 - rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events);

1205 - rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events);

1206 - rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events);

1207 - rrddim_set_by_pointer(st_events, rd_events_meter, (collected_number)statsd.meters.events);

1208 - rrddim_set_by_pointer(st_events, rd_events_histogram, (collected_number)statsd.histograms.events);

1209 - rrddim_set_by_pointer(st_events, rd_events_set, (collected_number)statsd.sets.events);

1210 - rrddim_set_by_pointer(st_events, rd_events_unknown, (collected_number)statsd.unknown);

1382 + rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events);

1383 + rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events);

1384 + rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events);

1385 + rrddim_set_by_pointer(st_events, rd_events_meter, (collected_number)statsd.meters.events);

1386 + rrddim_set_by_pointer(st_events, rd_events_histogram, (collected_number)statsd.histograms.events);

1387 + rrddim_set_by_pointer(st_events, rd_events_set, (collected_number)statsd.sets.events);

1388 + rrddim_set_by_pointer(st_events, rd_events_unknown, (collected_number)statsd.unknown_types);

1389 + rrddim_set_by_pointer(st_events, rd_events_errors, (collected_number)statsd.socket_errors);

1390 +

1391 + rrddim_set_by_pointer(st_reads, rd_reads_tcp, (collected_number)statsd.tcp_socket_reads);

1392 + rrddim_set_by_pointer(st_reads, rd_reads_udp, (collected_number)statsd.udp_socket_reads);

1393 +

1394 + rrddim_set_by_pointer(st_bytes, rd_bytes_tcp, (collected_number)statsd.tcp_bytes_read);

1395 + rrddim_set_by_pointer(st_bytes, rd_bytes_udp, (collected_number)statsd.udp_bytes_read);

1396 +

1397 + rrddim_set_by_pointer(st_packets, rd_packets_tcp, (collected_number)statsd.tcp_packets_received);

1398 + rrddim_set_by_pointer(st_packets, rd_packets_udp, (collected_number)statsd.udp_packets_received);

1399

1400 rrddim_set_by_pointer(st_pcharts, rd_pcharts, (collected_number)statsd.private_charts);

1401

@@ -1216,6 +1404,9 @@ void *statsd_main(void *ptr) {

1404

1405 rrdset_done(st_metrics);

1406 rrdset_done(st_events);

1407 + rrdset_done(st_reads);

1408 + rrdset_done(st_bytes);

1409 + rrdset_done(st_packets);

1410 rrdset_done(st_pcharts);

1411

1412 if(unlikely(netdata_exit))