fixes for statsd parsing and statsd optimizations
Costa Tsaousis (ktsaou) committed
May 13, 2017 at 01:59 UTC
3b5a5cf44e447993eee94b7ebf4ca45f8a347003
1 file changed
+72
-94
src/statsd.c
+72
-94
@@ -380,7 +380,7 @@ static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, cons
380
// statsd parsing numbers
381
382
static inline long double statsd_parse_float(const char *v, long double def) {
383
- long double value = def;
383
+ long double value;
384
385
if(likely(v && *v)) {
386
char *e = NULL;
@@ -388,12 +388,14 @@ static inline long double statsd_parse_float(const char *v, long double def) {
388
if(unlikely(e && *e))
389
error("STATSD: excess data '%s' after value '%s'", e, v);
390
}
391
+ else
392
+ value = def;
393
394
return value;
395
}
396
397
static inline long long statsd_parse_int(const char *v, long long def) {
396
- long long value = def;
398
+ long long value;
399
400
if(likely(v && *v)) {
401
char *e = NULL;
@@ -401,6 +403,8 @@ static inline long long statsd_parse_int(const char *v, long long def) {
403
if(unlikely(e && *e))
404
error("STATSD: excess data '%s' after value '%s'", e, v);
405
}
406
+ else
407
+ value = def;
408
409
return value;
410
}
@@ -518,88 +522,55 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
522
// statsd parsing
523
524
static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling) {
521
- debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', rate '%s'", name, value, type, sampling);
525
+ debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', rate '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)");
526
527
if(unlikely(!name || !*name)) return;
528
if(unlikely(!type || !*type)) type = "m";
529
526
- switch (*type) {
527
- case 'g':
528
- statsd_process_gauge(
529
- statsd_find_or_add_metric(&statsd.gauges, name, STATSD_METRIC_TYPE_GAUGE),
530
- value, sampling);
531
- break;
532
-
533
- case 'c': // etsy/statsd, but brubeck uses it as 'meter' - sorry brubeck, this is stupid
534
- case 'C': // brubeck
535
- statsd_process_counter(
536
- statsd_find_or_add_metric(&statsd.counters, name, STATSD_METRIC_TYPE_COUNTER),
537
- value, sampling);
538
- break;
530
+ char t0 = type[0], t1 = type[1];
531
540
- case 'm':
541
- if (type[1] == 's')
542
- statsd_process_timer(
543
- statsd_find_or_add_metric(&statsd.timers, name, STATSD_METRIC_TYPE_TIMER),
544
- value, sampling);
545
- else if (type[1] == '\0')
546
- statsd_process_meter(
547
- statsd_find_or_add_metric(&statsd.meters, name, STATSD_METRIC_TYPE_METER),
548
- value, sampling);
549
- else
550
- statsd.unknown_types++;
551
- break;
552
-
553
- case 'h':
554
- statsd_process_histogram(
555
- statsd_find_or_add_metric(&statsd.histograms, name, STATSD_METRIC_TYPE_HISTOGRAM),
556
- value, sampling);
557
- break;
558
-
559
- case 's':
560
- statsd_process_set(
561
- statsd_find_or_add_metric(&statsd.sets, name, STATSD_METRIC_TYPE_SET),
562
- value);
563
- break;
564
-
565
- default:
566
- statsd.unknown_types++;
567
- break;
532
+ if(unlikely(t0 == 'g' && t1 == '\0')) {
533
+ statsd_process_gauge(
534
+ statsd_find_or_add_metric(&statsd.gauges, name, STATSD_METRIC_TYPE_GAUGE),
535
+ value, sampling);
536
+ }
537
+ else if(unlikely((t0 == 'c' || t0 == 'C') && t1 == '\0')) {
538
+ // etsy/statsd uses 'c'
539
+ // brubeck uses 'C'
540
+ statsd_process_counter(
541
+ statsd_find_or_add_metric(&statsd.counters, name, STATSD_METRIC_TYPE_COUNTER),
542
+ value, sampling);
543
+ }
544
+ else if(unlikely(t0 == 'm' && t1 == '\0')) {
545
+ statsd_process_meter(
546
+ statsd_find_or_add_metric(&statsd.meters, name, STATSD_METRIC_TYPE_METER),
547
+ value, sampling);
548
+ }
549
+ else if(unlikely(t0 == 'h' && t1 == '\0')) {
550
+ statsd_process_histogram(
551
+ statsd_find_or_add_metric(&statsd.histograms, name, STATSD_METRIC_TYPE_HISTOGRAM),
552
+ value, sampling);
553
+ }
554
+ else if(unlikely(t0 == 's' && t1 == '\0')) {
555
+ statsd_process_set(
556
+ statsd_find_or_add_metric(&statsd.sets, name, STATSD_METRIC_TYPE_SET),
557
+ value);
558
+ }
559
+ else if(unlikely(t0 == 'm' && t1 == 's' && type[2] == '\0')) {
560
+ statsd_process_timer(
561
+ statsd_find_or_add_metric(&statsd.timers, name, STATSD_METRIC_TYPE_TIMER),
562
+ value, sampling);
563
+ }
564
+ else {
565
+ statsd.unknown_types++;
566
+ error("STATSD: metric '%s' with value '%s' is sent with unknown metric type '%s'", name, value?value:"", type);
567
}
568
}
569
571
-static inline const char *statsd_parse_name(const char *s, const char **name) {
572
- char c;
573
-
574
- *name = s;
575
- for(c = *s; c && c != ':' && c != '|' && c != '\n'; c = *++s) ;
576
-
577
- return s;
578
-}
579
-
580
-static inline const char *statsd_parse_value(const char *s, const char **value) {
581
- char c;
582
-
583
- *value = s;
584
- for(c = *s; c && c != '|' && c != '\n'; c = *++s) ;
585
-
586
- return s;
587
-}
588
-
589
-static inline const char *statsd_parse_type(const char *s, const char **type) {
590
- char c;
591
-
592
- *type = s;
593
- for(c = *s; c && c != '|' && c != '@' && c != '\n'; c = *++s) ;
594
-
595
- return s;
596
-}
597
-
598
-static inline const char *statsd_parse_sampling(const char *s, const char **sampling) {
570
+static inline const char *statsd_parse_skip_up_to(const char *s, char d1, char d2) {
571
char c;
572
601
- *sampling = s;
602
- for(c = *s; c && c != '\n'; c = *++s) ;
573
+ for(c = *s; c && c != d1 && c != d2 && c != '\r' && c != '\n'; c = *++s) ;
574
575
return s;
576
}
@@ -613,8 +584,6 @@ const char *statsd_parse_skip_spaces(const char *s) {
584
}
585
586
static inline const char *statsd_parse_field_trim(const char *start, char *end) {
616
- *end = '\0';
617
-
587
if(unlikely(!start)) {
588
start = end;
589
return start;
@@ -623,6 +592,7 @@ static inline const char *statsd_parse_field_trim(const char *start, char *end)
592
while(start <= end && (*start == ' ' || *start == '\t'))
593
start++;
594
595
+ *end = '\0';
596
end--;
597
while(end >= start && (*end == ' ' || *end == '\t'))
598
*end-- = '\0';
@@ -637,26 +607,24 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
607
const char *s = buffer;
608
while(*s) {
609
const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL;
640
- char *name_end, *value_end, *type_end, *sampling_end;
610
+ char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL;
611
642
- s = statsd_parse_name(s, &name);
643
- if(name == s || !*name) {
612
+ s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '|');
613
+ if(name == name_end) {
614
s = statsd_parse_skip_spaces(s);
615
continue;
616
}
647
- name_end = (char *)s;
617
649
- if(likely(*s == ':')) s = statsd_parse_value(++s, &value);
650
- value_end = (char *)s;
618
+ if(likely(*s == ':'))
619
+ s = value_end = (char *) statsd_parse_skip_up_to(value = ++s, '|', '|');
620
652
- if(likely(*s == '|')) s = statsd_parse_type(++s, &type);
653
- type_end = (char *)s;
621
+ if(likely(*s == '|'))
622
+ s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@');
623
655
- if(unlikely(*s == '|' || *s == '@')) {
656
- s = statsd_parse_sampling(++s, &sampling);
624
+ if(likely(*s == '|' || *s == '@')) {
625
+ s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '\r', '\n');
626
if(*sampling == '@') sampling++;
627
}
659
- sampling_end = (char *)s;
628
629
// skip everything until the end of the line
630
while(*s && *s != '\n') s++;
@@ -667,6 +635,8 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
635
memmove(buffer, name, size);
636
return size;
637
}
638
+ else
639
+ s = statsd_parse_skip_spaces(s);
640
641
statsd_process_metric(
642
statsd_parse_field_trim(name, name_end)
@@ -733,7 +703,8 @@ static void statsd_del_callback(int fd, void *data) {
703
if(t->type == STATSD_SOCKET_DATA_TYPE_TCP) {
704
if(t->len != 0) {
705
statsd.socket_errors++;
736
- error("STATSD: client is probably sending unterminated metrics. Closed socket left with '%s'", &t->buffer[t->len]);
706
+ error("STATSD: client is probably sending unterminated metrics. Closed socket left with '%s'. Trying to process it.", t->buffer);
707
+ statsd_process(t->buffer, t->len, 0);
708
}
709
}
710
else
@@ -759,8 +730,8 @@ static int statsd_rcv_callback(int fd, int socktype, void *data, short int *even
730
}
731
732
#ifdef NETDATA_INTERNAL_CHECKS
762
- if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) {
763
- error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_TCP);
733
+ if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_TCP)) {
734
+ error("STATSD: internal error: socket data type should be %d, but it is %d", (int)STATSD_SOCKET_DATA_TYPE_TCP, (int)d->type);
735
statsd.socket_errors++;
736
return -1;
737
}
@@ -780,7 +751,7 @@ static int statsd_rcv_callback(int fd, int socktype, void *data, short int *even
751
}
752
else if (!rc) {
753
// connection closed
783
- error("STATSD: client disconnected.");
754
+ debug(D_STATSD, "STATSD: client disconnected.");
755
ret = -1;
756
}
757
else {
@@ -846,7 +817,7 @@ static int statsd_rcv_callback(int fd, int socktype, void *data, short int *even
817
#else // !HAVE_RECVMMSG
818
ssize_t rc;
819
do {
849
- rc = recv(fd, d->buffer, STATSD_UDP_BUFFER_SIZE, MSG_DONTWAIT);
820
+ rc = recv(fd, d->buffer, STATSD_UDP_BUFFER_SIZE - 1, MSG_DONTWAIT);
821
if (rc < 0) {
822
// read failed
823
if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
@@ -1146,6 +1117,9 @@ int statsd_readfile(const char *path, const char *filename) {
1117
error("STATSD: invalid divider value '%s' at line %zu of file '%s/%s'. Using 1.", divider, line, path, filename);
1118
dim->divider = 1;
1119
}
1120
+
1121
+ debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divider " COLLECTED_NUMBER_FORMAT,
1122
+ dim->name, chart->name, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divider);
1123
}
1124
else {
1125
error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id);
@@ -1498,6 +1472,9 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char
1472
m->histogram.ext->last_stddev = (collected_number)roundl(standard_deviation(series, len) * STATSD_DECIMAL_DETAIL);
1473
m->histogram.ext->last_sum = (collected_number)roundl(sum(series, len) * STATSD_DECIMAL_DETAIL);
1474
1475
+ debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT,
1476
+ dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum);
1477
+
1478
m->reset = 1;
1479
updated = 1;
1480
}
@@ -1649,10 +1626,11 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
1626
1627
STATSD_APP_CHART_DIM *dim;
1628
for(dim = chart->dimensions; dim ;dim = dim->next) {
1652
- if(dim->value_ptr) {
1629
+ if(unlikely(dim->value_ptr)) {
1630
if(unlikely(!dim->rd))
1654
- dim->rd = rrddim_add(chart->st, dim->metric, dim->name, dim->multiplier, dim->divider, dim->algorithm);
1631
+ dim->rd = rrddim_add(chart->st, dim->name, NULL, dim->multiplier, dim->divider, dim->algorithm);
1632
1633
+ debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr);
1634
rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr);
1635
}
1636
}