support statsd synthetic charts with pattern based dimensions; fixes #2785
Costa Tsaousis (ktsaou) committed
Dec 9, 2017 at 17:56 UTC
53151d5b72c6c2281ea095eccb90173a9218f53f
1 file changed
+193
-132
src/statsd.c
+193
-132
@@ -167,18 +167,21 @@ typedef enum statsd_app_chart_dimension_value_type {
167
} STATSD_APP_CHART_DIM_VALUE_TYPE;
168
169
typedef struct statsd_app_chart_dimension {
170
- const char *name;
171
- const char *metric;
172
- uint32_t metric_hash;
173
- collected_number multiplier;
174
- collected_number divisor;
175
- STATSD_APP_CHART_DIM_VALUE_TYPE value_type;
170
+ const char *name; // the name of this dimension
171
+ const char *metric; // the source metric name of this dimension
172
+ uint32_t metric_hash; // hash for fast string comparisons
173
+
174
+ SIMPLE_PATTERN *metric_pattern; // set when the 'metric' is a simple pattern
175
177
- RRDDIM *rd;
178
- collected_number *value_ptr;
179
- RRD_ALGORITHM algorithm;
176
+ collected_number multiplier; // the multipler of the dimension
177
+ collected_number divisor; // the divisor of the dimension
178
+ STATSD_APP_CHART_DIM_VALUE_TYPE value_type; // which value to use of the source metric
179
181
- struct statsd_app_chart_dimension *next;
180
+ RRDDIM *rd; // a pointer to the RRDDIM that has been created for this dimension
181
+ collected_number *value_ptr; // a pointer to the source metric value
182
+ RRD_ALGORITHM algorithm; // the algorithm of this dimension
183
+
184
+ struct statsd_app_chart_dimension *next; // the next dimension for this chart
185
} STATSD_APP_CHART_DIM;
186
187
typedef struct statsd_app_chart {
@@ -933,6 +936,67 @@ void *statsd_collector_thread(void *ptr) {
936
937
#define STATSD_CONF_LINE_MAX 8192
938
939
+static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *path, const char *filename) {
940
+ if(!type || !*type) type = "last";
941
+
942
+ if(!strcmp(type, "events")) return STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS;
943
+ else if(!strcmp(type, "last")) return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
944
+ else if(!strcmp(type, "min")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MIN;
945
+ else if(!strcmp(type, "max")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MAX;
946
+ else if(!strcmp(type, "sum")) return STATSD_APP_CHART_DIM_VALUE_TYPE_SUM;
947
+ else if(!strcmp(type, "average")) return STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE;
948
+ else if(!strcmp(type, "median")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN;
949
+ else if(!strcmp(type, "stddev")) return STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV;
950
+ else if(!strcmp(type, "percentile")) return STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE;
951
+
952
+ error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename);
953
+ return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
954
+}
955
+
956
+static STATSD_APP_CHART_DIM *add_dimension_to_app_chart(
957
+ STATSD_APP *app
958
+ , STATSD_APP_CHART *chart
959
+ , const char *metric_name
960
+ , const char *dim_name
961
+ , collected_number multiplier
962
+ , collected_number divisor
963
+ , STATSD_APP_CHART_DIM_VALUE_TYPE value_type
964
+) {
965
+ STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1);
966
+
967
+ dim->metric = strdupz(metric_name);
968
+ dim->metric_hash = simple_hash(dim->metric);
969
+
970
+ dim->name = strdupz((dim_name && *dim_name)?dim_name:metric_name);
971
+ dim->multiplier = multiplier;
972
+ dim->divisor = divisor;
973
+ dim->value_type = value_type;
974
+
975
+ if(!dim->multiplier)
976
+ dim->multiplier = 1;
977
+
978
+ if(!dim->divisor)
979
+ dim->divisor = 1;
980
+
981
+ // append it to the list of dimension
982
+ STATSD_APP_CHART_DIM *tdim;
983
+ for(tdim = chart->dimensions; tdim && tdim->next ; tdim = tdim->next) ;
984
+ if(!tdim) {
985
+ dim->next = chart->dimensions;
986
+ chart->dimensions = dim;
987
+ }
988
+ else {
989
+ dim->next = tdim->next;
990
+ tdim->next = dim;
991
+ }
992
+ chart->dimensions_count++;
993
+
994
+ debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT,
995
+ dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor);
996
+
997
+ return dim;
998
+}
999
+
1000
int statsd_readfile(const char *path, const char *filename) {
1001
debug(D_STATSD, "STATSD configuration reading file '%s/%s'", path, filename);
1002
@@ -1091,63 +1155,35 @@ int statsd_readfile(const char *path, const char *filename) {
1155
}
1156
else if (!strcmp(name, "dimension")) {
1157
// metric [name [type [multiplier [divisor]]]]
1094
- char *words[5];
1095
- pluginsd_split_words(value, words, 5);
1096
-
1097
- char *metric_name = words[0];
1098
- char *dim_name = words[1];
1099
- char *type = words[2];
1100
- char *multipler = words[3];
1101
- char *divisor = words[4];
1102
-
1103
- STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1);
1104
-
1105
- dim->metric = strdupz(metric_name);
1106
- dim->metric_hash = simple_hash(dim->metric);
1107
-
1108
- dim->name = strdupz((dim_name && *dim_name)?dim_name:metric_name);
1109
- dim->multiplier = (multipler && *multipler)?str2l(multipler):1;
1110
- dim->divisor = (divisor && *divisor)?str2l(divisor):1;
1111
-
1112
- if(!type || !*type) type = "last";
1113
- if(!strcmp(type, "events")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS;
1114
- else if(!strcmp(type, "last")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
1115
- else if(!strcmp(type, "min")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MIN;
1116
- else if(!strcmp(type, "max")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MAX;
1117
- else if(!strcmp(type, "sum")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_SUM;
1118
- else if(!strcmp(type, "average")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE;
1119
- else if(!strcmp(type, "median")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN;
1120
- else if(!strcmp(type, "stddev")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV;
1121
- else if(!strcmp(type, "percentile")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE;
1122
- else {
1123
- error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename);
1124
- dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
1125
- }
1158
+ char *words[10];
1159
+ pluginsd_split_words(value, words, 10);
1160
1127
- if(!dim->multiplier) {
1128
- error("STATSD: invalid multiplier value '%s' at line %zu of file '%s/%s'. Using 1.", multipler, line, path, filename);
1129
- dim->multiplier = 1;
1130
- }
1131
- if(!dim->divisor) {
1132
- error("STATSD: invalid divisor value '%s' at line %zu of file '%s/%s'. Using 1.", divisor, line, path, filename);
1133
- dim->divisor = 1;
1134
- }
1161
+ int pattern = 0;
1162
+ size_t i = 0;
1163
+ char *metric_name = words[i++];
1164
1136
- // append it to the list of dimension
1137
- STATSD_APP_CHART_DIM *tdim;
1138
- for(tdim = chart->dimensions; tdim && tdim->next ; tdim = tdim->next) ;
1139
- if(!tdim) {
1140
- dim->next = chart->dimensions;
1141
- chart->dimensions = dim;
1165
+ if(strcmp(metric_name, "pattern") == 0) {
1166
+ metric_name = words[i++];
1167
+ pattern = 1;
1168
}
1143
- else {
1144
- dim->next = tdim->next;
1145
- tdim->next = dim;
1146
- }
1147
- chart->dimensions_count++;
1169
1149
- debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT,
1150
- dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor);
1170
+ char *dim_name = words[i++];
1171
+ char *type = words[i++];
1172
+ char *multipler = words[i++];
1173
+ char *divisor = words[i++];
1174
+
1175
+ STATSD_APP_CHART_DIM *dim = add_dimension_to_app_chart(
1176
+ app
1177
+ , chart
1178
+ , metric_name
1179
+ , dim_name
1180
+ , (multipler && *multipler)?str2l(multipler):1
1181
+ , (divisor && *divisor)?str2l(divisor):1
1182
+ , string2valuetype(type, line, path, filename)
1183
+ );
1184
+
1185
+ if(pattern)
1186
+ dim->metric_pattern = simple_pattern_create(dim->metric, SIMPLE_PATTERN_EXACT);
1187
}
1188
else {
1189
error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id);
@@ -1563,6 +1599,71 @@ static inline RRD_ALGORITHM statsd_algorithm_for_metric(STATSD_METRIC *m) {
1599
}
1600
}
1601
1602
+static inline void link_metric_to_app_dimension(STATSD_APP *app, STATSD_METRIC *m, STATSD_APP_CHART *chart, STATSD_APP_CHART_DIM *dim) {
1603
+ if(dim->value_type == STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS) {
1604
+ dim->value_ptr = &m->events;
1605
+ dim->algorithm = RRD_ALGORITHM_INCREMENTAL;
1606
+ }
1607
+ else if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
1608
+ dim->algorithm = RRD_ALGORITHM_ABSOLUTE;
1609
+ dim->divisor *= statsd.decimal_detail;
1610
+
1611
+ switch(dim->value_type) {
1612
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS:
1613
+ // will never match - added to avoid warning
1614
+ break;
1615
+
1616
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST:
1617
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE:
1618
+ dim->value_ptr = &m->last;
1619
+ break;
1620
+
1621
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM:
1622
+ dim->value_ptr = &m->histogram.ext->last_sum;
1623
+ break;
1624
+
1625
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN:
1626
+ dim->value_ptr = &m->histogram.ext->last_min;
1627
+ break;
1628
+
1629
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX:
1630
+ dim->value_ptr = &m->histogram.ext->last_max;
1631
+ break;
1632
+
1633
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN:
1634
+ dim->value_ptr = &m->histogram.ext->last_median;
1635
+ break;
1636
+
1637
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE:
1638
+ dim->value_ptr = &m->histogram.ext->last_percentile;
1639
+ break;
1640
+
1641
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV:
1642
+ dim->value_ptr = &m->histogram.ext->last_stddev;
1643
+ break;
1644
+ }
1645
+ }
1646
+ else {
1647
+ if (dim->value_type != STATSD_APP_CHART_DIM_VALUE_TYPE_LAST)
1648
+ error("STATSD: unsupported value type for dimension '%s' of chart '%s' of app '%s' on metric '%s'", dim->name, chart->id, app->name, m->name);
1649
+
1650
+ dim->value_ptr = &m->last;
1651
+ dim->algorithm = statsd_algorithm_for_metric(m);
1652
+
1653
+ if(m->type == STATSD_METRIC_TYPE_GAUGE)
1654
+ dim->divisor *= statsd.decimal_detail;
1655
+ }
1656
+
1657
+ if(unlikely(chart->st && dim->rd)) {
1658
+ rrddim_set_algorithm(chart->st, dim->rd, dim->algorithm);
1659
+ rrddim_set_multiplier(chart->st, dim->rd, dim->multiplier);
1660
+ rrddim_set_divisor(chart->st, dim->rd, dim->divisor);
1661
+ }
1662
+
1663
+ chart->dimensions_linked_count++;
1664
+ debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm));
1665
+}
1666
+
1667
static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC *m) {
1668
(void)index;
1669
@@ -1588,75 +1689,33 @@ static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC
1689
// check if there is a chart in this app, willing to get this metric
1690
STATSD_APP_CHART *chart;
1691
for(chart = app->charts; chart; chart = chart->next) {
1692
+
1693
STATSD_APP_CHART_DIM *dim;
1694
for(dim = chart->dimensions; dim ; dim = dim->next) {
1593
- if(!dim->value_ptr && dim->metric_hash == m->hash && !strcmp(dim->metric, m->name)) {
1594
- // we have a match - this metric should be linked to this dimension
1595
-
1596
- if(dim->value_type == STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS) {
1597
- dim->value_ptr = &m->events;
1598
- dim->algorithm = RRD_ALGORITHM_INCREMENTAL;
1599
- }
1600
- else if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
1601
- dim->algorithm = RRD_ALGORITHM_ABSOLUTE;
1602
- dim->divisor *= statsd.decimal_detail;
1603
-
1604
- switch(dim->value_type) {
1605
- case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS:
1606
- // will never match - added to avoid warning
1607
- break;
1608
-
1609
- case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST:
1610
- case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE:
1611
- dim->value_ptr = &m->last;
1612
- break;
1613
-
1614
- case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM:
1615
- dim->value_ptr = &m->histogram.ext->last_sum;
1616
- break;
1617
-
1618
- case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN:
1619
- dim->value_ptr = &m->histogram.ext->last_min;
1620
- break;
1621
-
1622
- case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX:
1623
- dim->value_ptr = &m->histogram.ext->last_max;
1624
- break;
1625
-
1626
- case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN:
1627
- dim->value_ptr = &m->histogram.ext->last_median;
1628
- break;
1629
-
1630
- case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE:
1631
- dim->value_ptr = &m->histogram.ext->last_percentile;
1632
- break;
1633
-
1634
- case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV:
1635
- dim->value_ptr = &m->histogram.ext->last_stddev;
1636
- break;
1637
- }
1638
- }
1639
- else {
1640
- if (dim->value_type != STATSD_APP_CHART_DIM_VALUE_TYPE_LAST)
1641
- error("STATSD: unsupported value type for dimension '%s' of chart '%s' of app '%s' on metric '%s'", dim->name, chart->id, app->name, m->name);
1642
-
1643
- dim->value_ptr = &m->last;
1644
- dim->algorithm = statsd_algorithm_for_metric(m);
1645
-
1646
- if(m->type == STATSD_METRIC_TYPE_GAUGE)
1647
- dim->divisor *= statsd.decimal_detail;
1648
- }
1649
-
1650
- if(unlikely(chart->st && dim->rd)) {
1651
- rrddim_set_algorithm(chart->st, dim->rd, dim->algorithm);
1652
- rrddim_set_multiplier(chart->st, dim->rd, dim->multiplier);
1653
- rrddim_set_divisor(chart->st, dim->rd, dim->divisor);
1695
+ if(unlikely(dim->metric_pattern)) {
1696
+ size_t len = strlen(m->name) + 1;
1697
+ char name[len];
1698
+ if(simple_pattern_matches_extract(dim->metric_pattern, m->name, name, len)) {
1699
+ add_dimension_to_app_chart(
1700
+ app
1701
+ , chart
1702
+ , m->name
1703
+ , name
1704
+ , dim->multiplier
1705
+ , dim->divisor
1706
+ , dim->value_type
1707
+ );
1708
+
1709
+ // the new dimension is appended to the list
1710
+ // so, it will be matched and linked later too
1711
}
1655
-
1656
- chart->dimensions_linked_count++;
1657
- debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm));
1712
+ }
1713
+ else if(!dim->value_ptr && dim->metric_hash == m->hash && !strcmp(dim->metric, m->name)) {
1714
+ // we have a match - this metric should be linked to this dimension
1715
+ link_metric_to_app_dimension(app, m, chart, dim);
1716
}
1717
}
1718
+
1719
}
1720
}
1721
}
@@ -1691,12 +1750,14 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
1750
1751
STATSD_APP_CHART_DIM *dim;
1752
for(dim = chart->dimensions; dim ;dim = dim->next) {
1694
- if(unlikely(!dim->rd))
1695
- dim->rd = rrddim_add(chart->st, dim->metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm);
1753
+ if(likely(!dim->metric_pattern)) {
1754
+ if (unlikely(!dim->rd))
1755
+ dim->rd = rrddim_add(chart->st, dim->metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm);
1756
1697
- if(unlikely(dim->value_ptr)) {
1698
- debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr);
1699
- rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr);
1757
+ if (unlikely(dim->value_ptr)) {
1758
+ debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr);
1759
+ rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr);
1760
+ }
1761
}
1762
}
1763