@cryptotaxi247 / netdata-1 / commits / 5f6dfc9ee

added option to enable/disable statsd; added statistical functions (some of which) are required by statsd metrics

Costa Tsaousis (ktsaou) committed Apr 25, 2017 at 02:18 UTC 5f6dfc9ee8cd469c33e647a6f4957e97619faa67
6 files changed +398 -7
CMakeLists.txt
+1 -1
@@ -151,7 +151,7 @@ set(NETDATA_SOURCE_FILES
151 src/web_client.h
152 src/web_server.c
153 src/web_server.h
154 - src/locks.h src/statsd.c src/statsd.h)
154 + src/locks.h src/statsd.c src/statsd.h src/statistical.c src/statistical.h)
155
156 set(APPS_PLUGIN_SOURCE_FILES
157 src/appconfig.c
src/Makefile.am
+2
@@ -119,6 +119,8 @@ netdata_SOURCES = \
119 simple_pattern.h \
120 socket.c \
121 socket.h \
122 + statistical.c \
123 + statistical.h \
124 statsd.c \
125 statsd.h \
126 storage_number.c \
src/common.h
+1
@@ -202,6 +202,7 @@
202 #define NETDATA_OS_TYPE "linux"
203 #endif /* __FreeBSD__, __APPLE__*/
204
205 +#include "statistical.h"
206 #include "socket.h"
207 #include "eval.h"
208 #include "health.h"
src/statistical.c new
+375
@@ -0,0 +1,375 @@
1 +#include "common.h"
2 +
3 +// --------------------------------------------------------------------------------------------------------------------
4 +
5 +long double average(long double *series, size_t entries) {
6 + size_t i, count = 0;
7 + long double sum = 0;
8 +
9 + for(i = 0; i < entries ; i++) {
10 + long double value = series[i];
11 + if(unlikely(isnan(value) || isinf(value))) continue;
12 + count++;
13 + sum += value;
14 + }
15 +
16 + return sum / (long double)count;
17 +}
18 +
19 +// --------------------------------------------------------------------------------------------------------------------
20 +
21 +long double moving_average(long double *series, size_t entries, size_t period) {
22 + size_t i, count = 0;
23 + long double sum = 0, avg = 0;
24 + long double p[period];
25 +
26 + for(i = 0; i < entries; i++) {
27 + long double value = series[i];
28 + if(unlikely(isnan(value) || isinf(value))) continue;
29 +
30 + if(count < period) {
31 + sum += value;
32 + avg = (count == period - 1) ? sum / (long double)period : 0;
33 + }
34 + else {
35 + sum = sum - p[count % period] + value;
36 + avg = sum / (long double)period;
37 + }
38 +
39 + p[count % period] = value;
40 + count++;
41 + }
42 + return avg;
43 +}
44 +
45 +// --------------------------------------------------------------------------------------------------------------------
46 +
47 +static int qsort_compare(const void *a, const void *b) {
48 + long double *p1 = (long double *)a, *p2 = (long double *)b;
49 + long double n1 = *p1, n2 = *p2;
50 +
51 + if(unlikely(isnan(n1) || isnan(n2))) {
52 + if(isnan(n1) && !isnan(n2)) return -1;
53 + if(!isnan(n1) && isnan(n2)) return 1;
54 + return 0;
55 + }
56 + if(unlikely(isinf(n1) || isinf(n2))) {
57 + if(!isinf(n1) && isinf(n2)) return -1;
58 + if(isinf(n1) && !isinf(n2)) return 1;
59 + return 0;
60 + }
61 +
62 + if(unlikely(n1 < n2)) return -1;
63 + if(unlikely(n1 > n2)) return 1;
64 + return 0;
65 +}
66 +
67 +long double median(long double *series, size_t entries) {
68 + if(unlikely(entries == 0))
69 + return NAN;
70 +
71 + if(unlikely(entries == 1))
72 + return series[0];
73 +
74 + if(unlikely(entries == 2))
75 + return (series[0] + series[1]) / 2;
76 +
77 + long double *copy = mallocz(sizeof(long double) * entries);
78 + memcpy(copy, series, sizeof(long double) * entries);
79 + qsort(copy, entries, sizeof(long double), qsort_compare);
80 +
81 + long double avg;
82 + if(entries % 2 == 0) {
83 + size_t m = entries / 2;
84 + avg = (copy[m] + copy[m + 1]) / 2;
85 + }
86 + else {
87 + avg = copy[entries / 2];
88 + }
89 +
90 + freez(copy);
91 + return avg;
92 +}
93 +
94 +// --------------------------------------------------------------------------------------------------------------------
95 +
96 +long double moving_median(long double *series, size_t entries, size_t period) {
97 + if(entries <= period)
98 + return median(series, entries);
99 +
100 + size_t len = entries - period;
101 + long double *data = mallocz(sizeof(long double) * len);
102 +
103 + size_t i;
104 + for(i = period; i < entries; i++) {
105 + data[i - period] = median(&series[i - period], period);
106 + }
107 +
108 + long double avg = median(data, entries - period);
109 + freez(data);
110 + return avg;
111 +}
112 +
113 +// --------------------------------------------------------------------------------------------------------------------
114 +
115 +// http://stackoverflow.com/a/15150143/4525767
116 +long double running_median_estimate(long double *series, size_t entries) {
117 + long double median = 0.0f;
118 + long double average = 0.0f;
119 + size_t i;
120 +
121 + for(i = 0; i < entries ; i++) {
122 + long double value = series[i];
123 + if(unlikely(isnan(value) || isinf(value))) continue;
124 +
125 + average += ( value - average ) * 0.1f; // rough running average.
126 + median += copysignl( average * 0.01, value - median );
127 + }
128 +
129 + return median;
130 +}
131 +
132 +// --------------------------------------------------------------------------------------------------------------------
133 +
134 +long double standard_deviation(long double *series, size_t entries) {
135 + size_t i, count = 0;
136 + long double sum = 0;
137 +
138 + for(i = 0; i < entries ; i++) {
139 + long double value = series[i];
140 + if(unlikely(isnan(value) || isinf(value))) continue;
141 + count++;
142 +
143 + sum += value;
144 + }
145 + long double average = sum / (long double)count;
146 +
147 + for(i = 0, count = 0, sum = 0; i < entries ; i++) {
148 + long double value = series[i];
149 + if(unlikely(isnan(value) || isinf(value))) continue;
150 + count++;
151 +
152 + sum += powl(value - average, 2);
153 + }
154 + long double variance = sum / (long double)(count - 1); // remove -1 to have a population stddev
155 +
156 + long double stddev = sqrtl(variance);
157 + return stddev;
158 +}
159 +
160 +// --------------------------------------------------------------------------------------------------------------------
161 +
162 +long double single_exponential_smoothing(long double *series, size_t entries, long double alpha) {
163 + size_t i, count = 0;
164 + long double level = 0, sum = 0;
165 +
166 + if(unlikely(isnan(alpha)))
167 + alpha = 0.3;
168 +
169 + for(i = 0; i < entries ; i++) {
170 + long double value = series[i];
171 + if(unlikely(isnan(value) || isinf(value))) continue;
172 + count++;
173 +
174 + sum += value;
175 +
176 + long double last_level = level;
177 + level = alpha * value + (1.0 - alpha) * last_level;
178 + }
179 +
180 + return level;
181 +}
182 +
183 +// --------------------------------------------------------------------------------------------------------------------
184 +
185 +// http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/
186 +long double double_exponential_smoothing(long double *series, size_t entries, long double alpha, long double beta, long double *forecast) {
187 + size_t i, count = 0;
188 + long double level = series[0], trend, sum;
189 +
190 + if(unlikely(isnan(alpha)))
191 + alpha = 0.3;
192 +
193 + if(unlikely(isnan(beta)))
194 + beta = 0.05;
195 +
196 + if(likely(entries > 1))
197 + trend = series[1] - series[0];
198 + else
199 + trend = 0;
200 +
201 + sum = series[0];
202 +
203 + for(i = 1; i < entries ; i++) {
204 + long double value = series[i];
205 + if(unlikely(isnan(value) || isinf(value))) continue;
206 + count++;
207 +
208 + sum += value;
209 +
210 + long double last_level = level;
211 +
212 + level = alpha * value + (1.0 - alpha) * (level + trend);
213 + trend = beta * (level - last_level) + (1.0 - beta) * trend;
214 + }
215 +
216 + if(forecast)
217 + *forecast = level + trend;
218 +
219 + return level;
220 +}
221 +
222 +// --------------------------------------------------------------------------------------------------------------------
223 +
224 +/*
225 + * Based on th R implementation
226 + *
227 + * a: level component
228 + * b: trend component
229 + * s: seasonal component
230 + *
231 + * Additive:
232 + *
233 + * Yhat[t+h] = a[t] + h * b[t] + s[t + 1 + (h - 1) mod p],
234 + * a[t] = α (Y[t] - s[t-p]) + (1-α) (a[t-1] + b[t-1])
235 + * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1]
236 + * s[t] = γ (Y[t] - a[t]) + (1-γ) s[t-p]
237 + *
238 + * Multiplicative:
239 + *
240 + * Yhat[t+h] = (a[t] + h * b[t]) * s[t + 1 + (h - 1) mod p],
241 + * a[t] = α (Y[t] / s[t-p]) + (1-α) (a[t-1] + b[t-1])
242 + * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1]
243 + * s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p]
244 + */
245 +static int __HoltWinters(
246 + long double *series,
247 + int entries, // start_time + h
248 +
249 + long double alpha, // alpha parameter of Holt-Winters Filter.
250 + long double beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing.
251 + long double gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted.
252 +
253 + int *seasonal,
254 + int *period,
255 + long double *a, // Start value for level (a[0]).
256 + long double *b, // Start value for trend (b[0]).
257 + long double *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0])
258 +
259 + /* return values */
260 + long double *SSE, // The final sum of squared errors achieved in optimizing
261 + long double *level, // Estimated values for the level component (size entries - t + 2)
262 + long double *trend, // Estimated values for the trend component (size entries - t + 2)
263 + long double *season // Estimated values for the seasonal component (size entries - t + 2)
264 +)
265 +{
266 + if(unlikely(entries < 4))
267 + return 0;
268 +
269 + int start_time = 2;
270 +
271 + long double res = 0, xhat = 0, stmp = 0;
272 + int i, i0, s0;
273 +
274 + /* copy start values to the beginning of the vectors */
275 + level[0] = *a;
276 + if(beta > 0) trend[0] = *b;
277 + if(gamma > 0) memcpy(season, s, *period * sizeof(long double));
278 +
279 + for(i = start_time - 1; i < entries; i++) {
280 + /* indices for period i */
281 + i0 = i - start_time + 2;
282 + s0 = i0 + *period - 1;
283 +
284 + /* forecast *for* period i */
285 + xhat = level[i0 - 1] + (beta > 0 ? trend[i0 - 1] : 0);
286 + stmp = gamma > 0 ? season[s0 - *period] : (*seasonal != 1);
287 + if (*seasonal == 1)
288 + xhat += stmp;
289 + else
290 + xhat *= stmp;
291 +
292 + /* Sum of Squared Errors */
293 + res = series[i] - xhat;
294 + *SSE += res * res;
295 +
296 + /* estimate of level *in* period i */
297 + if (*seasonal == 1)
298 + level[i0] = alpha * (series[i] - stmp)
299 + + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]);
300 + else
301 + level[i0] = alpha * (series[i] / stmp)
302 + + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]);
303 +
304 + /* estimate of trend *in* period i */
305 + if (beta > 0)
306 + trend[i0] = beta * (level[i0] - level[i0 - 1])
307 + + (1 - beta) * trend[i0 - 1];
308 +
309 + /* estimate of seasonal component *in* period i */
310 + if (gamma > 0) {
311 + if (*seasonal == 1)
312 + season[s0] = gamma * (series[i] - level[i0])
313 + + (1 - gamma) * stmp;
314 + else
315 + season[s0] = gamma * (series[i] / level[i0])
316 + + (1 - gamma) * stmp;
317 + }
318 + }
319 +
320 + return 1;
321 +}
322 +
323 +long double holtwinters(long double *series, size_t entries, long double alpha, long double beta, long double gamma, long double *forecast) {
324 + if(unlikely(isnan(alpha)))
325 + alpha = 0.3;
326 +
327 + if(unlikely(isnan(beta)))
328 + beta = 0.05;
329 +
330 + if(unlikely(isnan(gamma)))
331 + gamma = 0;
332 +
333 + int seasonal = 0;
334 + int period = 0;
335 + long double a0 = series[0];
336 + long double b0 = 0;
337 + long double s[] = {};
338 +
339 + long double errors;
340 + size_t nb_computations = entries;
341 + long double *estimated_level = callocz(nb_computations, sizeof(long double));
342 + long double *estimated_trend = callocz(nb_computations, sizeof(long double));
343 + long double *estimated_season = callocz(nb_computations, sizeof(long double));
344 +
345 + int ret = __HoltWinters(
346 + series,
347 + (int)entries,
348 + alpha,
349 + beta,
350 + gamma,
351 + &seasonal,
352 + &period,
353 + &a0,
354 + &b0,
355 + s,
356 + &errors,
357 + estimated_level,
358 + estimated_trend,
359 + estimated_season
360 + );
361 +
362 + long double value = estimated_level[nb_computations - 1];
363 +
364 + if(forecast)
365 + *forecast = 0.0;
366 +
367 + freez(estimated_level);
368 + freez(estimated_trend);
369 + freez(estimated_season);
370 +
371 + if(!ret)
372 + return 0.0;
373 +
374 + return value;
375 +}
src/statistical.h new
+14
@@ -0,0 +1,14 @@
1 +#ifndef NETDATA_STATISTICAL_H
2 +#define NETDATA_STATISTICAL_H
3 +
4 +extern long double average(long double *series, size_t entries);
5 +extern long double moving_average(long double *series, size_t entries, size_t period);
6 +extern long double median(long double *series, size_t entries);
7 +extern long double moving_median(long double *series, size_t entries, size_t period);
8 +extern long double running_median_estimate(long double *series, size_t entries);
9 +extern long double standard_deviation(long double *series, size_t entries);
10 +extern long double single_exponential_smoothing(long double *series, size_t entries, long double alpha);
11 +extern long double double_exponential_smoothing(long double *series, size_t entries, long double alpha, long double beta, long double *forecast);
12 +extern long double holtwinters(long double *series, size_t entries, long double alpha, long double beta, long double gamma, long double *forecast);
13 +
14 +#endif //NETDATA_STATISTICAL_H
src/statsd.c
+5 -6
@@ -167,12 +167,6 @@ static int statsd_metric_compare(void* a, void* b) {
167 else return strcmp(((STATSD_METRIC *)a)->name, ((STATSD_METRIC *)b)->name);
168 }
169
170 -static int statsd_metric_set_compare(void* a, void* b) {
171 - if(((STATSD_METRIC_SET_VALUE *)a)->hash < ((STATSD_METRIC_SET_VALUE *)b)->hash) return -1;
172 - else if(((STATSD_METRIC_SET_VALUE *)a)->hash > ((STATSD_METRIC_SET_VALUE *)b)->hash) return 1;
173 - else return strcmp(((STATSD_METRIC_SET_VALUE *)a)->value, ((STATSD_METRIC_SET_VALUE *)b)->value);
174 -}
175 -
170 static inline STATSD_METRIC *stasd_metric_index_find(STATSD_INDEX *index, const char *name, uint32_t hash) {
171 STATSD_METRIC tmp;
172 tmp.name = name;
@@ -316,6 +310,8 @@ static inline void statsd_process_timer(STATSD_METRIC *m, char *v, char *r) {
310 }
311
312 static inline void statsd_process_set(STATSD_METRIC *m, char *v, char *r) {
313 + (void)r;
314 +
315 if(unlikely(!v || !*v)) {
316 error("STATSD: metric of type set, with empty value is ignored.");
317 return;
@@ -650,6 +646,9 @@ void *statsd_main(void *ptr) {
646 if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
647 error("Cannot set pthread cancel state to ENABLE.");
648
649 + int enabled = config_get_boolean(CONFIG_SECTION_STATSD, "enabled", 1);
650 + if(!enabled) return NULL;
651 +
652 statsd_listen_sockets_setup();
653 if(!statsd.sockets.opened) {
654 error("STATSD: No statsd sockets to listen to.");