master
c 547 lines 19.6 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "libnetdata.h"
4
5 #define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2)
6 #define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED)
7 #define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED)
8
9 struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };
10
11 // --------------------------------------------------------------------------------------------------------------------
12
13 void json_escape_string(char *dst, const char *src, size_t size) {
14 const char *t;
15 char *d = dst, *e = &dst[size - 1];
16
17 for(t = src; *t && d < e ;t++) {
18 if(unlikely(*t == '\\' || *t == '"')) {
19 if(unlikely(d + 1 >= e)) break;
20 *d++ = '\\';
21 }
22 *d++ = *t;
23 }
24
25 *d = '\0';
26 }
27
28 char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {
29 char *s = fgets(buf, (int)buf_size, fp);
30 if (!s) return NULL;
31
32 char *t = s;
33 if (*t != '\0') {
34 // find the string end
35 while (*++t != '\0');
36
37 // trim trailing spaces/newlines/tabs
38 while (--t > s && *t == '\n')
39 *t = '\0';
40 }
41
42 if (len)
43 *len = t - s + 1;
44
45 return s;
46 }
47
48 // vsnprintfz() returns the number of bytes actually written - after possible truncation
49 int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args) {
50 if(unlikely(!dst || !n)) return 0;
51
52 if(unlikely(!fmt)) {
53 dst[0] = '\0';
54 return 0;
55 }
56
57 int size = vsnprintf(dst, n, fmt, args);
58 dst[n - 1] = '\0';
59
60 if (unlikely((size_t) size >= n)) size = (int)(n - 1);
61
62 return size;
63 }
64
65 // snprintfz() returns the number of bytes actually written - after possible truncation
66 int snprintfz(char *dst, size_t n, const char *fmt, ...) {
67 va_list args;
68
69 va_start(args, fmt);
70 int ret = vsnprintfz(dst, n, fmt, args);
71 va_end(args);
72
73 return ret;
74 }
75
76 // Returns the number of bytes read from the file if file_size is not NULL.
77 // The actual buffer has an extra byte set to zero (not included in the count).
78 char *read_by_filename(const char *filename, long *file_size)
79 {
80 FILE *f = fopen(filename, "r");
81 if (!f)
82 return NULL;
83
84 if (fseek(f, 0, SEEK_END) < 0) {
85 fclose(f);
86 return NULL;
87 }
88
89 long size = ftell(f);
90 if (size <= 0 || fseek(f, 0, SEEK_END) < 0) {
91 fclose(f);
92 return NULL;
93 }
94
95 char *contents = callocz(size + 1, 1);
96 if (fseek(f, 0, SEEK_SET) < 0) {
97 fclose(f);
98 freez(contents);
99 return NULL;
100 }
101
102 size_t res = fread(contents, 1, size, f);
103 if ( res != (size_t)size) {
104 freez(contents);
105 fclose(f);
106 return NULL;
107 }
108
109 fclose(f);
110
111 if (file_size)
112 *file_size = size;
113
114 return contents;
115 }
116
117 char *find_and_replace(const char *src, const char *find, const char *replace, const char *where)
118 {
119 size_t size = strlen(src) + 1;
120 size_t find_len = strlen(find);
121 size_t repl_len = strlen(replace);
122 char *value, *dst;
123
124 if (likely(where))
125 size += (repl_len - find_len);
126
127 value = mallocz(size);
128 dst = value;
129
130 if (likely(where)) {
131 size_t count = where - src;
132
133 memmove(dst, src, count);
134 src += count;
135 dst += count;
136
137 memmove(dst, replace, repl_len);
138 src += find_len;
139 dst += repl_len;
140 }
141
142 strcpy(dst, src);
143
144 return value;
145 }
146
147 static inline bool run_command_validate_max_line_length(const char *command, int max_line_length) {
148 if (likely(max_line_length > 0))
149 return true;
150
151 netdata_log_error("Invalid max_line_length %d for command '%s'.",
152 max_line_length, command ? command : "(null)");
153 return false;
154 }
155
156 BUFFER *run_command_and_get_output_to_buffer(const char *command, int max_line_length) {
157 if (unlikely(!run_command_validate_max_line_length(command, max_line_length)))
158 return NULL;
159
160 BUFFER *wb = buffer_create(0, NULL);
161
162 POPEN_INSTANCE *pi = spawn_popen_run(command);
163 if(pi) {
164 size_t buffer_size = (size_t)max_line_length + 1;
165 CLEAN_CHAR_P *buffer = mallocz(buffer_size);
166 while (fgets(buffer, max_line_length, spawn_popen_stdout(pi))) {
167 buffer[max_line_length] = '\0';
168 buffer_strcat(wb, buffer);
169 }
170 spawn_popen_kill(pi, 0);
171 }
172 else {
173 buffer_free(wb);
174 netdata_log_error("Failed to execute command '%s'.", command);
175 return NULL;
176 }
177
178 return wb;
179 }
180
181 bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length) {
182 if (unlikely(!run_command_validate_max_line_length(command, max_line_length)))
183 return false;
184
185 POPEN_INSTANCE *pi = spawn_popen_run(command);
186 if(pi) {
187 size_t buffer_size = (size_t)max_line_length + 1;
188 CLEAN_CHAR_P *buffer = mallocz(buffer_size);
189
190 while (fgets(buffer, max_line_length, spawn_popen_stdout(pi)))
191 fprintf(stdout, "%s", buffer);
192
193 spawn_popen_kill(pi, 0);
194 }
195 else {
196 netdata_log_error("Failed to execute command '%s'.", command);
197 return false;
198 }
199
200 return true;
201 }
202
203 struct timing_steps {
204 const char *name;
205 usec_t time;
206 size_t count;
207 } timing_steps[TIMING_STEP_MAX + 1] = {
208 [TIMING_STEP_INTERNAL] = { .name = "internal", .time = 0, },
209
210 [TIMING_STEP_BEGIN2_PREPARE] = { .name = "BEGIN2 prepare", .time = 0, },
211 [TIMING_STEP_BEGIN2_FIND_CHART] = { .name = "BEGIN2 find chart", .time = 0, },
212 [TIMING_STEP_BEGIN2_PARSE] = { .name = "BEGIN2 parse", .time = 0, },
213 [TIMING_STEP_BEGIN2_ML] = { .name = "BEGIN2 ml", .time = 0, },
214 [TIMING_STEP_BEGIN2_PROPAGATE] = { .name = "BEGIN2 propagate", .time = 0, },
215 [TIMING_STEP_BEGIN2_STORE] = { .name = "BEGIN2 store", .time = 0, },
216
217 [TIMING_STEP_SET2_PREPARE] = { .name = "SET2 prepare", .time = 0, },
218 [TIMING_STEP_SET2_LOOKUP_DIMENSION] = { .name = "SET2 find dimension", .time = 0, },
219 [TIMING_STEP_SET2_PARSE] = { .name = "SET2 parse", .time = 0, },
220 [TIMING_STEP_SET2_ML] = { .name = "SET2 ml", .time = 0, },
221 [TIMING_STEP_SET2_PROPAGATE] = { .name = "SET2 propagate", .time = 0, },
222 [TIMING_STEP_RRDSET_STORE_METRIC] = { .name = "SET2 rrdset store", .time = 0, },
223 [TIMING_STEP_DBENGINE_FIRST_CHECK] = { .name = "db 1st check", .time = 0, },
224 [TIMING_STEP_DBENGINE_CHECK_DATA] = { .name = "db check data", .time = 0, },
225 [TIMING_STEP_DBENGINE_PACK] = { .name = "db pack", .time = 0, },
226 [TIMING_STEP_DBENGINE_PAGE_FIN] = { .name = "db page fin", .time = 0, },
227 [TIMING_STEP_DBENGINE_MRG_UPDATE] = { .name = "db mrg update", .time = 0, },
228 [TIMING_STEP_DBENGINE_PAGE_ALLOC] = { .name = "db page alloc", .time = 0, },
229 [TIMING_STEP_DBENGINE_CREATE_NEW_PAGE] = { .name = "db new page", .time = 0, },
230 [TIMING_STEP_DBENGINE_FLUSH_PAGE] = { .name = "db page flush", .time = 0, },
231 [TIMING_STEP_SET2_STORE] = { .name = "SET2 store", .time = 0, },
232
233 [TIMING_STEP_END2_PREPARE] = { .name = "END2 prepare", .time = 0, },
234 [TIMING_STEP_END2_PUSH_V1] = { .name = "END2 push v1", .time = 0, },
235 [TIMING_STEP_END2_ML] = { .name = "END2 ml", .time = 0, },
236 [TIMING_STEP_END2_RRDSET] = { .name = "END2 rrdset", .time = 0, },
237 [TIMING_STEP_END2_PROPAGATE] = { .name = "END2 propagate", .time = 0, },
238 [TIMING_STEP_END2_STORE] = { .name = "END2 store", .time = 0, },
239
240 [TIMING_STEP_DBENGINE_EVICT_LOCK] = { .name = "EVC_LOCK", .time = 0, },
241 [TIMING_STEP_DBENGINE_EVICT_SELECT] = { .name = "EVC_SELECT", .time = 0, },
242 [TIMING_STEP_DBENGINE_EVICT_SELECT_PAGE ] = { .name = "EVT_SELECT_PAGE", .time = 0, },
243 [TIMING_STEP_DBENGINE_EVICT_RELOCATE_PAGE ] = { .name = "EVT_RELOCATE_PAGE", .time = 0, },
244 [TIMING_STEP_DBENGINE_EVICT_SORT] = { .name = "EVC_SORT", .time = 0, },
245 [TIMING_STEP_DBENGINE_EVICT_DEINDEX] = { .name = "EVC_DEINDEX", .time = 0, },
246 [TIMING_STEP_DBENGINE_EVICT_DEINDEX_PAGE] = { .name = "EVC_DEINDEX_PAGE", .time = 0, },
247 [TIMING_STEP_DBENGINE_EVICT_FINISHED] = { .name = "EVC_FINISHED", .time = 0, },
248 [TIMING_STEP_DBENGINE_EVICT_FREE_LOOP] = { .name = "EVC_FREE_LOOP", .time = 0, },
249 [TIMING_STEP_DBENGINE_EVICT_FREE_PAGE] = { .name = "EVC_FREE_PAGE", .time = 0, },
250 [TIMING_STEP_DBENGINE_EVICT_FREE_ATOMICS] = { .name = "EVC_FREE_ATOMICS", .time = 0, },
251 [TIMING_STEP_DBENGINE_EVICT_FREE_CB] = { .name = "EVC_FREE_CB", .time = 0, },
252 [TIMING_STEP_DBENGINE_EVICT_FREE_ATOMICS2] = { .name = "EVC_FREE_ATOMICS2", .time = 0, },
253 [TIMING_STEP_DBENGINE_EVICT_FREE_ARAL] = { .name = "EVC_FREE_ARAL", .time = 0, },
254 [TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_DATA] = { .name = "EVC_FREE_PGD_DATA", .time = 0, },
255 [TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_ARAL] = { .name = "EVC_FREE_PGD_ARAL", .time = 0, },
256 [TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_TIER1_ARAL] = { .name = "EVC_FREE_MAIN_T1ARL", .time = 0, },
257 [TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_GLIVE] = { .name = "EVC_FREE_MAIN_GLIVE", .time = 0, },
258 [TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_GWORKER] = { .name = "EVC_FREE_MAIN_GWORK", .time = 0, },
259 [TIMING_STEP_DBENGINE_EVICT_FREE_OPEN] = { .name = "EVC_FREE_OPEN", .time = 0, },
260 [TIMING_STEP_DBENGINE_EVICT_FREE_EXTENT] = { .name = "EVC_FREE_EXTENT", .time = 0, },
261
262 // terminator
263 [TIMING_STEP_MAX] = { .name = NULL, .time = 0, },
264 };
265
266 void timing_action(TIMING_ACTION action, TIMING_STEP step) {
267 static __thread usec_t last_action_time = 0;
268 static struct timing_steps timings2[TIMING_STEP_MAX + 1] = {};
269
270 switch(action) {
271 case TIMING_ACTION_INIT:
272 last_action_time = now_monotonic_usec();
273 break;
274
275 case TIMING_ACTION_STEP: {
276 if(!last_action_time)
277 return;
278
279 usec_t now = now_monotonic_usec();
280 __atomic_add_fetch(&timing_steps[step].time, now - last_action_time, __ATOMIC_RELAXED);
281 __atomic_add_fetch(&timing_steps[step].count, 1, __ATOMIC_RELAXED);
282 last_action_time = now;
283 break;
284 }
285
286 case TIMING_ACTION_FINISH: {
287 if(!last_action_time)
288 return;
289
290 usec_t expected = __atomic_load_n(&timing_steps[TIMING_STEP_INTERNAL].time, __ATOMIC_RELAXED);
291 if(last_action_time - expected < 10 * USEC_PER_SEC) {
292 last_action_time = 0;
293 return;
294 }
295
296 if(!__atomic_compare_exchange_n(&timing_steps[TIMING_STEP_INTERNAL].time, &expected, last_action_time, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
297 last_action_time = 0;
298 return;
299 }
300
301 struct timing_steps timings3[TIMING_STEP_MAX + 1];
302 memcpy(timings3, timing_steps, sizeof(timings3));
303
304 size_t total_reqs = 0;
305 usec_t total_usec = 0;
306 for(size_t t = 1; t < TIMING_STEP_MAX ; t++) {
307 total_usec += timings3[t].time - timings2[t].time;
308 total_reqs += timings3[t].count - timings2[t].count;
309 }
310
311 BUFFER *wb = buffer_create(1024, NULL);
312
313 for(size_t t = 1; t < TIMING_STEP_MAX ; t++) {
314 size_t requests = timings3[t].count - timings2[t].count;
315 if(!requests) continue;
316
317 buffer_sprintf(wb, "TIMINGS REPORT: [%3zu. %-20s]: # %10zu, t %11.2f ms (%6.2f %%), avg %6.2f usec/run\n",
318 t,
319 timing_steps[t].name ? timing_steps[t].name : "x",
320 requests,
321 (double) (timings3[t].time - timings2[t].time) / (double)USEC_PER_MS,
322 (double) (timings3[t].time - timings2[t].time) * 100.0 / (double) total_usec,
323 (double) (timings3[t].time - timings2[t].time) / (double)requests
324 );
325 }
326
327 netdata_log_info("TIMINGS REPORT:\n%sTIMINGS REPORT: total # %10zu, t %11.2f ms",
328 buffer_tostring(wb), total_reqs, (double)total_usec / USEC_PER_MS);
329
330 memcpy(timings2, timings3, sizeof(timings2));
331
332 last_action_time = 0;
333 buffer_free(wb);
334 }
335 }
336 }
337
338 int hash256_string(const unsigned char *string, size_t size, char *hash) {
339 EVP_MD_CTX *ctx;
340 ctx = EVP_MD_CTX_create();
341
342 if (!ctx)
343 return 0;
344
345 if (!EVP_DigestInit(ctx, EVP_sha256())) {
346 EVP_MD_CTX_destroy(ctx);
347 return 0;
348 }
349
350 if (!EVP_DigestUpdate(ctx, string, size)) {
351 EVP_MD_CTX_destroy(ctx);
352 return 0;
353 }
354
355 if (!EVP_DigestFinal(ctx, (unsigned char *)hash, NULL)) {
356 EVP_MD_CTX_destroy(ctx);
357 return 0;
358 }
359 EVP_MD_CTX_destroy(ctx);
360 return 1;
361 }
362
363
364 bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now) {
365 if(!now) now = now_realtime_sec();
366
367 int absolute_period_requested = -1;
368 time_t before_requested = *before;
369 time_t after_requested = *after;
370
371 // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
372 if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
373 // if the user asked for a positive relative time,
374 // flip it to a negative
375 if(before_requested > 0)
376 before_requested = -before_requested;
377
378 before_requested = now + before_requested;
379 absolute_period_requested = 0;
380 }
381
382 // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
383 if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
384 if(after_requested > 0)
385 after_requested = -after_requested;
386
387 // if the user didn't give an after, use the number of points
388 // to give a sane default
389 if(after_requested == 0)
390 after_requested = -600;
391
392 // since the query engine now returns inclusive timestamps
393 // it is awkward to return 6 points when after=-5 is given
394 // so for relative queries we add 1 second, to give
395 // more predictable results to users.
396 after_requested = before_requested + after_requested + 1;
397 absolute_period_requested = 0;
398 }
399
400 if(absolute_period_requested == -1)
401 absolute_period_requested = 1;
402
403 // check if the parameters are flipped
404 if(after_requested > before_requested) {
405 long long t = before_requested;
406 before_requested = after_requested;
407 after_requested = t;
408 }
409
410 // if the query requests future data
411 // shift the query back to be in the present time
412 // (this may also happen because of the rules above)
413 if(before_requested > now) {
414 time_t delta = before_requested - now;
415 before_requested -= delta;
416 after_requested -= delta;
417 }
418
419 *before = before_requested;
420 *after = after_requested;
421
422 return (absolute_period_requested != 1);
423 }
424
425 // Returns 1 if an absolute period was requested or 0 if it was a relative period
426 bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest) {
427 time_t now = now_realtime_sec() - 1;
428
429 if(now_ptr)
430 *now_ptr = now;
431
432 time_t before_requested = *before;
433 time_t after_requested = *after;
434
435 int absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested, now);
436
437 time_t absolute_minimum_time = now - (10 * 365 * 86400);
438 time_t absolute_maximum_time = now + (1 * 365 * 86400);
439
440 if (after_requested < absolute_minimum_time && !unittest)
441 after_requested = absolute_minimum_time;
442
443 if (after_requested > absolute_maximum_time && !unittest)
444 after_requested = absolute_maximum_time;
445
446 if (before_requested < absolute_minimum_time && !unittest)
447 before_requested = absolute_minimum_time;
448
449 if (before_requested > absolute_maximum_time && !unittest)
450 before_requested = absolute_maximum_time;
451
452 *before = before_requested;
453 *after = after_requested;
454
455 return (absolute_period_requested != 1);
456 }
457
458
459 #if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
460 static inline EVP_ENCODE_CTX *EVP_ENCODE_CTX_new(void)
461 {
462 EVP_ENCODE_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
463
464 if (ctx != NULL) {
465 memset(ctx, 0, sizeof(*ctx));
466 }
467 return ctx;
468 }
469
470 static void EVP_ENCODE_CTX_free(EVP_ENCODE_CTX *ctx)
471 {
472 OPENSSL_free(ctx);
473 }
474 #endif
475
476 int netdata_base64_decode(unsigned char *out, const unsigned char *in, const int in_len)
477 {
478 int outl;
479 unsigned char remaining_data[256];
480
481 EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new();
482 EVP_DecodeInit(ctx);
483 EVP_DecodeUpdate(ctx, out, &outl, in, in_len);
484 int remainder = 0;
485 EVP_DecodeFinal(ctx, remaining_data, &remainder);
486 EVP_ENCODE_CTX_free(ctx);
487 if (remainder)
488 return -1;
489
490 return outl;
491 }
492
493 int netdata_base64_encode(unsigned char *encoded, const unsigned char *input, size_t input_size)
494 {
495 return EVP_EncodeBlock(encoded, input, input_size);
496 }
497
498 // Keep internal implementation
499 // int netdata_base64_decode_internal(const char *encoded, char *decoded, size_t decoded_size) {
500 // static const unsigned char base64_table[256] = {
501 // ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7,
502 // ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
503 // ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
504 // ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31,
505 // ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39,
506 // ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47,
507 // ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55,
508 // ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63,
509 // [0 ... '+' - 1] = 255,
510 // ['+' + 1 ... '/' - 1] = 255,
511 // ['9' + 1 ... 'A' - 1] = 255,
512 // ['Z' + 1 ... 'a' - 1] = 255,
513 // ['z' + 1 ... 255] = 255
514 // };
515 //
516 // size_t count = 0;
517 // unsigned int tmp = 0;
518 // int i, bit;
519 //
520 // if (decoded_size < 1)
521 // return 0; // Buffer size must be at least 1 for null termination
522 //
523 // for (i = 0, bit = 0; encoded[i]; i++) {
524 // unsigned char value = base64_table[(unsigned char)encoded[i]];
525 // if (value > 63)
526 // return -1; // Invalid character in input
527 //
528 // tmp = tmp << 6 | value;
529 // if (++bit == 4) {
530 // if (count + 3 >= decoded_size) break; // Stop decoding if buffer is full
531 // decoded[count++] = (tmp >> 16) & 0xFF;
532 // decoded[count++] = (tmp >> 8) & 0xFF;
533 // decoded[count++] = tmp & 0xFF;
534 // tmp = 0;
535 // bit = 0;
536 // }
537 // }
538 //
539 // if (bit > 0 && count + 1 < decoded_size) {
540 // tmp <<= 6 * (4 - bit);
541 // if (bit > 2 && count + 1 < decoded_size) decoded[count++] = (tmp >> 16) & 0xFF;
542 // if (bit > 3 && count + 1 < decoded_size) decoded[count++] = (tmp >> 8) & 0xFF;
543 // }
544 //
545 // decoded[count] = '\0'; // Null terminate the output string
546 // return count;
547 // }