@cryptotaxi247 / netdata-1 / commits / 0d61c11b5

use gperf for the pluginsd/streaming parser hashtable (#15251)

* use gperf for the pluginsd parser * simplify pluginsd_parser by removing void pointers to user * pluginsd_split_words() with inlined pluginsd_space() * quoted_string_splitter() now uses a map instead of a function for determining spaces * add stress test for pluginsd parser * optimized BITMAP256 * optimized rrdpush receiver reception * optimized rrdpush sender compression * renames and cleanup * remove wrong negation * unify handshake and disconnection reasons * use parser_find_keyword * register job names only for the current repertoire

Costa Tsaousis committed Jun 26, 2023 at 14:00 UTC 0d61c11b5f4772a4762ede1d8204290b94bb08e7
29 files changed +1321 -1450
CMakeLists.txt
-2
@@ -491,8 +491,6 @@ set(LIBNETDATA_FILES
491 libnetdata/string/utf8.h
492 libnetdata/worker_utilization/worker_utilization.c
493 libnetdata/worker_utilization/worker_utilization.h
494 - libnetdata/parser/parser.h
495 - libnetdata/parser/parser.c
494 libnetdata/http/http_defs.h
495 )
496
Makefile.am
+1 -2
@@ -168,8 +168,6 @@ LIBNETDATA_FILES = \
168 libnetdata/log/log.h \
169 libnetdata/onewayalloc/onewayalloc.c \
170 libnetdata/onewayalloc/onewayalloc.h \
171 - libnetdata/parser/parser.c \
172 - libnetdata/parser/parser.h \
171 libnetdata/popen/popen.c \
172 libnetdata/popen/popen.h \
173 libnetdata/procfile/procfile.c \
@@ -425,6 +423,7 @@ PLUGINSD_PLUGIN_FILES = \
423 collectors/plugins.d/plugins_d.h \
424 collectors/plugins.d/pluginsd_parser.c \
425 collectors/plugins.d/pluginsd_parser.h \
426 + collectors/plugins.d/gperf-hashtable.h \
427 $(NULL)
428
429 RRD_PLUGIN_FILES = \
collectors/apps.plugin/apps_plugin.c
+2 -2
@@ -4372,7 +4372,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
4372 struct pid_stat *p;
4373
4374 char *words[PLUGINSD_MAX_WORDS] = { NULL };
4375 - size_t num_words = pluginsd_split_words(function, words, PLUGINSD_MAX_WORDS);
4375 + size_t num_words = quoted_strings_splitter_pluginsd(function, words, PLUGINSD_MAX_WORDS);
4376
4377 struct target *category = NULL, *user = NULL, *group = NULL;
4378 const char *process_name = NULL;
@@ -5219,7 +5219,7 @@ void *reader_main(void *arg __maybe_unused) {
5219 while(!apps_plugin_exit && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) {
5220
5221 char *words[PLUGINSD_MAX_WORDS] = { NULL };
5222 - size_t num_words = pluginsd_split_words(buffer, words, PLUGINSD_MAX_WORDS);
5222 + size_t num_words = quoted_strings_splitter_pluginsd(buffer, words, PLUGINSD_MAX_WORDS);
5223
5224 const char *keyword = get_word(words, num_words, 0);
5225
collectors/plugins.d/Makefile.am
+1
@@ -7,5 +7,6 @@ SUBDIRS = \
7 $(NULL)
8
9 dist_noinst_DATA = \
10 + gperf-config.txt \
11 README.md \
12 $(NULL)
collectors/plugins.d/gperf-config.txt new
+43
@@ -0,0 +1,43 @@
1 +PARSER_KEYWORD;
2 +%%
3 +#
4 +# Plugins Only Keywords
5 +#
6 +FLUSH, pluginsd_flush, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1
7 +DISABLE, pluginsd_disable, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2
8 +EXIT, pluginsd_exit, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3
9 +HOST, pluginsd_host, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 4
10 +HOST_DEFINE, pluginsd_host_define, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 5
11 +HOST_DEFINE_END, pluginsd_host_define_end, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 6
12 +HOST_LABEL, pluginsd_host_labels, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 7
13 +#
14 +# Common keywords
15 +#
16 +BEGIN, pluginsd_begin, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8
17 +CHART, pluginsd_chart, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 9
18 +CLABEL, pluginsd_clabel, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 10
19 +CLABEL_COMMIT, pluginsd_clabel_commit, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 11
20 +DIMENSION, pluginsd_dimension, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 12
21 +END, pluginsd_end, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13
22 +FUNCTION, pluginsd_function, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 14
23 +FUNCTION_RESULT_BEGIN, pluginsd_function_result_begin, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15
24 +LABEL, pluginsd_label, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16
25 +OVERWRITE, pluginsd_overwrite, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 17
26 +SET, pluginsd_set, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18
27 +VARIABLE, pluginsd_variable, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19
28 +#
29 +# Streaming only keywords
30 +#
31 +CLAIMED_ID, streaming_claimed_id, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20
32 +BEGIN2, pluginsd_begin_v2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21
33 +SET2, pluginsd_set_v2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22
34 +END2, pluginsd_end_v2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23
35 +#
36 +# Streaming Replication keywords
37 +#
38 +CHART_DEFINITION_END, pluginsd_chart_definition_end, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24
39 +RBEGIN, pluginsd_replay_begin, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25
40 +RDSTATE, pluginsd_replay_rrddim_collection_state, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26
41 +REND, pluginsd_replay_end, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27
42 +RSET, pluginsd_replay_set, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28
43 +RSSTATE, pluginsd_replay_rrdset_collection_state, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29
collectors/plugins.d/gperf-hashtable.h new
+164
@@ -0,0 +1,164 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +/* ANSI-C code produced by gperf version 3.1 */
3 +/* Command-line: gperf --multiple-iterations=1000 --hash-function-name=gperf_keyword_hash_function --lookup-function-name=gperf_lookup_keyword --word-array-name=gperf_keywords --constants-prefix=GPERF_PARSER_ --struct-type --slot-name=keyword --global-table --null-strings --omit-struct-type --output-file=gperf-hashtable.h gperf-config.txt */
4 +/* Computed positions: -k'1-2' */
5 +
6 +#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
7 + && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
8 + && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
9 + && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
10 + && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
11 + && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
12 + && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
13 + && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
14 + && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
15 + && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
16 + && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
17 + && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
18 + && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
19 + && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
20 + && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
21 + && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
22 + && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
23 + && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
24 + && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
25 + && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
26 + && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
27 + && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
28 + && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
29 +/* The character set is not based on ISO-646. */
30 +#error "gperf generated tables don't work with this execution character set. Please report a bug to <bug-gperf@gnu.org>."
31 +#endif
32 +
33 +
34 +#define GPERF_PARSER_TOTAL_KEYWORDS 29
35 +#define GPERF_PARSER_MIN_WORD_LENGTH 3
36 +#define GPERF_PARSER_MAX_WORD_LENGTH 21
37 +#define GPERF_PARSER_MIN_HASH_VALUE 4
38 +#define GPERF_PARSER_MAX_HASH_VALUE 36
39 +/* maximum key range = 33, duplicates = 0 */
40 +
41 +#ifdef __GNUC__
42 +__inline
43 +#else
44 +#ifdef __cplusplus
45 +inline
46 +#endif
47 +#endif
48 +static unsigned int
49 +gperf_keyword_hash_function (register const char *str, register size_t len)
50 +{
51 + static unsigned char asso_values[] =
52 + {
53 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
54 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
55 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
56 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
57 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
58 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
59 + 37, 37, 37, 37, 37, 15, 10, 1, 1, 9,
60 + 4, 37, 0, 20, 37, 37, 9, 37, 14, 0,
61 + 37, 37, 1, 0, 37, 7, 13, 37, 18, 37,
62 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
63 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
64 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
65 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
66 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
67 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
68 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
69 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
70 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
71 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
72 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
73 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
74 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
75 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
76 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
77 + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
78 + 37, 37, 37, 37, 37, 37
79 + };
80 + return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];
81 +}
82 +
83 +static PARSER_KEYWORD gperf_keywords[] =
84 + {
85 + {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0},
86 +#line 9 "gperf-config.txt"
87 + {"HOST", pluginsd_host, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 4},
88 +#line 42 "gperf-config.txt"
89 + {"RSET", pluginsd_replay_set, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28},
90 +#line 17 "gperf-config.txt"
91 + {"CHART", pluginsd_chart, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 9},
92 + {(char*)0},
93 +#line 43 "gperf-config.txt"
94 + {"RSSTATE", pluginsd_replay_rrdset_collection_state, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29},
95 +#line 40 "gperf-config.txt"
96 + {"RDSTATE", pluginsd_replay_rrddim_collection_state, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26},
97 +#line 12 "gperf-config.txt"
98 + {"HOST_LABEL", pluginsd_host_labels, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 7},
99 +#line 10 "gperf-config.txt"
100 + {"HOST_DEFINE", pluginsd_host_define, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 5},
101 +#line 26 "gperf-config.txt"
102 + {"SET", pluginsd_set, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18},
103 +#line 33 "gperf-config.txt"
104 + {"SET2", pluginsd_set_v2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22},
105 +#line 41 "gperf-config.txt"
106 + {"REND", pluginsd_replay_end, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27},
107 +#line 11 "gperf-config.txt"
108 + {"HOST_DEFINE_END", pluginsd_host_define_end, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 6},
109 +#line 18 "gperf-config.txt"
110 + {"CLABEL", pluginsd_clabel, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 10},
111 +#line 39 "gperf-config.txt"
112 + {"RBEGIN", pluginsd_replay_begin, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25},
113 +#line 6 "gperf-config.txt"
114 + {"FLUSH", pluginsd_flush, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
115 +#line 22 "gperf-config.txt"
116 + {"FUNCTION", pluginsd_function, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 14},
117 +#line 31 "gperf-config.txt"
118 + {"CLAIMED_ID", streaming_claimed_id, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20},
119 +#line 38 "gperf-config.txt"
120 + {"CHART_DEFINITION_END", pluginsd_chart_definition_end, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24},
121 +#line 25 "gperf-config.txt"
122 + {"OVERWRITE", pluginsd_overwrite, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 17},
123 +#line 19 "gperf-config.txt"
124 + {"CLABEL_COMMIT", pluginsd_clabel_commit, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 11},
125 +#line 16 "gperf-config.txt"
126 + {"BEGIN", pluginsd_begin, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8},
127 +#line 32 "gperf-config.txt"
128 + {"BEGIN2", pluginsd_begin_v2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21},
129 +#line 21 "gperf-config.txt"
130 + {"END", pluginsd_end, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13},
131 +#line 34 "gperf-config.txt"
132 + {"END2", pluginsd_end_v2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23},
133 +#line 7 "gperf-config.txt"
134 + {"DISABLE", pluginsd_disable, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2},
135 +#line 24 "gperf-config.txt"
136 + {"LABEL", pluginsd_label, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16},
137 +#line 20 "gperf-config.txt"
138 + {"DIMENSION", pluginsd_dimension, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 12},
139 +#line 8 "gperf-config.txt"
140 + {"EXIT", pluginsd_exit, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3},
141 +#line 23 "gperf-config.txt"
142 + {"FUNCTION_RESULT_BEGIN", pluginsd_function_result_begin, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15},
143 + {(char*)0}, {(char*)0}, {(char*)0},
144 +#line 27 "gperf-config.txt"
145 + {"VARIABLE", pluginsd_variable, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19}
146 + };
147 +
148 +PARSER_KEYWORD *
149 +gperf_lookup_keyword (register const char *str, register size_t len)
150 +{
151 + if (len <= GPERF_PARSER_MAX_WORD_LENGTH && len >= GPERF_PARSER_MIN_WORD_LENGTH)
152 + {
153 + register unsigned int key = gperf_keyword_hash_function (str, len);
154 +
155 + if (key <= GPERF_PARSER_MAX_HASH_VALUE)
156 + {
157 + register const char *s = gperf_keywords[key].keyword;
158 +
159 + if (s && *str == *s && !strcmp (str + 1, s + 1))
160 + return &gperf_keywords[key];
161 + }
162 + }
163 + return 0;
164 +}
collectors/plugins.d/plugins_d.c
+1 -1
@@ -18,7 +18,7 @@ inline size_t pluginsd_initialize_plugin_directories()
18 }
19
20 // Parse it and store it to plugin directories
21 - return quoted_strings_splitter(plugins_dir_list, plugin_directories, PLUGINSD_MAX_DIRECTORIES, config_isspace);
21 + return quoted_strings_splitter_config(plugins_dir_list, plugin_directories, PLUGINSD_MAX_DIRECTORIES);
22 }
23
24 static inline void plugin_set_disabled(struct plugind *cd) {
collectors/plugins.d/pluginsd_parser.c
+471 -450
@@ -68,8 +68,8 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
68 return -4;
69 }
70
71 -static inline RRDHOST *pluginsd_require_host_from_parent(void *user, const char *cmd) {
72 - RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
71 +static inline RRDHOST *pluginsd_require_host_from_parent(PARSER *parser, const char *cmd) {
72 + RRDHOST *host = parser->user.host;
73
74 if(unlikely(!host))
75 error("PLUGINSD: command %s requires a host, but is not set.", cmd);
@@ -77,8 +77,8 @@ static inline RRDHOST *pluginsd_require_host_from_parent(void *user, const char
77 return host;
78 }
79
80 -static inline RRDSET *pluginsd_require_chart_from_parent(void *user, const char *cmd, const char *parent_cmd) {
81 - RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
80 +static inline RRDSET *pluginsd_require_chart_from_parent(PARSER *parser, const char *cmd, const char *parent_cmd) {
81 + RRDSET *st = parser->user.st;
82
83 if(unlikely(!st))
84 error("PLUGINSD: command %s requires a chart defined via command %s, but is not set.", cmd, parent_cmd);
@@ -86,23 +86,21 @@ static inline RRDSET *pluginsd_require_chart_from_parent(void *user, const char
86 return st;
87 }
88
89 -static inline RRDSET *pluginsd_get_chart_from_parent(void *user) {
90 - return ((PARSER_USER_OBJECT *) user)->st;
89 +static inline RRDSET *pluginsd_get_chart_from_parent(PARSER *parser) {
90 + return parser->user.st;
91 }
92
93 -static inline void pluginsd_lock_rrdset_data_collection(void *user) {
94 - PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
95 - if(u->st && !u->v2.locked_data_collection) {
96 - spinlock_lock(&u->st->data_collection_lock);
97 - u->v2.locked_data_collection = true;
93 +static inline void pluginsd_lock_rrdset_data_collection(PARSER *parser) {
94 + if(parser->user.st && !parser->user.v2.locked_data_collection) {
95 + spinlock_lock(&parser->user.st->data_collection_lock);
96 + parser->user.v2.locked_data_collection = true;
97 }
98 }
99
101 -static inline bool pluginsd_unlock_rrdset_data_collection(void *user) {
102 - PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
103 - if(u->st && u->v2.locked_data_collection) {
104 - spinlock_unlock(&u->st->data_collection_lock);
105 - u->v2.locked_data_collection = false;
100 +static inline bool pluginsd_unlock_rrdset_data_collection(PARSER *parser) {
101 + if(parser->user.st && parser->user.v2.locked_data_collection) {
102 + spinlock_unlock(&parser->user.st->data_collection_lock);
103 + parser->user.v2.locked_data_collection = false;
104 return true;
105 }
106
@@ -123,29 +121,25 @@ void pluginsd_rrdset_cleanup(RRDSET *st) {
121 st->pluginsd.pos = 0;
122 }
123
126 -static inline void pluginsd_unlock_previous_chart(void *user, const char *keyword, bool stale) {
127 - PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
128 -
129 - if(unlikely(pluginsd_unlock_rrdset_data_collection(user))) {
124 +static inline void pluginsd_unlock_previous_chart(PARSER *parser, const char *keyword, bool stale) {
125 + if(unlikely(pluginsd_unlock_rrdset_data_collection(parser))) {
126 if(stale)
127 error("PLUGINSD: 'host:%s/chart:%s/' stale data collection lock found during %s; it has been unlocked",
132 - rrdhost_hostname(u->st->rrdhost), rrdset_id(u->st), keyword);
128 + rrdhost_hostname(parser->user.st->rrdhost), rrdset_id(parser->user.st), keyword);
129 }
130
135 - if(unlikely(u->v2.ml_locked)) {
136 - ml_chart_update_end(u->st);
137 - u->v2.ml_locked = false;
131 + if(unlikely(parser->user.v2.ml_locked)) {
132 + ml_chart_update_end(parser->user.st);
133 + parser->user.v2.ml_locked = false;
134
135 if(stale)
136 error("PLUGINSD: 'host:%s/chart:%s/' stale ML lock found during %s, it has been unlocked",
141 - rrdhost_hostname(u->st->rrdhost), rrdset_id(u->st), keyword);
137 + rrdhost_hostname(parser->user.st->rrdhost), rrdset_id(parser->user.st), keyword);
138 }
139 }
140
145 -static inline void pluginsd_set_chart_from_parent(void *user, RRDSET *st, const char *keyword) {
146 - PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
147 -
148 - pluginsd_unlock_previous_chart(user, keyword, true);
141 +static inline void pluginsd_set_chart_from_parent(PARSER *parser, RRDSET *st, const char *keyword) {
142 + pluginsd_unlock_previous_chart(parser, keyword, true);
143
144 if(st) {
145 size_t dims = dictionary_entries(st->rrddim_root_index);
@@ -160,7 +154,7 @@ static inline void pluginsd_set_chart_from_parent(void *user, RRDSET *st, const
154 st->pluginsd.pos = 0;
155 }
156
163 - u->st = st;
157 + parser->user.st = st;
158 }
159
160 static inline RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, const char *cmd) {
@@ -214,8 +208,8 @@ static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, cons
208 return st;
209 }
210
217 -static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(void *user, const char *keyword, const char *msg) {
218 - ((PARSER_USER_OBJECT *) user)->enabled = 0;
211 +static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg) {
212 + parser->user.enabled = 0;
213
214 if(keyword && msg) {
215 error_limit_static_global_var(erl, 1, 0);
@@ -225,19 +219,18 @@ static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(void *user, const char *keyword,
219 return PARSER_RC_ERROR;
220 }
221
228 -PARSER_RC pluginsd_set(char **words, size_t num_words, void *user)
229 -{
222 +static inline PARSER_RC pluginsd_set(char **words, size_t num_words, PARSER *parser) {
223 char *dimension = get_word(words, num_words, 1);
224 char *value = get_word(words, num_words, 2);
225
233 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_SET);
234 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
226 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_SET);
227 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
228
236 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_SET, PLUGINSD_KEYWORD_CHART);
237 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
229 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_SET, PLUGINSD_KEYWORD_CHART);
230 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
231
232 RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_SET);
240 - if(!rd) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
233 + if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
234
235 if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
236 debug(D_PLUGINSD, "PLUGINSD: 'host:%s/chart:%s/dim:%s' SET is setting value to '%s'",
@@ -249,18 +242,17 @@ PARSER_RC pluginsd_set(char **words, size_t num_words, void *user)
242 return PARSER_RC_OK;
243 }
244
252 -PARSER_RC pluginsd_begin(char **words, size_t num_words, void *user)
253 -{
245 +static inline PARSER_RC pluginsd_begin(char **words, size_t num_words, PARSER *parser) {
246 char *id = get_word(words, num_words, 1);
247 char *microseconds_txt = get_word(words, num_words, 2);
248
257 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_BEGIN);
258 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
249 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_BEGIN);
250 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
251
252 RRDSET *st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_BEGIN);
261 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
253 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
254
263 - pluginsd_set_chart_from_parent(user, st, PLUGINSD_KEYWORD_BEGIN);
255 + pluginsd_set_chart_from_parent(parser, st, PLUGINSD_KEYWORD_BEGIN);
256
257 usec_t microseconds = 0;
258 if (microseconds_txt && *microseconds_txt) {
@@ -285,7 +277,7 @@ PARSER_RC pluginsd_begin(char **words, size_t num_words, void *user)
277
278 if (likely(st->counter_done)) {
279 if (likely(microseconds)) {
288 - if (((PARSER_USER_OBJECT *)user)->trust_durations)
280 + if (parser->user.trust_durations)
281 rrdset_next_usec_unfiltered(st, microseconds);
282 else
283 rrdset_next_usec(st, microseconds);
@@ -296,22 +288,21 @@ PARSER_RC pluginsd_begin(char **words, size_t num_words, void *user)
288 return PARSER_RC_OK;
289 }
290
299 -PARSER_RC pluginsd_end(char **words, size_t num_words, void *user)
300 -{
291 +static inline PARSER_RC pluginsd_end(char **words, size_t num_words, PARSER *parser) {
292 UNUSED(words);
293 UNUSED(num_words);
294
304 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_END);
305 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
295 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_END);
296 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
297
307 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_END, PLUGINSD_KEYWORD_BEGIN);
308 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
298 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_END, PLUGINSD_KEYWORD_BEGIN);
299 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
300
301 if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
302 debug(D_PLUGINSD, "requested an END on chart '%s'", rrdset_id(st));
303
313 - pluginsd_set_chart_from_parent(user, NULL, PLUGINSD_KEYWORD_END);
314 - ((PARSER_USER_OBJECT *) user)->data_collections_count++;
304 + pluginsd_set_chart_from_parent(parser, NULL, PLUGINSD_KEYWORD_END);
305 + parser->user.data_collections_count++;
306
307 struct timeval now;
308 now_realtime_timeval(&now);
@@ -320,15 +311,13 @@ PARSER_RC pluginsd_end(char **words, size_t num_words, void *user)
311 return PARSER_RC_OK;
312 }
313
323 -static void pluginsd_host_define_cleanup(void *user) {
324 - PARSER_USER_OBJECT *u = user;
325 -
326 - string_freez(u->host_define.hostname);
327 - dictionary_destroy(u->host_define.rrdlabels);
314 +static void pluginsd_host_define_cleanup(PARSER *parser) {
315 + string_freez(parser->user.host_define.hostname);
316 + dictionary_destroy(parser->user.host_define.rrdlabels);
317
329 - u->host_define.hostname = NULL;
330 - u->host_define.rrdlabels = NULL;
331 - u->host_define.parsing_host = false;
318 + parser->user.host_define.hostname = NULL;
319 + parser->user.host_define.rrdlabels = NULL;
320 + parser->user.host_define.parsing_host = false;
321 }
322
323 static inline bool pluginsd_validate_machine_guid(const char *guid, uuid_t *uuid, char *output) {
@@ -340,61 +329,56 @@ static inline bool pluginsd_validate_machine_guid(const char *guid, uuid_t *uuid
329 return true;
330 }
331
343 -static PARSER_RC pluginsd_host_define(char **words, size_t num_words, void *user) {
344 - PARSER_USER_OBJECT *u = user;
345 -
332 +static inline PARSER_RC pluginsd_host_define(char **words, size_t num_words, PARSER *parser) {
333 char *guid = get_word(words, num_words, 1);
334 char *hostname = get_word(words, num_words, 2);
335
336 if(unlikely(!guid || !*guid || !hostname || !*hostname))
350 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_HOST_DEFINE, "missing parameters");
337 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE, "missing parameters");
338
352 - if(unlikely(u->host_define.parsing_host))
353 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_HOST_DEFINE,
339 + if(unlikely(parser->user.host_define.parsing_host))
340 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE,
341 "another host definition is already open - did you send " PLUGINSD_KEYWORD_HOST_DEFINE_END "?");
342
356 - if(!pluginsd_validate_machine_guid(guid, &u->host_define.machine_guid, u->host_define.machine_guid_str))
357 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_HOST_DEFINE, "cannot parse MACHINE_GUID - is it a valid UUID?");
343 + if(!pluginsd_validate_machine_guid(guid, &parser->user.host_define.machine_guid, parser->user.host_define.machine_guid_str))
344 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE, "cannot parse MACHINE_GUID - is it a valid UUID?");
345
359 - u->host_define.hostname = string_strdupz(hostname);
360 - u->host_define.rrdlabels = rrdlabels_create();
361 - u->host_define.parsing_host = true;
346 + parser->user.host_define.hostname = string_strdupz(hostname);
347 + parser->user.host_define.rrdlabels = rrdlabels_create();
348 + parser->user.host_define.parsing_host = true;
349
350 return PARSER_RC_OK;
351 }
352
366 -static inline PARSER_RC pluginsd_host_dictionary(char **words, size_t num_words, void *user, DICTIONARY *dict, const char *keyword) {
367 - PARSER_USER_OBJECT *u = user;
368 -
353 +static inline PARSER_RC pluginsd_host_dictionary(char **words, size_t num_words, PARSER *parser, DICTIONARY *dict, const char *keyword) {
354 char *name = get_word(words, num_words, 1);
355 char *value = get_word(words, num_words, 2);
356
357 if(!name || !*name || !value)
373 - return PLUGINSD_DISABLE_PLUGIN(user, keyword, "missing parameters");
358 + return PLUGINSD_DISABLE_PLUGIN(parser, keyword, "missing parameters");
359
375 - if(!u->host_define.parsing_host || !dict)
376 - return PLUGINSD_DISABLE_PLUGIN(user, keyword, "host is not defined, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this");
360 + if(!parser->user.host_define.parsing_host || !dict)
361 + return PLUGINSD_DISABLE_PLUGIN(parser, keyword, "host is not defined, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this");
362
363 rrdlabels_add(dict, name, value, RRDLABEL_SRC_CONFIG);
364
365 return PARSER_RC_OK;
366 }
367
383 -static PARSER_RC pluginsd_host_labels(char **words, size_t num_words, void *user) {
384 - PARSER_USER_OBJECT *u = user;
385 - return pluginsd_host_dictionary(words, num_words, user, u->host_define.rrdlabels, PLUGINSD_KEYWORD_HOST_LABEL);
368 +static inline PARSER_RC pluginsd_host_labels(char **words, size_t num_words, PARSER *parser) {
369 + return pluginsd_host_dictionary(words, num_words, parser,
370 + parser->user.host_define.rrdlabels,
371 + PLUGINSD_KEYWORD_HOST_LABEL);
372 }
373
388 -static PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t num_words __maybe_unused, void *user) {
389 - PARSER_USER_OBJECT *u = user;
390 -
391 - if(!u->host_define.parsing_host)
392 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_HOST_DEFINE_END, "missing initialization, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this");
374 +static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
375 + if(!parser->user.host_define.parsing_host)
376 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE_END, "missing initialization, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this");
377
378 RRDHOST *host = rrdhost_find_or_create(
395 - string2str(u->host_define.hostname),
396 - string2str(u->host_define.hostname),
397 - u->host_define.machine_guid_str,
379 + string2str(parser->user.host_define.hostname),
380 + string2str(parser->user.host_define.hostname),
381 + parser->user.host_define.machine_guid_str,
382 "Netdata Virtual Host 1.0",
383 netdata_configured_timezone,
384 netdata_configured_abbrev_timezone,
@@ -413,24 +397,24 @@ static PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t nu
397 default_rrdpush_enable_replication,
398 default_rrdpush_seconds_to_replicate,
399 default_rrdpush_replication_step,
416 - rrdhost_labels_to_system_info(u->host_define.rrdlabels),
400 + rrdhost_labels_to_system_info(parser->user.host_define.rrdlabels),
401 false
402 );
403
404 rrdhost_option_set(host, RRDHOST_OPTION_VIRTUAL_HOST);
405
406 if(host->rrdlabels) {
423 - rrdlabels_migrate_to_these(host->rrdlabels, u->host_define.rrdlabels);
407 + rrdlabels_migrate_to_these(host->rrdlabels, parser->user.host_define.rrdlabels);
408 }
409 else {
426 - host->rrdlabels = u->host_define.rrdlabels;
427 - u->host_define.rrdlabels = NULL;
410 + host->rrdlabels = parser->user.host_define.rrdlabels;
411 + parser->user.host_define.rrdlabels = NULL;
412 }
413
430 - pluginsd_host_define_cleanup(user);
414 + pluginsd_host_define_cleanup(parser);
415
432 - u->host = host;
433 - pluginsd_set_chart_from_parent(user, NULL, PLUGINSD_KEYWORD_HOST_DEFINE_END);
416 + parser->user.host = host;
417 + pluginsd_set_chart_from_parent(parser, NULL, PLUGINSD_KEYWORD_HOST_DEFINE_END);
418
419 rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN);
420 rrdcontext_host_child_connected(host);
@@ -439,34 +423,31 @@ static PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t nu
423 return PARSER_RC_OK;
424 }
425
442 -static PARSER_RC pluginsd_host(char **words, size_t num_words, void *user) {
443 - PARSER_USER_OBJECT *u = user;
444 -
426 +static inline PARSER_RC pluginsd_host(char **words, size_t num_words, PARSER *parser) {
427 char *guid = get_word(words, num_words, 1);
428
429 if(!guid || !*guid || strcmp(guid, "localhost") == 0) {
448 - u->host = localhost;
430 + parser->user.host = localhost;
431 return PARSER_RC_OK;
432 }
433
434 uuid_t uuid;
435 char uuid_str[UUID_STR_LEN];
436 if(!pluginsd_validate_machine_guid(guid, &uuid, uuid_str))
455 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_HOST, "cannot parse MACHINE_GUID - is it a valid UUID?");
437 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST, "cannot parse MACHINE_GUID - is it a valid UUID?");
438
439 RRDHOST *host = rrdhost_find_by_guid(uuid_str);
440 if(unlikely(!host))
459 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_HOST, "cannot find a host with this machine guid - have you created it?");
441 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST, "cannot find a host with this machine guid - have you created it?");
442
461 - u->host = host;
443 + parser->user.host = host;
444
445 return PARSER_RC_OK;
446 }
447
466 -PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user)
467 -{
468 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_CHART);
469 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
448 +static inline PARSER_RC pluginsd_chart(char **words, size_t num_words, PARSER *parser) {
449 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_CHART);
450 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
451
452 char *type = get_word(words, num_words, 1);
453 char *name = get_word(words, num_words, 2);
@@ -490,7 +471,7 @@ PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user)
471
472 // make sure we have the required variables
473 if (unlikely((!type || !*type || !id || !*id)))
493 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_CHART, "missing parameters");
474 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_CHART, "missing parameters");
475
476 // parse the name, and make sure it does not include 'type.'
477 if (unlikely(name && *name)) {
@@ -511,11 +492,11 @@ PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user)
492 if (likely(priority_s && *priority_s))
493 priority = str2i(priority_s);
494
514 - int update_every = ((PARSER_USER_OBJECT *) user)->cd->update_every;
495 + int update_every = parser->user.cd->update_every;
496 if (likely(update_every_s && *update_every_s))
497 update_every = str2i(update_every_s);
498 if (unlikely(!update_every))
518 - update_every = ((PARSER_USER_OBJECT *) user)->cd->update_every;
499 + update_every = parser->user.cd->update_every;
500
501 RRDSET_TYPE chart_type = RRDSET_TYPE_LINE;
502 if (unlikely(chart))
@@ -542,7 +523,7 @@ PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user)
523
524 st = rrdset_create(
525 host, type, id, name, family, context, title, units,
545 - (plugin && *plugin) ? plugin : ((PARSER_USER_OBJECT *)user)->cd->filename,
526 + (plugin && *plugin) ? plugin : parser->user.cd->filename,
527 module, priority, update_every,
528 chart_type);
529
@@ -573,22 +554,21 @@ PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user)
554 rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
555 }
556 }
576 - pluginsd_set_chart_from_parent(user, st, PLUGINSD_KEYWORD_CHART);
557 + pluginsd_set_chart_from_parent(parser, st, PLUGINSD_KEYWORD_CHART);
558
559 return PARSER_RC_OK;
560 }
561
581 -PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *user)
582 -{
562 +static inline PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, PARSER *parser) {
563 const char *first_entry_txt = get_word(words, num_words, 1);
564 const char *last_entry_txt = get_word(words, num_words, 2);
565 const char *wall_clock_time_txt = get_word(words, num_words, 3);
566
587 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_CHART_DEFINITION_END);
588 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
567 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_CHART_DEFINITION_END);
568 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
569
590 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_CHART_DEFINITION_END, PLUGINSD_KEYWORD_CHART);
591 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
570 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_CHART_DEFINITION_END, PLUGINSD_KEYWORD_CHART);
571 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
572
573 time_t first_entry_child = (first_entry_txt && *first_entry_txt) ? (time_t)str2ul(first_entry_txt) : 0;
574 time_t last_entry_child = (last_entry_txt && *last_entry_txt) ? (time_t)str2ul(last_entry_txt) : 0;
@@ -607,7 +587,6 @@ PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *us
587 rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
588 rrdhost_receiver_replicating_charts_plus_one(st->rrdhost);
589
610 - PARSER *parser = ((PARSER_USER_OBJECT *)user)->parser;
590 ok = replicate_chart_request(send_to_plugin, parser, host, st,
591 first_entry_child, last_entry_child, child_wall_clock_time,
592 0, 0);
@@ -622,8 +601,7 @@ PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *us
601 return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
602 }
603
625 -PARSER_RC pluginsd_dimension(char **words, size_t num_words, void *user)
626 -{
604 +static inline PARSER_RC pluginsd_dimension(char **words, size_t num_words, PARSER *parser) {
605 char *id = get_word(words, num_words, 1);
606 char *name = get_word(words, num_words, 2);
607 char *algorithm = get_word(words, num_words, 3);
@@ -631,14 +609,14 @@ PARSER_RC pluginsd_dimension(char **words, size_t num_words, void *user)
609 char *divisor_s = get_word(words, num_words, 5);
610 char *options = get_word(words, num_words, 6);
611
634 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_DIMENSION);
635 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
612 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_DIMENSION);
613 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
614
637 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_DIMENSION, PLUGINSD_KEYWORD_CHART);
638 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
615 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_DIMENSION, PLUGINSD_KEYWORD_CHART);
616 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
617
618 if (unlikely(!id))
641 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_DIMENSION, "missing dimension id");
619 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DIMENSION, "missing dimension id");
620
621 long multiplier = 1;
622 if (multiplier_s && *multiplier_s) {
@@ -842,8 +820,7 @@ static int pluginsd_execute_function_callback(BUFFER *destination_wb, int timeou
820 return HTTP_RESP_OK;
821 }
822
845 -PARSER_RC pluginsd_function(char **words, size_t num_words, void *user)
846 -{
823 +static inline PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser) {
824 bool global = false;
825 size_t i = 1;
826 if(num_words >= 2 && strcmp(get_word(words, num_words, 1), "GLOBAL") == 0) {
@@ -855,10 +832,10 @@ PARSER_RC pluginsd_function(char **words, size_t num_words, void *user)
832 char *timeout_s = get_word(words, num_words, i++);
833 char *help = get_word(words, num_words, i++);
834
858 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_FUNCTION);
835 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_FUNCTION);
836 if(!host) return PARSER_RC_ERROR;
837
861 - RRDSET *st = (global)?NULL:pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_FUNCTION, PLUGINSD_KEYWORD_CHART);
838 + RRDSET *st = (global)?NULL:pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_FUNCTION, PLUGINSD_KEYWORD_CHART);
839 if(!st) global = true;
840
841 if (unlikely(!timeout_s || !name || !help || (!global && !st))) {
@@ -880,7 +857,6 @@ PARSER_RC pluginsd_function(char **words, size_t num_words, void *user)
857 timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
858 }
859
883 - PARSER *parser = ((PARSER_USER_OBJECT *) user)->parser;
860 rrd_collector_add_function(host, st, name, timeout, help, false, pluginsd_execute_function_callback, parser);
861
862 return PARSER_RC_OK;
@@ -893,8 +869,7 @@ static void pluginsd_function_result_end(struct parser *parser, void *action_dat
869 string_freez(key);
870 }
871
896 -PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, void *user)
897 -{
872 +static inline PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER *parser) {
873 char *key = get_word(words, num_words, 1);
874 char *status = get_word(words, num_words, 2);
875 char *format = get_word(words, num_words, 3);
@@ -915,8 +890,6 @@ PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, void *u
890
891 time_t expiration = (expires && *expires) ? str2l(expires) : 0;
892
918 - PARSER *parser = ((PARSER_USER_OBJECT *) user)->parser;
919 -
893 struct inflight_function *pf = NULL;
894
895 if(key && *key)
@@ -949,16 +922,15 @@ PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, void *u
922
923 // ----------------------------------------------------------------------------
924
952 -PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user)
953 -{
925 +static inline PARSER_RC pluginsd_variable(char **words, size_t num_words, PARSER *parser) {
926 char *name = get_word(words, num_words, 1);
927 char *value = get_word(words, num_words, 2);
928 NETDATA_DOUBLE v;
929
958 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_VARIABLE);
959 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
930 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_VARIABLE);
931 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
932
961 - RRDSET *st = pluginsd_get_chart_from_parent(user);
933 + RRDSET *st = pluginsd_get_chart_from_parent(parser);
934
935 int global = (st) ? 0 : 1;
936
@@ -975,7 +947,7 @@ PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user)
947 }
948
949 if (unlikely(!name || !*name))
978 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_VARIABLE, "missing variable name");
950 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_VARIABLE, "missing variable name");
951
952 if (unlikely(!value || !*value))
953 value = NULL;
@@ -990,7 +962,7 @@ PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user)
962 }
963
964 if (!global && !st)
993 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_VARIABLE, "no chart is defined and no GLOBAL is given");
965 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_VARIABLE, "no chart is defined and no GLOBAL is given");
966
967 char *endptr = NULL;
968 v = (NETDATA_DOUBLE) str2ndd_encoded(value, &endptr);
@@ -1034,32 +1006,29 @@ PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user)
1006 return PARSER_RC_OK;
1007 }
1008
1037 -PARSER_RC pluginsd_flush(char **words __maybe_unused, size_t num_words __maybe_unused, void *user)
1038 -{
1009 +static inline PARSER_RC pluginsd_flush(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
1010 debug(D_PLUGINSD, "requested a " PLUGINSD_KEYWORD_FLUSH);
1040 - pluginsd_set_chart_from_parent(user, NULL, PLUGINSD_KEYWORD_FLUSH);
1041 - ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
1042 - ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
1043 - ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
1044 - ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
1011 + pluginsd_set_chart_from_parent(parser, NULL, PLUGINSD_KEYWORD_FLUSH);
1012 + parser->user.replay.start_time = 0;
1013 + parser->user.replay.end_time = 0;
1014 + parser->user.replay.start_time_ut = 0;
1015 + parser->user.replay.end_time_ut = 0;
1016 return PARSER_RC_OK;
1017 }
1018
1048 -PARSER_RC pluginsd_disable(char **words __maybe_unused, size_t num_words __maybe_unused, void *user __maybe_unused)
1049 -{
1019 +static inline PARSER_RC pluginsd_disable(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
1020 info("PLUGINSD: plugin called DISABLE. Disabling it.");
1051 - ((PARSER_USER_OBJECT *) user)->enabled = 0;
1021 + parser->user.enabled = 0;
1022 return PARSER_RC_STOP;
1023 }
1024
1055 -PARSER_RC pluginsd_label(char **words, size_t num_words, void *user)
1056 -{
1025 +static inline PARSER_RC pluginsd_label(char **words, size_t num_words, PARSER *parser) {
1026 const char *name = get_word(words, num_words, 1);
1027 const char *label_source = get_word(words, num_words, 2);
1028 const char *value = get_word(words, num_words, 3);
1029
1030 if (!name || !label_source || !value)
1062 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_LABEL, "missing parameters");
1031 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_LABEL, "missing parameters");
1032
1033 char *store = (char *)value;
1034 bool allocated_store = false;
@@ -1088,13 +1057,10 @@ PARSER_RC pluginsd_label(char **words, size_t num_words, void *user)
1057 }
1058 }
1059
1091 - if(unlikely(!((PARSER_USER_OBJECT *) user)->new_host_labels))
1092 - ((PARSER_USER_OBJECT *) user)->new_host_labels = rrdlabels_create();
1060 + if(unlikely(!(parser->user.new_host_labels)))
1061 + parser->user.new_host_labels = rrdlabels_create();
1062
1094 - rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_host_labels,
1095 - name,
1096 - store,
1097 - str2l(label_source));
1063 + rrdlabels_add(parser->user.new_host_labels, name, store, str2l(label_source));
1064
1065 if (allocated_store)
1066 freez(store);
@@ -1102,90 +1068,84 @@ PARSER_RC pluginsd_label(char **words, size_t num_words, void *user)
1068 return PARSER_RC_OK;
1069 }
1070
1105 -PARSER_RC pluginsd_overwrite(char **words __maybe_unused, size_t num_words __maybe_unused, void *user)
1106 -{
1107 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_OVERWRITE);
1108 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1071 +static inline PARSER_RC pluginsd_overwrite(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
1072 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_OVERWRITE);
1073 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1074
1075 debug(D_PLUGINSD, "requested to OVERWRITE host labels");
1076
1077 if(unlikely(!host->rrdlabels))
1078 host->rrdlabels = rrdlabels_create();
1079
1115 - rrdlabels_migrate_to_these(host->rrdlabels, (DICTIONARY *) (((PARSER_USER_OBJECT *)user)->new_host_labels));
1080 + rrdlabels_migrate_to_these(host->rrdlabels, parser->user.new_host_labels);
1081 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
1082
1118 - rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_host_labels);
1119 - ((PARSER_USER_OBJECT *)user)->new_host_labels = NULL;
1083 + rrdlabels_destroy(parser->user.new_host_labels);
1084 + parser->user.new_host_labels = NULL;
1085 return PARSER_RC_OK;
1086 }
1087
1123 -
1124 -PARSER_RC pluginsd_clabel(char **words, size_t num_words, void *user)
1125 -{
1088 +static inline PARSER_RC pluginsd_clabel(char **words, size_t num_words, PARSER *parser) {
1089 const char *name = get_word(words, num_words, 1);
1090 const char *value = get_word(words, num_words, 2);
1091 const char *label_source = get_word(words, num_words, 3);
1092
1093 if (!name || !value || !*label_source) {
1094 error("Ignoring malformed or empty CHART LABEL command.");
1132 - return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1095 + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1096 }
1097
1135 - if(unlikely(!((PARSER_USER_OBJECT *) user)->chart_rrdlabels_linked_temporarily)) {
1136 - RRDSET *st = pluginsd_get_chart_from_parent(user);
1137 - ((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily = st->rrdlabels;
1138 - rrdlabels_unmark_all(((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily);
1098 + if(unlikely(!parser->user.chart_rrdlabels_linked_temporarily)) {
1099 + RRDSET *st = pluginsd_get_chart_from_parent(parser);
1100 + parser->user.chart_rrdlabels_linked_temporarily = st->rrdlabels;
1101 + rrdlabels_unmark_all(parser->user.chart_rrdlabels_linked_temporarily);
1102 }
1103
1141 - rrdlabels_add(((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily,
1142 - name, value, str2l(label_source));
1104 + rrdlabels_add(parser->user.chart_rrdlabels_linked_temporarily, name, value, str2l(label_source));
1105
1106 return PARSER_RC_OK;
1107 }
1108
1147 -PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size_t num_words __maybe_unused, void *user)
1148 -{
1149 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_CLABEL_COMMIT);
1150 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1109 +static inline PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
1110 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT);
1111 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1112
1152 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_CLABEL_COMMIT, PLUGINSD_KEYWORD_BEGIN);
1153 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1113 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT, PLUGINSD_KEYWORD_BEGIN);
1114 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1115
1116 debug(D_PLUGINSD, "requested to commit chart labels");
1117
1157 - if(!((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily) {
1158 - error("PLUGINSD: 'host:%s' got CLABEL_COMMIT, without a CHART or BEGIN. Ignoring it.",
1159 - rrdhost_hostname(host));
1160 - return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1118 + if(!parser->user.chart_rrdlabels_linked_temporarily) {
1119 + error("PLUGINSD: 'host:%s' got CLABEL_COMMIT, without a CHART or BEGIN. Ignoring it.", rrdhost_hostname(host));
1120 + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1121 }
1122
1163 - rrdlabels_remove_all_unmarked(((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily);
1123 + rrdlabels_remove_all_unmarked(parser->user.chart_rrdlabels_linked_temporarily);
1124
1125 rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
1126 rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
1127
1168 - ((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily = NULL;
1128 + parser->user.chart_rrdlabels_linked_temporarily = NULL;
1129 return PARSER_RC_OK;
1130 }
1131
1172 -PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user) {
1132 +static inline PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser) {
1133 char *id = get_word(words, num_words, 1);
1134 char *start_time_str = get_word(words, num_words, 2);
1135 char *end_time_str = get_word(words, num_words, 3);
1136 char *child_now_str = get_word(words, num_words, 4);
1137
1178 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1179 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1138 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1139 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1140
1141 RRDSET *st;
1142 if (likely(!id || !*id))
1183 - st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1143 + st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1144 else
1145 st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1146
1187 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1188 - pluginsd_set_chart_from_parent(user, st, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1147 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1148 + pluginsd_set_chart_from_parent(parser, st, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1149
1150 if(start_time_str && end_time_str) {
1151 time_t start_time = (time_t) str2ull_encoded(start_time_str);
@@ -1237,12 +1197,12 @@ PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user) {
1197 if(st->db.current_entry >= st->db.entries)
1198 st->db.current_entry -= st->db.entries;
1199
1240 - ((PARSER_USER_OBJECT *) user)->replay.start_time = start_time;
1241 - ((PARSER_USER_OBJECT *) user)->replay.end_time = end_time;
1242 - ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC;
1243 - ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC;
1244 - ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = wall_clock_time;
1245 - ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = true;
1200 + parser->user.replay.start_time = start_time;
1201 + parser->user.replay.end_time = end_time;
1202 + parser->user.replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC;
1203 + parser->user.replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC;
1204 + parser->user.replay.wall_clock_time = wall_clock_time;
1205 + parser->user.replay.rset_enabled = true;
1206
1207 return PARSER_RC_OK;
1208 }
@@ -1257,12 +1217,12 @@ PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user) {
1217 // the child sends an RBEGIN without any parameters initially
1218 // setting rset_enabled to false, means the RSET should not store any metrics
1219 // to store metrics, the RBEGIN needs to have timestamps
1260 - ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
1261 - ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
1262 - ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
1263 - ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
1264 - ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = 0;
1265 - ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = false;
1220 + parser->user.replay.start_time = 0;
1221 + parser->user.replay.end_time = 0;
1222 + parser->user.replay.start_time_ut = 0;
1223 + parser->user.replay.end_time_ut = 0;
1224 + parser->user.replay.wall_clock_time = 0;
1225 + parser->user.replay.rset_enabled = false;
1226 return PARSER_RC_OK;
1227 }
1228
@@ -1293,20 +1253,18 @@ static inline SN_FLAGS pluginsd_parse_storage_number_flags(const char *flags_str
1253 return flags;
1254 }
1255
1296 -PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
1297 -{
1256 +static inline PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser) {
1257 char *dimension = get_word(words, num_words, 1);
1258 char *value_str = get_word(words, num_words, 2);
1259 char *flags_str = get_word(words, num_words, 3);
1260
1302 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_SET);
1303 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1261 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_SET);
1262 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1263
1305 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1306 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1264 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1265 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1266
1308 - PARSER_USER_OBJECT *u = user;
1309 - if(!u->replay.rset_enabled) {
1267 + if(!parser->user.replay.rset_enabled) {
1268 error_limit_static_thread_var(erl, 1, 0);
1269 error_limit(&erl, "PLUGINSD: 'host:%s/chart:%s' got a %s but it is disabled by %s errors",
1270 rrdhost_hostname(host), rrdset_id(st), PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
@@ -1316,18 +1274,18 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
1274 }
1275
1276 RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_REPLAY_SET);
1319 - if(!rd) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1277 + if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1278
1321 - if (unlikely(!u->replay.start_time || !u->replay.end_time)) {
1279 + if (unlikely(!parser->user.replay.start_time || !parser->user.replay.end_time)) {
1280 error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s with invalid timestamps %ld to %ld from a %s. Disabling it.",
1281 rrdhost_hostname(host),
1282 rrdset_id(st),
1283 dimension,
1284 PLUGINSD_KEYWORD_REPLAY_SET,
1327 - u->replay.start_time,
1328 - u->replay.end_time,
1285 + parser->user.replay.start_time,
1286 + parser->user.replay.end_time,
1287 PLUGINSD_KEYWORD_REPLAY_BEGIN);
1330 - return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1288 + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1289 }
1290
1291 if (unlikely(!value_str || !*value_str))
@@ -1348,8 +1306,8 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
1306 flags = SN_EMPTY_SLOT;
1307 }
1308
1351 - rrddim_store_metric(rd, u->replay.end_time_ut, value, flags);
1352 - rd->last_collected_time.tv_sec = u->replay.end_time;
1309 + rrddim_store_metric(rd, parser->user.replay.end_time_ut, value, flags);
1310 + rd->last_collected_time.tv_sec = parser->user.replay.end_time;
1311 rd->last_collected_time.tv_usec = 0;
1312 rd->collections_counter++;
1313 }
@@ -1363,9 +1321,8 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
1321 return PARSER_RC_OK;
1322 }
1323
1366 -PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, void *user)
1367 -{
1368 - if(((PARSER_USER_OBJECT *) user)->replay.rset_enabled == false)
1324 +static inline PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, PARSER *parser) {
1325 + if(parser->user.replay.rset_enabled == false)
1326 return PARSER_RC_OK;
1327
1328 char *dimension = get_word(words, num_words, 1);
@@ -1374,14 +1331,14 @@ PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words
1331 char *last_calculated_value_str = get_word(words, num_words, 4);
1332 char *last_stored_value_str = get_word(words, num_words, 5);
1333
1377 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1378 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1334 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1335 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1336
1380 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1381 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1337 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1338 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1339
1340 RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1384 - if(!rd) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1341 + if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1342
1343 usec_t dim_last_collected_ut = (usec_t)rd->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->last_collected_time.tv_usec;
1344 usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
@@ -1397,19 +1354,18 @@ PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words
1354 return PARSER_RC_OK;
1355 }
1356
1400 -PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, void *user)
1401 -{
1402 - if(((PARSER_USER_OBJECT *) user)->replay.rset_enabled == false)
1357 +static inline PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, PARSER *parser) {
1358 + if(parser->user.replay.rset_enabled == false)
1359 return PARSER_RC_OK;
1360
1361 char *last_collected_ut_str = get_word(words, num_words, 1);
1362 char *last_updated_ut_str = get_word(words, num_words, 2);
1363
1408 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE);
1409 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1364 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE);
1365 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1366
1411 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1412 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1367 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1368 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1369
1370 usec_t chart_last_collected_ut = (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec;
1371 usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
@@ -1431,8 +1387,7 @@ PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words
1387 return PARSER_RC_OK;
1388 }
1389
1434 -PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1435 -{
1390 +static inline PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser) {
1391 if (num_words < 7) { // accepts 7, but the 7th is optional
1392 error("REPLAY: malformed " PLUGINSD_KEYWORD_REPLAY_END " command");
1393 return PARSER_RC_ERROR;
@@ -1458,13 +1413,11 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1413 time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t) str2ull_encoded(
1414 child_world_time_txt) : now_realtime_sec();
1415
1461 - PARSER_USER_OBJECT *user_object = user;
1462 -
1463 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_END);
1464 - if(!host) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1416 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_END);
1417 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1418
1466 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_END, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1467 - if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1419 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_REPLAY_END, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1420 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1421
1422 #ifdef NETDATA_LOG_REPLICATION_REQUESTS
1423 internal_error(true,
@@ -1477,12 +1430,12 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1430 );
1431 #endif
1432
1480 - ((PARSER_USER_OBJECT *) user)->data_collections_count++;
1433 + parser->user.data_collections_count++;
1434
1482 - if(((PARSER_USER_OBJECT *) user)->replay.rset_enabled && st->rrdhost->receiver) {
1435 + if(parser->user.replay.rset_enabled && st->rrdhost->receiver) {
1436 time_t now = now_realtime_sec();
1437 time_t started = st->rrdhost->receiver->replication_first_time_t;
1485 - time_t current = ((PARSER_USER_OBJECT *) user)->replay.end_time;
1438 + time_t current = parser->user.replay.end_time;
1439
1440 if(started && current > started) {
1441 host->rrdpush_receiver_replication_percent = (NETDATA_DOUBLE) (current - started) * 100.0 / (NETDATA_DOUBLE) (now - started);
@@ -1491,12 +1444,12 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1444 }
1445 }
1446
1494 - ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
1495 - ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
1496 - ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
1497 - ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
1498 - ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = 0;
1499 - ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = false;
1447 + parser->user.replay.start_time = 0;
1448 + parser->user.replay.end_time = 0;
1449 + parser->user.replay.start_time_ut = 0;
1450 + parser->user.replay.end_time_ut = 0;
1451 + parser->user.replay.wall_clock_time = 0;
1452 + parser->user.replay.rset_enabled = false;
1453
1454 st->counter++;
1455 st->counter_done++;
@@ -1526,7 +1479,7 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1479 rrdhost_hostname(host), rrdset_id(st));
1480 #endif
1481
1529 - pluginsd_set_chart_from_parent(user, NULL, PLUGINSD_KEYWORD_REPLAY_END);
1482 + pluginsd_set_chart_from_parent(parser, NULL, PLUGINSD_KEYWORD_REPLAY_END);
1483
1484 host->rrdpush_receiver_replication_percent = 100.0;
1485 worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, host->rrdpush_receiver_replication_percent);
@@ -1534,17 +1487,17 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1487 return PARSER_RC_OK;
1488 }
1489
1537 - pluginsd_set_chart_from_parent(user, NULL, PLUGINSD_KEYWORD_REPLAY_END);
1490 + pluginsd_set_chart_from_parent(parser, NULL, PLUGINSD_KEYWORD_REPLAY_END);
1491
1492 rrdcontext_updated_retention_rrdset(st);
1493
1541 - bool ok = replicate_chart_request(send_to_plugin, user_object->parser, host, st,
1494 + bool ok = replicate_chart_request(send_to_plugin, parser, host, st,
1495 first_entry_child, last_entry_child, child_world_time,
1496 first_entry_requested, last_entry_requested);
1497 return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
1498 }
1499
1547 -PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
1500 +static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER *parser) {
1501 timing_init();
1502
1503 char *id = get_word(words, num_words, 1);
@@ -1553,17 +1506,17 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
1506 char *wall_clock_time_str = get_word(words, num_words, 4);
1507
1508 if(unlikely(!id || !update_every_str || !end_time_str || !wall_clock_time_str))
1556 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_BEGIN_V2, "missing parameters");
1509 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_BEGIN_V2, "missing parameters");
1510
1558 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_BEGIN_V2);
1559 - if(unlikely(!host)) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1511 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_BEGIN_V2);
1512 + if(unlikely(!host)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1513
1514 timing_step(TIMING_STEP_BEGIN2_PREPARE);
1515
1516 RRDSET *st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_BEGIN_V2);
1564 - if(unlikely(!st)) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1517 + if(unlikely(!st)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1518
1566 - pluginsd_set_chart_from_parent(user, st, PLUGINSD_KEYWORD_BEGIN_V2);
1519 + pluginsd_set_chart_from_parent(parser, st, PLUGINSD_KEYWORD_BEGIN_V2);
1520
1521 if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE | RRDSET_FLAG_ARCHIVED)))
1522 rrdset_isnot_obsolete(st);
@@ -1590,32 +1543,31 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
1543 // ------------------------------------------------------------------------
1544 // prepare our state
1545
1593 - pluginsd_lock_rrdset_data_collection(user);
1546 + pluginsd_lock_rrdset_data_collection(parser);
1547
1595 - PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
1596 - u->v2.update_every = update_every;
1597 - u->v2.end_time = end_time;
1598 - u->v2.wall_clock_time = wall_clock_time;
1599 - u->v2.ml_locked = ml_chart_update_begin(st);
1548 + parser->user.v2.update_every = update_every;
1549 + parser->user.v2.end_time = end_time;
1550 + parser->user.v2.wall_clock_time = wall_clock_time;
1551 + parser->user.v2.ml_locked = ml_chart_update_begin(st);
1552
1553 timing_step(TIMING_STEP_BEGIN2_ML);
1554
1555 // ------------------------------------------------------------------------
1556 // propagate it forward in v2
1557
1606 - if(!u->v2.stream_buffer.wb && rrdhost_has_rrdpush_sender_enabled(st->rrdhost))
1607 - u->v2.stream_buffer = rrdset_push_metric_initialize(u->st, wall_clock_time);
1558 + if(!parser->user.v2.stream_buffer.wb && rrdhost_has_rrdpush_sender_enabled(st->rrdhost))
1559 + parser->user.v2.stream_buffer = rrdset_push_metric_initialize(parser->user.st, wall_clock_time);
1560
1609 - if(u->v2.stream_buffer.v2 && u->v2.stream_buffer.wb) {
1561 + if(parser->user.v2.stream_buffer.v2 && parser->user.v2.stream_buffer.wb) {
1562 // check if receiver and sender have the same number parsing capabilities
1611 - bool can_copy = stream_has_capability(u, STREAM_CAP_IEEE754) == stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754);
1612 - NUMBER_ENCODING encoding = stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
1563 + bool can_copy = stream_has_capability(&parser->user, STREAM_CAP_IEEE754) == stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754);
1564 + NUMBER_ENCODING encoding = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
1565
1614 - BUFFER *wb = u->v2.stream_buffer.wb;
1566 + BUFFER *wb = parser->user.v2.stream_buffer.wb;
1567
1568 buffer_need_bytes(wb, 1024);
1569
1618 - if(unlikely(u->v2.stream_buffer.begin_v2_added))
1570 + if(unlikely(parser->user.v2.stream_buffer.begin_v2_added))
1571 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_END_V2 "\n", sizeof(PLUGINSD_KEYWORD_END_V2) - 1 + 1);
1572
1573 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2 " '", sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1 + 2);
@@ -1643,8 +1595,8 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
1595
1596 buffer_fast_strcat(wb, "\n", 1);
1597
1646 - u->v2.stream_buffer.last_point_end_time_s = end_time;
1647 - u->v2.stream_buffer.begin_v2_added = true;
1598 + parser->user.v2.stream_buffer.last_point_end_time_s = end_time;
1599 + parser->user.v2.stream_buffer.begin_v2_added = true;
1600 }
1601
1602 timing_step(TIMING_STEP_BEGIN2_PROPAGATE);
@@ -1669,7 +1621,7 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
1621 return PARSER_RC_OK;
1622 }
1623
1672 -PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
1624 +static inline PARSER_RC pluginsd_set_v2(char **words, size_t num_words, PARSER *parser) {
1625 timing_init();
1626
1627 char *dimension = get_word(words, num_words, 1);
@@ -1678,20 +1630,18 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
1630 char *flags_str = get_word(words, num_words, 4);
1631
1632 if(unlikely(!dimension || !collected_str || !value_str || !flags_str))
1681 - return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_SET_V2, "missing parameters");
1682 -
1683 - PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
1633 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_SET_V2, "missing parameters");
1634
1685 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_SET_V2);
1686 - if(unlikely(!host)) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1635 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_SET_V2);
1636 + if(unlikely(!host)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1637
1688 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_SET_V2, PLUGINSD_KEYWORD_BEGIN_V2);
1689 - if(unlikely(!st)) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1638 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_SET_V2, PLUGINSD_KEYWORD_BEGIN_V2);
1639 + if(unlikely(!st)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1640
1641 timing_step(TIMING_STEP_SET2_PREPARE);
1642
1643 RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_SET_V2);
1694 - if(unlikely(!rd)) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1644 + if(unlikely(!rd)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1645
1646 if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED)))
1647 rrddim_isnot_obsolete(st, rd);
@@ -1720,11 +1670,11 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
1670 value = NAN;
1671 flags = SN_EMPTY_SLOT;
1672
1723 - if(u->v2.ml_locked)
1724 - ml_dimension_is_anomalous(rd, u->v2.end_time, 0, false);
1673 + if(parser->user.v2.ml_locked)
1674 + ml_dimension_is_anomalous(rd, parser->user.v2.end_time, 0, false);
1675 }
1726 - else if(u->v2.ml_locked) {
1727 - if (ml_dimension_is_anomalous(rd, u->v2.end_time, value, true)) {
1676 + else if(parser->user.v2.ml_locked) {
1677 + if (ml_dimension_is_anomalous(rd, parser->user.v2.end_time, value, true)) {
1678 // clear anomaly bit: 0 -> is anomalous, 1 -> not anomalous
1679 flags &= ~((storage_number) SN_FLAG_NOT_ANOMALOUS);
1680 }
@@ -1737,13 +1687,13 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
1687 // ------------------------------------------------------------------------
1688 // propagate it forward in v2
1689
1740 - if(u->v2.stream_buffer.v2 && u->v2.stream_buffer.begin_v2_added && u->v2.stream_buffer.wb) {
1690 + if(parser->user.v2.stream_buffer.v2 && parser->user.v2.stream_buffer.begin_v2_added && parser->user.v2.stream_buffer.wb) {
1691 // check if receiver and sender have the same number parsing capabilities
1742 - bool can_copy = stream_has_capability(u, STREAM_CAP_IEEE754) == stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754);
1743 - NUMBER_ENCODING integer_encoding = stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
1744 - NUMBER_ENCODING doubles_encoding = stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
1692 + bool can_copy = stream_has_capability(&parser->user, STREAM_CAP_IEEE754) == stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754);
1693 + NUMBER_ENCODING integer_encoding = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
1694 + NUMBER_ENCODING doubles_encoding = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
1695
1746 - BUFFER *wb = u->v2.stream_buffer.wb;
1696 + BUFFER *wb = parser->user.v2.stream_buffer.wb;
1697 buffer_need_bytes(wb, 1024);
1698 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2 " '", sizeof(PLUGINSD_KEYWORD_SET_V2) - 1 + 2);
1699 buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
@@ -1767,8 +1717,8 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
1717 // ------------------------------------------------------------------------
1718 // store it
1719
1770 - rrddim_store_metric(rd, u->v2.end_time * USEC_PER_SEC, value, flags);
1771 - rd->last_collected_time.tv_sec = u->v2.end_time;
1720 + rrddim_store_metric(rd, parser->user.v2.end_time * USEC_PER_SEC, value, flags);
1721 + rd->last_collected_time.tv_sec = parser->user.v2.end_time;
1722 rd->last_collected_time.tv_usec = 0;
1723 rd->last_collected_value = collected_value;
1724 rd->last_stored_value = value;
@@ -1781,37 +1731,36 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
1731 return PARSER_RC_OK;
1732 }
1733
1784 -void pluginsd_cleanup_v2(void *user) {
1734 +void pluginsd_cleanup_v2(PARSER *parser) {
1735 // this is called when the thread is stopped while processing
1786 - pluginsd_set_chart_from_parent(user, NULL, "THREAD CLEANUP");
1736 + pluginsd_set_chart_from_parent(parser, NULL, "THREAD CLEANUP");
1737 }
1738
1789 -PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_unused, void *user) {
1739 +static inline PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
1740 timing_init();
1741
1792 - RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_END_V2);
1793 - if(unlikely(!host)) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1742 + RRDHOST *host = pluginsd_require_host_from_parent(parser, PLUGINSD_KEYWORD_END_V2);
1743 + if(unlikely(!host)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1744
1795 - RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_END_V2, PLUGINSD_KEYWORD_BEGIN_V2);
1796 - if(unlikely(!st)) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
1745 + RRDSET *st = pluginsd_require_chart_from_parent(parser, PLUGINSD_KEYWORD_END_V2, PLUGINSD_KEYWORD_BEGIN_V2);
1746 + if(unlikely(!st)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1747
1798 - PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
1799 - u->data_collections_count++;
1748 + parser->user.data_collections_count++;
1749
1750 timing_step(TIMING_STEP_END2_PREPARE);
1751
1752 // ------------------------------------------------------------------------
1753 // propagate the whole chart update in v1
1754
1806 - if(unlikely(!u->v2.stream_buffer.v2 && !u->v2.stream_buffer.begin_v2_added && u->v2.stream_buffer.wb))
1807 - rrdset_push_metrics_v1(&u->v2.stream_buffer, st);
1755 + if(unlikely(!parser->user.v2.stream_buffer.v2 && !parser->user.v2.stream_buffer.begin_v2_added && parser->user.v2.stream_buffer.wb))
1756 + rrdset_push_metrics_v1(&parser->user.v2.stream_buffer, st);
1757
1758 timing_step(TIMING_STEP_END2_PUSH_V1);
1759
1760 // ------------------------------------------------------------------------
1761 // unblock data collection
1762
1814 - pluginsd_unlock_previous_chart(user, PLUGINSD_KEYWORD_END_V2, false);
1763 + pluginsd_unlock_previous_chart(parser, PLUGINSD_KEYWORD_END_V2, false);
1764 rrdcontext_collected_rrdset(st);
1765 store_metric_collection_completed();
1766
@@ -1820,7 +1769,7 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_
1769 // ------------------------------------------------------------------------
1770 // propagate it forward
1771
1823 - rrdset_push_metrics_finished(&u->v2.stream_buffer, st);
1772 + rrdset_push_metrics_finished(&parser->user.v2.stream_buffer, st);
1773
1774 timing_step(TIMING_STEP_END2_PROPAGATE);
1775
@@ -1829,16 +1778,16 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_
1778
1779 RRDDIM *rd;
1780 rrddim_foreach_read(rd, st) {
1832 - rd->calculated_value = 0;
1833 - rd->collected_value = 0;
1834 - rrddim_clear_updated(rd);
1835 - }
1781 + rd->calculated_value = 0;
1782 + rd->collected_value = 0;
1783 + rrddim_clear_updated(rd);
1784 + }
1785 rrddim_foreach_done(rd);
1786
1787 // ------------------------------------------------------------------------
1788 // reset state
1789
1841 - u->v2 = (struct parser_user_object_v2){ 0 };
1790 + parser->user.v2 = (struct parser_user_object_v2){ 0 };
1791
1792 timing_step(TIMING_STEP_END2_STORE);
1793 timing_report();
@@ -1846,19 +1795,143 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_
1795 return PARSER_RC_OK;
1796 }
1797
1798 +static inline PARSER_RC pluginsd_exit(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
1799 + info("PLUGINSD: plugin called EXIT.");
1800 + return PARSER_RC_STOP;
1801 +}
1802 +
1803 +static inline PARSER_RC streaming_claimed_id(char **words, size_t num_words, PARSER *parser)
1804 +{
1805 + const char *host_uuid_str = get_word(words, num_words, 1);
1806 + const char *claim_id_str = get_word(words, num_words, 2);
1807 +
1808 + if (!host_uuid_str || !claim_id_str) {
1809 + error("Command CLAIMED_ID came malformed, uuid = '%s', claim_id = '%s'",
1810 + host_uuid_str ? host_uuid_str : "[unset]",
1811 + claim_id_str ? claim_id_str : "[unset]");
1812 + return PARSER_RC_ERROR;
1813 + }
1814 +
1815 + uuid_t uuid;
1816 + RRDHOST *host = parser->user.host;
1817 +
1818 + // We don't need the parsed UUID
1819 + // just do it to check the format
1820 + if(uuid_parse(host_uuid_str, uuid)) {
1821 + error("1st parameter (host GUID) to CLAIMED_ID command is not valid GUID. Received: \"%s\".", host_uuid_str);
1822 + return PARSER_RC_ERROR;
1823 + }
1824 + if(uuid_parse(claim_id_str, uuid) && strcmp(claim_id_str, "NULL") != 0) {
1825 + error("2nd parameter (Claim ID) to CLAIMED_ID command is not valid GUID. Received: \"%s\".", claim_id_str);
1826 + return PARSER_RC_ERROR;
1827 + }
1828 +
1829 + if(strcmp(host_uuid_str, host->machine_guid) != 0) {
1830 + error("Claim ID is for host \"%s\" but it came over connection for \"%s\"", host_uuid_str, host->machine_guid);
1831 + return PARSER_RC_OK; //the message is OK problem must be somewhere else
1832 + }
1833 +
1834 + rrdhost_aclk_state_lock(host);
1835 +
1836 + if (host->aclk_state.claimed_id)
1837 + freez(host->aclk_state.claimed_id);
1838 +
1839 + host->aclk_state.claimed_id = strcmp(claim_id_str, "NULL") ? strdupz(claim_id_str) : NULL;
1840 +
1841 + rrdhost_aclk_state_unlock(host);
1842 +
1843 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID |RRDHOST_FLAG_METADATA_UPDATE);
1844 +
1845 + rrdpush_send_claimed_id(host);
1846 +
1847 + return PARSER_RC_OK;
1848 +}
1849 +
1850 +// ----------------------------------------------------------------------------
1851 +
1852 +typedef enum {
1853 + PARSER_FGETS_RESULT_OK,
1854 + PARSER_FGETS_RESULT_TIMEOUT,
1855 + PARSER_FGETS_RESULT_ERROR,
1856 + PARSER_FGETS_RESULT_EOF,
1857 +} PARSER_FGETS_RESULT;
1858 +
1859 +static inline PARSER_FGETS_RESULT parser_fgets(char *s, int size, FILE *stream) {
1860 + errno = 0;
1861 +
1862 + struct pollfd fds[1];
1863 + int timeout_msecs = 2 * 60 * MSEC_PER_SEC;
1864 +
1865 + fds[0].fd = fileno(stream);
1866 + fds[0].events = POLLIN;
1867 +
1868 + int ret = poll(fds, 1, timeout_msecs);
1869 +
1870 + if (ret > 0) {
1871 + /* There is data to read */
1872 + if (fds[0].revents & POLLIN) {
1873 + char *tmp = fgets(s, size, stream);
1874 +
1875 + if(unlikely(!tmp)) {
1876 + if (feof(stream)) {
1877 + error("PARSER: read failed: end of file.");
1878 + return PARSER_FGETS_RESULT_EOF;
1879 + }
1880 +
1881 + else if (ferror(stream)) {
1882 + error("PARSER: read failed: input error.");
1883 + return PARSER_FGETS_RESULT_ERROR;
1884 + }
1885 +
1886 + error("PARSER: read failed: unknown error.");
1887 + return PARSER_FGETS_RESULT_ERROR;
1888 + }
1889 +
1890 + return PARSER_FGETS_RESULT_OK;
1891 + }
1892 + else if(fds[0].revents & POLLERR) {
1893 + error("PARSER: read failed: POLLERR.");
1894 + return PARSER_FGETS_RESULT_ERROR;
1895 + }
1896 + else if(fds[0].revents & POLLHUP) {
1897 + error("PARSER: read failed: POLLHUP.");
1898 + return PARSER_FGETS_RESULT_ERROR;
1899 + }
1900 + else if(fds[0].revents & POLLNVAL) {
1901 + error("PARSER: read failed: POLLNVAL.");
1902 + return PARSER_FGETS_RESULT_ERROR;
1903 + }
1904 +
1905 + error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set.");
1906 + return PARSER_FGETS_RESULT_ERROR;
1907 + }
1908 + else if (ret == 0) {
1909 + error("PARSER: timeout while waiting for data.");
1910 + return PARSER_FGETS_RESULT_TIMEOUT;
1911 + }
1912 +
1913 + error("PARSER: poll() failed with code %d.", ret);
1914 + return PARSER_FGETS_RESULT_ERROR;
1915 +}
1916 +
1917 +static int parser_next(PARSER *parser, char *buffer, size_t buffer_size) {
1918 + if(likely(parser_fgets(buffer, (int)buffer_size, (FILE *)parser->fp_input) == PARSER_FGETS_RESULT_OK))
1919 + return 0;
1920 +
1921 + return 1;
1922 +}
1923 +
1924 void pluginsd_process_thread_cleanup(void *ptr) {
1925 PARSER *parser = (PARSER *)ptr;
1926
1852 - pluginsd_cleanup_v2(parser->user);
1853 - pluginsd_host_define_cleanup(parser->user);
1927 + pluginsd_cleanup_v2(parser);
1928 + pluginsd_host_define_cleanup(parser);
1929
1930 rrd_collector_finished();
1931
1932 parser_destroy(parser);
1933 }
1934
1860 -// New plugins.d parser
1861 -
1935 inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugin_input, FILE *fp_plugin_output, int trust_durations)
1936 {
1937 int enabled = cd->unsafe.enabled;
@@ -1883,16 +1956,18 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi
1956 clearerr(fp_plugin_input);
1957 clearerr(fp_plugin_output);
1958
1886 - PARSER_USER_OBJECT user = {
1887 - .enabled = cd->unsafe.enabled,
1888 - .host = host,
1889 - .cd = cd,
1890 - .trust_durations = trust_durations
1891 - };
1959 + PARSER *parser;
1960 + {
1961 + PARSER_USER_OBJECT user = {
1962 + .enabled = cd->unsafe.enabled,
1963 + .host = host,
1964 + .cd = cd,
1965 + .trust_durations = trust_durations
1966 + };
1967
1893 - // fp_plugin_output = our input; fp_plugin_input = our output
1894 - PARSER *parser = parser_init(&user, fp_plugin_output, fp_plugin_input, -1,
1895 - PARSER_INPUT_SPLIT, NULL);
1968 + // fp_plugin_output = our input; fp_plugin_input = our output
1969 + parser = parser_init(&user, fp_plugin_output, fp_plugin_input, -1, PARSER_INPUT_SPLIT, NULL);
1970 + }
1971
1972 pluginsd_keywords_init(parser, PARSER_INIT_PLUGINSD);
1973
@@ -1902,7 +1977,6 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi
1977 // so, parser needs to be allocated before pushing it
1978 netdata_thread_cleanup_push(pluginsd_process_thread_cleanup, parser);
1979
1905 - user.parser = parser;
1980 char buffer[PLUGINSD_LINE_MAX + 1];
1981
1982 while (likely(!parser_next(parser, buffer, PLUGINSD_LINE_MAX))) {
@@ -1913,8 +1987,8 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi
1987 // free parser with the pop function
1988 netdata_thread_cleanup_pop(1);
1989
1916 - cd->unsafe.enabled = user.enabled;
1917 - size_t count = user.data_collections_count;
1990 + cd->unsafe.enabled = parser->user.enabled;
1991 + size_t count = parser->user.data_collections_count;
1992
1993 if (likely(count)) {
1994 cd->successful_collections += count;
@@ -1926,140 +2000,87 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi
2000 return count;
2001 }
2002
1929 -PARSER_RC pluginsd_exit(char **words __maybe_unused, size_t num_words __maybe_unused, void *user __maybe_unused)
1930 -{
1931 - info("PLUGINSD: plugin called EXIT.");
1932 - return PARSER_RC_STOP;
1933 -}
1934 -
1935 -static void pluginsd_keywords_init_internal(PARSER *parser, PLUGINSD_KEYWORDS types, void (*add_func)(PARSER *parser, char *keyword, keyword_function func)) {
1936 -
1937 - if (types & PARSER_INIT_PLUGINSD) {
1938 - add_func(parser, PLUGINSD_KEYWORD_FLUSH, pluginsd_flush);
1939 - add_func(parser, PLUGINSD_KEYWORD_DISABLE, pluginsd_disable);
1940 -
1941 - add_func(parser, PLUGINSD_KEYWORD_HOST_DEFINE, pluginsd_host_define);
1942 - add_func(parser, PLUGINSD_KEYWORD_HOST_DEFINE_END, pluginsd_host_define_end);
1943 - add_func(parser, PLUGINSD_KEYWORD_HOST_LABEL, pluginsd_host_labels);
1944 - add_func(parser, PLUGINSD_KEYWORD_HOST, pluginsd_host);
1945 -
1946 - add_func(parser, PLUGINSD_KEYWORD_EXIT, pluginsd_exit);
1947 - }
1948 -
1949 - if (types & (PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING)) {
1950 - // plugins.d plugins and streaming
1951 - add_func(parser, PLUGINSD_KEYWORD_CHART, pluginsd_chart);
1952 - add_func(parser, PLUGINSD_KEYWORD_DIMENSION, pluginsd_dimension);
1953 - add_func(parser, PLUGINSD_KEYWORD_VARIABLE, pluginsd_variable);
1954 - add_func(parser, PLUGINSD_KEYWORD_LABEL, pluginsd_label);
1955 - add_func(parser, PLUGINSD_KEYWORD_OVERWRITE, pluginsd_overwrite);
1956 - add_func(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT, pluginsd_clabel_commit);
1957 - add_func(parser, PLUGINSD_KEYWORD_CLABEL, pluginsd_clabel);
1958 - add_func(parser, PLUGINSD_KEYWORD_FUNCTION, pluginsd_function);
1959 - add_func(parser, PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN, pluginsd_function_result_begin);
1960 -
1961 - add_func(parser, PLUGINSD_KEYWORD_BEGIN, pluginsd_begin);
1962 - add_func(parser, PLUGINSD_KEYWORD_SET, pluginsd_set);
1963 - add_func(parser, PLUGINSD_KEYWORD_END, pluginsd_end);
2003 +void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire) {
2004 + parser_init_repertoire(parser, repertoire);
2005
2006 + if (repertoire & (PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING))
2007 inflight_functions_init(parser);
1966 - }
1967 -
1968 - if (types & PARSER_INIT_STREAMING) {
1969 - add_func(parser, PLUGINSD_KEYWORD_CHART_DEFINITION_END, pluginsd_chart_definition_end);
2008 +}
2009
1971 - // replication
1972 - add_func(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN, pluginsd_replay_begin);
1973 - add_func(parser, PLUGINSD_KEYWORD_REPLAY_SET, pluginsd_replay_set);
1974 - add_func(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, pluginsd_replay_rrddim_collection_state);
1975 - add_func(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE, pluginsd_replay_rrdset_collection_state);
1976 - add_func(parser, PLUGINSD_KEYWORD_REPLAY_END, pluginsd_replay_end);
2010 +PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_output, int fd,
2011 + PARSER_INPUT_TYPE flags, void *ssl __maybe_unused) {
2012 + PARSER *parser;
2013
1978 - // streaming metrics v2
1979 - add_func(parser, PLUGINSD_KEYWORD_BEGIN_V2, pluginsd_begin_v2);
1980 - add_func(parser, PLUGINSD_KEYWORD_SET_V2, pluginsd_set_v2);
1981 - add_func(parser, PLUGINSD_KEYWORD_END_V2, pluginsd_end_v2);
1982 - }
1983 -}
2014 + parser = callocz(1, sizeof(*parser));
2015 + if(user)
2016 + parser->user = *user;
2017 + parser->fd = fd;
2018 + parser->fp_input = fp_input;
2019 + parser->fp_output = fp_output;
2020 +#ifdef ENABLE_HTTPS
2021 + parser->ssl_output = ssl;
2022 +#endif
2023 + parser->flags = flags;
2024
1985 -void pluginsd_keywords_init(PARSER *parser, PLUGINSD_KEYWORDS types) {
1986 - pluginsd_keywords_init_internal(parser, types, parser_add_keyword);
2025 + spinlock_init(&parser->writer.spinlock);
2026 + return parser;
2027 }
2028
1989 -struct pluginsd_user_unittest {
1990 - size_t size;
1991 - const char **hashtable;
1992 - uint32_t (*hash)(const char *s);
1993 - size_t collisions;
1994 -};
2029 +#include "gperf-hashtable.h"
2030
1996 -void pluginsd_keyword_collision_check(PARSER *parser, char *keyword, keyword_function func __maybe_unused) {
1997 - struct pluginsd_user_unittest *u = parser->user;
2031 +void parser_init_repertoire(PARSER *parser, PARSER_REPERTOIRE repertoire) {
2032 + parser->repertoire = repertoire;
2033
1999 - uint32_t hash = u->hash(keyword);
2000 - uint32_t slot = hash % u->size;
2034 + for(size_t i = GPERF_PARSER_MIN_HASH_VALUE ; i <= GPERF_PARSER_MAX_HASH_VALUE ;i++) {
2035 + if(gperf_keywords[i].keyword && *gperf_keywords[i].keyword && (parser->repertoire & gperf_keywords[i].repertoire))
2036 + worker_register_job_name(gperf_keywords[i].worker_job_id, gperf_keywords[i].keyword);
2037 + }
2038 +}
2039
2002 - if(u->hashtable[slot])
2003 - u->collisions++;
2040 +void parser_destroy(PARSER *parser) {
2041 + if (unlikely(!parser))
2042 + return;
2043
2005 - u->hashtable[slot] = keyword;
2044 + dictionary_destroy(parser->inflight.functions);
2045 + freez(parser);
2046 }
2047
2008 -static struct {
2009 - const char *name;
2010 - uint32_t (*hash)(const char *s);
2011 - size_t slots_needed;
2012 -} hashers[] = {
2013 - { .name = "djb2_hash32(s)", djb2_hash32, .slots_needed = 0, },
2014 - { .name = "fnv1_hash32(s)", fnv1_hash32, .slots_needed = 0, },
2015 - { .name = "fnv1a_hash32(s)", fnv1a_hash32, .slots_needed = 0, },
2016 - { .name = "larson_hash32(s)", larson_hash32, .slots_needed = 0, },
2017 - { .name = "pluginsd_parser_hash32(s)", pluginsd_parser_hash32, .slots_needed = 0, },
2018 -
2019 - // terminator
2020 - { .name = NULL, NULL, .slots_needed = 0, },
2021 -};
2022 -
2048 int pluginsd_parser_unittest(void) {
2024 - PARSER *p;
2025 - size_t slots_to_check = 1000;
2026 - size_t i, h;
2027 -
2028 - // check for hashtable collisions
2029 - for(h = 0; hashers[h].name ;h++) {
2030 - hashers[h].slots_needed = slots_to_check * 1000000;
2031 -
2032 - for (i = 10; i < slots_to_check; i++) {
2033 - struct pluginsd_user_unittest user = {
2034 - .hash = hashers[h].hash,
2035 - .size = i,
2036 - .hashtable = callocz(i, sizeof(const char *)),
2037 - .collisions = 0,
2038 - };
2039 -
2040 - p = parser_init(&user, NULL, NULL, -1, PARSER_INPUT_SPLIT, NULL);
2041 - pluginsd_keywords_init_internal(p, PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING,
2042 - pluginsd_keyword_collision_check);
2043 - parser_destroy(p);
2044 -
2045 - freez(user.hashtable);
2046 -
2047 - if (!user.collisions) {
2048 - hashers[h].slots_needed = i;
2049 - break;
2050 - }
2049 + PARSER *p = parser_init(NULL, NULL, NULL, -1, PARSER_INPUT_SPLIT, NULL);
2050 + pluginsd_keywords_init(p, PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING);
2051 +
2052 + char *lines[] = {
2053 + "BEGIN2 abcdefghijklmnopqr 123",
2054 + "SET2 abcdefg 0x12345678 0 0",
2055 + "SET2 hijklmnoqr 0x12345678 0 0",
2056 + "SET2 stuvwxyz 0x12345678 0 0",
2057 + "END2",
2058 + NULL,
2059 + };
2060 +
2061 + char *words[PLUGINSD_MAX_WORDS];
2062 + size_t iterations = 1000000;
2063 + size_t count = 0;
2064 + char input[PLUGINSD_LINE_MAX + 1];
2065 +
2066 + usec_t started = now_realtime_usec();
2067 + while(--iterations) {
2068 + for(size_t line = 0; lines[line] ;line++) {
2069 + strncpyz(input, lines[line], PLUGINSD_LINE_MAX);
2070 + size_t num_words = quoted_strings_splitter_pluginsd(input, words, PLUGINSD_MAX_WORDS);
2071 + const char *command = get_word(words, num_words, 0);
2072 + PARSER_KEYWORD *keyword = parser_find_keyword(p, command);
2073 + if(unlikely(!keyword))
2074 + fatal("Cannot parse the line '%s'", lines[line]);
2075 + count++;
2076 }
2077 }
2078 + usec_t ended = now_realtime_usec();
2079
2054 - for(h = 0; hashers[h].name ;h++) {
2055 - if(hashers[h].slots_needed > 1000)
2056 - info("PARSER: hash function '%s' cannot be used without collisions under %zu slots", hashers[h].name, slots_to_check);
2057 - else
2058 - info("PARSER: hash function '%s' needs PARSER_KEYWORDS_HASHTABLE_SIZE (in parser.h) set to %zu", hashers[h].name, hashers[h].slots_needed);
2059 - }
2080 + info("Parsed %zu lines in %0.2f secs, %0.2f klines/sec", count,
2081 + (double)(ended - started) / (double)USEC_PER_SEC,
2082 + (double)count / ((double)(ended - started) / (double)USEC_PER_SEC) / 1000.0);
2083
2061 - p = parser_init(NULL, NULL, NULL, -1, PARSER_INPUT_SPLIT, NULL);
2062 - pluginsd_keywords_init(p, PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING);
2084 parser_destroy(p);
2085 return 0;
2086 }
collectors/plugins.d/pluginsd_parser.h
+160 -5
@@ -5,13 +5,39 @@
5
6 #include "daemon/common.h"
7
8 +#define WORKER_PARSER_FIRST_JOB 3
9 +
10 +// this has to be in-sync with the same at receiver.c
11 +#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)
12 +
13 +// PARSER return codes
14 +typedef enum __attribute__ ((__packed__)) parser_rc {
15 + PARSER_RC_OK, // Callback was successful, go on
16 + PARSER_RC_STOP, // Callback says STOP
17 + PARSER_RC_ERROR // Callback failed (abort rest of callbacks)
18 +} PARSER_RC;
19 +
20 +typedef enum __attribute__ ((__packed__)) parser_input_type {
21 + PARSER_INPUT_SPLIT = (1 << 1),
22 + PARSER_DEFER_UNTIL_KEYWORD = (1 << 2),
23 +} PARSER_INPUT_TYPE;
24 +
25 typedef enum __attribute__ ((__packed__)) {
26 PARSER_INIT_PLUGINSD = (1 << 1),
27 PARSER_INIT_STREAMING = (1 << 2),
11 -} PLUGINSD_KEYWORDS;
28 +} PARSER_REPERTOIRE;
29 +
30 +struct parser;
31 +typedef PARSER_RC (*keyword_function)(char **words, size_t num_words, struct parser *parser);
32 +
33 +typedef struct parser_keyword {
34 + char *keyword;
35 + keyword_function func;
36 + PARSER_REPERTOIRE repertoire;
37 + size_t worker_job_id;
38 +} PARSER_KEYWORD;
39
40 typedef struct parser_user_object {
14 - PARSER *parser;
41 RRDSET *st;
42 RRDHOST *host;
43 void *opaque;
@@ -54,9 +80,138 @@ typedef struct parser_user_object {
80 } v2;
81 } PARSER_USER_OBJECT;
82
57 -PARSER_RC pluginsd_function(char **words, size_t num_words, void *user);
58 -PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, void *user);
83 +typedef struct parser {
84 + uint8_t version; // Parser version
85 + PARSER_REPERTOIRE repertoire;
86 + uint32_t flags;
87 + int fd; // Socket
88 + size_t line;
89 + FILE *fp_input; // Input source e.g. stream
90 + FILE *fp_output; // Stream to send commands to plugin
91 +
92 +#ifdef ENABLE_HTTPS
93 + NETDATA_SSL *ssl_output;
94 +#endif
95 +
96 + PARSER_USER_OBJECT user; // User defined structure to hold extra state between calls
97 +
98 + struct {
99 + const char *end_keyword;
100 + BUFFER *response;
101 + void (*action)(struct parser *parser, void *action_data);
102 + void *action_data;
103 + } defer;
104 +
105 + struct {
106 + DICTIONARY *functions;
107 + usec_t smaller_timeout;
108 + } inflight;
109 +
110 + struct {
111 + SPINLOCK spinlock;
112 + } writer;
113 +
114 +} PARSER;
115 +
116 +static inline int find_first_keyword(const char *src, char *dst, int dst_size, bool *isspace_map) {
117 + const char *s = src, *keyword_start;
118 +
119 + while (unlikely(isspace_map[(uint8_t)*s])) s++;
120 + keyword_start = s;
121 +
122 + while (likely(*s && !isspace_map[(uint8_t)*s]) && dst_size > 1) {
123 + *dst++ = *s++;
124 + dst_size--;
125 + }
126 + *dst = '\0';
127 + return dst_size == 0 ? 0 : (int) (s - keyword_start);
128 +}
129 +
130 +PARSER_KEYWORD *gperf_lookup_keyword(register const char *str, register size_t len);
131 +
132 +static inline PARSER_KEYWORD *parser_find_keyword(PARSER *parser, const char *command) {
133 + PARSER_KEYWORD *t = gperf_lookup_keyword(command, strlen(command));
134 + if(t && (t->repertoire & parser->repertoire))
135 + return t;
136 +
137 + return NULL;
138 +}
139 +
140 +static inline int parser_action(PARSER *parser, char *input) {
141 + parser->line++;
142 +
143 + if(unlikely(parser->flags & PARSER_DEFER_UNTIL_KEYWORD)) {
144 + char command[PLUGINSD_LINE_MAX + 1];
145 + bool has_keyword = find_first_keyword(input, command, PLUGINSD_LINE_MAX, isspace_map_pluginsd);
146 +
147 + if(!has_keyword || strcmp(command, parser->defer.end_keyword) != 0) {
148 + if(parser->defer.response) {
149 + buffer_strcat(parser->defer.response, input);
150 + if(buffer_strlen(parser->defer.response) > 10 * 1024 * 1024) {
151 + // more than 10MB of data
152 + // a bad plugin that did not send the end_keyword
153 + internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));
154 + return 1;
155 + }
156 + }
157 + return 0;
158 + }
159 + else {
160 + // call the action
161 + parser->defer.action(parser, parser->defer.action_data);
162 +
163 + // empty everything
164 + parser->defer.action = NULL;
165 + parser->defer.action_data = NULL;
166 + parser->defer.end_keyword = NULL;
167 + parser->defer.response = NULL;
168 + parser->flags &= ~PARSER_DEFER_UNTIL_KEYWORD;
169 + }
170 + return 0;
171 + }
172 +
173 + char *words[PLUGINSD_MAX_WORDS];
174 + size_t num_words = quoted_strings_splitter_pluginsd(input, words, PLUGINSD_MAX_WORDS);
175 + const char *command = get_word(words, num_words, 0);
176 +
177 + if(unlikely(!command))
178 + return 0;
179 +
180 + PARSER_RC rc;
181 + PARSER_KEYWORD *t = parser_find_keyword(parser, command);
182 + if(likely(t)) {
183 + worker_is_busy(t->worker_job_id);
184 + rc = (*t->func)(words, num_words, parser);
185 + worker_is_idle();
186 + }
187 + else
188 + rc = PARSER_RC_ERROR;
189 +
190 + if(rc == PARSER_RC_ERROR) {
191 + BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
192 + for(size_t i = 0; i < num_words ;i++) {
193 + if(i) buffer_fast_strcat(wb, " ", 1);
194 +
195 + buffer_fast_strcat(wb, "\"", 1);
196 + const char *s = get_word(words, num_words, i);
197 + buffer_strcat(wb, s?s:"");
198 + buffer_fast_strcat(wb, "\"", 1);
199 + }
200 +
201 + error("PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)",
202 + command, parser->line, buffer_tostring(wb));
203 +
204 + buffer_free(wb);
205 + }
206 +
207 + return (rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP);
208 +}
209 +
210 +PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_output, int fd, PARSER_INPUT_TYPE flags, void *ssl);
211 +void parser_init_repertoire(PARSER *parser, PARSER_REPERTOIRE repertoire);
212 +void parser_destroy(PARSER *working_parser);
213 +void pluginsd_cleanup_v2(PARSER *parser);
214 void inflight_functions_init(PARSER *parser);
60 -void pluginsd_keywords_init(PARSER *parser, PLUGINSD_KEYWORDS types);
215 +void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire);
216
217 #endif //NETDATA_PLUGINSD_PARSER_H
collectors/statsd.plugin/statsd.c
+1 -1
@@ -1456,7 +1456,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1456 else if (!strcmp(name, "dimension")) {
1457 // metric [name [type [multiplier [divisor]]]]
1458 char *words[10] = { NULL };
1459 - size_t num_words = pluginsd_split_words(value, words, 10);
1459 + size_t num_words = quoted_strings_splitter_pluginsd(value, words, 10);
1460
1461 int pattern = 0;
1462 size_t i = 0;
configure.ac
-1
@@ -1877,7 +1877,6 @@ AC_CONFIG_FILES([
1877 libnetdata/locks/Makefile
1878 libnetdata/log/Makefile
1879 libnetdata/onewayalloc/Makefile
1880 - libnetdata/parser/Makefile
1880 libnetdata/popen/Makefile
1881 libnetdata/procfile/Makefile
1882 libnetdata/simple_pattern/Makefile
daemon/main.c
+6 -2
@@ -110,10 +110,10 @@ bool service_running(SERVICE_TYPE service) {
110 if(unlikely(!sth))
111 sth = service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, NULL, NULL, false);
112
113 - if(netdata_exit)
113 + if(unlikely(netdata_exit))
114 __atomic_store_n(&service_globals.running, 0, __ATOMIC_RELAXED);
115
116 - if(service == 0)
116 + if(unlikely(service == 0))
117 service = sth->services;
118
119 sth->services |= service;
@@ -1555,6 +1555,10 @@ int main(int argc, char **argv) {
1555 unittest_running = true;
1556 return julytest();
1557 }
1558 + else if(strcmp(optarg, "parsertest") == 0) {
1559 + unittest_running = true;
1560 + return pluginsd_parser_unittest();
1561 + }
1562 else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
1563 optarg += strlen(createdataset_string);
1564 unsigned history_seconds = strtoul(optarg, NULL, 0);
database/contexts/api_v2.c
+5 -6
@@ -309,7 +309,7 @@ static void rrdhost_receiver_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *
309
310 if(s->ingest.type == RRDHOST_INGEST_TYPE_CHILD) {
311 if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE)
312 - buffer_json_member_add_string(wb, "reason", s->ingest.reason);
312 + buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->ingest.reason));
313
314 if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING) {
315 buffer_json_member_add_object(wb, "replication");
@@ -353,7 +353,7 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke
353 buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since);
354
355 if (s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE)
356 - buffer_json_member_add_string(wb, "reason", s->stream.reason);
356 + buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->stream.reason));
357
358 if (s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) {
359 buffer_json_member_add_object(wb, "replication");
@@ -400,10 +400,9 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke
400 else
401 buffer_json_member_add_string(wb, "destination", string2str(d->destination));
402
403 - buffer_json_member_add_time_t(wb, "last_check", d->last_attempt);
404 - buffer_json_member_add_time_t(wb, "age", s->now - d->last_attempt);
405 - buffer_json_member_add_string_or_omit(wb, "last_error", d->last_error);
406 - buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->last_handshake));
403 + buffer_json_member_add_time_t(wb, "since", d->since);
404 + buffer_json_member_add_time_t(wb, "age", s->now - d->since);
405 + buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->reason));
406 if(d->postpone_reconnection_until > s->now) {
407 buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until);
408 buffer_json_member_add_time_t(wb, "next_in", d->postpone_reconnection_until - s->now);
database/rrd.h
+1 -1
@@ -1189,7 +1189,7 @@ struct rrdhost {
1189 struct rrdpush_destinations *destination; // the current destination from the above list
1190 SIMPLE_PATTERN *rrdpush_send_charts_matching; // pattern to match the charts to be sent
1191
1192 - const char *rrdpush_last_receiver_exit_reason;
1192 + int32_t rrdpush_last_receiver_exit_reason;
1193 time_t rrdpush_seconds_to_replicate; // max time we want to replicate from the child
1194 time_t rrdpush_replication_step; // seconds per replication step
1195 size_t rrdpush_receiver_replicating_charts; // the number of charts currently being replicated from a child
database/rrdfunctions.c
+5 -5
@@ -884,7 +884,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
884 buffer_json_add_array_item_string(wb, NULL); // InSince
885 buffer_json_add_array_item_string(wb, NULL); // InAge
886 }
887 - buffer_json_add_array_item_string(wb, s.ingest.reason); // InReason
887 + buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(s.ingest.reason)); // InReason
888 buffer_json_add_array_item_uint64(wb, s.ingest.hops); // InHops
889 buffer_json_add_array_item_double(wb, s.ingest.replication.completion); // InReplCompletion
890 buffer_json_add_array_item_uint64(wb, s.ingest.replication.instances); // InReplInstances
@@ -904,7 +904,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
904 buffer_json_add_array_item_string(wb, NULL); // OutSince
905 buffer_json_add_array_item_string(wb, NULL); // OutAge
906 }
907 - buffer_json_add_array_item_string(wb, s.stream.reason); // OutReason
907 + buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(s.stream.reason)); // OutReason
908 buffer_json_add_array_item_uint64(wb, s.stream.hops); // OutHops
909 buffer_json_add_array_item_double(wb, s.stream.replication.completion); // OutReplCompletion
910 buffer_json_add_array_item_uint64(wb, s.stream.replication.instances); // OutReplInstances
@@ -923,10 +923,10 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
923 buffer_json_add_array_item_array(wb); // OutAttemptHandshake
924 time_t last_attempt = 0;
925 for(struct rrdpush_destinations *d = host->destinations; d ; d = d->next) {
926 - if(d->last_attempt > last_attempt)
927 - last_attempt = d->last_attempt;
926 + if(d->since > last_attempt)
927 + last_attempt = d->since;
928
929 - buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(d->last_handshake));
929 + buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(d->reason));
930 }
931 buffer_json_array_close(wb); // // OutAttemptHandshake
932
database/rrdhost.c
+7 -16
@@ -1118,10 +1118,8 @@ static void rrdhost_streaming_sender_structures_init(RRDHOST *host)
1118 host->sender->rrdpush_sender_socket = -1;
1119
1120 #ifdef ENABLE_COMPRESSION
1121 - if(default_compression_enabled) {
1121 + if(default_compression_enabled)
1122 host->sender->flags |= SENDER_FLAG_COMPRESSION;
1123 - host->sender->compressor = create_compressor();
1124 - }
1123 else
1124 host->sender->flags &= ~SENDER_FLAG_COMPRESSION;
1125 #endif
@@ -1137,11 +1135,10 @@ static void rrdhost_streaming_sender_structures_free(RRDHOST *host)
1135 if (unlikely(!host->sender))
1136 return;
1137
1140 - rrdpush_sender_thread_stop(host, "HOST CLEANUP", true); // stop a possibly running thread
1138 + rrdpush_sender_thread_stop(host, STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, true); // stop a possibly running thread
1139 cbuffer_free(host->sender->buffer);
1140 #ifdef ENABLE_COMPRESSION
1143 - if (host->sender->compressor)
1144 - host->sender->compressor->destroy(&host->sender->compressor);
1141 + rrdpush_compressor_destroy(&host->sender->compressor);
1142 #endif
1143 replication_cleanup_sender(host->sender);
1144
@@ -1174,7 +1171,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1171 rrdhost_streaming_sender_structures_free(host);
1172
1173 if (netdata_exit || force)
1177 - stop_streaming_receiver(host, "HOST CLEANUP");
1174 + stop_streaming_receiver(host, STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP);
1175
1176
1177 // ------------------------------------------------------------------------
@@ -1761,7 +1758,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1758 // --- ingest ---
1759
1760 s->ingest.since = MAX(host->child_connect_time, host->child_disconnected_time);
1764 - s->ingest.reason = (online) ? "" : host->rrdpush_last_receiver_exit_reason;
1761 + s->ingest.reason = (online) ? STREAM_HANDSHAKE_NEVER : host->rrdpush_last_receiver_exit_reason;
1762
1763 netdata_mutex_lock(&host->receiver_lock);
1764 s->ingest.hops = (host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);
@@ -1819,9 +1816,6 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1816 if(!s->ingest.since)
1817 s->ingest.since = netdata_start_time;
1818
1822 - if(!s->ingest.reason)
1823 - s->ingest.reason = "";
1824 -
1819 if(s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE)
1820 s->db.liveness = RRDHOST_DB_LIVENESS_LIVE;
1821 else
@@ -1847,7 +1841,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1841
1842 if (rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED)) {
1843 s->stream.hops = host->sender->hops;
1850 - s->stream.reason = "";
1844 + s->stream.reason = STREAM_HANDSHAKE_NEVER;
1845 s->stream.capabilities = host->sender->capabilities;
1846
1847 s->stream.replication.completion = rrdhost_sender_replication_completion_unsafe(host, now, &s->stream.replication.instances);
@@ -1859,7 +1853,7 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1853 s->stream.status = RRDHOST_STREAM_STATUS_ONLINE;
1854
1855 #ifdef ENABLE_COMPRESSION
1862 - s->stream.compression = (stream_has_capability(host->sender, STREAM_CAP_COMPRESSION) && host->sender->compressor);
1856 + s->stream.compression = (stream_has_capability(host->sender, STREAM_CAP_COMPRESSION) && host->sender->compressor.initialized);
1857 #endif
1858 }
1859 else {
@@ -1876,9 +1870,6 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1870 if(!s->stream.since)
1871 s->stream.since = netdata_start_time;
1872
1879 - if(!s->stream.reason)
1880 - s->stream.reason = "";
1881 -
1873 // --- ml ---
1874
1875 if(ml_host_get_host_status(host, &s->ml.metrics)) {
libnetdata/Makefile.am
-1
@@ -20,7 +20,6 @@ SUBDIRS = \
20 locks \
21 log \
22 onewayalloc \
23 - parser \
23 popen \
24 procfile \
25 simple_pattern \
libnetdata/libnetdata.c
+15 -93
@@ -1684,7 +1684,7 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c
1684 return value;
1685 }
1686
1687 -inline int pluginsd_space(char c) {
1687 +inline int pluginsd_isspace(char c) {
1688 switch(c) {
1689 case ' ':
1690 case '\t':
@@ -1698,8 +1698,7 @@ inline int pluginsd_space(char c) {
1698 }
1699 }
1700
1701 -inline int config_isspace(char c)
1702 -{
1701 +inline int config_isspace(char c) {
1702 switch (c) {
1703 case ' ':
1704 case '\t':
@@ -1713,100 +1712,23 @@ inline int config_isspace(char c)
1712 }
1713 }
1714
1716 -// split a text into words, respecting quotes
1717 -inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char))
1718 -{
1719 - char *s = str, quote = 0;
1720 - size_t i = 0;
1721 -
1722 - // skip all white space
1723 - while (unlikely(custom_isspace(*s)))
1724 - s++;
1725 -
1726 - if(unlikely(!*s)) {
1727 - words[i] = NULL;
1728 - return 0;
1729 - }
1730 -
1731 - // check for quote
1732 - if (unlikely(*s == '\'' || *s == '"')) {
1733 - quote = *s; // remember the quote
1734 - s++; // skip the quote
1735 - }
1736 -
1737 - // store the first word
1738 - words[i++] = s;
1739 -
1740 - // while we have something
1741 - while (likely(*s)) {
1742 - // if it is an escape
1743 - if (unlikely(*s == '\\' && s[1])) {
1744 - s += 2;
1745 - continue;
1746 - }
1747 -
1748 - // if it is a quote
1749 - else if (unlikely(*s == quote)) {
1750 - quote = 0;
1751 - *s = ' ';
1752 - continue;
1753 - }
1754 -
1755 - // if it is a space
1756 - else if (unlikely(quote == 0 && custom_isspace(*s))) {
1757 - // terminate the word
1758 - *s++ = '\0';
1759 -
1760 - // skip all white space
1761 - while (likely(custom_isspace(*s)))
1762 - s++;
1763 -
1764 - // check for a quote
1765 - if (unlikely(*s == '\'' || *s == '"')) {
1766 - quote = *s; // remember the quote
1767 - s++; // skip the quote
1768 - }
1769 -
1770 - // if we reached the end, stop
1771 - if (unlikely(!*s))
1772 - break;
1773 -
1774 - // store the next word
1775 - if (likely(i < max_words))
1776 - words[i++] = s;
1777 - else
1778 - break;
1779 - }
1780 -
1781 - // anything else
1782 - else
1783 - s++;
1784 - }
1785 -
1786 - if (i < max_words)
1787 - words[i] = NULL;
1715 +inline int group_by_label_isspace(char c) {
1716 + if(c == ',' || c == '|')
1717 + return 1;
1718
1789 - return i;
1719 + return 0;
1720 }
1721
1792 -inline size_t pluginsd_split_words(char *str, char **words, size_t max_words)
1793 -{
1794 - return quoted_strings_splitter(str, words, max_words, pluginsd_space);
1795 -}
1722 +bool isspace_map_pluginsd[256] = {};
1723 +bool isspace_map_config[256] = {};
1724 +bool isspace_map_group_by_label[256] = {};
1725
1797 -bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx) {
1798 - if (unlikely(!ptr))
1799 - return false;
1800 - return (ptr->data[idx / 64] & (1ULL << (idx % 64)));
1801 -}
1802 -
1803 -void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) {
1804 - if (unlikely(!ptr))
1805 - return;
1806 - if (likely(value))
1807 - ptr->data[idx / 64] |= (1ULL << (idx % 64));
1808 - else
1809 - ptr->data[idx / 64] &= ~(1ULL << (idx % 64));
1726 +__attribute__((constructor)) void initialize_is_space_arrays(void) {
1727 + for(int c = 0; c < 256 ; c++) {
1728 + isspace_map_pluginsd[c] = pluginsd_isspace((char) c);
1729 + isspace_map_config[c] = config_isspace((char) c);
1730 + isspace_map_group_by_label[c] = group_by_label_isspace((char) c);
1731 + }
1732 }
1733
1734 bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length) {
libnetdata/libnetdata.h
+112 -7
@@ -591,23 +591,129 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c
591 // Taken from linux kernel
592 #define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
593
594 +#ifdef ENV32BIT
595 +typedef struct bitmap256 {
596 + uint32_t data[8];
597 +} BITMAP256;
598 +
599 +#define bitmap256_get_bit(ptr, idx) (((ptr)->data[(idx) >> 5] & (1ULL << ((idx) & 31))) != 0)
600 +static inline void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) {
601 + uint32_t mask = 1U << (idx & 31);
602 + if (value)
603 + ptr->data[idx >> 5] |= mask;
604 + else
605 + ptr->data[idx >> 5] &= ~mask;
606 +}
607 +#else // 64bit
608 typedef struct bitmap256 {
609 uint64_t data[4];
610 } BITMAP256;
611
598 -bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx);
599 -void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value);
612 +#define bitmap256_get_bit(ptr, idx) ((ptr)->data[(idx) >> 6] & (1ULL << ((idx) & 63)))
613 +static inline void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) {
614 + uint64_t mask = 1ULL << (idx & 63);
615 + if (value)
616 + ptr->data[idx >> 6] |= mask;
617 + else
618 + ptr->data[idx >> 6] &= ~mask;
619 +}
620 +#endif // 64bit
621
622 #define COMPRESSION_MAX_MSG_SIZE 0x4000
623 #define PLUGINSD_LINE_MAX (COMPRESSION_MAX_MSG_SIZE - 1024)
624 +int pluginsd_isspace(char c);
625 int config_isspace(char c);
604 -int pluginsd_space(char c);
626 +int group_by_label_isspace(char c);
627 +
628 +extern bool isspace_map_pluginsd[256];
629 +extern bool isspace_map_config[256];
630 +extern bool isspace_map_group_by_label[256];
631 +
632 +static inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, bool *isspace_map) {
633 + char *s = str, quote = 0;
634 + size_t i = 0;
635 +
636 + // skip all white space
637 + while (unlikely(isspace_map[(uint8_t)*s]))
638 + s++;
639 +
640 + if(unlikely(!*s)) {
641 + words[i] = NULL;
642 + return 0;
643 + }
644 +
645 + // check for quote
646 + if (unlikely(*s == '\'' || *s == '"')) {
647 + quote = *s; // remember the quote
648 + s++; // skip the quote
649 + }
650 +
651 + // store the first word
652 + words[i++] = s;
653 +
654 + // while we have something
655 + while (likely(*s)) {
656 + // if it is an escape
657 + if (unlikely(*s == '\\' && s[1])) {
658 + s += 2;
659 + continue;
660 + }
661 +
662 + // if it is a quote
663 + else if (unlikely(*s == quote)) {
664 + quote = 0;
665 + *s = ' ';
666 + continue;
667 + }
668 +
669 + // if it is a space
670 + else if (unlikely(quote == 0 && isspace_map[(uint8_t)*s])) {
671 + // terminate the word
672 + *s++ = '\0';
673 +
674 + // skip all white space
675 + while (likely(isspace_map[(uint8_t)*s]))
676 + s++;
677 +
678 + // check for a quote
679 + if (unlikely(*s == '\'' || *s == '"')) {
680 + quote = *s; // remember the quote
681 + s++; // skip the quote
682 + }
683 +
684 + // if we reached the end, stop
685 + if (unlikely(!*s))
686 + break;
687 +
688 + // store the next word
689 + if (likely(i < max_words))
690 + words[i++] = s;
691 + else
692 + break;
693 + }
694 +
695 + // anything else
696 + else
697 + s++;
698 + }
699 +
700 + if (likely(i < max_words))
701 + words[i] = NULL;
702 +
703 + return i;
704 +}
705 +
706 +#define quoted_strings_splitter_query_group_by_label(str, words, max_words) \
707 + quoted_strings_splitter(str, words, max_words, isspace_map_group_by_label)
708 +
709 +#define quoted_strings_splitter_config(str, words, max_words) \
710 + quoted_strings_splitter(str, words, max_words, isspace_map_config)
711
606 -size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char));
607 -size_t pluginsd_split_words(char *str, char **words, size_t max_words);
712 +#define quoted_strings_splitter_pluginsd(str, words, max_words) \
713 + quoted_strings_splitter(str, words, max_words, isspace_map_pluginsd)
714
715 static inline char *get_word(char **words, size_t num_words, size_t index) {
610 - if (index >= num_words)
716 + if (unlikely(index >= num_words))
717 return NULL;
718
719 return words[index];
@@ -664,7 +770,6 @@ extern char *netdata_configured_host_prefix;
770 #include "libnetdata/aral/aral.h"
771 #include "onewayalloc/onewayalloc.h"
772 #include "worker_utilization/worker_utilization.h"
667 -#include "parser/parser.h"
773 #include "yaml.h"
774 #include "http/http_defs.h"
775 #include "gorilla/gorilla.h"
libnetdata/parser/Makefile.am deleted
-9
@@ -1,9 +0,0 @@
1 -# SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -AUTOMAKE_OPTIONS = subdir-objects
4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 -
6 -dist_noinst_DATA = \
7 - README.md \
8 - $(NULL)
9 -
libnetdata/parser/README.md deleted
-28
@@ -1,28 +0,0 @@
1 -<!--
2 -title: "Parser"
3 -custom_edit_url: https://github.com/netdata/netdata/blob/master/parser/README.md
4 -sidebar_label: "Parser"
5 -learn_status: "Published"
6 -learn_topic_type: "References"
7 -learn_rel_path: "Developers/Database"
8 --->
9 -
10 -# Parser
11 -
12 -## Introduction
13 -
14 -Generic parser that is used to register keywords and a corresponding function that will be executed when that
15 -keyword is encountered in the command stream (either from plugins or via streaming)
16 -
17 -To use a parser do the following:
18 -
19 -1. Define a structure that will be used to share user state across calls (user defined `void *user`)
20 -2. Initialize the parser using `parser_init`
21 -3. Register keywords with their associated callback function using `parser_add_keyword`
22 -4. Start a loop for as long there is input (or parser_action returns error)
23 - 1. Fetch the next line using `parser_next` (if needed)
24 - 2. Process the line using `parser_action`
25 -5. Release the parser using `parser_destroy`
26 -6. Release the user structure
27 -
28 -See examples in receiver.c / pluginsd_parser.c
libnetdata/parser/parser.c deleted
-277
@@ -1,277 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -#include <poll.h>
3 -#include <stdio.h>
4 -
5 -#include "parser.h"
6 -#include "collectors/plugins.d/pluginsd_parser.h"
7 -
8 -static inline int find_first_keyword(const char *src, char *dst, int dst_size, int (*custom_isspace)(char)) {
9 - const char *s = src, *keyword_start;
10 -
11 - while (unlikely(custom_isspace(*s))) s++;
12 - keyword_start = s;
13 -
14 - while (likely(*s && !custom_isspace(*s)) && dst_size > 1) {
15 - *dst++ = *s++;
16 - dst_size--;
17 - }
18 - *dst = '\0';
19 - return dst_size == 0 ? 0 : (int) (s - keyword_start);
20 -}
21 -
22 -/*
23 - * Initialize a parser
24 - * user : as defined by the user, will be shared across calls
25 - * input : main input stream (auto detect stream -- file, socket, pipe)
26 - * buffer : This is the buffer to be used (if null a buffer of size will be allocated)
27 - * size : buffer size either passed or will be allocated
28 - * If the buffer is auto allocated, it will auto freed when the parser is destroyed
29 - *
30 - *
31 - */
32 -
33 -PARSER *parser_init(void *user, FILE *fp_input, FILE *fp_output, int fd,
34 - PARSER_INPUT_TYPE flags, void *ssl __maybe_unused)
35 -{
36 - PARSER *parser;
37 -
38 - parser = callocz(1, sizeof(*parser));
39 - parser->user = user;
40 - parser->fd = fd;
41 - parser->fp_input = fp_input;
42 - parser->fp_output = fp_output;
43 -#ifdef ENABLE_HTTPS
44 - parser->ssl_output = ssl;
45 -#endif
46 - parser->flags = flags;
47 - parser->worker_job_next_id = WORKER_PARSER_FIRST_JOB;
48 -
49 - spinlock_init(&parser->writer.spinlock);
50 - return parser;
51 -}
52 -
53 -
54 -static inline PARSER_KEYWORD *parser_find_keyword(PARSER *parser, const char *command) {
55 - uint32_t hash = parser_hash_function(command);
56 - uint32_t slot = hash % PARSER_KEYWORDS_HASHTABLE_SIZE;
57 - PARSER_KEYWORD *t = parser->keywords.hashtable[slot];
58 -
59 - if(likely(t && strcmp(t->keyword, command) == 0))
60 - return t;
61 -
62 - return NULL;
63 -}
64 -
65 -/*
66 - * Add a keyword and the corresponding function that will be called
67 - * Multiple functions may be added
68 - * Input : keyword
69 - * : callback function
70 - * : flags
71 - * Output: > 0 registered function number
72 - * : 0 Error
73 - */
74 -
75 -void parser_add_keyword(PARSER *parser, char *keyword, keyword_function func) {
76 - if(unlikely(!parser || !keyword || !*keyword || !func))
77 - fatal("PARSER: invalid parameters");
78 -
79 - PARSER_KEYWORD *t = callocz(1, sizeof(*t));
80 - t->worker_job_id = parser->worker_job_next_id++;
81 - t->keyword = strdupz(keyword);
82 - t->func = func;
83 -
84 - uint32_t hash = parser_hash_function(keyword);
85 - uint32_t slot = hash % PARSER_KEYWORDS_HASHTABLE_SIZE;
86 -
87 - if(unlikely(parser->keywords.hashtable[slot]))
88 - fatal("PARSER: hashtable collision between keyword '%s' and '%s' on slot %u. "
89 - "Change the hashtable size and / or the hashing function. "
90 - "Run the unit test to find the optimal values.",
91 - parser->keywords.hashtable[slot]->keyword,
92 - t->keyword,
93 - slot
94 - );
95 -
96 - parser->keywords.hashtable[slot] = t;
97 -
98 - worker_register_job_name(t->worker_job_id, t->keyword);
99 -}
100 -
101 -/*
102 - * Cleanup a previously allocated parser
103 - */
104 -
105 -void parser_destroy(PARSER *parser)
106 -{
107 - if (unlikely(!parser))
108 - return;
109 -
110 - dictionary_destroy(parser->inflight.functions);
111 -
112 - // Remove keywords
113 - for(size_t i = 0 ; i < PARSER_KEYWORDS_HASHTABLE_SIZE; i++) {
114 - PARSER_KEYWORD *t = parser->keywords.hashtable[i];
115 - if (t) {
116 - freez(t->keyword);
117 - freez(t);
118 - }
119 - }
120 -
121 - freez(parser);
122 -}
123 -
124 -
125 -/*
126 - * Fetch the next line to process
127 - *
128 - */
129 -
130 -typedef enum {
131 - PARSER_FGETS_RESULT_OK,
132 - PARSER_FGETS_RESULT_TIMEOUT,
133 - PARSER_FGETS_RESULT_ERROR,
134 - PARSER_FGETS_RESULT_EOF,
135 -} PARSER_FGETS_RESULT;
136 -
137 -static inline PARSER_FGETS_RESULT parser_fgets(char *s, int size, FILE *stream) {
138 - errno = 0;
139 -
140 - struct pollfd fds[1];
141 - int timeout_msecs = 2 * 60 * MSEC_PER_SEC;
142 -
143 - fds[0].fd = fileno(stream);
144 - fds[0].events = POLLIN;
145 -
146 - int ret = poll(fds, 1, timeout_msecs);
147 -
148 - if (ret > 0) {
149 - /* There is data to read */
150 - if (fds[0].revents & POLLIN) {
151 - char *tmp = fgets(s, size, stream);
152 -
153 - if(unlikely(!tmp)) {
154 - if (feof(stream)) {
155 - error("PARSER: read failed: end of file.");
156 - return PARSER_FGETS_RESULT_EOF;
157 - }
158 -
159 - else if (ferror(stream)) {
160 - error("PARSER: read failed: input error.");
161 - return PARSER_FGETS_RESULT_ERROR;
162 - }
163 -
164 - error("PARSER: read failed: unknown error.");
165 - return PARSER_FGETS_RESULT_ERROR;
166 - }
167 -
168 - return PARSER_FGETS_RESULT_OK;
169 - }
170 - else if(fds[0].revents & POLLERR) {
171 - error("PARSER: read failed: POLLERR.");
172 - return PARSER_FGETS_RESULT_ERROR;
173 - }
174 - else if(fds[0].revents & POLLHUP) {
175 - error("PARSER: read failed: POLLHUP.");
176 - return PARSER_FGETS_RESULT_ERROR;
177 - }
178 - else if(fds[0].revents & POLLNVAL) {
179 - error("PARSER: read failed: POLLNVAL.");
180 - return PARSER_FGETS_RESULT_ERROR;
181 - }
182 -
183 - error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set.");
184 - return PARSER_FGETS_RESULT_ERROR;
185 - }
186 - else if (ret == 0) {
187 - error("PARSER: timeout while waiting for data.");
188 - return PARSER_FGETS_RESULT_TIMEOUT;
189 - }
190 -
191 - error("PARSER: poll() failed with code %d.", ret);
192 - return PARSER_FGETS_RESULT_ERROR;
193 -}
194 -
195 -int parser_next(PARSER *parser, char *buffer, size_t buffer_size) {
196 - if(likely(parser_fgets(buffer, (int)buffer_size, (FILE *)parser->fp_input) == PARSER_FGETS_RESULT_OK))
197 - return 0;
198 -
199 - return 1;
200 -}
201 -
202 -/*
203 -* Takes an initialized parser object that has an unprocessed entry (by calling parser_next)
204 -* and if it contains a valid keyword, it will execute all the callbacks
205 -*
206 -*/
207 -
208 -inline int parser_action(PARSER *parser, char *input)
209 -{
210 - parser->line++;
211 -
212 - if(unlikely(parser->flags & PARSER_DEFER_UNTIL_KEYWORD)) {
213 - char command[PLUGINSD_LINE_MAX + 1];
214 - bool has_keyword = find_first_keyword(input, command, PLUGINSD_LINE_MAX, pluginsd_space);
215 -
216 - if(!has_keyword || strcmp(command, parser->defer.end_keyword) != 0) {
217 - if(parser->defer.response) {
218 - buffer_strcat(parser->defer.response, input);
219 - if(buffer_strlen(parser->defer.response) > 10 * 1024 * 1024) {
220 - // more than 10MB of data
221 - // a bad plugin that did not send the end_keyword
222 - internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));
223 - return 1;
224 - }
225 - }
226 - return 0;
227 - }
228 - else {
229 - // call the action
230 - parser->defer.action(parser, parser->defer.action_data);
231 -
232 - // empty everything
233 - parser->defer.action = NULL;
234 - parser->defer.action_data = NULL;
235 - parser->defer.end_keyword = NULL;
236 - parser->defer.response = NULL;
237 - parser->flags &= ~PARSER_DEFER_UNTIL_KEYWORD;
238 - }
239 - return 0;
240 - }
241 -
242 - char *words[PLUGINSD_MAX_WORDS];
243 - size_t num_words = pluginsd_split_words(input, words, PLUGINSD_MAX_WORDS);
244 - const char *command = get_word(words, num_words, 0);
245 -
246 - if(unlikely(!command))
247 - return 0;
248 -
249 - PARSER_RC rc;
250 - PARSER_KEYWORD *t = parser_find_keyword(parser, command);
251 - if(likely(t)) {
252 - worker_is_busy(t->worker_job_id);
253 - rc = (*t->func)(words, num_words, parser->user);
254 - worker_is_idle();
255 - }
256 - else
257 - rc = PARSER_RC_ERROR;
258 -
259 - if(rc == PARSER_RC_ERROR) {
260 - BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
261 - for(size_t i = 0; i < num_words ;i++) {
262 - if(i) buffer_fast_strcat(wb, " ", 1);
263 -
264 - buffer_fast_strcat(wb, "\"", 1);
265 - const char *s = get_word(words, num_words, i);
266 - buffer_strcat(wb, s?s:"");
267 - buffer_fast_strcat(wb, "\"", 1);
268 - }
269 -
270 - error("PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)",
271 - command, parser->line, buffer_tostring(wb));
272 -
273 - buffer_free(wb);
274 - }
275 -
276 - return (rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP);
277 -}
libnetdata/parser/parser.h deleted
-105
@@ -1,105 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef NETDATA_INCREMENTAL_PARSER_H
4 -#define NETDATA_INCREMENTAL_PARSER_H 1
5 -
6 -#include "../libnetdata.h"
7 -
8 -#define WORKER_PARSER_FIRST_JOB 3
9 -
10 -// this has to be in-sync with the same at receiver.c
11 -#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)
12 -
13 -#define PARSER_KEYWORDS_HASHTABLE_SIZE 73 // unittest finds this magic number
14 -//#define parser_hash_function(s) djb2_hash32(s)
15 -//#define parser_hash_function(s) fnv1_hash32(s)
16 -//#define parser_hash_function(s) fnv1a_hash32(s)
17 -//#define parser_hash_function(s) larson_hash32(s)
18 -#define parser_hash_function(s) pluginsd_parser_hash32(s)
19 -
20 -// PARSER return codes
21 -typedef enum __attribute__ ((__packed__)) parser_rc {
22 - PARSER_RC_OK, // Callback was successful, go on
23 - PARSER_RC_STOP, // Callback says STOP
24 - PARSER_RC_ERROR // Callback failed (abort rest of callbacks)
25 -} PARSER_RC;
26 -
27 -typedef enum __attribute__ ((__packed__)) parser_input_type {
28 - PARSER_INPUT_SPLIT = (1 << 1),
29 - PARSER_DEFER_UNTIL_KEYWORD = (1 << 2),
30 -} PARSER_INPUT_TYPE;
31 -
32 -typedef PARSER_RC (*keyword_function)(char **words, size_t num_words, void *user_data);
33 -
34 -typedef struct parser_keyword {
35 - size_t worker_job_id;
36 - char *keyword;
37 - keyword_function func;
38 -} PARSER_KEYWORD;
39 -
40 -typedef struct parser {
41 - size_t worker_job_next_id;
42 - uint8_t version; // Parser version
43 - int fd; // Socket
44 - FILE *fp_input; // Input source e.g. stream
45 - FILE *fp_output; // Stream to send commands to plugin
46 -#ifdef ENABLE_HTTPS
47 - NETDATA_SSL *ssl_output;
48 -#endif
49 - void *user; // User defined structure to hold extra state between calls
50 - uint32_t flags;
51 - size_t line;
52 -
53 - struct {
54 - PARSER_KEYWORD *hashtable[PARSER_KEYWORDS_HASHTABLE_SIZE];
55 - } keywords;
56 -
57 - struct {
58 - const char *end_keyword;
59 - BUFFER *response;
60 - void (*action)(struct parser *parser, void *action_data);
61 - void *action_data;
62 - } defer;
63 -
64 - struct {
65 - DICTIONARY *functions;
66 - usec_t smaller_timeout;
67 - } inflight;
68 -
69 - struct {
70 - SPINLOCK spinlock;
71 - } writer;
72 -} PARSER;
73 -
74 -PARSER *parser_init(void *user, FILE *fp_input, FILE *fp_output, int fd, PARSER_INPUT_TYPE flags, void *ssl);
75 -void parser_add_keyword(PARSER *working_parser, char *keyword, keyword_function func);
76 -int parser_next(PARSER *working_parser, char *buffer, size_t buffer_size);
77 -int parser_action(PARSER *working_parser, char *input);
78 -void parser_destroy(PARSER *working_parser);
79 -
80 -PARSER_RC pluginsd_set(char **words, size_t num_words, void *user);
81 -PARSER_RC pluginsd_begin(char **words, size_t num_words, void *user);
82 -PARSER_RC pluginsd_end(char **words, size_t num_words, void *user);
83 -PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user);
84 -PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *user);
85 -PARSER_RC pluginsd_dimension(char **words, size_t num_words, void *user);
86 -PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user);
87 -PARSER_RC pluginsd_flush(char **words, size_t num_words, void *user);
88 -PARSER_RC pluginsd_disable(char **words, size_t num_words, void *user);
89 -PARSER_RC pluginsd_label(char **words, size_t num_words, void *user);
90 -PARSER_RC pluginsd_overwrite(char **words, size_t num_words, void *user);
91 -PARSER_RC pluginsd_clabel_commit(char **words, size_t num_words, void *user);
92 -PARSER_RC pluginsd_clabel(char **words, size_t num_words, void *user);
93 -
94 -PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user);
95 -PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, void *user);
96 -PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, void *user);
97 -PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user);
98 -PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user);
99 -
100 -PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user);
101 -PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user);
102 -PARSER_RC pluginsd_end_v2(char **words, size_t num_words, void *user);
103 -void pluginsd_cleanup_v2(void *user);
104 -
105 -#endif
streaming/compression.c
+79 -220
@@ -5,55 +5,40 @@
5
6 #define STREAM_COMPRESSION_MSG "STREAM_COMPRESSION"
7
8 -// signature MUST end with a newline
9 -#define SIGNATURE ((uint32_t)('z' | 0x80) | (0x80 << 8) | (0x80 << 16) | ('\n' << 24))
10 -#define SIGNATURE_MASK ((uint32_t)0xff | (0x80 << 8) | (0x80 << 16) | (0xff << 24))
11 -#define SIGNATURE_SIZE 4
12 -
13 -
14 -/*
15 - * LZ4 streaming API compressor specific data
16 - */
17 -struct compressor_data {
18 - LZ4_stream_t *stream;
19 - char *input_ring_buffer;
20 - size_t input_ring_buffer_size;
21 - size_t input_ring_buffer_pos;
22 -};
23 -
24 -
8 /*
9 * Reset compressor state for a new stream
10 */
28 -static void lz4_compressor_reset(struct compressor_state *state)
29 -{
30 - if (state->data) {
31 - if (state->data->stream) {
32 - LZ4_resetStream_fast(state->data->stream);
33 - internal_error(true, "%s: compressor reset", STREAM_COMPRESSION_MSG);
34 - }
35 - state->data->input_ring_buffer_pos = 0;
11 +void rrdpush_compressor_reset(struct compressor_state *state) {
12 + if(!state->initialized) {
13 + state->initialized = true;
14 +
15 + state->stream.lz4_stream = LZ4_createStream();
16 + state->stream.input_ring_buffer_size = LZ4_DECODER_RING_BUFFER_SIZE(COMPRESSION_MAX_MSG_SIZE * 2);
17 + state->stream.input_ring_buffer = callocz(1, state->stream.input_ring_buffer_size);
18 + state->compression_result_buffer_size = 0;
19 }
20 +
21 + LZ4_resetStream_fast(state->stream.lz4_stream);
22 +
23 + state->stream.input_ring_buffer_pos = 0;
24 }
25
26 /*
27 * Destroy compressor state and all related data
28 */
42 -static void lz4_compressor_destroy(struct compressor_state **state)
43 -{
44 - if (state && *state) {
45 - struct compressor_state *s = *state;
46 - if (s->data) {
47 - if (s->data->stream)
48 - LZ4_freeStream(s->data->stream);
49 - freez(s->data->input_ring_buffer);
50 - freez(s->data);
51 - }
52 - freez(s->compression_result_buffer);
53 - freez(s);
54 - *state = NULL;
55 - debug(D_STREAM, "%s: Compressor Destroyed.", STREAM_COMPRESSION_MSG);
29 +void rrdpush_compressor_destroy(struct compressor_state *state) {
30 + if (state->stream.lz4_stream) {
31 + LZ4_freeStream(state->stream.lz4_stream);
32 + state->stream.lz4_stream = NULL;
33 }
34 +
35 + freez(state->stream.input_ring_buffer);
36 + state->stream.input_ring_buffer = NULL;
37 +
38 + freez(state->compression_result_buffer);
39 + state->compression_result_buffer = NULL;
40 +
41 + state->initialized = false;
42 }
43
44 /*
@@ -62,18 +47,18 @@ static void lz4_compressor_destroy(struct compressor_state **state)
47 * Return the size of compressed data block as result and the pointer to internal buffer using the last argument
48 * or 0 in case of error
49 */
65 -static size_t lz4_compressor_compress(struct compressor_state *state, const char *data, size_t size, char **out)
66 -{
50 +size_t rrdpush_compress(struct compressor_state *state, const char *data, size_t size, char **out) {
51 if(unlikely(!state || !size || !out))
52 return 0;
53
54 if(unlikely(size > COMPRESSION_MAX_MSG_SIZE)) {
71 - error("%s: Compression Failed - Message size %lu above compression buffer limit: %d", STREAM_COMPRESSION_MSG, (long unsigned int)size, COMPRESSION_MAX_MSG_SIZE);
55 + error("RRDPUSH COMPRESS: Compression Failed - Message size %lu above compression buffer limit: %d",
56 + (long unsigned int)size, COMPRESSION_MAX_MSG_SIZE);
57 return 0;
58 }
59
60 size_t max_dst_size = LZ4_COMPRESSBOUND(size);
76 - size_t data_size = max_dst_size + SIGNATURE_SIZE;
61 + size_t data_size = max_dst_size + RRDPUSH_COMPRESSION_SIGNATURE_SIZE;
62
63 if (!state->compression_result_buffer) {
64 state->compression_result_buffer = mallocz(data_size);
@@ -85,16 +70,16 @@ static size_t lz4_compressor_compress(struct compressor_state *state, const char
70 }
71
72 // the ring buffer always has space for LZ4_MAX_MSG_SIZE
88 - memcpy(state->data->input_ring_buffer + state->data->input_ring_buffer_pos, data, size);
73 + memcpy(state->stream.input_ring_buffer + state->stream.input_ring_buffer_pos, data, size);
74
75 // this call needs the last 64K of our previous data
76 // they are available in the ring buffer
77 long int compressed_data_size = LZ4_compress_fast_continue(
93 - state->data->stream,
94 - state->data->input_ring_buffer + state->data->input_ring_buffer_pos,
95 - state->compression_result_buffer + SIGNATURE_SIZE,
96 - size,
97 - max_dst_size,
78 + state->stream.lz4_stream,
79 + state->stream.input_ring_buffer + state->stream.input_ring_buffer_pos,
80 + state->compression_result_buffer + RRDPUSH_COMPRESSION_SIGNATURE_SIZE,
81 + (int)size,
82 + (int)max_dst_size,
83 1);
84
85 if (compressed_data_size < 0) {
@@ -103,220 +88,94 @@ static size_t lz4_compressor_compress(struct compressor_state *state, const char
88 }
89
90 // update the next writing position of the ring buffer
106 - state->data->input_ring_buffer_pos += size;
107 - if(unlikely(state->data->input_ring_buffer_pos >= state->data->input_ring_buffer_size - COMPRESSION_MAX_MSG_SIZE))
108 - state->data->input_ring_buffer_pos = 0;
91 + state->stream.input_ring_buffer_pos += size;
92 + if(unlikely(state->stream.input_ring_buffer_pos >= state->stream.input_ring_buffer_size - COMPRESSION_MAX_MSG_SIZE))
93 + state->stream.input_ring_buffer_pos = 0;
94
95 // update the signature header
96 uint32_t len = ((compressed_data_size & 0x7f) | 0x80 | (((compressed_data_size & (0x7f << 7)) << 1) | 0x8000)) << 8;
112 - *(uint32_t *)state->compression_result_buffer = len | SIGNATURE;
97 + *(uint32_t *)state->compression_result_buffer = len | RRDPUSH_COMPRESSION_SIGNATURE;
98 *out = state->compression_result_buffer;
99 debug(D_STREAM, "%s: Compressed data header: %ld", STREAM_COMPRESSION_MSG, compressed_data_size);
115 - return compressed_data_size + SIGNATURE_SIZE;
116 -}
117 -
118 -/*
119 - * Create and initialize compressor state
120 - * Return the pointer to compressor_state structure created
121 - */
122 -struct compressor_state *create_compressor()
123 -{
124 - struct compressor_state *state = callocz(1, sizeof(struct compressor_state));
125 -
126 - state->reset = lz4_compressor_reset;
127 - state->compress = lz4_compressor_compress;
128 - state->destroy = lz4_compressor_destroy;
129 -
130 - state->data = callocz(1, sizeof(struct compressor_data));
131 - state->data->stream = LZ4_createStream();
132 - state->data->input_ring_buffer_size = LZ4_DECODER_RING_BUFFER_SIZE(COMPRESSION_MAX_MSG_SIZE * 2);
133 - state->data->input_ring_buffer = callocz(1, state->data->input_ring_buffer_size);
134 - state->compression_result_buffer_size = 0;
135 - state->reset(state);
136 - debug(D_STREAM, "%s: Initialize streaming compression!", STREAM_COMPRESSION_MSG);
137 - return state;
138 -}
139 -
140 -/*
141 - * LZ4 streaming API decompressor specific data
142 - */
143 -struct decompressor_stream {
144 - LZ4_streamDecode_t *lz4_stream;
145 - char *buffer;
146 - size_t size;
147 - size_t write_at;
148 - size_t read_at;
149 -};
150 -
151 -/*
152 - * Reset decompressor state for a new stream
153 - */
154 -static void lz4_decompressor_reset(struct decompressor_state *state)
155 -{
156 - if (state->stream) {
157 - if (state->stream->lz4_stream)
158 - LZ4_setStreamDecode(state->stream->lz4_stream, NULL, 0);
159 -
160 - state->stream->write_at = 0;
161 - state->stream->read_at = 0;
162 - }
163 -}
164 -
165 -/*
166 - * Destroy decompressor state and all related data
167 - */
168 -static void lz4_decompressor_destroy(struct decompressor_state **state)
169 -{
170 - if (state && *state) {
171 - struct decompressor_state *s = *state;
172 - if (s->stream) {
173 - debug(D_STREAM, "%s: Destroying decompressor.", STREAM_COMPRESSION_MSG);
174 - if (s->stream->lz4_stream)
175 - LZ4_freeStreamDecode(s->stream->lz4_stream);
176 - freez(s->stream->buffer);
177 - freez(s->stream);
178 - }
179 - freez(s);
180 - *state = NULL;
181 - }
182 -}
183 -
184 -static size_t decode_compress_header(const char *data, size_t data_size) {
185 - if (unlikely(!data || !data_size))
186 - return 0;
187 -
188 - if (unlikely(data_size != SIGNATURE_SIZE))
189 - return 0;
190 -
191 - uint32_t sign = *(uint32_t *)data;
192 - if (unlikely((sign & SIGNATURE_MASK) != SIGNATURE))
193 - return 0;
194 -
195 - size_t length = ((sign >> 8) & 0x7f) | ((sign >> 9) & (0x7f << 7));
196 - return length;
197 -}
198 -
199 -/*
200 - * Start the collection of compressed data in an internal buffer
201 - * Return the size of compressed data or 0 for uncompressed data
202 - */
203 -static size_t lz4_decompressor_start(struct decompressor_state *state __maybe_unused, const char *header, size_t header_size) {
204 - if(unlikely(state->stream->read_at != state->stream->write_at))
205 - fatal("%s: asked to decompress new data, while there are unread data in the decompression buffer!"
206 - , STREAM_COMPRESSION_MSG);
207 -
208 - return decode_compress_header(header, header_size);
100 + return compressed_data_size + RRDPUSH_COMPRESSION_SIGNATURE_SIZE;
101 }
102
103 /*
104 * Decompress the compressed data in the internal buffer
105 * Return the size of uncompressed data or 0 for error
106 */
215 -static size_t lz4_decompressor_decompress(struct decompressor_state *state, const char *compressed_data, size_t compressed_size) {
107 +size_t rrdpush_decompress(struct decompressor_state *state, const char *compressed_data, size_t compressed_size) {
108 if (unlikely(!state || !compressed_data || !compressed_size))
109 return 0;
110
219 - if(unlikely(state->stream->read_at != state->stream->write_at))
220 - fatal("%s: asked to decompress new data, while there are unread data in the decompression buffer!"
221 - , STREAM_COMPRESSION_MSG);
111 + if(unlikely(state->stream.read_at != state->stream.write_at))
112 + fatal("RRDPUSH_DECOMPRESS: asked to decompress new data, while there are unread data in the decompression buffer!");
113
223 - if (unlikely(state->stream->write_at >= state->stream->size / 2)) {
224 - state->stream->write_at = 0;
225 - state->stream->read_at = 0;
114 + if (unlikely(state->stream.write_at >= state->stream.size / 2)) {
115 + state->stream.write_at = 0;
116 + state->stream.read_at = 0;
117 }
118
119 long int decompressed_size = LZ4_decompress_safe_continue(
229 - state->stream->lz4_stream
120 + state->stream.lz4_stream
121 , compressed_data
231 - , state->stream->buffer + state->stream->write_at
122 + , state->stream.buffer + state->stream.write_at
123 , (int)compressed_size
233 - , (int)(state->stream->size - state->stream->write_at)
124 + , (int)(state->stream.size - state->stream.write_at)
125 );
126
127 if (unlikely(decompressed_size < 0)) {
237 - error("%s: decompressor returned negative decompressed bytes: %ld", STREAM_COMPRESSION_MSG, decompressed_size);
128 + error("RRDPUSH DECOMPRESS: decompressor returned negative decompressed bytes: %ld", decompressed_size);
129 return 0;
130 }
131
241 - if(unlikely(decompressed_size + state->stream->write_at > state->stream->size))
242 - fatal("%s: decompressor overflown the stream_buffer. size: %zu, pos: %zu, added: %ld, exceeding the buffer by %zu"
243 - , STREAM_COMPRESSION_MSG
244 - , state->stream->size
245 - , state->stream->write_at
132 + if(unlikely(decompressed_size + state->stream.write_at > state->stream.size))
133 + fatal("RRDPUSH DECOMPRESS: decompressor overflown the stream_buffer. size: %zu, pos: %zu, added: %ld, "
134 + "exceeding the buffer by %zu"
135 + , state->stream.size
136 + , state->stream.write_at
137 , decompressed_size
247 - , (size_t)(state->stream->write_at + decompressed_size - state->stream->size)
138 + , (size_t)(state->stream.write_at + decompressed_size - state->stream.size)
139 );
140
250 - state->stream->write_at += decompressed_size;
141 + state->stream.write_at += decompressed_size;
142
143 // statistics
253 - state->total_compressed += compressed_size + SIGNATURE_SIZE;
144 + state->total_compressed += compressed_size + RRDPUSH_COMPRESSION_SIGNATURE_SIZE;
145 state->total_uncompressed += decompressed_size;
146 state->packet_count++;
147
148 return decompressed_size;
149 }
150
260 -/*
261 - * Return the size of uncompressed data left in the internal buffer or 0 for error
262 - */
263 -static size_t lz4_decompressor_decompressed_bytes_in_buffer(struct decompressor_state *state) {
264 - if(unlikely(state->stream->read_at > state->stream->write_at))
265 - fatal("%s: invalid read/write stream positions"
266 - , STREAM_COMPRESSION_MSG);
267 -
268 - return state->stream->write_at - state->stream->read_at;
269 -}
151 +void rrdpush_decompressor_reset(struct decompressor_state *state) {
152 + if(!state->initialized) {
153 + state->initialized = true;
154 + state->stream.lz4_stream = LZ4_createStreamDecode();
155 + state->stream.size = LZ4_decoderRingBufferSize(COMPRESSION_MAX_MSG_SIZE) * 2;
156 + state->stream.buffer = mallocz(state->stream.size);
157 + }
158
271 -/*
272 - * Fill the buffer provided with uncompressed data from the internal buffer
273 - * Return the size of uncompressed data copied or 0 for error
274 - */
275 -static size_t lz4_decompressor_get(struct decompressor_state *state, char *dst, size_t size) {
276 - if (unlikely(!state || !size || !dst))
277 - return 0;
159 + LZ4_setStreamDecode(state->stream.lz4_stream, NULL, 0);
160
279 - size_t remaining = lz4_decompressor_decompressed_bytes_in_buffer(state);
280 - if(unlikely(!remaining))
281 - return 0;
161 + state->signature_size = RRDPUSH_COMPRESSION_SIGNATURE_SIZE;
162 + state->stream.write_at = 0;
163 + state->stream.read_at = 0;
164 +}
165
283 - size_t bytes_to_return = size;
284 - if(bytes_to_return > remaining)
285 - bytes_to_return = remaining;
166 +void rrdpush_decompressor_destroy(struct decompressor_state *state) {
167 + if(unlikely(!state->initialized))
168 + return;
169
287 - memcpy(dst, state->stream->buffer + state->stream->read_at, bytes_to_return);
288 - state->stream->read_at += bytes_to_return;
170 + if (state->stream.lz4_stream) {
171 + LZ4_freeStreamDecode(state->stream.lz4_stream);
172 + state->stream.lz4_stream = NULL;
173 + }
174
290 - if(unlikely(state->stream->read_at > state->stream->write_at))
291 - fatal("%s: invalid read/write stream positions"
292 - , STREAM_COMPRESSION_MSG);
175 + freez(state->stream.buffer);
176 + state->stream.buffer = NULL;
177
294 - return bytes_to_return;
178 + state->initialized = false;
179 }
180
297 -/*
298 - * Create and initialize decompressor state
299 - * Return the pointer to decompressor_state structure created
300 - */
301 -struct decompressor_state *create_decompressor()
302 -{
303 - struct decompressor_state *state = callocz(1, sizeof(struct decompressor_state));
304 - state->signature_size = SIGNATURE_SIZE;
305 - state->reset = lz4_decompressor_reset;
306 - state->start = lz4_decompressor_start;
307 - state->decompress = lz4_decompressor_decompress;
308 - state->get = lz4_decompressor_get;
309 - state->decompressed_bytes_in_buffer = lz4_decompressor_decompressed_bytes_in_buffer;
310 - state->destroy = lz4_decompressor_destroy;
311 -
312 - state->stream = callocz(1, sizeof(struct decompressor_stream));
313 - fatal_assert(state->stream);
314 - state->stream->lz4_stream = LZ4_createStreamDecode();
315 - state->stream->size = LZ4_decoderRingBufferSize(COMPRESSION_MAX_MSG_SIZE) * 2;
316 - state->stream->buffer = mallocz(state->stream->size);
317 - fatal_assert(state->stream->buffer);
318 - state->reset(state);
319 - debug(D_STREAM, "%s: Initialize streaming decompression!", STREAM_COMPRESSION_MSG);
320 - return state;
321 -}
181 #endif
streaming/receiver.c
+92 -139
@@ -2,17 +2,6 @@
2
3 #include "rrdpush.h"
4
5 -// IMPORTANT: to add workers, you have to edit WORKER_PARSER_FIRST_JOB accordingly
6 -#define WORKER_RECEIVER_JOB_BYTES_READ (WORKER_PARSER_FIRST_JOB - 1)
7 -#define WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED (WORKER_PARSER_FIRST_JOB - 2)
8 -
9 -// this has to be the same at parser.h
10 -#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)
11 -
12 -#if WORKER_PARSER_FIRST_JOB < 1
13 -#error The define WORKER_PARSER_FIRST_JOB needs to be at least 1
14 -#endif
15 -
5 extern struct config stream_config;
6
7 void receiver_state_free(struct receiver_state *rpt) {
@@ -40,12 +29,11 @@ void receiver_state_free(struct receiver_state *rpt) {
29 }
30
31 #ifdef ENABLE_COMPRESSION
43 - if (rpt->decompressor)
44 - rpt->decompressor->destroy(&rpt->decompressor);
32 + rrdpush_decompressor_destroy(&rpt->decompressor);
33 #endif
34
35 if(rpt->system_info)
48 - rrdhost_system_info_free(rpt->system_info);
36 + rrdhost_system_info_free(rpt->system_info);
37
38 __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_receivers, sizeof(*rpt), __ATOMIC_RELAXED);
39
@@ -54,51 +42,18 @@ void receiver_state_free(struct receiver_state *rpt) {
42
43 #include "collectors/plugins.d/pluginsd_parser.h"
44
57 -PARSER_RC streaming_claimed_id(char **words, size_t num_words, void *user)
58 -{
59 - const char *host_uuid_str = get_word(words, num_words, 1);
60 - const char *claim_id_str = get_word(words, num_words, 2);
61 -
62 - if (!host_uuid_str || !claim_id_str) {
63 - error("Command CLAIMED_ID came malformed, uuid = '%s', claim_id = '%s'",
64 - host_uuid_str ? host_uuid_str : "[unset]",
65 - claim_id_str ? claim_id_str : "[unset]");
66 - return PARSER_RC_ERROR;
67 - }
68 -
69 - uuid_t uuid;
70 - RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host;
71 -
72 - // We don't need the parsed UUID
73 - // just do it to check the format
74 - if(uuid_parse(host_uuid_str, uuid)) {
75 - error("1st parameter (host GUID) to CLAIMED_ID command is not valid GUID. Received: \"%s\".", host_uuid_str);
76 - return PARSER_RC_ERROR;
77 - }
78 - if(uuid_parse(claim_id_str, uuid) && strcmp(claim_id_str, "NULL")) {
79 - error("2nd parameter (Claim ID) to CLAIMED_ID command is not valid GUID. Received: \"%s\".", claim_id_str);
80 - return PARSER_RC_ERROR;
81 - }
82 -
83 - if(strcmp(host_uuid_str, host->machine_guid)) {
84 - error("Claim ID is for host \"%s\" but it came over connection for \"%s\"", host_uuid_str, host->machine_guid);
85 - return PARSER_RC_OK; //the message is OK problem must be somewhere else
86 - }
87 -
88 - rrdhost_aclk_state_lock(host);
89 - if (host->aclk_state.claimed_id)
90 - freez(host->aclk_state.claimed_id);
91 - host->aclk_state.claimed_id = strcmp(claim_id_str, "NULL") ? strdupz(claim_id_str) : NULL;
92 - rrdhost_aclk_state_unlock(host);
93 -
94 - rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID |RRDHOST_FLAG_METADATA_UPDATE);
45 +// IMPORTANT: to add workers, you have to edit WORKER_PARSER_FIRST_JOB accordingly
46 +#define WORKER_RECEIVER_JOB_BYTES_READ (WORKER_PARSER_FIRST_JOB - 1)
47 +#define WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED (WORKER_PARSER_FIRST_JOB - 2)
48
96 - rrdpush_send_claimed_id(host);
49 +// this has to be the same at parser.h
50 +#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)
51
98 - return PARSER_RC_OK;
99 -}
52 +#if WORKER_PARSER_FIRST_JOB < 1
53 +#error The define WORKER_PARSER_FIRST_JOB needs to be at least 1
54 +#endif
55
101 -static int read_stream(struct receiver_state *r, char* buffer, size_t size) {
56 +static inline int read_stream(struct receiver_state *r, char* buffer, size_t size) {
57 if(unlikely(!size)) {
58 internal_error(true, "%s() asked to read zero bytes", __FUNCTION__);
59 return 0;
@@ -137,7 +92,7 @@ static int read_stream(struct receiver_state *r, char* buffer, size_t size) {
92 return (int)bytes_read;
93 }
94
140 -static bool receiver_read_uncompressed(struct receiver_state *r) {
95 +static inline bool receiver_read_uncompressed(struct receiver_state *r) {
96 #ifdef NETDATA_INTERNAL_CHECKS
97 if(r->read_buffer[r->read_len] != '\0')
98 fatal("%s(): read_buffer does not start with zero", __FUNCTION__ );
@@ -157,19 +112,17 @@ static bool receiver_read_uncompressed(struct receiver_state *r) {
112 }
113
114 #ifdef ENABLE_COMPRESSION
160 -static bool receiver_read_compressed(struct receiver_state *r) {
115 +static inline bool receiver_read_compressed(struct receiver_state *r) {
116
162 -#ifdef NETDATA_INTERNAL_CHECKS
163 - if(r->read_buffer[r->read_len] != '\0')
164 - fatal("%s: read_buffer does not start with zero #2", __FUNCTION__ );
165 -#endif
117 + internal_fatal(r->read_buffer[r->read_len] != '\0',
118 + "%s: read_buffer does not start with zero #2", __FUNCTION__ );
119
120 // first use any available uncompressed data
168 - if (r->decompressor->decompressed_bytes_in_buffer(r->decompressor)) {
121 + if (likely(rrdpush_decompressed_bytes_in_buffer(&r->decompressor))) {
122 size_t available = sizeof(r->read_buffer) - r->read_len - 1;
170 - if (available) {
171 - size_t len = r->decompressor->get(r->decompressor, r->read_buffer + r->read_len, available);
172 - if (!len) {
123 + if (likely(available)) {
124 + size_t len = rrdpush_decompressor_get(&r->decompressor, r->read_buffer + r->read_len, available);
125 + if (unlikely(!len)) {
126 internal_error(true, "decompressor returned zero length #1");
127 return false;
128 }
@@ -178,7 +131,7 @@ static bool receiver_read_compressed(struct receiver_state *r) {
131 r->read_buffer[r->read_len] = '\0';
132 }
133 else
181 - internal_error(true, "The line to read is too big! Already have %d bytes in read_buffer.", r->read_len);
134 + internal_fatal(true, "The line to read is too big! Already have %d bytes in read_buffer.", r->read_len);
135
136 return true;
137 }
@@ -186,8 +139,9 @@ static bool receiver_read_compressed(struct receiver_state *r) {
139 // no decompressed data available
140 // read the compression signature of the next block
141
189 - if(unlikely(r->read_len + r->decompressor->signature_size > sizeof(r->read_buffer) - 1)) {
190 - internal_error(true, "The last incomplete line does not leave enough room for the next compression header! Already have %d bytes in read_buffer.", r->read_len);
142 + if(unlikely(r->read_len + r->decompressor.signature_size > sizeof(r->read_buffer) - 1)) {
143 + internal_error(true, "The last incomplete line does not leave enough room for the next compression header! "
144 + "Already have %d bytes in read_buffer.", r->read_len);
145 return false;
146 }
147
@@ -195,19 +149,19 @@ static bool receiver_read_compressed(struct receiver_state *r) {
149 // we have to do a loop here, because read_stream() may return less than the data we need
150 int bytes_read = 0;
151 do {
198 - int ret = read_stream(r, r->read_buffer + r->read_len + bytes_read, r->decompressor->signature_size - bytes_read);
152 + int ret = read_stream(r, r->read_buffer + r->read_len + bytes_read, r->decompressor.signature_size - bytes_read);
153 if (unlikely(ret <= 0))
154 return false;
155
156 bytes_read += ret;
203 - } while(unlikely(bytes_read < (int)r->decompressor->signature_size));
157 + } while(unlikely(bytes_read < (int)r->decompressor.signature_size));
158
159 worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)bytes_read);
160
207 - if(unlikely(bytes_read != (int)r->decompressor->signature_size))
208 - fatal("read %d bytes, but expected compression signature of size %zu", bytes_read, r->decompressor->signature_size);
161 + if(unlikely(bytes_read != (int)r->decompressor.signature_size))
162 + fatal("read %d bytes, but expected compression signature of size %zu", bytes_read, r->decompressor.signature_size);
163
210 - size_t compressed_message_size = r->decompressor->start(r->decompressor, r->read_buffer + r->read_len, bytes_read);
164 + size_t compressed_message_size = rrdpush_decompressor_start(&r->decompressor, r->read_buffer + r->read_len, bytes_read);
165 if (unlikely(!compressed_message_size)) {
166 internal_error(true, "multiplexed uncompressed data in compressed stream!");
167 r->read_len += bytes_read;
@@ -244,8 +198,8 @@ static bool receiver_read_compressed(struct receiver_state *r) {
198 worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)compressed_bytes_read);
199
200 // decompress the compressed block
247 - size_t bytes_to_parse = r->decompressor->decompress(r->decompressor, compressed, compressed_bytes_read);
248 - if (!bytes_to_parse) {
201 + size_t bytes_to_parse = rrdpush_decompress(&r->decompressor, compressed, compressed_bytes_read);
202 + if (unlikely(!bytes_to_parse)) {
203 internal_error(true, "no bytes to parse.");
204 return false;
205 }
@@ -253,8 +207,8 @@ static bool receiver_read_compressed(struct receiver_state *r) {
207 worker_set_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, (NETDATA_DOUBLE)bytes_to_parse);
208
209 // fill read buffer with decompressed data
256 - size_t len = (int)r->decompressor->get(r->decompressor, r->read_buffer + r->read_len, sizeof(r->read_buffer) - r->read_len - 1);
257 - if (!len) {
210 + size_t len = (int) rrdpush_decompressor_get(&r->decompressor, r->read_buffer + r->read_len, sizeof(r->read_buffer) - r->read_len - 1);
211 + if (unlikely(!len)) {
212 internal_error(true, "decompressor returned zero length #2");
213 return false;
214 }
@@ -264,7 +218,7 @@ static bool receiver_read_compressed(struct receiver_state *r) {
218 return true;
219 }
220 #else // !ENABLE_COMPRESSION
267 -static bool receiver_read_compressed(struct receiver_state *r) {
221 +static inline bool receiver_read_compressed(struct receiver_state *r) {
222 return receiver_read_uncompressed(r);
223 }
224 #endif // ENABLE_COMPRESSION
@@ -272,7 +226,7 @@ static bool receiver_read_compressed(struct receiver_state *r) {
226 /* Produce a full line if one exists, statefully return where we start next time.
227 * When we hit the end of the buffer with a partial line move it to the beginning for the next fill.
228 */
275 -static char *receiver_next_line(struct receiver_state *r, char *buffer, size_t buffer_length, size_t *pos) {
229 +static inline char *receiver_next_line(struct receiver_state *r, char *buffer, size_t buffer_length, size_t *pos) {
230 size_t start = *pos;
231
232 char *ss = &r->read_buffer[start];
@@ -323,20 +277,50 @@ static char *receiver_next_line(struct receiver_state *r, char *buffer, size_t b
277
278 bool plugin_is_enabled(struct plugind *cd);
279
280 +static void receiver_set_exit_reason(struct receiver_state *rpt, STREAM_HANDSHAKE reason, bool force) {
281 + if(force || !rpt->exit.reason)
282 + rpt->exit.reason = reason;
283 +}
284 +
285 +static inline bool receiver_should_continue(struct receiver_state *rpt) {
286 + static __thread size_t counter = 0;
287 +
288 + if(unlikely(rpt->exit.shutdown)) {
289 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, false);
290 + return false;
291 + }
292 +
293 + // check every 1000 lines read
294 + if((counter++ % 1000) != 0) return true;
295 +
296 + if(unlikely(!service_running(SERVICE_STREAMING))) {
297 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT, false);
298 + return false;
299 + }
300 +
301 + netdata_thread_testcancel();
302 +
303 + rpt->last_msg_t = now_monotonic_sec();
304 +
305 + return true;
306 +}
307 +
308 static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, int fd, void *ssl) {
309 size_t result;
310
329 - PARSER_USER_OBJECT user = {
330 - .enabled = plugin_is_enabled(cd),
331 - .host = rpt->host,
332 - .opaque = rpt,
333 - .cd = cd,
334 - .trust_durations = 1,
335 - .capabilities = rpt->capabilities,
336 - };
337 -
338 - PARSER *parser = parser_init(&user, NULL, NULL, fd,
339 - PARSER_INPUT_SPLIT, ssl);
311 + PARSER *parser = NULL;
312 + {
313 + PARSER_USER_OBJECT user = {
314 + .enabled = plugin_is_enabled(cd),
315 + .host = rpt->host,
316 + .opaque = rpt,
317 + .cd = cd,
318 + .trust_durations = 1,
319 + .capabilities = rpt->capabilities,
320 + };
321 +
322 + parser = parser_init(&user, NULL, NULL, fd, PARSER_INPUT_SPLIT, ssl);
323 + }
324
325 pluginsd_keywords_init(parser, PARSER_INIT_STREAMING);
326
@@ -346,20 +330,15 @@ static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, i
330 // so, parser needs to be allocated before pushing it
331 netdata_thread_cleanup_push(pluginsd_process_thread_cleanup, parser);
332
349 - parser_add_keyword(parser, "CLAIMED_ID", streaming_claimed_id);
350 -
351 - user.parser = parser;
352 -
333 bool compressed_connection = false;
334 +
335 #ifdef ENABLE_COMPRESSION
336 if(stream_has_capability(rpt, STREAM_CAP_COMPRESSION)) {
337 compressed_connection = true;
357 -
358 - if (!rpt->decompressor)
359 - rpt->decompressor = create_decompressor();
360 - else
361 - rpt->decompressor->reset(rpt->decompressor);
338 + rrdpush_decompressor_reset(&rpt->decompressor);
339 }
340 + else
341 + rrdpush_decompressor_destroy(&rpt->decompressor);
342 #endif
343
344 rpt->read_buffer[0] = '\0';
@@ -367,51 +346,27 @@ static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, i
346
347 size_t read_buffer_start = 0;
348 char buffer[PLUGINSD_LINE_MAX + 2] = "";
370 - while(service_running(SERVICE_STREAMING)) {
371 - netdata_thread_testcancel();
349 + while(receiver_should_continue(rpt)) {
350
351 if(!receiver_next_line(rpt, buffer, PLUGINSD_LINE_MAX + 2, &read_buffer_start)) {
374 - bool have_new_data;
375 - if(likely(compressed_connection))
376 - have_new_data = receiver_read_compressed(rpt);
377 - else
378 - have_new_data = receiver_read_uncompressed(rpt);
352 + bool have_new_data = compressed_connection ? receiver_read_compressed(rpt) : receiver_read_uncompressed(rpt);
353
354 if(unlikely(!have_new_data)) {
381 - if(!rpt->exit.reason)
382 - rpt->exit.reason = "SOCKET READ ERROR";
383 -
355 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_ERROR, false);
356 break;
357 }
358
387 - rpt->last_msg_t = now_realtime_sec();
359 continue;
360 }
361
391 - if(unlikely(!service_running(SERVICE_STREAMING))) {
392 - if(!rpt->exit.reason)
393 - rpt->exit.reason = "NETDATA EXIT";
394 - goto done;
395 - }
396 - if(unlikely(rpt->exit.shutdown)) {
397 - if(!rpt->exit.reason)
398 - rpt->exit.reason = "SHUTDOWN REQUESTED";
399 -
400 - goto done;
401 - }
402 -
362 if (unlikely(parser_action(parser, buffer))) {
363 internal_error(true, "parser_action() failed on keyword '%s'.", buffer);
405 -
406 - if(!rpt->exit.reason)
407 - rpt->exit.reason = "PARSER FAILED";
408 -
364 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, false);
365 break;
366 }
367 }
368
413 -done:
414 - result = user.data_collections_count;
369 + result = parser ? parser->user.data_collections_count : 0;
370
371 // free parser with the pop function
372 netdata_thread_cleanup_pop(1);
@@ -501,7 +456,7 @@ static void rrdhost_clear_receiver(struct receiver_state *rpt) {
456 if (rpt->config.health_enabled == CONFIG_BOOLEAN_AUTO)
457 host->health.health_enabled = 0;
458
504 - rrdpush_sender_thread_stop(host, "RECEIVER LEFT", false);
459 + rrdpush_sender_thread_stop(host, STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, false);
460
461 signal_rrdcontext = true;
462 rrdpush_receiver_replication_reset(host);
@@ -520,7 +475,7 @@ static void rrdhost_clear_receiver(struct receiver_state *rpt) {
475 }
476 }
477
523 -bool stop_streaming_receiver(RRDHOST *host, const char *reason) {
478 +bool stop_streaming_receiver(RRDHOST *host, STREAM_HANDSHAKE reason) {
479 bool ret = false;
480
481 netdata_mutex_lock(&host->receiver_lock);
@@ -528,7 +483,7 @@ bool stop_streaming_receiver(RRDHOST *host, const char *reason) {
483 if(host->receiver) {
484 if(!host->receiver->exit.shutdown) {
485 host->receiver->exit.shutdown = true;
531 - host->receiver->exit.reason = reason;
486 + receiver_set_exit_reason(host->receiver, reason, true);
487 shutdown(host->receiver->fd, SHUT_RDWR);
488 }
489
@@ -586,9 +541,9 @@ void rrdpush_receive_log_status(struct receiver_state *rpt, const char *msg, con
541 , rpt->client_ip, rpt->client_port
542 , msg
543 , status
589 - , rpt->exit.reason?" (":""
590 - , rpt->exit.reason?rpt->exit.reason:""
591 - , rpt->exit.reason?")":""
544 + , rpt->exit.reason != STREAM_HANDSHAKE_NEVER?" (":""
545 + , stream_handshake_error_to_string(rpt->exit.reason)
546 + , rpt->exit.reason != STREAM_HANDSHAKE_NEVER?")":""
547 );
548
549 }
@@ -661,7 +616,6 @@ static void rrdpush_receive(struct receiver_state *rpt)
616 rpt->config.rrdpush_compression = default_compression_enabled;
617 rpt->config.rrdpush_compression = appconfig_get_boolean(&stream_config, rpt->key, "enable compression", rpt->config.rrdpush_compression);
618 rpt->config.rrdpush_compression = appconfig_get_boolean(&stream_config, rpt->machine_guid, "enable compression", rpt->config.rrdpush_compression);
664 - rpt->rrdpush_compression = (rpt->config.rrdpush_compression && default_compression_enabled);
619 #endif //ENABLE_COMPRESSION
620
621 (void)appconfig_set_default(&stream_config, rpt->machine_guid, "host tags", (rpt->tags)?rpt->tags:"");
@@ -758,7 +712,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
712
713 #ifdef ENABLE_COMPRESSION
714 if (stream_has_capability(rpt, STREAM_CAP_COMPRESSION)) {
761 - if (!rpt->rrdpush_compression)
715 + if (!rpt->config.rrdpush_compression)
716 rpt->capabilities &= ~STREAM_CAP_COMPRESSION;
717 }
718 #endif
@@ -839,8 +793,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
793 #endif
794 );
795
842 - if(!rpt->exit.reason)
843 - rpt->exit.reason = "PARSER EXIT";
796 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT, false);
797
798 {
799 char msg[100 + 1];
streaming/rrdpush.c
+17 -7
@@ -597,7 +597,7 @@ int connect_to_one_of_destinations(
597 if (reconnects_counter)
598 *reconnects_counter += 1;
599
600 - d->last_attempt = now;
600 + d->since = now;
601 d->attempts++;
602 sock = connect_to_this(string2str(d->destination), default_port, timeout);
603
@@ -683,7 +683,7 @@ void rrdpush_destinations_free(RRDHOST *host) {
683
684 // Either the receiver lost the connection or the host is being destroyed.
685 // The sender mutex guards thread creation, any spurious data is wiped on reconnection.
686 -void rrdpush_sender_thread_stop(RRDHOST *host, const char *reason, bool wait) {
686 +void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wait) {
687 if (!host->sender)
688 return;
689
@@ -784,7 +784,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri
784 return rrdpush_receiver_too_busy_now(w);
785
786 struct receiver_state *rpt = callocz(1, sizeof(*rpt));
787 - rpt->last_msg_t = now_realtime_sec();
787 + rpt->last_msg_t = now_monotonic_sec();
788 rpt->capabilities = STREAM_CAP_INVALID;
789 rpt->hops = 1;
790
@@ -1140,7 +1140,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri
1140 if (host) {
1141 netdata_mutex_lock(&host->receiver_lock);
1142 if (host->receiver) {
1143 - age = now_realtime_sec() - host->receiver->last_msg_t;
1143 + age = now_monotonic_sec() - host->receiver->last_msg_t;
1144
1145 if (age < 30)
1146 receiver_working = true;
@@ -1151,7 +1151,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri
1151 }
1152 rrd_unlock();
1153
1154 - if (receiver_stale && stop_streaming_receiver(host, "STALE RECEIVER")) {
1154 + if (receiver_stale && stop_streaming_receiver(host, STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER)) {
1155 // we stopped the receiver
1156 // we can proceed with this connection
1157 receiver_stale = false;
@@ -1224,7 +1224,7 @@ static struct {
1224 { STREAM_HANDSHAKE_OK_V3, "CONNECTED" },
1225 { STREAM_HANDSHAKE_OK_V2, "CONNECTED" },
1226 { STREAM_HANDSHAKE_OK_V1, "CONNECTED" },
1227 - { STREAM_HANDSHAKE_NEVER, "NOT_TRIED_YET" },
1227 + { STREAM_HANDSHAKE_NEVER, "" },
1228 { STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE, "BAD HANDSHAKE" },
1229 { STREAM_HANDSHAKE_ERROR_LOCALHOST, "LOCALHOST" },
1230 { STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED, "ALREADY CONNECTED" },
@@ -1236,7 +1236,17 @@ static struct {
1236 { STREAM_HANDSHAKE_ERROR_CANT_CONNECT, "CANT CONNECT" },
1237 { STREAM_HANDSHAKE_BUSY_TRY_LATER, "BUSY TRY LATER" },
1238 { STREAM_HANDSHAKE_INTERNAL_ERROR, "INTERNAL ERROR" },
1239 - { STREAM_HANDSHAKE_INITIALIZATION, "INITIALIZING" },
1239 + { STREAM_HANDSHAKE_INITIALIZATION, "REMOTE IS INITIALIZING" },
1240 + { STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, "DISCONNECTED HOST CLEANUP" },
1241 + { STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER, "DISCONNECTED STALE RECEIVER" },
1242 + { STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, "DISCONNECTED SHUTDOWN REQUESTED" },
1243 + { STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT, "DISCONNECTED NETDATA EXIT" },
1244 + { STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT, "DISCONNECTED PARSE ENDED" },
1245 + { STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_ERROR, "DISCONNECTED SOCKET READ ERROR" },
1246 + { STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, "DISCONNECTED PARSE ERROR" },
1247 + { STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, "DISCONNECTED RECEIVER LEFT" },
1248 + { STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST, "DISCONNECTED ORPHAN HOST" },
1249 + { STREAM_HANDSHAKE_NON_STREAMABLE_HOST, "NON STREAMABLE HOST" },
1250 { 0, NULL },
1251 };
1252
streaming/rrdpush.h
+105 -26
@@ -94,6 +94,17 @@ typedef enum {
94 STREAM_HANDSHAKE_BUSY_TRY_LATER = -10,
95 STREAM_HANDSHAKE_INTERNAL_ERROR = -11,
96 STREAM_HANDSHAKE_INITIALIZATION = -12,
97 + STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP = -13,
98 + STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER = -14,
99 + STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN = -15,
100 + STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT = -16,
101 + STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT = -17,
102 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_ERROR = -18,
103 + STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED = -19,
104 + STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT = -20,
105 + STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST = -21,
106 + STREAM_HANDSHAKE_NON_STREAMABLE_HOST = -22,
107 +
108 } STREAM_HANDSHAKE;
109
110
@@ -108,28 +119,98 @@ typedef struct {
119 } stream_encoded_t;
120
121 #ifdef ENABLE_COMPRESSION
122 +// signature MUST end with a newline
123 +#define RRDPUSH_COMPRESSION_SIGNATURE ((uint32_t)('z' | 0x80) | (0x80 << 8) | (0x80 << 16) | ('\n' << 24))
124 +#define RRDPUSH_COMPRESSION_SIGNATURE_MASK ((uint32_t)0xff | (0x80 << 8) | (0x80 << 16) | (0xff << 24))
125 +#define RRDPUSH_COMPRESSION_SIGNATURE_SIZE 4
126 +
127 struct compressor_state {
128 + bool initialized;
129 char *compression_result_buffer;
130 size_t compression_result_buffer_size;
114 - struct compressor_data *data; // Compression API specific data
115 - void (*reset)(struct compressor_state *state);
131 + struct {
132 + void *lz4_stream;
133 + char *input_ring_buffer;
134 + size_t input_ring_buffer_size;
135 + size_t input_ring_buffer_pos;
136 + } stream;
137 size_t (*compress)(struct compressor_state *state, const char *data, size_t size, char **buffer);
138 void (*destroy)(struct compressor_state **state);
139 };
140
141 +void rrdpush_compressor_reset(struct compressor_state *state);
142 +void rrdpush_compressor_destroy(struct compressor_state *state);
143 +size_t rrdpush_compress(struct compressor_state *state, const char *data, size_t size, char **out);
144 +
145 struct decompressor_state {
146 + bool initialized;
147 size_t signature_size;
148 size_t total_compressed;
149 size_t total_uncompressed;
150 size_t packet_count;
125 - struct decompressor_stream *stream; // Decompression API specific data
126 - void (*reset)(struct decompressor_state *state);
127 - size_t (*start)(struct decompressor_state *state, const char *header, size_t header_size);
128 - size_t (*decompress)(struct decompressor_state *state, const char *compressed_data, size_t compressed_size);
129 - size_t (*decompressed_bytes_in_buffer)(struct decompressor_state *state);
130 - size_t (*get)(struct decompressor_state *state, char *data, size_t size);
131 - void (*destroy)(struct decompressor_state **state);
151 + struct {
152 + void *lz4_stream;
153 + char *buffer;
154 + size_t size;
155 + size_t write_at;
156 + size_t read_at;
157 + } stream;
158 };
159 +
160 +void rrdpush_decompressor_destroy(struct decompressor_state *state);
161 +void rrdpush_decompressor_reset(struct decompressor_state *state);
162 +size_t rrdpush_decompress(struct decompressor_state *state, const char *compressed_data, size_t compressed_size);
163 +
164 +static inline size_t rrdpush_decompress_decode_header(const char *data, size_t data_size) {
165 + if (unlikely(!data || !data_size))
166 + return 0;
167 +
168 + if (unlikely(data_size != RRDPUSH_COMPRESSION_SIGNATURE_SIZE))
169 + return 0;
170 +
171 + uint32_t sign = *(uint32_t *)data;
172 + if (unlikely((sign & RRDPUSH_COMPRESSION_SIGNATURE_MASK) != RRDPUSH_COMPRESSION_SIGNATURE))
173 + return 0;
174 +
175 + size_t length = ((sign >> 8) & 0x7f) | ((sign >> 9) & (0x7f << 7));
176 + return length;
177 +}
178 +
179 +static inline size_t rrdpush_decompressor_start(struct decompressor_state *state, const char *header, size_t header_size) {
180 + if(unlikely(state->stream.read_at != state->stream.write_at))
181 + fatal("RRDPUSH DECOMPRESS: asked to decompress new data, while there are unread data in the decompression buffer!");
182 +
183 + return rrdpush_decompress_decode_header(header, header_size);
184 +}
185 +
186 +static inline size_t rrdpush_decompressed_bytes_in_buffer(struct decompressor_state *state) {
187 + if(unlikely(state->stream.read_at > state->stream.write_at))
188 + fatal("RRDPUSH DECOMPRESS: invalid read/write stream positions");
189 +
190 + return state->stream.write_at - state->stream.read_at;
191 +}
192 +
193 +static inline size_t rrdpush_decompressor_get(struct decompressor_state *state, char *dst, size_t size) {
194 + if (unlikely(!state || !size || !dst))
195 + return 0;
196 +
197 + size_t remaining = rrdpush_decompressed_bytes_in_buffer(state);
198 +
199 + if(unlikely(!remaining))
200 + return 0;
201 +
202 + size_t bytes_to_return = size;
203 + if(bytes_to_return > remaining)
204 + bytes_to_return = remaining;
205 +
206 + memcpy(dst, state->stream.buffer + state->stream.read_at, bytes_to_return);
207 + state->stream.read_at += bytes_to_return;
208 +
209 + if(unlikely(state->stream.read_at > state->stream.write_at))
210 + fatal("RRDPUSH DECOMPRESS: invalid read/write stream positions");
211 +
212 + return bytes_to_return;
213 +}
214 #endif
215
216 // Thread-local storage
@@ -171,7 +252,7 @@ struct sender_state {
252 netdata_mutex_t mutex;
253 struct circular_buffer *buffer;
254 char read_buffer[PLUGINSD_LINE_MAX + 1];
174 - int read_len;
255 + ssize_t read_len;
256 STREAM_CAPABILITIES capabilities;
257
258 size_t sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_MAX];
@@ -182,15 +263,16 @@ struct sender_state {
263 uint16_t hops;
264
265 #ifdef ENABLE_COMPRESSION
185 - struct compressor_state *compressor;
266 + struct compressor_state compressor;
267 #endif
268 +
269 #ifdef ENABLE_HTTPS
270 NETDATA_SSL ssl; // structure used to encrypt the connection
271 #endif
272
273 struct {
274 bool shutdown;
193 - const char *reason;
275 + STREAM_HANDSHAKE reason;
276 } exit;
277
278 struct {
@@ -293,7 +375,7 @@ struct receiver_state {
375
376 struct {
377 bool shutdown; // signal the streaming parser to exit
296 - const char *reason; // the reason of disconnection to log
378 + STREAM_HANDSHAKE reason;
379 } exit;
380
381 struct {
@@ -315,13 +397,12 @@ struct receiver_state {
397 #ifdef ENABLE_HTTPS
398 NETDATA_SSL ssl;
399 #endif
318 -#ifdef ENABLE_COMPRESSION
319 - unsigned int rrdpush_compression;
320 - struct decompressor_state *decompressor;
321 -#endif
400
401 time_t replication_first_time_t;
402
403 +#ifdef ENABLE_COMPRESSION
404 + struct decompressor_state decompressor;
405 +#endif
406 /*
407 struct {
408 uint32_t count;
@@ -334,11 +415,9 @@ struct rrdpush_destinations {
415 STRING *destination;
416 bool ssl;
417 uint32_t attempts;
337 -
338 - const char *last_error;
339 - time_t last_attempt;
418 + time_t since;
419 time_t postpone_reconnection_until;
341 - STREAM_HANDSHAKE last_handshake;
420 + STREAM_HANDSHAKE reason;
421
422 struct rrdpush_destinations *prev;
423 struct rrdpush_destinations *next;
@@ -390,7 +469,7 @@ void rrdpush_send_global_functions(RRDHOST *host);
469 #define THREAD_TAG_STREAM_SENDER "SNDR" // "[host]" is appended
470
471 int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string);
393 -void rrdpush_sender_thread_stop(RRDHOST *host, const char *reason, bool wait);
472 +void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wait);
473
474 void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva);
475 void log_stream_connection(const char *client_ip, const char *client_port, const char *api_key, const char *machine_guid, const char *host, const char *msg);
@@ -407,8 +486,8 @@ void rrdpush_signal_sender_to_wake_up(struct sender_state *s);
486
487 #ifdef ENABLE_COMPRESSION
488 struct compressor_state *create_compressor();
410 -struct decompressor_state *create_decompressor();
489 #endif
490 +
491 void rrdpush_reset_destinations_postpone_time(RRDHOST *host);
492 const char *stream_handshake_error_to_string(STREAM_HANDSHAKE handshake_error);
493 void stream_capabilities_to_json_array(BUFFER *wb, STREAM_CAPABILITIES caps, const char *key);
@@ -419,7 +498,7 @@ STREAM_CAPABILITIES convert_stream_version_to_capabilities(int32_t version, RRDH
498 int32_t stream_capabilities_to_vn(uint32_t caps);
499
500 void receiver_state_free(struct receiver_state *rpt);
422 -bool stop_streaming_receiver(RRDHOST *host, const char *reason);
501 +bool stop_streaming_receiver(RRDHOST *host, STREAM_HANDSHAKE reason);
502
503 void sender_thread_buffer_free(void);
504
@@ -623,7 +702,7 @@ typedef struct rrdhost_status {
702 STREAM_CAPABILITIES capabilities;
703 uint32_t id;
704 time_t since;
626 - const char *reason;
705 + STREAM_HANDSHAKE reason;
706
707 struct {
708 bool in_progress;
@@ -641,7 +720,7 @@ typedef struct rrdhost_status {
720 STREAM_CAPABILITIES capabilities;
721 uint32_t id;
722 time_t since;
644 - const char *reason;
723 + STREAM_HANDSHAKE reason;
724
725 struct {
726 bool in_progress;
streaming/sender.c
+31 -35
@@ -11,7 +11,7 @@
11 #define WORKER_SENDER_JOB_DISCONNECT_OVERFLOW 6
12 #define WORKER_SENDER_JOB_DISCONNECT_TIMEOUT 7
13 #define WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR 8
14 -#define WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR 9
14 +#define WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR 9
15 #define WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR 10
16 #define WORKER_SENDER_JOB_DISCONNECT_PARENT_CLOSED 11
17 #define WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR 12
@@ -118,7 +118,7 @@ void sender_commit(struct sender_state *s, BUFFER *wb, STREAM_TRAFFIC_TYPE type)
118 }
119
120 #ifdef ENABLE_COMPRESSION
121 - if (stream_has_capability(s, STREAM_CAP_COMPRESSION) && s->compressor) {
121 + if (stream_has_capability(s, STREAM_CAP_COMPRESSION) && s->compressor.initialized) {
122 while(src_len) {
123 size_t size_to_compress = src_len;
124
@@ -144,13 +144,13 @@ void sender_commit(struct sender_state *s, BUFFER *wb, STREAM_TRAFFIC_TYPE type)
144 }
145
146 char *dst;
147 - size_t dst_len = s->compressor->compress(s->compressor, src, size_to_compress, &dst);
147 + size_t dst_len = rrdpush_compress(&s->compressor, src, size_to_compress, &dst);
148 if (!dst_len) {
149 error("STREAM %s [send to %s]: COMPRESSION failed. Resetting compressor and re-trying",
150 rrdhost_hostname(s->host), s->connected_to);
151
152 - s->compressor->reset(s->compressor);
153 - dst_len = s->compressor->compress(s->compressor, src, size_to_compress, &dst);
152 + rrdpush_compressor_reset(&s->compressor);
153 + dst_len = rrdpush_compress(&s->compressor, src, size_to_compress, &dst);
154 if(!dst_len) {
155 error("STREAM %s [send to %s]: COMPRESSION failed again. Deactivating compression",
156 rrdhost_hostname(s->host), s->connected_to);
@@ -492,8 +492,7 @@ static inline bool rrdpush_sender_validate_response(RRDHOST *host, struct sender
492 }
493
494 if(version >= STREAM_HANDSHAKE_OK_V1) {
495 - host->destination->last_error = NULL;
496 - host->destination->last_handshake = version;
495 + host->destination->reason = version;
496 host->destination->postpone_reconnection_until = now_realtime_sec() + s->reconnect_delay;
497 s->capabilities = convert_stream_version_to_capabilities(version, host, true);
498 return true;
@@ -505,8 +504,7 @@ static inline bool rrdpush_sender_validate_response(RRDHOST *host, struct sender
504
505 worker_is_busy(worker_job_id);
506 rrdpush_sender_thread_close_socket(host);
508 - host->destination->last_error = error;
509 - host->destination->last_handshake = version;
507 + host->destination->reason = version;
508 host->destination->postpone_reconnection_until = now_realtime_sec() + delay;
509
510 char buf[LOG_DATE_LENGTH];
@@ -533,8 +531,7 @@ static bool rrdpush_sender_connect_ssl(struct sender_state *s) {
531
532 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);
533 rrdpush_sender_thread_close_socket(host);
536 - host->destination->last_error = "SSL error";
537 - host->destination->last_handshake = STREAM_HANDSHAKE_ERROR_SSL_ERROR;
534 + host->destination->reason = STREAM_HANDSHAKE_ERROR_SSL_ERROR;
535 host->destination->postpone_reconnection_until = now_realtime_sec() + 5 * 60;
536 return false;
537 }
@@ -546,8 +543,7 @@ static bool rrdpush_sender_connect_ssl(struct sender_state *s) {
543 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);
544 error("SSL: closing the stream connection, because the server SSL certificate is not valid.");
545 rrdpush_sender_thread_close_socket(host);
549 - host->destination->last_error = "invalid SSL certificate";
550 - host->destination->last_handshake = STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE;
546 + host->destination->reason = STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE;
547 host->destination->postpone_reconnection_until = now_realtime_sec() + 5 * 60;
548 return false;
549 }
@@ -709,7 +705,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
705 if(!rrdpush_sender_connect_ssl(s))
706 return false;
707
712 - ssize_t bytes, len = strlen(http);
708 + ssize_t bytes, len = (ssize_t)strlen(http);
709
710 bytes = send_timeout(
711 #ifdef ENABLE_HTTPS
@@ -725,8 +721,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
721 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
722 rrdpush_sender_thread_close_socket(host);
723 error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", rrdhost_hostname(host), s->connected_to);
728 - host->destination->last_error = "timeout while sending request";
729 - host->destination->last_handshake = STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT;
724 + host->destination->reason = STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT;
725 host->destination->postpone_reconnection_until = now_realtime_sec() + 1 * 60;
726 return false;
727 }
@@ -745,8 +740,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
740 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
741 rrdpush_sender_thread_close_socket(host);
742 error("STREAM %s [send to %s]: remote netdata does not respond.", rrdhost_hostname(host), s->connected_to);
748 - host->destination->last_error = "timeout while expecting first response";
749 - host->destination->last_handshake = STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT;
743 + host->destination->reason = STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT;
744 host->destination->postpone_reconnection_until = now_realtime_sec() + 30;
745 return false;
746 }
@@ -763,12 +757,10 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
757 return false;
758
759 #ifdef ENABLE_COMPRESSION
766 - if(stream_has_capability(s, STREAM_CAP_COMPRESSION)) {
767 - if(!s->compressor)
768 - s->compressor = create_compressor();
769 - else
770 - s->compressor->reset(s->compressor);
771 - }
760 + if(stream_has_capability(s, STREAM_CAP_COMPRESSION))
761 + rrdpush_compressor_reset(&s->compressor);
762 + else
763 + rrdpush_compressor_destroy(&s->compressor);
764 #endif //ENABLE_COMPRESSION
765
766 log_sender_capabilities(s);
@@ -821,9 +813,9 @@ static bool attempt_to_connect(struct sender_state *state)
813 return false;
814 }
815
824 -// TCP window is open and we have data to transmit.
816 +// TCP window is open, and we have data to transmit.
817 static ssize_t attempt_to_send(struct sender_state *s) {
826 - ssize_t ret = 0;
818 + ssize_t ret;
819
820 #ifdef NETDATA_INTERNAL_CHECKS
821 struct circular_buffer *cb = s->buffer;
@@ -958,7 +950,7 @@ void execute_commands(struct sender_state *s) {
950 // internal_error(true, "STREAM %s [send to %s] received command over connection: %s", rrdhost_hostname(s->host), s->connected_to, start);
951
952 char *words[PLUGINSD_MAX_WORDS] = { NULL };
961 - size_t num_words = pluginsd_split_words(start, words, PLUGINSD_MAX_WORDS);
953 + size_t num_words = quoted_strings_splitter_pluginsd(start, words, PLUGINSD_MAX_WORDS);
954
955 const char *keyword = get_word(words, num_words, 0);
956
@@ -1108,7 +1100,7 @@ static bool rrdhost_set_sender(RRDHOST *host) {
1100 host->rrdpush_sender_connection_counter++;
1101 host->sender->tid = gettid();
1102 host->sender->last_state_since_t = now_realtime_sec();
1111 - host->sender->exit.reason = NULL;
1103 + host->sender->exit.reason = STREAM_HANDSHAKE_NEVER;
1104 ret = true;
1105 }
1106 netdata_mutex_unlock(&host->sender->mutex);
@@ -1126,6 +1118,10 @@ static void rrdhost_clear_sender___while_having_sender_mutex(RRDHOST *host) {
1118 host->sender->exit.shutdown = false;
1119 rrdhost_flag_clear(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN | RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED | RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
1120 host->sender->last_state_since_t = now_realtime_sec();
1121 + if(host->destination) {
1122 + host->destination->since = host->sender->last_state_since_t;
1123 + host->destination->reason = host->sender->exit.reason;
1124 + }
1125 }
1126
1127 rrdpush_reset_destinations_postpone_time(host);
@@ -1137,25 +1133,25 @@ static bool rrdhost_sender_should_exit(struct sender_state *s) {
1133
1134 if(unlikely(!service_running(SERVICE_STREAMING))) {
1135 if(!s->exit.reason)
1140 - s->exit.reason = "NETDATA EXIT";
1136 + s->exit.reason = STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT;
1137 return true;
1138 }
1139
1140 if(unlikely(!rrdhost_has_rrdpush_sender_enabled(s->host))) {
1141 if(!s->exit.reason)
1146 - s->exit.reason = "NON STREAMABLE HOST";
1142 + s->exit.reason = STREAM_HANDSHAKE_NON_STREAMABLE_HOST;
1143 return true;
1144 }
1145
1146 if(unlikely(s->exit.shutdown)) {
1147 if(!s->exit.reason)
1152 - s->exit.reason = "SENDER SHUTDOWN REQUESTED";
1148 + s->exit.reason = STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN;
1149 return true;
1150 }
1151
1152 if(unlikely(rrdhost_flag_check(s->host, RRDHOST_FLAG_ORPHAN))) {
1153 if(!s->exit.reason)
1158 - s->exit.reason = "RECEIVER LEFT (ORPHAN HOST)";
1154 + s->exit.reason = STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST;
1155 return true;
1156 }
1157
@@ -1171,7 +1167,7 @@ static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
1167 netdata_mutex_lock(&host->sender->mutex);
1168 info("STREAM %s [send]: sending thread exits %s",
1169 rrdhost_hostname(host),
1174 - host->sender->exit.reason ? host->sender->exit.reason : "");
1170 + host->sender->exit.reason != STREAM_HANDSHAKE_NEVER ? stream_handshake_error_to_string(host->sender->exit.reason) : "");
1171
1172 rrdpush_sender_thread_close_socket(host);
1173 rrdpush_sender_pipe_close(host, host->sender->rrdpush_sender_pipe, false);
@@ -1220,7 +1216,7 @@ void *rrdpush_sender_thread(void *ptr) {
1216 // disconnection reasons
1217 worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT, "disconnect timeout");
1218 worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR, "disconnect poll error");
1223 - worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR, "disconnect socket error");
1219 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR, "disconnect socket error");
1220 worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW, "disconnect overflow");
1221 worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR, "disconnect ssl error");
1222 worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_PARENT_CLOSED, "disconnect parent closed");
@@ -1481,7 +1477,7 @@ void *rrdpush_sender_thread(void *ptr) {
1477 error = "connection is invalid (POLLNVAL)";
1478
1479 if(unlikely(error)) {
1484 - worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR);
1480 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR);
1481 error("STREAM %s [send to %s]: restarting connection: %s - %zu bytes transmitted.",
1482 rrdhost_hostname(s->host), s->connected_to, error, s->sent_bytes_on_this_connection);
1483 rrdpush_sender_thread_close_socket(s->host);
web/api/queries/query.c
+2 -9
@@ -2516,13 +2516,6 @@ void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTION
2516 buffer_json_object_close(wb); // key
2517 }
2518
2519 -static int group_by_label_is_space(char c) {
2520 - if(c == ',' || c == '|')
2521 - return 1;
2522 -
2523 - return 0;
2524 -}
2525 -
2519 static void rrd2rrdr_set_timestamps(RRDR *r) {
2520 QUERY_TARGET *qt = r->internal.qt;
2521
@@ -2755,9 +2748,9 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2748 for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
2749 if (qt->request.group_by[g].group_by & RRDR_GROUP_BY_LABEL &&
2750 qt->request.group_by[g].group_by_label && *qt->request.group_by[g].group_by_label)
2758 - qt->group_by[g].used = quoted_strings_splitter(
2751 + qt->group_by[g].used = quoted_strings_splitter_query_group_by_label(
2752 qt->request.group_by[g].group_by_label, qt->group_by[g].label_keys,
2760 - GROUP_BY_MAX_LABEL_KEYS, group_by_label_is_space);
2753 + GROUP_BY_MAX_LABEL_KEYS);
2754
2755 if (!qt->group_by[g].used)
2756 qt->request.group_by[g].group_by &= ~RRDR_GROUP_BY_LABEL;