@cryptotaxi247 / netdata-1 / commits / 532e9b3d8

fix missing labels from parents (#16296)

* maintain in /tmp/stream-receiver-X.txt a copy of metadata received * stream log metadata to /tmp/stream-sender-localhost.txt * log the stream of all senders * cleanup use of X_update_metadata() functions * fix for last commit * rrdlabel unmark/mark/delete unmarked restored

Costa Tsaousis committed Oct 28, 2023 at 23:32 UTC 532e9b3d8d6494c69243fb9b51b0e74ffff9d2bd
12 files changed +164 -111
collectors/plugins.d/gperf-config.txt
+35 -35
@@ -12,47 +12,47 @@ PARSER_KEYWORD;
12 #
13 # Plugins Only Keywords
14 #
15 -FLUSH, 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1
16 -DISABLE, 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2
17 -EXIT, 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3
18 -HOST, 71, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 4
19 -HOST_DEFINE, 72, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 5
20 -HOST_DEFINE_END, 73, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 6
21 -HOST_LABEL, 74, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 7
15 +FLUSH, 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1
16 +DISABLE, 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2
17 +EXIT, 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3
18 +HOST, 71, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 4
19 +HOST_DEFINE, 72, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 5
20 +HOST_DEFINE_END, 73, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 6
21 +HOST_LABEL, 74, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 7
22 #
23 # Common keywords
24 #
25 -BEGIN, 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8
26 -CHART, 32, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 9
27 -CLABEL, 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 10
28 -CLABEL_COMMIT, 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 11
29 -DIMENSION, 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 12
30 -END, 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13
31 -FUNCTION, 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 14
32 -FUNCTION_RESULT_BEGIN, 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15
33 -LABEL, 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16
34 -OVERWRITE, 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 17
35 -SET, 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18
36 -VARIABLE, 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19
37 -DYNCFG_ENABLE, 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20
38 -DYNCFG_REGISTER_MODULE, 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21
39 -DYNCFG_REGISTER_JOB, 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22
40 -DYNCFG_RESET, 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23
41 -REPORT_JOB_STATUS, 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24
42 -DELETE_JOB, 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25
25 +BEGIN, 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8
26 +CHART, 32, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 9
27 +CLABEL, 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 10
28 +CLABEL_COMMIT, 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 11
29 +DIMENSION, 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 12
30 +END, 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13
31 +FUNCTION, 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14
32 +FUNCTION_RESULT_BEGIN, 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15
33 +LABEL, 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 16
34 +OVERWRITE, 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17
35 +SET, 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18
36 +VARIABLE, 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 19
37 +DYNCFG_ENABLE, 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20
38 +DYNCFG_REGISTER_MODULE, 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21
39 +DYNCFG_REGISTER_JOB, 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22
40 +DYNCFG_RESET, 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23
41 +REPORT_JOB_STATUS, 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24
42 +DELETE_JOB, 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25
43 #
44 # Streaming only keywords
45 #
46 -CLAIMED_ID, 61, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26
47 -BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27
48 -SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28
49 -END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29
46 +CLAIMED_ID, 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26
47 +BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27
48 +SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28
49 +END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29
50 #
51 # Streaming Replication keywords
52 #
53 -CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30
54 -RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31
55 -RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32
56 -REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33
57 -RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34
58 -RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
53 +CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30
54 +RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31
55 +RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32
56 +REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33
57 +RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34
58 +RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
collectors/plugins.d/gperf-hashtable.h
+35 -35
@@ -83,79 +83,79 @@ static PARSER_KEYWORD gperf_keywords[] =
83 {
84 {(char*)0}, {(char*)0}, {(char*)0},
85 #line 30 "gperf-config.txt"
86 - {"END", 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13},
86 + {"END", 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13},
87 #line 49 "gperf-config.txt"
88 - {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29},
88 + {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29},
89 #line 56 "gperf-config.txt"
90 - {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33},
90 + {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33},
91 #line 17 "gperf-config.txt"
92 - {"EXIT", 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3},
92 + {"EXIT", 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3},
93 #line 16 "gperf-config.txt"
94 - {"DISABLE", 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2},
94 + {"DISABLE", 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2},
95 #line 55 "gperf-config.txt"
96 - {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32},
96 + {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32},
97 #line 29 "gperf-config.txt"
98 - {"DIMENSION", 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 12},
98 + {"DIMENSION", 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 12},
99 #line 42 "gperf-config.txt"
100 - {"DELETE_JOB", 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25},
100 + {"DELETE_JOB", 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25},
101 {(char*)0},
102 #line 40 "gperf-config.txt"
103 - {"DYNCFG_RESET", 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23},
103 + {"DYNCFG_RESET", 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23},
104 #line 37 "gperf-config.txt"
105 - {"DYNCFG_ENABLE", 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20},
105 + {"DYNCFG_ENABLE", 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20},
106 #line 26 "gperf-config.txt"
107 - {"CHART", 32, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 9},
107 + {"CHART", 32, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 9},
108 #line 35 "gperf-config.txt"
109 - {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18},
109 + {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18},
110 #line 48 "gperf-config.txt"
111 - {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28},
111 + {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28},
112 #line 57 "gperf-config.txt"
113 - {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34},
113 + {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34},
114 #line 41 "gperf-config.txt"
115 - {"REPORT_JOB_STATUS", 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24},
115 + {"REPORT_JOB_STATUS", 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24},
116 #line 39 "gperf-config.txt"
117 - {"DYNCFG_REGISTER_JOB", 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22},
117 + {"DYNCFG_REGISTER_JOB", 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22},
118 #line 58 "gperf-config.txt"
119 - {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
119 + {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
120 #line 18 "gperf-config.txt"
121 - {"HOST", 71, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 4},
121 + {"HOST", 71, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 4},
122 #line 38 "gperf-config.txt"
123 - {"DYNCFG_REGISTER_MODULE", 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21},
123 + {"DYNCFG_REGISTER_MODULE", 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21},
124 #line 25 "gperf-config.txt"
125 - {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8},
125 + {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8},
126 #line 47 "gperf-config.txt"
127 - {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27},
127 + {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27},
128 #line 54 "gperf-config.txt"
129 - {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31},
129 + {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31},
130 #line 27 "gperf-config.txt"
131 - {"CLABEL", 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 10},
131 + {"CLABEL", 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 10},
132 #line 21 "gperf-config.txt"
133 - {"HOST_LABEL", 74, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 7},
133 + {"HOST_LABEL", 74, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 7},
134 #line 19 "gperf-config.txt"
135 - {"HOST_DEFINE", 72, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 5},
135 + {"HOST_DEFINE", 72, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 5},
136 #line 53 "gperf-config.txt"
137 - {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30},
137 + {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30},
138 #line 46 "gperf-config.txt"
139 - {"CLAIMED_ID", 61, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26},
139 + {"CLAIMED_ID", 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26},
140 #line 15 "gperf-config.txt"
141 - {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
141 + {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
142 #line 20 "gperf-config.txt"
143 - {"HOST_DEFINE_END", 73, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 6},
143 + {"HOST_DEFINE_END", 73, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 6},
144 #line 28 "gperf-config.txt"
145 - {"CLABEL_COMMIT", 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 11},
145 + {"CLABEL_COMMIT", 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 11},
146 #line 31 "gperf-config.txt"
147 - {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 14},
147 + {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14},
148 #line 34 "gperf-config.txt"
149 - {"OVERWRITE", 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 17},
149 + {"OVERWRITE", 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17},
150 #line 33 "gperf-config.txt"
151 - {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16},
151 + {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 16},
152 #line 36 "gperf-config.txt"
153 - {"VARIABLE", 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19},
153 + {"VARIABLE", 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 19},
154 {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0},
155 {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0},
156 {(char*)0},
157 #line 32 "gperf-config.txt"
158 - {"FUNCTION_RESULT_BEGIN", 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15}
158 + {"FUNCTION_RESULT_BEGIN", 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15}
159 };
160
161 PARSER_KEYWORD *
collectors/plugins.d/pluginsd_parser.c
+7
@@ -2890,6 +2890,13 @@ void pluginsd_process_thread_cleanup(void *ptr) {
2890
2891 rrd_collector_finished();
2892
2893 +#ifdef NETDATA_LOG_STREAM_RECEIVE
2894 + if(parser->user.stream_log_fp) {
2895 + fclose(parser->user.stream_log_fp);
2896 + parser->user.stream_log_fp = NULL;
2897 + }
2898 +#endif
2899 +
2900 parser_destroy(parser);
2901 }
2902
collectors/plugins.d/pluginsd_parser.h
+17
@@ -30,6 +30,7 @@ typedef enum __attribute__ ((__packed__)) parser_input_type {
30 typedef enum __attribute__ ((__packed__)) {
31 PARSER_INIT_PLUGINSD = (1 << 1),
32 PARSER_INIT_STREAMING = (1 << 2),
33 + PARSER_REP_METADATA = (1 << 3),
34 } PARSER_REPERTOIRE;
35
36 struct parser;
@@ -54,6 +55,11 @@ typedef struct parser_user_object {
55 size_t data_collections_count;
56 int enabled;
57
58 +#ifdef NETDATA_LOG_STREAM_RECEIVE
59 + FILE *stream_log_fp;
60 + PARSER_REPERTOIRE stream_log_repertoire;
61 +#endif
62 +
63 STREAM_CAPABILITIES capabilities; // receiver capabilities
64
65 struct {
@@ -154,6 +160,11 @@ static inline PARSER_KEYWORD *parser_find_keyword(PARSER *parser, const char *co
160 }
161
162 static inline int parser_action(PARSER *parser, char *input) {
163 +#ifdef NETDATA_LOG_STREAM_RECEIVE
164 + static __thread char line[PLUGINSD_LINE_MAX + 1];
165 + strncpyz(line, input, sizeof(line) - 1);
166 +#endif
167 +
168 parser->line++;
169
170 if(unlikely(parser->flags & PARSER_DEFER_UNTIL_KEYWORD)) {
@@ -197,6 +208,12 @@ static inline int parser_action(PARSER *parser, char *input) {
208 PARSER_KEYWORD *t = parser_find_keyword(parser, command);
209 if(likely(t)) {
210 worker_is_busy(t->worker_job_id);
211 +
212 +#ifdef NETDATA_LOG_STREAM_RECEIVE
213 + if(parser->user.stream_log_fp && t->repertoire & parser->user.stream_log_repertoire)
214 + fprintf(parser->user.stream_log_fp, "%s", line);
215 +#endif
216 +
217 rc = parser_execute(parser, t, words, num_words);
218 // rc = (*t->func)(words, num_words, parser);
219 worker_is_idle();
database/rrd.h
+2 -6
@@ -901,9 +901,7 @@ static inline uint32_t rrdset_metadata_upstream_version(RRDSET *st) {
901 return __atomic_load_n(&st->rrdpush.sender.sent_version, __ATOMIC_RELAXED);
902 }
903
904 -static inline void rrdset_metadata_updated(RRDSET *st) {
905 - __atomic_add_fetch(&st->version, 1, __ATOMIC_RELAXED);
906 -}
904 +void rrdset_metadata_updated(RRDSET *st);
905
906 static inline void rrdset_metadata_exposed_upstream(RRDSET *st, uint32_t version) {
907 __atomic_store_n(&st->rrdpush.sender.sent_version, version, __ATOMIC_RELAXED);
@@ -922,9 +920,7 @@ static inline uint32_t rrddim_metadata_upstream_version(RRDDIM *rd) {
920 return __atomic_load_n(&rd->rrdpush.sender.sent_version, __ATOMIC_RELAXED);
921 }
922
925 -static inline void rrddim_metadata_updated(RRDDIM *rd) {
926 - rrdset_metadata_updated(rd->rrdset);
927 -}
923 +void rrddim_metadata_updated(RRDDIM *rd);
924
925 static inline void rrddim_metadata_exposed_upstream(RRDDIM *rd, uint32_t version) {
926 __atomic_store_n(&rd->rrdpush.sender.sent_version, version, __ATOMIC_RELAXED);
database/rrddim.c
+6 -3
@@ -4,6 +4,11 @@
4 #include "rrd.h"
5 #include "storage_engine.h"
6
7 +void rrddim_metadata_updated(RRDDIM *rd) {
8 + rrdcontext_updated_rrddim(rd);
9 + rrdset_metadata_updated(rd->rrdset);
10 +}
11 +
12 // ----------------------------------------------------------------------------
13 // RRDDIM index
14
@@ -157,7 +162,6 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
162
163 // let the chart resync
164 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
160 - rrdset_metadata_updated(st);
165
166 ml_dimension_new(rd);
167
@@ -285,10 +289,9 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
289 if(ctr->react_action == RRDDIM_REACT_UPDATED) {
290 // the chart needs to be updated to the parent
291 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
288 - rrdset_metadata_updated(st);
292 }
293
291 - rrdcontext_updated_rrddim(rd);
294 + rrddim_metadata_updated(rd);
295 }
296
297 size_t rrddim_size(void) {
database/rrdlabels.c
+15 -18
@@ -683,7 +683,7 @@ static RRDLABEL *rrdlabels_find_label_with_key_unsafe(RRDLABELS *labels, RRDLABE
683 RRDLABEL *found = NULL;
684 while ((PValue = JudyLFirstThenNext(labels->JudyL, &Index, &first_then_next))) {
685 RRDLABEL *lb = (RRDLABEL *)Index;
686 - if (lb->index.key == label->index.key) {
686 + if (lb->index.key == label->index.key && lb != label) {
687 found = (RRDLABEL *)Index;
688 break;
689 }
@@ -700,13 +700,7 @@ static void labels_add_already_sanitized(RRDLABELS *labels, const char *key, con
700
701 spinlock_lock(&labels->spinlock);
702
703 - RRDLABEL *old_label_with_key = rrdlabels_find_label_with_key_unsafe(labels, new_label);
704 -
705 - if (old_label_with_key == new_label) {
706 - spinlock_unlock(&labels->spinlock);
707 - delete_label(new_label);
708 - return;
709 - }
703 + RRDLABEL_SRC new_ls = (ls & ~(RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_OLD));
704
705 size_t mem_before_judyl = JudyLMemUsed(labels->JudyL);
706
@@ -714,24 +708,27 @@ static void labels_add_already_sanitized(RRDLABELS *labels, const char *key, con
708 if (!PValue || PValue == PJERR)
709 fatal("RRDLABELS: corrupted labels JudyL array");
710
717 - RRDLABEL_SRC new_ls = (ls & ~(RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_OLD));
718 - labels->version++;
719 -
720 - if (old_label_with_key) {
721 - (void)JudyLDel(&labels->JudyL, (Word_t)old_label_with_key, PJE0);
711 + if(*PValue) {
712 new_ls |= RRDLABEL_FLAG_OLD;
723 - } else
713 + delete_label(new_label);
714 + }
715 + else {
716 new_ls |= RRDLABEL_FLAG_NEW;
717
718 + RRDLABEL *old_label_with_same_key = rrdlabels_find_label_with_key_unsafe(labels, new_label);
719 + if (old_label_with_same_key) {
720 + (void) JudyLDel(&labels->JudyL, (Word_t) old_label_with_same_key, PJE0);
721 + delete_label(old_label_with_same_key);
722 + }
723 + }
724 +
725 + labels->version++;
726 *((RRDLABEL_SRC *)PValue) = new_ls;
727
728 size_t mem_after_judyl = JudyLMemUsed(labels->JudyL);
729 STATS_PLUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, mem_after_judyl - mem_before_judyl, 0);
730
731 spinlock_unlock(&labels->spinlock);
732 -
733 - if (old_label_with_key)
734 - delete_label((RRDLABEL *)old_label_with_key);
732 }
733
734 void rrdlabels_add(RRDLABELS *labels, const char *name, const char *value, RRDLABEL_SRC ls)
@@ -1043,7 +1040,7 @@ void rrdlabels_copy(RRDLABELS *dst, RRDLABELS *src)
1040 lfe_start_nolock(src, label, ls)
1041 {
1042 RRDLABEL *old_label_with_key = rrdlabels_find_label_with_key_unsafe(dst, label);
1046 - if (old_label_with_key && old_label_with_key == label)
1043 + if (old_label_with_key)
1044 continue;
1045
1046 Pvoid_t *PValue = JudyLIns(&dst->JudyL, (Word_t)label, PJE0);
database/rrdset.c
+7 -3
@@ -5,6 +5,12 @@
5 #include <sched.h>
6 #include "storage_engine.h"
7
8 +
9 +void rrdset_metadata_updated(RRDSET *st) {
10 + __atomic_add_fetch(&st->version, 1, __ATOMIC_RELAXED);
11 + rrdcontext_updated_rrdset(st);
12 +}
13 +
14 // ----------------------------------------------------------------------------
15 // RRDSET rrdpush send chart_slots
16
@@ -484,7 +490,6 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
490 rrdset_update_permanent_labels(st);
491
492 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
487 - rrdset_metadata_updated(st);
493
494 return ctr->react_action != RRDSET_REACT_NONE;
495 }
@@ -510,10 +515,9 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
515 }
516 rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
517 rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
513 - rrdset_metadata_updated(st);
518 }
519
516 - rrdcontext_updated_rrdset(st);
520 + rrdset_metadata_updated(st);
521 }
522
523 void rrdset_index_init(RRDHOST *host) {
streaming/receiver.c
+9
@@ -353,6 +353,15 @@ static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, i
353
354 buffered_reader_init(&rpt->reader);
355
356 +#ifdef NETDATA_LOG_STREAM_RECEIVE
357 + {
358 + char filename[FILENAME_MAX + 1];
359 + snprintfz(filename, FILENAME_MAX, "/tmp/stream-receiver-%s.txt", rpt->host ? rrdhost_hostname(rpt->host) : "unknown");
360 + parser->user.stream_log_fp = fopen(filename, "w");
361 + parser->user.stream_log_repertoire = PARSER_REP_METADATA;
362 + }
363 +#endif
364 +
365 BUFFER *buffer = buffer_create(sizeof(rpt->reader.read_buffer), NULL);
366 while(!receiver_should_stop(rpt)) {
367
streaming/rrdpush.c
+1 -1
@@ -374,7 +374,7 @@ static void rrdpush_send_chart_metrics(BUFFER *wb, RRDSET *st, struct sender_sta
374 internal_error(true, "STREAM: 'host:%s/chart:%s/dim:%s' flag 'exposed' is updated but not exposed",
375 rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd));
376 // we will include it in the next iteration
377 - rrdset_metadata_updated(st);
377 + rrddim_metadata_updated(rd);
378 }
379 }
380 rrddim_foreach_done(rd);
streaming/rrdpush.h
+4
@@ -215,6 +215,10 @@ struct sender_state {
215
216 struct compressor_state compressor;
217
218 +#ifdef NETDATA_LOG_STREAM_SENDER
219 + FILE *stream_log_fp;
220 +#endif
221 +
222 #ifdef ENABLE_HTTPS
223 NETDATA_SSL ssl; // structure used to encrypt the connection
224 #endif
streaming/sender.c
+26 -10
@@ -90,15 +90,22 @@ void sender_commit(struct sender_state *s, BUFFER *wb, STREAM_TRAFFIC_TYPE type)
90
91 sender_lock(s);
92
93 -// if(s->host == localhost && type == STREAM_TRAFFIC_TYPE_METADATA) {
94 -// FILE *fp = fopen("/tmp/stream.txt", "a");
95 -// fprintf(fp, "\n--- SEND MESSAGE START: %s ----\n"
96 -// "%s"
97 -// "--- SEND MESSAGE END ----------------------------------------\n"
98 -// , rrdhost_hostname(s->host), src
99 -// );
100 -// fclose(fp);
101 -// }
93 +#ifdef NETDATA_LOG_STREAM_SENDER
94 + if(type == STREAM_TRAFFIC_TYPE_METADATA) {
95 + if(!s->stream_log_fp) {
96 + char filename[FILENAME_MAX + 1];
97 + snprintfz(filename, FILENAME_MAX, "/tmp/stream-sender-%s.txt", s->host ? rrdhost_hostname(s->host) : "unknown");
98 +
99 + s->stream_log_fp = fopen(filename, "w");
100 + }
101 +
102 + fprintf(s->stream_log_fp, "\n--- SEND MESSAGE START: %s ----\n"
103 + "%s"
104 + "--- SEND MESSAGE END ----------------------------------------\n"
105 + , rrdhost_hostname(s->host), src
106 + );
107 + }
108 +#endif
109
110 if(unlikely(s->buffer->max_size < (src_len + 1) * SENDER_BUFFER_ADAPT_TO_TIMES_MAX_SIZE)) {
111 netdata_log_info("STREAM %s [send to %s]: max buffer size of %zu is too small for a data message of size %zu. Increasing the max buffer size to %d times the max data message size.",
@@ -255,12 +262,13 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {
262 rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
263
264 st->rrdpush.sender.resync_time_s = 0;
258 - rrdset_metadata_updated(st);
265
266 RRDDIM *rd;
267 rrddim_foreach_read(rd, st)
268 rrddim_metadata_exposed_upstream_clear(rd);
269 rrddim_foreach_done(rd);
270 +
271 + rrdset_metadata_updated(st);
272 }
273 rrdset_foreach_done(st);
274
@@ -1235,6 +1243,14 @@ static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
1243 rrdpush_sender_pipe_close(host, host->sender->rrdpush_sender_pipe, false);
1244
1245 rrdhost_clear_sender___while_having_sender_mutex(host);
1246 +
1247 +#ifdef NETDATA_LOG_STREAM_SENDER
1248 + if(host->sender->stream_log_fp) {
1249 + fclose(host->sender->stream_log_fp);
1250 + host->sender->stream_log_fp = NULL;
1251 + }
1252 +#endif
1253 +
1254 sender_unlock(host->sender);
1255
1256 freez(s->pipe_buffer);