@cryptotaxi247 / netdata-1 / commits / 2175104d4

Faster parents (#16127)

* cache ctx in collection handle * cache rd together with rda * do not repeatedy call rrdcontexts - cached collection status; optimize pluginsd_acquire_dimension() * fix unit tests * do the absolutely minimum while updating timestamps, ensure validity during reading them * when the stream is INTERPOLATED, buffer outstanding data for up to 50ms if the buffer contains DATA only. * remove the spinlock from mrg * remove the metric flags that are not used any more * mrg writers can be different threads * update first time when latest clean is also updated * cleanup * set hot page with a simple atomic operation * sender sets chart slot for every chart * work on senders without SLOT * enable SLOT capability * send slot at BEGIN when SLOT is enabled * fix slot generation and parsing * send slot while re-streaming * use the sender capabilities, not the receiver * cleanup * add slots support to all chart and dimension related plugin commands * fix condition * fix calculation * check sender capabilties * assign slots in constructors * we need the dimension slot at the DIMENSION keyword * more debug info in case of dimension mismatch * ensure the RRDDIM EXPOSED flag is multi-threaded and set it after the sender buffer has been committed, so that replication will not send dimensions prematurely * fix renumbering on child restart * reset rda caching when receiving a chart definition * optimize pluginsd_end_v2() * do not do zero sized allocations * trust the chart slot id of the child * cleanup charts on pluginsd thread exit * better cleanup * find the chart and put it in the slot, if it not already there * move slots array to host * initialize pluginsd slots properly * add slots to replay begin; do not cleanup slots that dont belong to a chart * cleanup on obsolete * cleanup slots on obsoletions * cleanup and renames about obsoletion * rewrite obsolation service code to remove race conditions * better service obsoletion log * added debugging * more debug * exposed flag now compares versions * removed debugging messages * respolve conflicts * fix replication check for unsent dimensions

Costa Tsaousis committed Oct 27, 2023 at 20:42 UTC 2175104d417b0c3208b28c713256caf56eec7dd9
40 files changed +1171 -634
collectors/cgroups.plugin/sys_fs_cgroup.c
+31 -31
@@ -1760,36 +1760,36 @@ char *cgroup_parse_resolved_name_and_labels(RRDLABELS *labels, char *data) {
1760 // ----------------------------------------------------------------------------
1761
1762 static inline void free_pressure(struct pressure *res) {
1763 - if (res->some.share_time.st) rrdset_is_obsolete(res->some.share_time.st);
1764 - if (res->some.total_time.st) rrdset_is_obsolete(res->some.total_time.st);
1765 - if (res->full.share_time.st) rrdset_is_obsolete(res->full.share_time.st);
1766 - if (res->full.total_time.st) rrdset_is_obsolete(res->full.total_time.st);
1763 + if (res->some.share_time.st) rrdset_is_obsolete___safe_from_collector_thread(res->some.share_time.st);
1764 + if (res->some.total_time.st) rrdset_is_obsolete___safe_from_collector_thread(res->some.total_time.st);
1765 + if (res->full.share_time.st) rrdset_is_obsolete___safe_from_collector_thread(res->full.share_time.st);
1766 + if (res->full.total_time.st) rrdset_is_obsolete___safe_from_collector_thread(res->full.total_time.st);
1767 freez(res->filename);
1768 }
1769
1770 static inline void cgroup_free(struct cgroup *cg) {
1771 netdata_log_debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available");
1772
1773 - if(cg->st_cpu) rrdset_is_obsolete(cg->st_cpu);
1774 - if(cg->st_cpu_limit) rrdset_is_obsolete(cg->st_cpu_limit);
1775 - if(cg->st_cpu_per_core) rrdset_is_obsolete(cg->st_cpu_per_core);
1776 - if(cg->st_cpu_nr_throttled) rrdset_is_obsolete(cg->st_cpu_nr_throttled);
1777 - if(cg->st_cpu_throttled_time) rrdset_is_obsolete(cg->st_cpu_throttled_time);
1778 - if(cg->st_cpu_shares) rrdset_is_obsolete(cg->st_cpu_shares);
1779 - if(cg->st_mem) rrdset_is_obsolete(cg->st_mem);
1780 - if(cg->st_writeback) rrdset_is_obsolete(cg->st_writeback);
1781 - if(cg->st_mem_activity) rrdset_is_obsolete(cg->st_mem_activity);
1782 - if(cg->st_pgfaults) rrdset_is_obsolete(cg->st_pgfaults);
1783 - if(cg->st_mem_usage) rrdset_is_obsolete(cg->st_mem_usage);
1784 - if(cg->st_mem_usage_limit) rrdset_is_obsolete(cg->st_mem_usage_limit);
1785 - if(cg->st_mem_utilization) rrdset_is_obsolete(cg->st_mem_utilization);
1786 - if(cg->st_mem_failcnt) rrdset_is_obsolete(cg->st_mem_failcnt);
1787 - if(cg->st_io) rrdset_is_obsolete(cg->st_io);
1788 - if(cg->st_serviced_ops) rrdset_is_obsolete(cg->st_serviced_ops);
1789 - if(cg->st_throttle_io) rrdset_is_obsolete(cg->st_throttle_io);
1790 - if(cg->st_throttle_serviced_ops) rrdset_is_obsolete(cg->st_throttle_serviced_ops);
1791 - if(cg->st_queued_ops) rrdset_is_obsolete(cg->st_queued_ops);
1792 - if(cg->st_merged_ops) rrdset_is_obsolete(cg->st_merged_ops);
1773 + if(cg->st_cpu) rrdset_is_obsolete___safe_from_collector_thread(cg->st_cpu);
1774 + if(cg->st_cpu_limit) rrdset_is_obsolete___safe_from_collector_thread(cg->st_cpu_limit);
1775 + if(cg->st_cpu_per_core) rrdset_is_obsolete___safe_from_collector_thread(cg->st_cpu_per_core);
1776 + if(cg->st_cpu_nr_throttled) rrdset_is_obsolete___safe_from_collector_thread(cg->st_cpu_nr_throttled);
1777 + if(cg->st_cpu_throttled_time) rrdset_is_obsolete___safe_from_collector_thread(cg->st_cpu_throttled_time);
1778 + if(cg->st_cpu_shares) rrdset_is_obsolete___safe_from_collector_thread(cg->st_cpu_shares);
1779 + if(cg->st_mem) rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem);
1780 + if(cg->st_writeback) rrdset_is_obsolete___safe_from_collector_thread(cg->st_writeback);
1781 + if(cg->st_mem_activity) rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem_activity);
1782 + if(cg->st_pgfaults) rrdset_is_obsolete___safe_from_collector_thread(cg->st_pgfaults);
1783 + if(cg->st_mem_usage) rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem_usage);
1784 + if(cg->st_mem_usage_limit) rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem_usage_limit);
1785 + if(cg->st_mem_utilization) rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem_utilization);
1786 + if(cg->st_mem_failcnt) rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem_failcnt);
1787 + if(cg->st_io) rrdset_is_obsolete___safe_from_collector_thread(cg->st_io);
1788 + if(cg->st_serviced_ops) rrdset_is_obsolete___safe_from_collector_thread(cg->st_serviced_ops);
1789 + if(cg->st_throttle_io) rrdset_is_obsolete___safe_from_collector_thread(cg->st_throttle_io);
1790 + if(cg->st_throttle_serviced_ops) rrdset_is_obsolete___safe_from_collector_thread(cg->st_throttle_serviced_ops);
1791 + if(cg->st_queued_ops) rrdset_is_obsolete___safe_from_collector_thread(cg->st_queued_ops);
1792 + if(cg->st_merged_ops) rrdset_is_obsolete___safe_from_collector_thread(cg->st_merged_ops);
1793
1794 freez(cg->filename_cpuset_cpus);
1795 freez(cg->filename_cpu_cfs_period);
@@ -3474,7 +3474,7 @@ void update_cgroup_charts(int update_every) {
3474 cpu_usage = (NETDATA_DOUBLE)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
3475 NETDATA_DOUBLE cpu_used = 100 * (cpu_usage - cg->prev_cpu_usage) / (value * update_every);
3476
3477 - rrdset_isnot_obsolete(cg->st_cpu_limit);
3477 + rrdset_isnot_obsolete___safe_from_collector_thread(cg->st_cpu_limit);
3478
3479 rrddim_set(cg->st_cpu_limit, "used", (cpu_used > 0)?cpu_used:0);
3480
@@ -3485,7 +3485,7 @@ void update_cgroup_charts(int update_every) {
3485 }
3486 else {
3487 if(unlikely(cg->st_cpu_limit)) {
3488 - rrdset_is_obsolete(cg->st_cpu_limit);
3488 + rrdset_is_obsolete___safe_from_collector_thread(cg->st_cpu_limit);
3489 cg->st_cpu_limit = NULL;
3490 }
3491 rrdsetvar_custom_chart_variable_set(cg->st_cpu, cg->chart_var_cpu_limit, NAN);
@@ -3854,7 +3854,7 @@ void update_cgroup_charts(int update_every) {
3854 rrddim_add(cg->st_mem_usage_limit, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
3855 }
3856
3857 - rrdset_isnot_obsolete(cg->st_mem_usage_limit);
3857 + rrdset_isnot_obsolete___safe_from_collector_thread(cg->st_mem_usage_limit);
3858
3859 rrddim_set(cg->st_mem_usage_limit, "available", memory_limit - cg->memory.usage_in_bytes);
3860 rrddim_set(cg->st_mem_usage_limit, "used", cg->memory.usage_in_bytes);
@@ -3884,7 +3884,7 @@ void update_cgroup_charts(int update_every) {
3884 }
3885
3886 if (memory_limit) {
3887 - rrdset_isnot_obsolete(cg->st_mem_utilization);
3887 + rrdset_isnot_obsolete___safe_from_collector_thread(cg->st_mem_utilization);
3888
3889 rrddim_set(
3890 cg->st_mem_utilization, "utilization", cg->memory.usage_in_bytes * 100 / memory_limit);
@@ -3894,12 +3894,12 @@ void update_cgroup_charts(int update_every) {
3894 }
3895 else {
3896 if(unlikely(cg->st_mem_usage_limit)) {
3897 - rrdset_is_obsolete(cg->st_mem_usage_limit);
3897 + rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem_usage_limit);
3898 cg->st_mem_usage_limit = NULL;
3899 }
3900
3901 if(unlikely(cg->st_mem_utilization)) {
3902 - rrdset_is_obsolete(cg->st_mem_utilization);
3902 + rrdset_is_obsolete___safe_from_collector_thread(cg->st_mem_utilization);
3903 cg->st_mem_utilization = NULL;
3904 }
3905 }
@@ -4624,7 +4624,7 @@ void *cgroups_main(void *ptr) {
4624 collector_error("CGROUP: cannot create thread worker. uv_thread_create(): %s", uv_strerror(error));
4625 goto exit;
4626 }
4627 - uv_thread_set_name_np(discovery_thread.thread, "PLUGIN[cgroups]");
4627 + uv_thread_set_name_np(discovery_thread.thread, "P[cgroups]");
4628
4629 heartbeat_t hb;
4630 heartbeat_init(&hb);
collectors/diskspace.plugin/plugin_diskspace.c
+1 -1
@@ -59,7 +59,7 @@ struct mount_point_metadata {
59
60 static DICTIONARY *dict_mountpoints = NULL;
61
62 -#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
62 +#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete___safe_from_collector_thread(st); (st) = NULL; } } while(st)
63
64 int mount_point_cleanup(const char *name, void *entry, int slow) {
65 (void)name;
collectors/plugins.d/plugins_d.h
+1 -3
@@ -21,9 +21,7 @@
21 #define PLUGINSD_KEYWORD_REPORT_JOB_STATUS "REPORT_JOB_STATUS"
22 #define PLUGINSD_KEYWORD_DELETE_JOB "DELETE_JOB"
23
24 -#define PLUGINSD_LINE_MAX_SSL_READ 512
25 -
26 -#define PLUGINSD_MAX_WORDS 20
24 +#define PLUGINSD_MAX_WORDS 30
25
26 #define PLUGINSD_MAX_DIRECTORIES 20
27 extern char *plugin_directories[PLUGINSD_MAX_DIRECTORIES];
collectors/plugins.d/pluginsd_parser.c
+329 -119
@@ -4,8 +4,8 @@
4
5 #define LOG_FUNCTIONS false
6
7 -#define SERVING_STREAMING(parser) (parser->repertoire == PARSER_INIT_STREAMING)
8 -#define SERVING_PLUGINSD(parser) (parser->repertoire == PARSER_INIT_PLUGINSD)
7 +#define SERVING_STREAMING(parser) ((parser)->repertoire == PARSER_INIT_STREAMING)
8 +#define SERVING_PLUGINSD(parser) ((parser)->repertoire == PARSER_INIT_PLUGINSD)
9
10 static ssize_t send_to_plugin(const char *txt, void *data) {
11 PARSER *parser = data;
@@ -110,23 +110,6 @@ static inline bool pluginsd_unlock_rrdset_data_collection(PARSER *parser) {
110 return false;
111 }
112
113 -void pluginsd_rrdset_cleanup(RRDSET *st) {
114 - spinlock_lock(&st->pluginsd.spinlock);
115 -
116 - for(size_t i = 0; i < st->pluginsd.size ; i++) {
117 - rrddim_acquired_release(st->pluginsd.rda[i]); // can be NULL
118 - st->pluginsd.rda[i] = NULL;
119 - }
120 -
121 - freez(st->pluginsd.rda);
122 - st->pluginsd.collector_tid = 0;
123 - st->pluginsd.rda = NULL;
124 - st->pluginsd.size = 0;
125 - st->pluginsd.pos = 0;
126 -
127 - spinlock_unlock(&st->pluginsd.spinlock);
128 -}
129 -
113 static inline void pluginsd_unlock_previous_scope_chart(PARSER *parser, const char *keyword, bool stale) {
114 if(unlikely(pluginsd_unlock_rrdset_data_collection(parser))) {
115 if(stale)
@@ -150,7 +133,12 @@ static inline void pluginsd_unlock_previous_scope_chart(PARSER *parser, const ch
133
134 static inline void pluginsd_clear_scope_chart(PARSER *parser, const char *keyword) {
135 pluginsd_unlock_previous_scope_chart(parser, keyword, true);
136 +
137 + if(parser->user.cleanup_slots && parser->user.st)
138 + rrdset_pluginsd_receive_unslot(parser->user.st);
139 +
140 parser->user.st = NULL;
141 + parser->user.cleanup_slots = false;
142 }
143
144 static inline bool pluginsd_set_scope_chart(PARSER *parser, RRDSET *st, const char *keyword) {
@@ -176,61 +164,139 @@ static inline bool pluginsd_set_scope_chart(PARSER *parser, RRDSET *st, const ch
164
165 pluginsd_clear_scope_chart(parser, keyword);
166
179 - size_t dims = dictionary_entries(st->rrddim_root_index);
180 - if(unlikely(st->pluginsd.size < dims)) {
181 - st->pluginsd.rda = reallocz(st->pluginsd.rda, dims * sizeof(RRDDIM_ACQUIRED *));
167 + st->pluginsd.pos = 0;
168 + parser->user.st = st;
169 + parser->user.cleanup_slots = false;
170 +
171 + return true;
172 +}
173 +
174 +static inline void pluginsd_rrddim_put_to_slot(PARSER *parser, RRDSET *st, RRDDIM *rd, ssize_t slot, bool obsolete) {
175 + size_t wanted_size = st->pluginsd.size;
176 +
177 + if(slot >= 1) {
178 + st->pluginsd.with_slots = true;
179 + wanted_size = slot;
180 + }
181 + else
182 + wanted_size = dictionary_entries(st->rrddim_root_index);
183 +
184 + if(wanted_size > st->pluginsd.size) {
185 + st->pluginsd.prd_array = reallocz(st->pluginsd.prd_array, wanted_size * sizeof(struct pluginsd_rrddim));
186
187 // initialize the empty slots
184 - for(ssize_t i = (ssize_t)dims - 1; i >= (ssize_t)st->pluginsd.size ;i--)
185 - st->pluginsd.rda[i] = NULL;
188 + for(ssize_t i = (ssize_t) wanted_size - 1; i >= (ssize_t) st->pluginsd.size; i--) {
189 + st->pluginsd.prd_array[i].rda = NULL;
190 + st->pluginsd.prd_array[i].rd = NULL;
191 + st->pluginsd.prd_array[i].id = NULL;
192 + }
193
187 - st->pluginsd.size = dims;
194 + st->pluginsd.size = wanted_size;
195 }
196
190 - st->pluginsd.pos = 0;
191 - parser->user.st = st;
197 + if(st->pluginsd.with_slots) {
198 + struct pluginsd_rrddim *prd = &st->pluginsd.prd_array[slot - 1];
199
193 - return true;
200 + if(prd->rd != rd) {
201 + prd->rda = rrddim_find_and_acquire(st, string2str(rd->id));
202 + prd->rd = rrddim_acquired_to_rrddim(prd->rda);
203 + prd->id = string2str(prd->rd->id);
204 + }
205 +
206 + if(obsolete)
207 + parser->user.cleanup_slots = true;
208 + }
209 }
210
196 -static inline RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, const char *cmd) {
211 +static inline RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, ssize_t slot, const char *cmd) {
212 if (unlikely(!dimension || !*dimension)) {
213 netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, without a dimension.",
214 rrdhost_hostname(host), rrdset_id(st), cmd);
215 return NULL;
216 }
217
203 - if(unlikely(st->pluginsd.pos >= st->pluginsd.size))
204 - st->pluginsd.pos = 0;
218 + if (unlikely(!st->pluginsd.size)) {
219 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, but the chart has no dimensions.",
220 + rrdhost_hostname(host), rrdset_id(st), cmd);
221 + return NULL;
222 + }
223 +
224 + struct pluginsd_rrddim *prd;
225 + RRDDIM *rd;
226
206 - RRDDIM_ACQUIRED *rda = st->pluginsd.rda[st->pluginsd.pos];
227 + if(likely(st->pluginsd.with_slots)) {
228 + // caching with slots
229
208 - if(likely(rda)) {
209 - RRDDIM *rd = rrddim_acquired_to_rrddim(rda);
210 - if (likely(rd && string_strcmp(rd->id, dimension) == 0)) {
211 - // we found a cached RDA
212 - st->pluginsd.pos++;
230 + if(unlikely(slot < 1 || slot > st->pluginsd.size)) {
231 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s with slot %zd, but slots in the range [1 - %u] are expected.",
232 + rrdhost_hostname(host), rrdset_id(st), cmd, slot, st->pluginsd.size);
233 + return NULL;
234 + }
235 +
236 + prd = &st->pluginsd.prd_array[slot - 1];
237 +
238 + rd = prd->rd;
239 + if(likely(rd)) {
240 +#ifdef NETDATA_INTERNAL_CHECKS
241 + if(strcmp(prd->id, dimension) != 0) {
242 + ssize_t t;
243 + for(t = 0; t < st->pluginsd.size ;t++) {
244 + if (strcmp(st->pluginsd.prd_array[t].id, dimension) == 0)
245 + break;
246 + }
247 + if(t >= st->pluginsd.size)
248 + t = -1;
249 +
250 + internal_fatal(true,
251 + "PLUGINSD: expected to find dimension '%s' on slot %zd, but found '%s', "
252 + "the right slot is %zd",
253 + dimension, slot, prd->id, t);
254 + }
255 +#endif
256 return rd;
257 }
215 - else {
216 - // the collector is sending dimensions in a different order
217 - // release the previous one, to reuse this slot
218 - rrddim_acquired_release(rda);
219 - st->pluginsd.rda[st->pluginsd.pos] = NULL;
258 + }
259 + else {
260 + // caching without slots
261 +
262 + if(unlikely(st->pluginsd.pos >= st->pluginsd.size))
263 + st->pluginsd.pos = 0;
264 +
265 + prd = &st->pluginsd.prd_array[st->pluginsd.pos++];
266 +
267 + rd = prd->rd;
268 + if(likely(rd)) {
269 + const char *id = prd->id;
270 +
271 + if(strcmp(id, dimension) == 0) {
272 + // we found it cached
273 + return rd;
274 + }
275 + else {
276 + // the cached one is not good for us
277 + rrddim_acquired_release(prd->rda);
278 + prd->rda = NULL;
279 + prd->rd = rd = NULL;
280 + prd->id = NULL;
281 + }
282 }
283 }
284
223 - rda = rrddim_find_and_acquire(st, dimension);
285 + // we need to find the dimension and set it to prd
286 +
287 + RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension);
288 if (unlikely(!rda)) {
289 netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s but dimension does not exist.",
226 - rrdhost_hostname(host), rrdset_id(st), dimension, cmd);
290 + rrdhost_hostname(host), rrdset_id(st), dimension, cmd);
291
292 return NULL;
293 }
294
231 - st->pluginsd.rda[st->pluginsd.pos++] = rda;
295 + prd->rda = rda;
296 + prd->rd = rd = rrddim_acquired_to_rrddim(rda);
297 + prd->id = string2str(rd->id);
298
233 - return rrddim_acquired_to_rrddim(rda);
299 + return rd;
300 }
301
302 static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, const char *cmd) {
@@ -248,6 +314,70 @@ static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, cons
314 return st;
315 }
316
317 +static inline ssize_t pluginsd_parse_rrd_slot(char **words, size_t num_words) {
318 + ssize_t slot = -1;
319 + char *id = get_word(words, num_words, 1);
320 + if(id && id[0] == PLUGINSD_KEYWORD_SLOT[0] && id[1] == PLUGINSD_KEYWORD_SLOT[1] &&
321 + id[2] == PLUGINSD_KEYWORD_SLOT[2] && id[3] == PLUGINSD_KEYWORD_SLOT[3] && id[4] == ':') {
322 + slot = (ssize_t) str2ull_encoded(&id[5]);
323 + if(slot < 0) slot = 0; // to make the caller increment its idx of the words
324 + }
325 +
326 + return slot;
327 +}
328 +
329 +static inline void pluginsd_rrdset_cache_put_to_slot(PARSER *parser, RRDSET *st, ssize_t slot, bool obsolete) {
330 + // clean possible old cached data
331 + rrdset_pluginsd_receive_unslot(st);
332 +
333 + if(unlikely(slot < 1 || slot >= INT32_MAX))
334 + return;
335 +
336 + RRDHOST *host = st->rrdhost;
337 +
338 + if(unlikely((size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size)) {
339 + spinlock_lock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
340 + size_t old_slots = host->rrdpush.receive.pluginsd_chart_slots.size;
341 + size_t new_slots = (old_slots < PLUGINSD_MIN_RRDSET_POINTERS_CACHE) ? PLUGINSD_MIN_RRDSET_POINTERS_CACHE : old_slots * 2;
342 +
343 + if(new_slots < (size_t)slot)
344 + new_slots = slot;
345 +
346 + host->rrdpush.receive.pluginsd_chart_slots.array =
347 + reallocz(host->rrdpush.receive.pluginsd_chart_slots.array, new_slots * sizeof(RRDSET *));
348 +
349 + for(size_t i = old_slots; i < new_slots ;i++)
350 + host->rrdpush.receive.pluginsd_chart_slots.array[i] = NULL;
351 +
352 + host->rrdpush.receive.pluginsd_chart_slots.size = new_slots;
353 + spinlock_unlock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
354 + }
355 +
356 + host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1] = st;
357 + st->pluginsd.last_slot = (int32_t)slot - 1;
358 + parser->user.cleanup_slots = obsolete;
359 +}
360 +
361 +static inline RRDSET *pluginsd_rrdset_cache_get_from_slot(PARSER *parser, RRDHOST *host, const char *id, ssize_t slot, const char *keyword) {
362 + if(unlikely(slot < 1 || (size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size))
363 + return pluginsd_find_chart(host, id, keyword);
364 +
365 + RRDSET *st = host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1];
366 +
367 + if(!st) {
368 + st = pluginsd_find_chart(host, id, keyword);
369 + if(st)
370 + pluginsd_rrdset_cache_put_to_slot(parser, st, slot, rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE));
371 + }
372 + else {
373 + internal_fatal(string_strcmp(st->id, id) != 0,
374 + "PLUGINSD: wrong chart in slot %zd, expected '%s', found '%s'",
375 + slot - 1, id, string2str(st->id));
376 + }
377 +
378 + return st;
379 +}
380 +
381 static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg) {
382 parser->user.enabled = 0;
383
@@ -260,8 +390,12 @@ static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyw
390 }
391
392 static inline PARSER_RC pluginsd_set(char **words, size_t num_words, PARSER *parser) {
263 - char *dimension = get_word(words, num_words, 1);
264 - char *value = get_word(words, num_words, 2);
393 + int idx = 1;
394 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
395 + if(slot >= 0) idx++;
396 +
397 + char *dimension = get_word(words, num_words, idx++);
398 + char *value = get_word(words, num_words, idx++);
399
400 RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_SET);
401 if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
@@ -269,7 +403,7 @@ static inline PARSER_RC pluginsd_set(char **words, size_t num_words, PARSER *par
403 RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_SET, PLUGINSD_KEYWORD_CHART);
404 if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
405
272 - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_SET);
406 + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_SET);
407 if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
408
409 st->pluginsd.set = true;
@@ -285,13 +419,17 @@ static inline PARSER_RC pluginsd_set(char **words, size_t num_words, PARSER *par
419 }
420
421 static inline PARSER_RC pluginsd_begin(char **words, size_t num_words, PARSER *parser) {
288 - char *id = get_word(words, num_words, 1);
289 - char *microseconds_txt = get_word(words, num_words, 2);
422 + int idx = 1;
423 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
424 + if(slot >= 0) idx++;
425 +
426 + char *id = get_word(words, num_words, idx++);
427 + char *microseconds_txt = get_word(words, num_words, idx++);
428
429 RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_BEGIN);
430 if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
431
294 - RRDSET *st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_BEGIN);
432 + RRDSET *st = pluginsd_rrdset_cache_get_from_slot(parser, host, id, slot, PLUGINSD_KEYWORD_BEGIN);
433 if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
434
435 if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_BEGIN))
@@ -492,18 +630,22 @@ static inline PARSER_RC pluginsd_chart(char **words, size_t num_words, PARSER *p
630 RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_CHART);
631 if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
632
495 - char *type = get_word(words, num_words, 1);
496 - char *name = get_word(words, num_words, 2);
497 - char *title = get_word(words, num_words, 3);
498 - char *units = get_word(words, num_words, 4);
499 - char *family = get_word(words, num_words, 5);
500 - char *context = get_word(words, num_words, 6);
501 - char *chart = get_word(words, num_words, 7);
502 - char *priority_s = get_word(words, num_words, 8);
503 - char *update_every_s = get_word(words, num_words, 9);
504 - char *options = get_word(words, num_words, 10);
505 - char *plugin = get_word(words, num_words, 11);
506 - char *module = get_word(words, num_words, 12);
633 + int idx = 1;
634 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
635 + if(slot >= 0) idx++;
636 +
637 + char *type = get_word(words, num_words, idx++);
638 + char *name = get_word(words, num_words, idx++);
639 + char *title = get_word(words, num_words, idx++);
640 + char *units = get_word(words, num_words, idx++);
641 + char *family = get_word(words, num_words, idx++);
642 + char *context = get_word(words, num_words, idx++);
643 + char *chart = get_word(words, num_words, idx++);
644 + char *priority_s = get_word(words, num_words, idx++);
645 + char *update_every_s = get_word(words, num_words, idx++);
646 + char *options = get_word(words, num_words, idx++);
647 + char *plugin = get_word(words, num_words, idx++);
648 + char *module = get_word(words, num_words, idx++);
649
650 // parse the id from type
651 char *id = NULL;
@@ -570,14 +712,15 @@ static inline PARSER_RC pluginsd_chart(char **words, size_t num_words, PARSER *p
712 module, priority, update_every,
713 chart_type);
714
715 + bool obsolete = false;
716 if (likely(st)) {
717 if (options && *options) {
718 if (strstr(options, "obsolete")) {
576 - pluginsd_rrdset_cleanup(st);
577 - rrdset_is_obsolete(st);
719 + rrdset_is_obsolete___safe_from_collector_thread(st);
720 + obsolete = true;
721 }
722 else
580 - rrdset_isnot_obsolete(st);
723 + rrdset_isnot_obsolete___safe_from_collector_thread(st);
724
725 if (strstr(options, "detail"))
726 rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
@@ -595,13 +738,15 @@ static inline PARSER_RC pluginsd_chart(char **words, size_t num_words, PARSER *p
738 rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
739 }
740 else {
598 - rrdset_isnot_obsolete(st);
741 + rrdset_isnot_obsolete___safe_from_collector_thread(st);
742 rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
743 rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
744 }
745
746 if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_CHART))
747 return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
748 +
749 + pluginsd_rrdset_cache_put_to_slot(parser, st, slot, obsolete);
750 }
751 else
752 pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_CHART);
@@ -652,12 +797,16 @@ static inline PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_w
797 }
798
799 static inline PARSER_RC pluginsd_dimension(char **words, size_t num_words, PARSER *parser) {
655 - char *id = get_word(words, num_words, 1);
656 - char *name = get_word(words, num_words, 2);
657 - char *algorithm = get_word(words, num_words, 3);
658 - char *multiplier_s = get_word(words, num_words, 4);
659 - char *divisor_s = get_word(words, num_words, 5);
660 - char *options = get_word(words, num_words, 6);
800 + int idx = 1;
801 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
802 + if(slot >= 0) idx++;
803 +
804 + char *id = get_word(words, num_words, idx++);
805 + char *name = get_word(words, num_words, idx++);
806 + char *algorithm = get_word(words, num_words, idx++);
807 + char *multiplier_s = get_word(words, num_words, idx++);
808 + char *divisor_s = get_word(words, num_words, idx++);
809 + char *options = get_word(words, num_words, idx++);
810
811 RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_DIMENSION);
812 if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
@@ -696,11 +845,14 @@ static inline PARSER_RC pluginsd_dimension(char **words, size_t num_words, PARSE
845 int unhide_dimension = 1;
846
847 rrddim_option_clear(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS);
848 + bool obsolete = false;
849 if (options && *options) {
700 - if (strstr(options, "obsolete") != NULL)
701 - rrddim_is_obsolete(st, rd);
850 + if (strstr(options, "obsolete") != NULL) {
851 + obsolete = true;
852 + rrddim_is_obsolete___safe_from_collector_thread(st, rd);
853 + }
854 else
703 - rrddim_isnot_obsolete(st, rd);
855 + rrddim_isnot_obsolete___safe_from_collector_thread(st, rd);
856
857 unhide_dimension = !strstr(options, "hidden");
858
@@ -708,8 +860,9 @@ static inline PARSER_RC pluginsd_dimension(char **words, size_t num_words, PARSE
860 rrddim_option_set(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS);
861 if (strstr(options, "nooverflow") != NULL)
862 rrddim_option_set(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS);
711 - } else
712 - rrddim_isnot_obsolete(st, rd);
863 + }
864 + else
865 + rrddim_isnot_obsolete___safe_from_collector_thread(st, rd);
866
867 bool should_update_dimension = false;
868
@@ -727,6 +880,8 @@ static inline PARSER_RC pluginsd_dimension(char **words, size_t num_words, PARSE
880 rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
881 }
882
883 + pluginsd_rrddim_put_to_slot(parser, st, rd, slot, obsolete);
884 +
885 return PARSER_RC_OK;
886 }
887
@@ -1317,10 +1472,14 @@ static inline PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size
1472 }
1473
1474 static inline PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser) {
1320 - char *id = get_word(words, num_words, 1);
1321 - char *start_time_str = get_word(words, num_words, 2);
1322 - char *end_time_str = get_word(words, num_words, 3);
1323 - char *child_now_str = get_word(words, num_words, 4);
1475 + int idx = 1;
1476 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
1477 + if(slot >= 0) idx++;
1478 +
1479 + char *id = get_word(words, num_words, idx++);
1480 + char *start_time_str = get_word(words, num_words, idx++);
1481 + char *end_time_str = get_word(words, num_words, idx++);
1482 + char *child_now_str = get_word(words, num_words, idx++);
1483
1484 RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1485 if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
@@ -1329,7 +1488,7 @@ static inline PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PA
1488 if (likely(!id || !*id))
1489 st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1490 else
1332 - st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1491 + st = pluginsd_rrdset_cache_get_from_slot(parser, host, id, slot, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1492
1493 if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1494
@@ -1444,9 +1603,13 @@ static inline SN_FLAGS pluginsd_parse_storage_number_flags(const char *flags_str
1603 }
1604
1605 static inline PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser) {
1447 - char *dimension = get_word(words, num_words, 1);
1448 - char *value_str = get_word(words, num_words, 2);
1449 - char *flags_str = get_word(words, num_words, 3);
1606 + int idx = 1;
1607 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
1608 + if(slot >= 0) idx++;
1609 +
1610 + char *dimension = get_word(words, num_words, idx++);
1611 + char *value_str = get_word(words, num_words, idx++);
1612 + char *flags_str = get_word(words, num_words, idx++);
1613
1614 RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_SET);
1615 if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
@@ -1463,7 +1626,7 @@ static inline PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARS
1626 return PARSER_RC_OK;
1627 }
1628
1466 - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_REPLAY_SET);
1629 + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_SET);
1630 if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1631
1632 st->pluginsd.set = true;
@@ -1517,11 +1680,15 @@ static inline PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, si
1680 if(parser->user.replay.rset_enabled == false)
1681 return PARSER_RC_OK;
1682
1520 - char *dimension = get_word(words, num_words, 1);
1521 - char *last_collected_ut_str = get_word(words, num_words, 2);
1522 - char *last_collected_value_str = get_word(words, num_words, 3);
1523 - char *last_calculated_value_str = get_word(words, num_words, 4);
1524 - char *last_stored_value_str = get_word(words, num_words, 5);
1683 + int idx = 1;
1684 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
1685 + if(slot >= 0) idx++;
1686 +
1687 + char *dimension = get_word(words, num_words, idx++);
1688 + char *last_collected_ut_str = get_word(words, num_words, idx++);
1689 + char *last_collected_value_str = get_word(words, num_words, idx++);
1690 + char *last_calculated_value_str = get_word(words, num_words, idx++);
1691 + char *last_stored_value_str = get_word(words, num_words, idx++);
1692
1693 RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1694 if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
@@ -1535,7 +1702,7 @@ static inline PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, si
1702 st->pluginsd.set = false;
1703 }
1704
1538 - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1705 + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1706 if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1707
1708 usec_t dim_last_collected_ut = (usec_t)rd->collector.last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->collector.last_collected_time.tv_usec;
@@ -1699,10 +1866,14 @@ static inline PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARS
1866 static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER *parser) {
1867 timing_init();
1868
1702 - char *id = get_word(words, num_words, 1);
1703 - char *update_every_str = get_word(words, num_words, 2);
1704 - char *end_time_str = get_word(words, num_words, 3);
1705 - char *wall_clock_time_str = get_word(words, num_words, 4);
1869 + int idx = 1;
1870 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
1871 + if(slot >= 0) idx++;
1872 +
1873 + char *id = get_word(words, num_words, idx++);
1874 + char *update_every_str = get_word(words, num_words, idx++);
1875 + char *end_time_str = get_word(words, num_words, idx++);
1876 + char *wall_clock_time_str = get_word(words, num_words, idx++);
1877
1878 if(unlikely(!id || !update_every_str || !end_time_str || !wall_clock_time_str))
1879 return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_BEGIN_V2, "missing parameters");
@@ -1712,14 +1883,15 @@ static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER
1883
1884 timing_step(TIMING_STEP_BEGIN2_PREPARE);
1885
1715 - RRDSET *st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_BEGIN_V2);
1886 + RRDSET *st = pluginsd_rrdset_cache_get_from_slot(parser, host, id, slot, PLUGINSD_KEYWORD_BEGIN_V2);
1887 +
1888 if(unlikely(!st)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1889
1890 if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_BEGIN_V2))
1891 return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1892
1893 if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))
1722 - rrdset_isnot_obsolete(st);
1894 + rrdset_isnot_obsolete___safe_from_collector_thread(st);
1895
1896 timing_step(TIMING_STEP_BEGIN2_FIND_CHART);
1897
@@ -1759,9 +1931,12 @@ static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER
1931 parser->user.v2.stream_buffer = rrdset_push_metric_initialize(parser->user.st, wall_clock_time);
1932
1933 if(parser->user.v2.stream_buffer.v2 && parser->user.v2.stream_buffer.wb) {
1762 - // check if receiver and sender have the same number parsing capabilities
1934 + // check receiver capabilities
1935 bool can_copy = stream_has_capability(&parser->user, STREAM_CAP_IEEE754) == stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754);
1764 - NUMBER_ENCODING encoding = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
1936 +
1937 + // check sender capabilities
1938 + bool with_slots = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_SLOTS) ? true : false;
1939 + NUMBER_ENCODING integer_encoding = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
1940
1941 BUFFER *wb = parser->user.v2.stream_buffer.wb;
1942
@@ -1770,28 +1945,35 @@ static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER
1945 if(unlikely(parser->user.v2.stream_buffer.begin_v2_added))
1946 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_END_V2 "\n", sizeof(PLUGINSD_KEYWORD_END_V2) - 1 + 1);
1947
1773 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2 " '", sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1 + 2);
1948 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2, sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1);
1949 +
1950 + if(with_slots) {
1951 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
1952 + buffer_print_uint64_encoded(wb, integer_encoding, st->rrdpush.sender.chart_slot);
1953 + }
1954 +
1955 + buffer_fast_strcat(wb, " '", 2);
1956 buffer_fast_strcat(wb, rrdset_id(st), string_strlen(st->id));
1957 buffer_fast_strcat(wb, "' ", 2);
1958
1959 if(can_copy)
1960 buffer_strcat(wb, update_every_str);
1961 else
1780 - buffer_print_uint64_encoded(wb, encoding, update_every);
1962 + buffer_print_uint64_encoded(wb, integer_encoding, update_every);
1963
1964 buffer_fast_strcat(wb, " ", 1);
1965
1966 if(can_copy)
1967 buffer_strcat(wb, end_time_str);
1968 else
1787 - buffer_print_uint64_encoded(wb, encoding, end_time);
1969 + buffer_print_uint64_encoded(wb, integer_encoding, end_time);
1970
1971 buffer_fast_strcat(wb, " ", 1);
1972
1973 if(can_copy)
1974 buffer_strcat(wb, wall_clock_time_str);
1975 else
1794 - buffer_print_uint64_encoded(wb, encoding, wall_clock_time);
1976 + buffer_print_uint64_encoded(wb, integer_encoding, wall_clock_time);
1977
1978 buffer_fast_strcat(wb, "\n", 1);
1979
@@ -1824,10 +2006,14 @@ static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER
2006 static inline PARSER_RC pluginsd_set_v2(char **words, size_t num_words, PARSER *parser) {
2007 timing_init();
2008
1827 - char *dimension = get_word(words, num_words, 1);
1828 - char *collected_str = get_word(words, num_words, 2);
1829 - char *value_str = get_word(words, num_words, 3);
1830 - char *flags_str = get_word(words, num_words, 4);
2009 + int idx = 1;
2010 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
2011 + if(slot >= 0) idx++;
2012 +
2013 + char *dimension = get_word(words, num_words, idx++);
2014 + char *collected_str = get_word(words, num_words, idx++);
2015 + char *value_str = get_word(words, num_words, idx++);
2016 + char *flags_str = get_word(words, num_words, idx++);
2017
2018 if(unlikely(!dimension || !collected_str || !value_str || !flags_str))
2019 return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_SET_V2, "missing parameters");
@@ -1840,13 +2026,13 @@ static inline PARSER_RC pluginsd_set_v2(char **words, size_t num_words, PARSER *
2026
2027 timing_step(TIMING_STEP_SET2_PREPARE);
2028
1843 - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_SET_V2);
2029 + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_SET_V2);
2030 if(unlikely(!rd)) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
2031
2032 st->pluginsd.set = true;
2033
2034 if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED)))
1849 - rrddim_isnot_obsolete(st, rd);
2035 + rrddim_isnot_obsolete___safe_from_collector_thread(st, rd);
2036
2037 timing_step(TIMING_STEP_SET2_LOOKUP_DIMENSION);
2038
@@ -1892,12 +2078,22 @@ static inline PARSER_RC pluginsd_set_v2(char **words, size_t num_words, PARSER *
2078 if(parser->user.v2.stream_buffer.v2 && parser->user.v2.stream_buffer.begin_v2_added && parser->user.v2.stream_buffer.wb) {
2079 // check if receiver and sender have the same number parsing capabilities
2080 bool can_copy = stream_has_capability(&parser->user, STREAM_CAP_IEEE754) == stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754);
2081 +
2082 + // check the sender capabilities
2083 + bool with_slots = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_SLOTS) ? true : false;
2084 NUMBER_ENCODING integer_encoding = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
2085 NUMBER_ENCODING doubles_encoding = stream_has_capability(&parser->user.v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
2086
2087 BUFFER *wb = parser->user.v2.stream_buffer.wb;
2088 buffer_need_bytes(wb, 1024);
1900 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2 " '", sizeof(PLUGINSD_KEYWORD_SET_V2) - 1 + 2);
2089 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2, sizeof(PLUGINSD_KEYWORD_SET_V2) - 1);
2090 +
2091 + if(with_slots) {
2092 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
2093 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
2094 + }
2095 +
2096 + buffer_fast_strcat(wb, " '", 2);
2097 buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
2098 buffer_fast_strcat(wb, "' ", 2);
2099 if(can_copy)
@@ -1978,13 +2174,27 @@ static inline PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_
2174 // ------------------------------------------------------------------------
2175 // cleanup RRDSET / RRDDIM
2176
1981 - RRDDIM *rd;
1982 - rrddim_foreach_read(rd, st) {
1983 - rd->collector.calculated_value = 0;
1984 - rd->collector.collected_value = 0;
1985 - rrddim_clear_updated(rd);
2177 + if(likely(st->pluginsd.with_slots)) {
2178 + for(size_t i = 0; i < st->pluginsd.size ;i++) {
2179 + RRDDIM *rd = st->pluginsd.prd_array[i].rd;
2180 +
2181 + if(!rd)
2182 + continue;
2183 +
2184 + rd->collector.calculated_value = 0;
2185 + rd->collector.collected_value = 0;
2186 + rrddim_clear_updated(rd);
2187 + }
2188 + }
2189 + else {
2190 + RRDDIM *rd;
2191 + rrddim_foreach_read(rd, st){
2192 + rd->collector.calculated_value = 0;
2193 + rd->collector.collected_value = 0;
2194 + rrddim_clear_updated(rd);
2195 + }
2196 + rrddim_foreach_done(rd);
2197 }
1987 - rrddim_foreach_done(rd);
2198
2199 // ------------------------------------------------------------------------
2200 // reset state
collectors/plugins.d/pluginsd_parser.h
+3
@@ -13,6 +13,8 @@
13 // this controls the max response size of a function
14 #define PLUGINSD_MAX_DEFERRED_SIZE (20 * 1024 * 1024)
15
16 +#define PLUGINSD_MIN_RRDSET_POINTERS_CACHE 1024
17 +
18 // PARSER return codes
19 typedef enum __attribute__ ((__packed__)) parser_rc {
20 PARSER_RC_OK, // Callback was successful, go on
@@ -41,6 +43,7 @@ typedef struct parser_keyword {
43 } PARSER_KEYWORD;
44
45 typedef struct parser_user_object {
46 + bool cleanup_slots;
47 RRDSET *st;
48 RRDHOST *host;
49 void *opaque;
collectors/proc.plugin/ipc.c
+6 -6
@@ -451,8 +451,8 @@ int do_ipc(int update_every, usec_t dt) {
451 msq->found = 0;
452 }
453 else {
454 - rrddim_is_obsolete(st_msq_messages, msq->rd_messages);
455 - rrddim_is_obsolete(st_msq_bytes, msq->rd_bytes);
454 + rrddim_is_obsolete___safe_from_collector_thread(st_msq_messages, msq->rd_messages);
455 + rrddim_is_obsolete___safe_from_collector_thread(st_msq_bytes, msq->rd_bytes);
456
457 // remove message queue from the linked list
458 if(!msq_prev)
@@ -480,19 +480,19 @@ int do_ipc(int update_every, usec_t dt) {
480 if(unlikely(dimensions_num > dimensions_limit)) {
481 collector_info("Message queue statistics has been disabled");
482 collector_info("There are %lld dimensions in memory but limit was set to %lld", dimensions_num, dimensions_limit);
483 - rrdset_is_obsolete(st_msq_messages);
484 - rrdset_is_obsolete(st_msq_bytes);
483 + rrdset_is_obsolete___safe_from_collector_thread(st_msq_messages);
484 + rrdset_is_obsolete___safe_from_collector_thread(st_msq_bytes);
485 st_msq_messages = NULL;
486 st_msq_bytes = NULL;
487 do_msg = CONFIG_BOOLEAN_NO;
488 }
489 else if(unlikely(!message_queue_root)) {
490 collector_info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_messages), rrdset_id(st_msq_messages));
491 - rrdset_is_obsolete(st_msq_messages);
491 + rrdset_is_obsolete___safe_from_collector_thread(st_msq_messages);
492 st_msq_messages = NULL;
493
494 collector_info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_bytes), rrdset_id(st_msq_bytes));
495 - rrdset_is_obsolete(st_msq_bytes);
495 + rrdset_is_obsolete___safe_from_collector_thread(st_msq_bytes);
496 st_msq_bytes = NULL;
497 }
498 }
collectors/proc.plugin/proc_diskstats.c
+1 -1
@@ -168,7 +168,7 @@ static struct disk {
168 struct disk *next;
169 } *disk_root = NULL;
170
171 -#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
171 +#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete___safe_from_collector_thread(st); (st) = NULL; } } while(st)
172
173 // static char *path_to_get_hw_sector_size = NULL;
174 // static char *path_to_get_hw_sector_size_partitions = NULL;
collectors/proc.plugin/proc_mdstat.c
+8 -8
@@ -73,7 +73,7 @@ static inline void make_chart_obsolete(char *name, const char *id_modifier)
73 snprintfz(id, 50, "mdstat.%s_%s", name, id_modifier);
74 st = rrdset_find_active_byname_localhost(id);
75 if (likely(st))
76 - rrdset_is_obsolete(st);
76 + rrdset_is_obsolete___safe_from_collector_thread(st);
77 }
78 }
79
@@ -409,7 +409,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
409 update_every,
410 RRDSET_TYPE_LINE);
411
412 - rrdset_isnot_obsolete(st_mdstat_health);
412 + rrdset_isnot_obsolete___safe_from_collector_thread(st_mdstat_health);
413 }
414
415 if (!redundant_num) {
@@ -457,7 +457,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
457 update_every,
458 RRDSET_TYPE_STACKED);
459
460 - rrdset_isnot_obsolete(raid->st_disks);
460 + rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_disks);
461
462 add_labels_to_mdstat(raid, raid->st_disks);
463 }
@@ -492,7 +492,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
492 update_every,
493 RRDSET_TYPE_LINE);
494
495 - rrdset_isnot_obsolete(raid->st_mismatch_cnt);
495 + rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_mismatch_cnt);
496
497 add_labels_to_mdstat(raid, raid->st_mismatch_cnt);
498 }
@@ -524,7 +524,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
524 update_every,
525 RRDSET_TYPE_LINE);
526
527 - rrdset_isnot_obsolete(raid->st_operation);
527 + rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_operation);
528
529 add_labels_to_mdstat(raid, raid->st_operation);
530 }
@@ -561,7 +561,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
561 NETDATA_CHART_PRIO_MDSTAT_FINISH + raid_idx * 10,
562 update_every, RRDSET_TYPE_LINE);
563
564 - rrdset_isnot_obsolete(raid->st_finish);
564 + rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_finish);
565
566 add_labels_to_mdstat(raid, raid->st_finish);
567 }
@@ -590,7 +590,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
590 update_every,
591 RRDSET_TYPE_LINE);
592
593 - rrdset_isnot_obsolete(raid->st_speed);
593 + rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_speed);
594
595 add_labels_to_mdstat(raid, raid->st_speed);
596 }
@@ -622,7 +622,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
622 update_every,
623 RRDSET_TYPE_LINE);
624
625 - rrdset_isnot_obsolete(raid->st_nonredundant);
625 + rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_nonredundant);
626
627 add_labels_to_mdstat(raid, raid->st_nonredundant);
628 }
collectors/proc.plugin/proc_net_dev.c
+12 -12
@@ -216,18 +216,18 @@ static size_t netdev_added = 0, netdev_found = 0;
216 // ----------------------------------------------------------------------------
217
218 static void netdev_charts_release(struct netdev *d) {
219 - if(d->st_bandwidth) rrdset_is_obsolete(d->st_bandwidth);
220 - if(d->st_packets) rrdset_is_obsolete(d->st_packets);
221 - if(d->st_errors) rrdset_is_obsolete(d->st_errors);
222 - if(d->st_drops) rrdset_is_obsolete(d->st_drops);
223 - if(d->st_fifo) rrdset_is_obsolete(d->st_fifo);
224 - if(d->st_compressed) rrdset_is_obsolete(d->st_compressed);
225 - if(d->st_events) rrdset_is_obsolete(d->st_events);
226 - if(d->st_speed) rrdset_is_obsolete(d->st_speed);
227 - if(d->st_duplex) rrdset_is_obsolete(d->st_duplex);
228 - if(d->st_operstate) rrdset_is_obsolete(d->st_operstate);
229 - if(d->st_carrier) rrdset_is_obsolete(d->st_carrier);
230 - if(d->st_mtu) rrdset_is_obsolete(d->st_mtu);
219 + if(d->st_bandwidth) rrdset_is_obsolete___safe_from_collector_thread(d->st_bandwidth);
220 + if(d->st_packets) rrdset_is_obsolete___safe_from_collector_thread(d->st_packets);
221 + if(d->st_errors) rrdset_is_obsolete___safe_from_collector_thread(d->st_errors);
222 + if(d->st_drops) rrdset_is_obsolete___safe_from_collector_thread(d->st_drops);
223 + if(d->st_fifo) rrdset_is_obsolete___safe_from_collector_thread(d->st_fifo);
224 + if(d->st_compressed) rrdset_is_obsolete___safe_from_collector_thread(d->st_compressed);
225 + if(d->st_events) rrdset_is_obsolete___safe_from_collector_thread(d->st_events);
226 + if(d->st_speed) rrdset_is_obsolete___safe_from_collector_thread(d->st_speed);
227 + if(d->st_duplex) rrdset_is_obsolete___safe_from_collector_thread(d->st_duplex);
228 + if(d->st_operstate) rrdset_is_obsolete___safe_from_collector_thread(d->st_operstate);
229 + if(d->st_carrier) rrdset_is_obsolete___safe_from_collector_thread(d->st_carrier);
230 + if(d->st_mtu) rrdset_is_obsolete___safe_from_collector_thread(d->st_mtu);
231
232 d->st_bandwidth = NULL;
233 d->st_compressed = NULL;
collectors/proc.plugin/proc_net_wireless.c
+7 -6
@@ -85,12 +85,13 @@ static struct netwireless {
85
86 static void netwireless_free_st(struct netwireless *wireless_dev)
87 {
88 - if (wireless_dev->st_status) rrdset_is_obsolete(wireless_dev->st_status);
89 - if (wireless_dev->st_link) rrdset_is_obsolete(wireless_dev->st_link);
90 - if (wireless_dev->st_level) rrdset_is_obsolete(wireless_dev->st_level);
91 - if (wireless_dev->st_noise) rrdset_is_obsolete(wireless_dev->st_noise);
92 - if (wireless_dev->st_discarded_packets) rrdset_is_obsolete(wireless_dev->st_discarded_packets);
93 - if (wireless_dev->st_missed_beacon) rrdset_is_obsolete(wireless_dev->st_missed_beacon);
88 + if (wireless_dev->st_status) rrdset_is_obsolete___safe_from_collector_thread(wireless_dev->st_status);
89 + if (wireless_dev->st_link) rrdset_is_obsolete___safe_from_collector_thread(wireless_dev->st_link);
90 + if (wireless_dev->st_level) rrdset_is_obsolete___safe_from_collector_thread(wireless_dev->st_level);
91 + if (wireless_dev->st_noise) rrdset_is_obsolete___safe_from_collector_thread(wireless_dev->st_noise);
92 + if (wireless_dev->st_discarded_packets)
93 + rrdset_is_obsolete___safe_from_collector_thread(wireless_dev->st_discarded_packets);
94 + if (wireless_dev->st_missed_beacon) rrdset_is_obsolete___safe_from_collector_thread(wireless_dev->st_missed_beacon);
95
96 wireless_dev->st_status = NULL;
97 wireless_dev->st_link = NULL;
collectors/proc.plugin/proc_spl_kstat_zfs.c
+1 -1
@@ -240,7 +240,7 @@ DICTIONARY *zfs_pools = NULL;
240 void disable_zfs_pool_state(struct zfs_pool *pool)
241 {
242 if (pool->st)
243 - rrdset_is_obsolete(pool->st);
243 + rrdset_is_obsolete___safe_from_collector_thread(pool->st);
244
245 pool->st = NULL;
246
collectors/proc.plugin/sys_block_zram.c
+1 -1
@@ -3,7 +3,7 @@
3 #include "plugin_proc.h"
4
5 #define PLUGIN_PROC_MODULE_ZRAM_NAME "/sys/block/zram"
6 -#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
6 +#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete___safe_from_collector_thread(st); (st) = NULL; } } while(st)
7
8 typedef struct mm_stat {
9 unsigned long long orig_data_size;
collectors/proc.plugin/sys_class_drm.c
+10 -10
@@ -707,7 +707,7 @@ static int do_rrd_util_gpu(struct card *const c){
707 else {
708 collector_error("Cannot read util_gpu for %s: [%s]", c->pathname, c->id.marketing_name);
709 freez((void *) c->pathname_util_gpu);
710 - rrdset_is_obsolete(c->st_util_gpu);
710 + rrdset_is_obsolete___safe_from_collector_thread(c->st_util_gpu);
711 return 1;
712 }
713 }
@@ -721,7 +721,7 @@ static int do_rrd_util_mem(struct card *const c){
721 else {
722 collector_error("Cannot read util_mem for %s: [%s]", c->pathname, c->id.marketing_name);
723 freez((void *) c->pathname_util_mem);
724 - rrdset_is_obsolete(c->st_util_mem);
724 + rrdset_is_obsolete___safe_from_collector_thread(c->st_util_mem);
725 return 1;
726 }
727 }
@@ -735,7 +735,7 @@ static int do_rrd_clk_gpu(struct card *const c){
735 else {
736 collector_error("Cannot read clk_gpu for %s: [%s]", c->pathname, c->id.marketing_name);
737 freez((void *) c->pathname_clk_gpu);
738 - rrdset_is_obsolete(c->st_clk_gpu);
738 + rrdset_is_obsolete___safe_from_collector_thread(c->st_clk_gpu);
739 return 1;
740 }
741 }
@@ -749,7 +749,7 @@ static int do_rrd_clk_mem(struct card *const c){
749 else {
750 collector_error("Cannot read clk_mem for %s: [%s]", c->pathname, c->id.marketing_name);
751 freez((void *) c->pathname_clk_mem);
752 - rrdset_is_obsolete(c->st_clk_mem);
752 + rrdset_is_obsolete___safe_from_collector_thread(c->st_clk_mem);
753 return 1;
754 }
755 }
@@ -771,8 +771,8 @@ static int do_rrd_vram(struct card *const c){
771 collector_error("Cannot read used_vram for %s: [%s]", c->pathname, c->id.marketing_name);
772 freez((void *) c->pathname_mem_used_vram);
773 freez((void *) c->pathname_mem_total_vram);
774 - rrdset_is_obsolete(c->st_mem_usage_perc_vram);
775 - rrdset_is_obsolete(c->st_mem_usage_vram);
774 + rrdset_is_obsolete___safe_from_collector_thread(c->st_mem_usage_perc_vram);
775 + rrdset_is_obsolete___safe_from_collector_thread(c->st_mem_usage_vram);
776 return 1;
777 }
778 }
@@ -794,8 +794,8 @@ static int do_rrd_vis_vram(struct card *const c){
794 collector_error("Cannot read used_vis_vram for %s: [%s]", c->pathname, c->id.marketing_name);
795 freez((void *) c->pathname_mem_used_vis_vram);
796 freez((void *) c->pathname_mem_total_vis_vram);
797 - rrdset_is_obsolete(c->st_mem_usage_perc_vis_vram);
798 - rrdset_is_obsolete(c->st_mem_usage_vis_vram);
797 + rrdset_is_obsolete___safe_from_collector_thread(c->st_mem_usage_perc_vis_vram);
798 + rrdset_is_obsolete___safe_from_collector_thread(c->st_mem_usage_vis_vram);
799 return 1;
800 }
801 }
@@ -817,8 +817,8 @@ static int do_rrd_gtt(struct card *const c){
817 collector_error("Cannot read used_gtt for %s: [%s]", c->pathname, c->id.marketing_name);
818 freez((void *) c->pathname_mem_used_gtt);
819 freez((void *) c->pathname_mem_total_gtt);
820 - rrdset_is_obsolete(c->st_mem_usage_perc_gtt);
821 - rrdset_is_obsolete(c->st_mem_usage_gtt);
820 + rrdset_is_obsolete___safe_from_collector_thread(c->st_mem_usage_perc_gtt);
821 + rrdset_is_obsolete___safe_from_collector_thread(c->st_mem_usage_gtt);
822 return 1;
823 }
824 }
collectors/proc.plugin/sys_class_power_supply.c
+2 -2
@@ -65,7 +65,7 @@ void power_supply_free(struct power_supply *ps) {
65
66 // free capacity structure
67 if(likely(ps->capacity)) {
68 - if(likely(ps->capacity->st)) rrdset_is_obsolete(ps->capacity->st);
68 + if(likely(ps->capacity->st)) rrdset_is_obsolete___safe_from_collector_thread(ps->capacity->st);
69 freez(ps->capacity->filename);
70 if(likely(ps->capacity->fd != -1)) close(ps->capacity->fd);
71 files_num--;
@@ -89,7 +89,7 @@ void power_supply_free(struct power_supply *ps) {
89 }
90
91 // free properties
92 - if(likely(pr->st)) rrdset_is_obsolete(pr->st);
92 + if(likely(pr->st)) rrdset_is_obsolete___safe_from_collector_thread(pr->st);
93 freez(pr->name);
94 freez(pr->title);
95 freez(pr->units);
collectors/proc.plugin/sys_fs_btrfs.c
+7 -7
@@ -196,8 +196,8 @@ static inline int collect_btrfs_commits_stats(BTRFS_NODE *node, int update_every
196
197 static inline void btrfs_free_commits_stats(BTRFS_NODE *node){
198 if(node->st_commits){
199 - rrdset_is_obsolete(node->st_commits);
200 - rrdset_is_obsolete(node->st_commit_timings);
199 + rrdset_is_obsolete___safe_from_collector_thread(node->st_commits);
200 + rrdset_is_obsolete___safe_from_collector_thread(node->st_commit_timings);
201 }
202 freez(node->commit_stats_filename);
203 node->commit_stats_filename = NULL;
@@ -211,7 +211,7 @@ static inline void btrfs_free_disk(BTRFS_DISK *d) {
211
212 static inline void btrfs_free_device(BTRFS_DEVICE *d) {
213 if(d->st_error_stats)
214 - rrdset_is_obsolete(d->st_error_stats);
214 + rrdset_is_obsolete___safe_from_collector_thread(d->st_error_stats);
215 freez(d->error_stats_filename);
216 freez(d);
217 }
@@ -220,16 +220,16 @@ static inline void btrfs_free_node(BTRFS_NODE *node) {
220 // collector_info("BTRFS: destroying '%s'", node->id);
221
222 if(node->st_allocation_disks)
223 - rrdset_is_obsolete(node->st_allocation_disks);
223 + rrdset_is_obsolete___safe_from_collector_thread(node->st_allocation_disks);
224
225 if(node->st_allocation_data)
226 - rrdset_is_obsolete(node->st_allocation_data);
226 + rrdset_is_obsolete___safe_from_collector_thread(node->st_allocation_data);
227
228 if(node->st_allocation_metadata)
229 - rrdset_is_obsolete(node->st_allocation_metadata);
229 + rrdset_is_obsolete___safe_from_collector_thread(node->st_allocation_metadata);
230
231 if(node->st_allocation_system)
232 - rrdset_is_obsolete(node->st_allocation_system);
232 + rrdset_is_obsolete___safe_from_collector_thread(node->st_allocation_system);
233
234 freez(node->allocation_data_bytes_used_filename);
235 freez(node->allocation_data_total_bytes_filename);
daemon/global_statistics.c
-6
@@ -1881,8 +1881,6 @@ static void dbengine2_statistics_charts(void) {
1881 static RRDDIM *rd_mrg_metrics = NULL;
1882 static RRDDIM *rd_mrg_acquired = NULL;
1883 static RRDDIM *rd_mrg_collected = NULL;
1884 - static RRDDIM *rd_mrg_with_retention = NULL;
1885 - static RRDDIM *rd_mrg_without_retention = NULL;
1884 static RRDDIM *rd_mrg_multiple_writers = NULL;
1885
1886 if (unlikely(!st_mrg_metrics)) {
@@ -1903,8 +1901,6 @@ static void dbengine2_statistics_charts(void) {
1901 rd_mrg_metrics = rrddim_add(st_mrg_metrics, "all", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1902 rd_mrg_acquired = rrddim_add(st_mrg_metrics, "acquired", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1903 rd_mrg_collected = rrddim_add(st_mrg_metrics, "collected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1906 - rd_mrg_with_retention = rrddim_add(st_mrg_metrics, "with retention", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1907 - rd_mrg_without_retention = rrddim_add(st_mrg_metrics, "without retention", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1904 rd_mrg_multiple_writers = rrddim_add(st_mrg_metrics, "multi-collected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1905 }
1906 priority++;
@@ -1912,8 +1908,6 @@ static void dbengine2_statistics_charts(void) {
1908 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_metrics, (collected_number)mrg_stats.entries);
1909 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_acquired, (collected_number)mrg_stats.entries_referenced);
1910 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_collected, (collected_number)mrg_stats.writers);
1915 - rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_with_retention, (collected_number)mrg_stats.entries_with_retention);
1916 - rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_without_retention, (collected_number)mrg_stats.entries - (collected_number)mrg_stats.entries_with_retention);
1911 rrddim_set_by_pointer(st_mrg_metrics, rd_mrg_multiple_writers, (collected_number)mrg_stats.writers_conflicts);
1912
1913 rrdset_done(st_mrg_metrics);
daemon/service.c
+83 -37
@@ -76,33 +76,48 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
76 rrddim_free(st, rd);
77 }
78
79 -static bool svc_rrdset_archive_obsolete_dimensions(RRDSET *st, bool all_dimensions) {
79 +static inline bool svc_rrdset_archive_obsolete_dimensions(RRDSET *st, bool all_dimensions) {
80 + if(!all_dimensions && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))
81 + return true;
82 +
83 worker_is_busy(WORKER_JOB_ARCHIVE_CHART_DIMENSIONS);
84
85 + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
86 +
87 RRDDIM *rd;
88 time_t now = now_realtime_sec();
89
85 - bool done_all_dimensions = true;
90 + size_t dim_candidates = 0;
91 + size_t dim_archives = 0;
92
93 dfe_start_write(st->rrddim_root_index, rd) {
88 - if(unlikely(
89 - all_dimensions ||
90 - (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && (rd->collector.last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now))
91 - )) {
92 -
93 - if(dictionary_acquired_item_references(rd_dfe.item) == 1) {
94 - netdata_log_info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rrddim_name(rd), rrddim_id(rd), rrdset_name(st), rrdset_id(st));
95 - svc_rrddim_obsolete_to_archive(rd);
94 + bool candidate = (all_dimensions || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE));
95 +
96 + if(candidate) {
97 + dim_candidates++;
98 +
99 + if(rd->collector.last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now) {
100 + size_t references = dictionary_acquired_item_references(rd_dfe.item);
101 + if(references == 1) {
102 +// netdata_log_info("Removing obsolete dimension 'host:%s/chart:%s/dim:%s'",
103 +// rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd));
104 + svc_rrddim_obsolete_to_archive(rd);
105 + dim_archives++;
106 + }
107 +// else
108 +// netdata_log_info("Cannot remove obsolete dimension 'host:%s/chart:%s/dim:%s'",
109 +// rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd));
110 }
97 - else
98 - done_all_dimensions = false;
111 }
100 - else
101 - done_all_dimensions = false;
112 }
113 dfe_done(rd);
114
105 - return done_all_dimensions;
115 + if(dim_archives != dim_candidates) {
116 + rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
117 + return false;
118 + }
119 +
120 + return true;
121 }
122
123 static void svc_rrdset_obsolete_to_free(RRDSET *st) {
@@ -132,34 +147,58 @@ static void svc_rrdset_obsolete_to_free(RRDSET *st) {
147 rrdset_free(st);
148 }
149
135 -static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
150 +static inline void svc_rrdhost_cleanup_charts_marked_obsolete(RRDHOST *host) {
151 + if(!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS))
152 + return;
153 +
154 worker_is_busy(WORKER_JOB_CLEANUP_OBSOLETE_CHARTS);
155
156 + rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS);
157 +
158 + size_t full_candidates = 0;
159 + size_t full_archives = 0;
160 + size_t partial_candidates = 0;
161 + size_t partial_archives = 0;
162 +
163 time_t now = now_realtime_sec();
164 RRDSET *st;
165 rrdset_foreach_reentrant(st, host) {
166 if(rrdset_is_replicating(st))
167 continue;
168
144 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)
145 - && st->last_accessed_time_s + rrdset_free_obsolete_time_s < now
146 - && st->last_updated.tv_sec + rrdset_free_obsolete_time_s < now
147 - && st->last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now
148 - )) {
149 - svc_rrdset_obsolete_to_free(st);
150 - }
151 - else if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) {
152 - rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
153 - svc_rrdset_archive_obsolete_dimensions(st, false);
169 + RRDSET_FLAGS flags = rrdset_flag_get(st);
170 + bool obsolete_chart = flags & RRDSET_FLAG_OBSOLETE;
171 + bool obsolete_dims = flags & RRDSET_FLAG_OBSOLETE_DIMENSIONS;
172 +
173 + if(obsolete_dims) {
174 + partial_candidates++;
175 +
176 + if(svc_rrdset_archive_obsolete_dimensions(st, false))
177 + partial_archives++;
178 }
155 - else if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
156 - rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS);
179 +
180 + if(obsolete_chart) {
181 + full_candidates++;
182 +
183 + if(unlikely( st->last_accessed_time_s + rrdset_free_obsolete_time_s < now
184 + && st->last_updated.tv_sec + rrdset_free_obsolete_time_s < now
185 + && st->last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now
186 + )) {
187 + svc_rrdset_obsolete_to_free(st);
188 + full_archives++;
189 + }
190 }
191 }
192 rrdset_foreach_done(st);
193 +
194 + if(partial_archives != partial_candidates)
195 + rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS);
196 +
197 + if(full_archives != full_candidates)
198 + rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS);
199 }
200
162 -static void svc_rrdset_check_obsoletion(RRDHOST *host) {
201 +static void svc_rrdhost_detect_obsolete_charts(RRDHOST *host) {
202 worker_is_busy(WORKER_JOB_CHILD_CHART_OBSOLETION_CHECK);
203
204 time_t now = now_realtime_sec();
@@ -171,11 +210,21 @@ static void svc_rrdset_check_obsoletion(RRDHOST *host) {
210
211 last_entry_t = rrdset_last_entry_s(st);
212
213 +// if(last_entry_t + st->update_every * 2 + 30 < now)
214 +// netdata_log_error("Possibly obsolete chart 'host:%s/chart:%s', last entry is %zu secs old "
215 +// "(replicating: %s, obsolete: %s, obsolete dims: %s)",
216 +// rrdhost_hostname(host), rrdset_id(st), now - last_entry_t,
217 +// rrdset_is_replicating(st) ? "true" : "false",
218 +// rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) ? "true" : "false",
219 +// rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS) ? "true" : "false"
220 +// );
221 +
222 +
223 if(last_entry_t && last_entry_t < host->child_connect_time &&
175 - host->child_connect_time + TIME_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT + ITERATIONS_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT * st->update_every
176 - < now)
224 + host->child_connect_time + TIME_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT +
225 + (ITERATIONS_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT * st->update_every) < now)
226
178 - rrdset_is_obsolete(st);
227 + rrdset_is_obsolete___safe_from_collector_thread(st);
228 }
229 rrdset_foreach_done(st);
230 }
@@ -190,10 +239,7 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
239 if(rrdhost_receiver_replicating_charts(host) || rrdhost_sender_replicating_charts(host))
240 continue;
241
193 - if(rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS)) {
194 - rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS);
195 - svc_rrdhost_cleanup_obsolete_charts(host);
196 - }
242 + svc_rrdhost_cleanup_charts_marked_obsolete(host);
243
244 if(host != localhost
245 && host->trigger_chart_obsoletion_check
@@ -205,7 +251,7 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
251 || (host->child_connect_time + TIME_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT < now_realtime_sec())
252 )
253 ) {
208 - svc_rrdset_check_obsoletion(host);
254 + svc_rrdhost_detect_obsolete_charts(host);
255 host->trigger_chart_obsoletion_check = 0;
256 }
257 }
daemon/unit_test.c
+11
@@ -2101,6 +2101,14 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
2101 return errors + value_errors + time_errors;
2102 }
2103
2104 +void test_dbengine_charts_and_dims_are_not_collected(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]) {
2105 + for(int c = 0; c < CHARTS ; c++) {
2106 + st[c]->rrdcontexts.collected = false;
2107 + for(int d = 0; d < DIMS ; d++)
2108 + rd[c][d]->rrdcontexts.collected = false;
2109 + }
2110 +}
2111 +
2112 int test_dbengine(void)
2113 {
2114 fprintf(stderr, "%s() running...\n", __FUNCTION__ );
@@ -2128,6 +2136,7 @@ int test_dbengine(void)
2136 time_end[current_region] = test_dbengine_create_metrics(st,rd, current_region, time_start[current_region]);
2137
2138 errors += test_dbengine_check_metrics(st, rd, current_region, time_start[current_region]);
2139 + test_dbengine_charts_and_dims_are_not_collected(st, rd);
2140
2141 current_region = 1; //this is the second region of data
2142 update_every = REGION_UPDATE_EVERY[current_region]; // set data collection frequency to 3 seconds
@@ -2145,6 +2154,7 @@ int test_dbengine(void)
2154 time_end[current_region] = test_dbengine_create_metrics(st,rd, current_region, time_start[current_region]);
2155
2156 errors += test_dbengine_check_metrics(st, rd, current_region, time_start[current_region]);
2157 + test_dbengine_charts_and_dims_are_not_collected(st, rd);
2158
2159 current_region = 2; //this is the third region of data
2160 update_every = REGION_UPDATE_EVERY[current_region]; // set data collection frequency to 1 seconds
@@ -2162,6 +2172,7 @@ int test_dbengine(void)
2172 time_end[current_region] = test_dbengine_create_metrics(st,rd, current_region, time_start[current_region]);
2173
2174 errors += test_dbengine_check_metrics(st, rd, current_region, time_start[current_region]);
2175 + test_dbengine_charts_and_dims_are_not_collected(st, rd);
2176
2177 for (current_region = 0 ; current_region < REGIONS ; ++current_region) {
2178 errors += test_dbengine_check_rrdr(st, rd, current_region, time_start[current_region], time_end[current_region]);
database/contexts/api_v2.c
+1
@@ -674,6 +674,7 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke
674 buffer_json_member_add_uint64(wb, "metadata", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA]);
675 buffer_json_member_add_uint64(wb, "functions", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS]);
676 buffer_json_member_add_uint64(wb, "replication", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION]);
677 + buffer_json_member_add_uint64(wb, "dyncfg", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DYNCFG]);
678 }
679 buffer_json_object_close(wb); // traffic
680
database/contexts/instance.c
+27 -15
@@ -329,11 +329,11 @@ inline void rrdinstance_from_rrdset(RRDSET *st) {
329
330 RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, string2str(tri.id), &tri, sizeof(tri));
331
332 - RRDCONTEXT_ACQUIRED *rca_old = st->rrdcontext;
333 - RRDINSTANCE_ACQUIRED *ria_old = st->rrdinstance;
332 + RRDCONTEXT_ACQUIRED *rca_old = st->rrdcontexts.rrdcontext;
333 + RRDINSTANCE_ACQUIRED *ria_old = st->rrdcontexts.rrdinstance;
334
335 - st->rrdcontext = rca;
336 - st->rrdinstance = ria;
335 + st->rrdcontexts.rrdcontext = rca;
336 + st->rrdcontexts.rrdinstance = ria;
337
338 if(rca == rca_old) {
339 rrdcontext_release(rca_old);
@@ -354,16 +354,16 @@ inline void rrdinstance_from_rrdset(RRDSET *st) {
354 // migrate all dimensions to the new metrics
355 RRDDIM *rd;
356 rrddim_foreach_read(rd, st) {
357 - if (!rd->rrdmetric) continue;
357 + if (!rd->rrdcontexts.rrdmetric) continue;
358
359 - RRDMETRIC *rm_old = rrdmetric_acquired_value(rd->rrdmetric);
359 + RRDMETRIC *rm_old = rrdmetric_acquired_value(rd->rrdcontexts.rrdmetric);
360 rrd_flags_replace(rm_old, RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
361 rm_old->rrddim = NULL;
362 rm_old->first_time_s = 0;
363 rm_old->last_time_s = 0;
364
365 - rrdmetric_release(rd->rrdmetric);
366 - rd->rrdmetric = NULL;
365 + rrdmetric_release(rd->rrdcontexts.rrdmetric);
366 + rd->rrdcontexts.rrdmetric = NULL;
367
368 rrdmetric_from_rrddim(rd);
369 }
@@ -406,12 +406,12 @@ inline void rrdinstance_from_rrdset(RRDSET *st) {
406
407 #define rrdset_get_rrdinstance(st) rrdset_get_rrdinstance_with_trace(st, __FUNCTION__);
408 static inline RRDINSTANCE *rrdset_get_rrdinstance_with_trace(RRDSET *st, const char *function) {
409 - if(unlikely(!st->rrdinstance)) {
409 + if(unlikely(!st->rrdcontexts.rrdinstance)) {
410 netdata_log_error("RRDINSTANCE: RRDSET '%s' is not linked to an RRDINSTANCE at %s()", rrdset_id(st), function);
411 return NULL;
412 }
413
414 - RRDINSTANCE *ri = rrdinstance_acquired_value(st->rrdinstance);
414 + RRDINSTANCE *ri = rrdinstance_acquired_value(st->rrdcontexts.rrdinstance);
415 if(unlikely(!ri)) {
416 netdata_log_error("RRDINSTANCE: RRDSET '%s' lost its link to an RRDINSTANCE at %s()", rrdset_id(st), function);
417 return NULL;
@@ -439,14 +439,17 @@ inline void rrdinstance_rrdset_is_freed(RRDSET *st) {
439
440 rrdinstance_trigger_updates(ri, __FUNCTION__ );
441
442 - rrdinstance_release(st->rrdinstance);
443 - st->rrdinstance = NULL;
442 + rrdinstance_release(st->rrdcontexts.rrdinstance);
443 + st->rrdcontexts.rrdinstance = NULL;
444
445 - rrdcontext_release(st->rrdcontext);
446 - st->rrdcontext = NULL;
445 + rrdcontext_release(st->rrdcontexts.rrdcontext);
446 + st->rrdcontexts.rrdcontext = NULL;
447 + st->rrdcontexts.collected = false;
448 }
449
450 inline void rrdinstance_rrdset_has_updated_retention(RRDSET *st) {
451 + st->rrdcontexts.collected = false;
452 +
453 RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
454 if(unlikely(!ri)) return;
455
@@ -455,8 +458,10 @@ inline void rrdinstance_rrdset_has_updated_retention(RRDSET *st) {
458 }
459
460 inline void rrdinstance_updated_rrdset_name(RRDSET *st) {
461 + st->rrdcontexts.collected = false;
462 +
463 // the chart may not be initialized when this is called
459 - if(unlikely(!st->rrdinstance)) return;
464 + if(unlikely(!st->rrdcontexts.rrdinstance)) return;
465
466 RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
467 if(unlikely(!ri)) return;
@@ -491,6 +496,8 @@ inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, RRDSET *
496 }
497
498 inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
499 + st->rrdcontexts.collected = false;
500 +
501 RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
502 if(unlikely(!ri)) return;
503
@@ -503,6 +510,11 @@ inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
510 }
511
512 inline void rrdinstance_collected_rrdset(RRDSET *st) {
513 + if(st->rrdcontexts.collected)
514 + return;
515 +
516 + st->rrdcontexts.collected = true;
517 +
518 RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
519 if(unlikely(!ri)) {
520 rrdcontext_updated_rrdset(st);
database/contexts/metric.c
+18 -10
@@ -239,10 +239,10 @@ void rrdmetric_from_rrddim(RRDDIM *rd) {
239 if(unlikely(!rd->rrdset->rrdhost))
240 fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rrdset_id(rd->rrdset));
241
242 - if(unlikely(!rd->rrdset->rrdinstance))
242 + if(unlikely(!rd->rrdset->rrdcontexts.rrdinstance))
243 fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rrdset_id(rd->rrdset));
244
245 - RRDINSTANCE *ri = rrdinstance_acquired_value(rd->rrdset->rrdinstance);
245 + RRDINSTANCE *ri = rrdinstance_acquired_value(rd->rrdset->rrdcontexts.rrdinstance);
246
247 RRDMETRIC trm = {
248 .id = string_dup(rd->id),
@@ -254,20 +254,21 @@ void rrdmetric_from_rrddim(RRDDIM *rd) {
254
255 RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)dictionary_set_and_acquire_item(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
256
257 - if(rd->rrdmetric)
258 - rrdmetric_release(rd->rrdmetric);
257 + if(rd->rrdcontexts.rrdmetric)
258 + rrdmetric_release(rd->rrdcontexts.rrdmetric);
259
260 - rd->rrdmetric = rma;
260 + rd->rrdcontexts.rrdmetric = rma;
261 + rd->rrdcontexts.collected = false;
262 }
263
264 #define rrddim_get_rrdmetric(rd) rrddim_get_rrdmetric_with_trace(rd, __FUNCTION__)
265 static inline RRDMETRIC *rrddim_get_rrdmetric_with_trace(RRDDIM *rd, const char *function) {
265 - if(unlikely(!rd->rrdmetric)) {
266 + if(unlikely(!rd->rrdcontexts.rrdmetric)) {
267 netdata_log_error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rrddim_id(rd), function);
268 return NULL;
269 }
270
270 - RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdmetric);
271 + RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdcontexts.rrdmetric);
272 if(unlikely(!rm)) {
273 netdata_log_error("RRDMETRIC: RRDDIM '%s' lost the link to its RRDMETRIC at %s()", rrddim_id(rd), function);
274 return NULL;
@@ -288,11 +289,14 @@ inline void rrdmetric_rrddim_is_freed(RRDDIM *rd) {
289
290 rm->rrddim = NULL;
291 rrdmetric_trigger_updates(rm, __FUNCTION__ );
291 - rrdmetric_release(rd->rrdmetric);
292 - rd->rrdmetric = NULL;
292 + rrdmetric_release(rd->rrdcontexts.rrdmetric);
293 + rd->rrdcontexts.rrdmetric = NULL;
294 + rd->rrdcontexts.collected = false;
295 }
296
297 inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) {
298 + rd->rrdcontexts.collected = false;
299 +
300 RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
301 if(unlikely(!rm)) return;
302
@@ -305,6 +309,11 @@ inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) {
309 }
310
311 inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
312 + if(rd->rrdcontexts.collected)
313 + return;
314 +
315 + rd->rrdcontexts.collected = true;
316 +
317 RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
318 if(unlikely(!rm)) return;
319
@@ -316,4 +325,3 @@ inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
325
326 rrdmetric_trigger_updates(rm, __FUNCTION__ );
327 }
319 -
database/contexts/query_target.c
+5 -5
@@ -835,8 +835,8 @@ static ssize_t query_context_add(void *data, RRDCONTEXT_ACQUIRED *rca, bool quer
835 if(query_instance_add(qtl, qn, qc, qt->request.ria, queryable_context, false))
836 added++;
837 }
838 - else if(unlikely(qtl->st && qtl->st->rrdcontext == rca && qtl->st->rrdinstance)) {
839 - if(query_instance_add(qtl, qn, qc, qtl->st->rrdinstance, queryable_context, false))
838 + else if(unlikely(qtl->st && qtl->st->rrdcontexts.rrdcontext == rca && qtl->st->rrdcontexts.rrdinstance)) {
839 + if(query_instance_add(qtl, qn, qc, qtl->st->rrdcontexts.rrdinstance, queryable_context, false))
840 added++;
841 }
842 else {
@@ -894,11 +894,11 @@ static ssize_t query_node_add(void *data, RRDHOST *host, bool queryable_host) {
894 qn->node_id[0] = '\0';
895
896 // is the chart given valid?
897 - if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
897 + if(unlikely(qtl->st && (!qtl->st->rrdcontexts.rrdinstance || !qtl->st->rrdcontexts.rrdcontext))) {
898 netdata_log_error("QUERY TARGET: RRDSET '%s' given, but it is not linked to rrdcontext structures. Linking it now.", rrdset_name(qtl->st));
899 rrdinstance_from_rrdset(qtl->st);
900
901 - if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
901 + if(unlikely(qtl->st && (!qtl->st->rrdcontexts.rrdinstance || !qtl->st->rrdcontexts.rrdcontext))) {
902 netdata_log_error("QUERY TARGET: RRDSET '%s' given, but failed to be linked to rrdcontext structures. Switching to context query.",
903 rrdset_name(qtl->st));
904
@@ -918,7 +918,7 @@ static ssize_t query_node_add(void *data, RRDHOST *host, bool queryable_host) {
918 }
919 else if(unlikely(qtl->st)) {
920 // single chart data queries
921 - if(query_context_add(qtl, qtl->st->rrdcontext, true))
921 + if(query_context_add(qtl, qtl->st->rrdcontexts.rrdcontext, true))
922 added++;
923 }
924 else {
database/engine/metric.c
+129 -201
@@ -3,28 +3,41 @@
3 typedef int32_t REFCOUNT;
4 #define REFCOUNT_DELETING (-100)
5
6 -typedef enum __attribute__ ((__packed__)) {
7 - METRIC_FLAG_HAS_RETENTION = (1 << 0),
8 -} METRIC_FLAGS;
9 -
6 struct metric {
7 uuid_t uuid; // never changes
8 Word_t section; // never changes
9
14 - time_t first_time_s; //
15 - time_t latest_time_s_clean; // archived pages latest time
16 - time_t latest_time_s_hot; // latest time of the currently collected page
17 - uint32_t latest_update_every_s; //
10 + time_t first_time_s; // the timestamp of the oldest point in the database
11 + time_t latest_time_s_clean; // the timestamp of the newest point in the database
12 + time_t latest_time_s_hot; // the timestamp of the latest point that has been collected (not yet stored)
13 + uint32_t latest_update_every_s; // the latest data collection frequency
14 pid_t writer;
15 uint8_t partition;
20 - METRIC_FLAGS flags;
16 REFCOUNT refcount;
22 - SPINLOCK spinlock; // protects all variable members
17
18 // THIS IS allocated with malloc()
19 // YOU HAVE TO INITIALIZE IT YOURSELF !
20 };
21
22 +#define set_metric_field_with_condition(field, value, condition) ({ \
23 + typeof(field) _current = __atomic_load_n(&(field), __ATOMIC_RELAXED); \
24 + typeof(field) _wanted = value; \
25 + bool did_it = true; \
26 + \
27 + do { \
28 + if((condition) && (_current != _wanted)) { \
29 + ; \
30 + } \
31 + else { \
32 + did_it = false; \
33 + break; \
34 + } \
35 + } while(!__atomic_compare_exchange_n(&(field), &_current, _wanted, \
36 + false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); \
37 + \
38 + did_it; \
39 +})
40 +
41 static struct aral_statistics mrg_aral_statistics;
42
43 struct mrg {
@@ -73,9 +86,6 @@ static inline void MRG_STATS_DELETE_MISS(MRG *mrg, size_t partition) {
86 #define mrg_index_write_lock(mrg, partition) rw_spinlock_write_lock(&(mrg)->index[partition].rw_spinlock)
87 #define mrg_index_write_unlock(mrg, partition) rw_spinlock_write_unlock(&(mrg)->index[partition].rw_spinlock)
88
76 -#define metric_lock(metric) spinlock_lock(&(metric)->spinlock)
77 -#define metric_unlock(metric) spinlock_unlock(&(metric)->spinlock)
78 -
89 static inline void mrg_stats_size_judyl_change(MRG *mrg, size_t mem_before_judyl, size_t mem_after_judyl, size_t partition) {
90 if(mem_after_judyl > mem_before_judyl)
91 __atomic_add_fetch(&mrg->index[partition].stats.size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED);
@@ -97,40 +107,34 @@ static inline size_t uuid_partition(MRG *mrg __maybe_unused, uuid_t *uuid) {
107 return *n % mrg->partitions;
108 }
109
100 -static inline bool metric_has_retention_unsafe(MRG *mrg __maybe_unused, METRIC *metric) {
101 - size_t partition = metric->partition;
110 +static inline time_t mrg_metric_get_first_time_s_smart(MRG *mrg __maybe_unused, METRIC *metric) {
111 + time_t first_time_s = __atomic_load_n(&metric->first_time_s, __ATOMIC_RELAXED);
112
103 - bool has_retention = (metric->first_time_s > 0 || metric->latest_time_s_clean > 0 || metric->latest_time_s_hot > 0);
113 + if(first_time_s <= 0) {
114 + first_time_s = __atomic_load_n(&metric->latest_time_s_clean, __ATOMIC_RELAXED);
115 + if(first_time_s <= 0)
116 + first_time_s = __atomic_load_n(&metric->latest_time_s_hot, __ATOMIC_RELAXED);
117
105 - if(has_retention && !(metric->flags & METRIC_FLAG_HAS_RETENTION)) {
106 - metric->flags |= METRIC_FLAG_HAS_RETENTION;
107 - __atomic_add_fetch(&mrg->index[partition].stats.entries_with_retention, 1, __ATOMIC_RELAXED);
108 - }
109 - else if(!has_retention && (metric->flags & METRIC_FLAG_HAS_RETENTION)) {
110 - metric->flags &= ~METRIC_FLAG_HAS_RETENTION;
111 - __atomic_sub_fetch(&mrg->index[partition].stats.entries_with_retention, 1, __ATOMIC_RELAXED);
118 + if(first_time_s <= 0)
119 + first_time_s = 0;
120 + else
121 + __atomic_store_n(&metric->first_time_s, first_time_s, __ATOMIC_RELAXED);
122 }
123
114 - return has_retention;
124 + return first_time_s;
125 }
126
117 -static inline REFCOUNT metric_acquire(MRG *mrg __maybe_unused, METRIC *metric, bool having_spinlock) {
127 +static inline REFCOUNT metric_acquire(MRG *mrg __maybe_unused, METRIC *metric) {
128 size_t partition = metric->partition;
129 + REFCOUNT expected = metric->refcount;
130 REFCOUNT refcount;
131
121 - if(!having_spinlock)
122 - metric_lock(metric);
123 -
124 - if(unlikely(metric->refcount < 0))
125 - fatal("METRIC: refcount is %d (negative) during acquire", metric->refcount);
126 -
127 - refcount = ++metric->refcount;
128 -
129 - // update its retention flags
130 - metric_has_retention_unsafe(mrg, metric);
132 + do {
133 + if(expected < 0)
134 + fatal("METRIC: refcount is %d (negative) during acquire", metric->refcount);
135
132 - if(!having_spinlock)
133 - metric_unlock(metric);
136 + refcount = expected + 1;
137 + } while(!__atomic_compare_exchange_n(&metric->refcount, &expected, refcount, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
138
139 if(refcount == 1)
140 __atomic_add_fetch(&mrg->index[partition].stats.entries_referenced, 1, __ATOMIC_RELAXED);
@@ -141,28 +145,25 @@ static inline REFCOUNT metric_acquire(MRG *mrg __maybe_unused, METRIC *metric, b
145 }
146
147 static inline bool metric_release_and_can_be_deleted(MRG *mrg __maybe_unused, METRIC *metric) {
144 - bool ret = true;
148 size_t partition = metric->partition;
149 + REFCOUNT expected = metric->refcount;
150 REFCOUNT refcount;
151
148 - metric_lock(metric);
149 -
150 - if(unlikely(metric->refcount <= 0))
151 - fatal("METRIC: refcount is %d (zero or negative) during release", metric->refcount);
152 -
153 - refcount = --metric->refcount;
154 -
155 - if(likely(metric_has_retention_unsafe(mrg, metric) || refcount != 0))
156 - ret = false;
152 + do {
153 + if(expected <= 0)
154 + fatal("METRIC: refcount is %d (zero or negative) during release", metric->refcount);
155
158 - metric_unlock(metric);
156 + refcount = expected - 1;
157 + } while(!__atomic_compare_exchange_n(&metric->refcount, &expected, refcount, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
158
159 if(unlikely(!refcount))
160 __atomic_sub_fetch(&mrg->index[partition].stats.entries_referenced, 1, __ATOMIC_RELAXED);
161
162 __atomic_sub_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED);
163
165 - return ret;
164 + time_t first, last, ue;
165 + mrg_metric_get_retention(mrg, metric, &first, &last, &ue);
166 + return (!first || !last || first > last);
167 }
168
169 static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {
@@ -192,7 +193,7 @@ static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *r
193 if(unlikely(*PValue != NULL)) {
194 METRIC *metric = *PValue;
195
195 - metric_acquire(mrg, metric, false);
196 + metric_acquire(mrg, metric);
197
198 MRG_STATS_DUPLICATE_ADD(mrg, partition);
199
@@ -215,10 +216,8 @@ static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *r
216 metric->latest_update_every_s = entry->latest_update_every_s;
217 metric->writer = 0;
218 metric->refcount = 0;
218 - metric->flags = 0;
219 metric->partition = partition;
220 - spinlock_init(&metric->spinlock);
221 - metric_acquire(mrg, metric, true); // no spinlock use required here
220 + metric_acquire(mrg, metric);
221 *PValue = metric;
222
223 MRG_STATS_ADDED_METRIC(mrg, partition);
@@ -252,7 +251,7 @@ static inline METRIC *metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t sect
251
252 METRIC *metric = *PValue;
253
255 - metric_acquire(mrg, metric, false);
254 + metric_acquire(mrg, metric);
255
256 mrg_index_read_unlock(mrg, partition);
257
@@ -363,7 +362,7 @@ inline bool mrg_metric_release_and_delete(MRG *mrg, METRIC *metric) {
362 }
363
364 inline METRIC *mrg_metric_dup(MRG *mrg, METRIC *metric) {
366 - metric_acquire(mrg, metric, false);
365 + metric_acquire(mrg, metric);
366 return metric;
367 }
368
@@ -389,10 +388,7 @@ inline bool mrg_metric_set_first_time_s(MRG *mrg __maybe_unused, METRIC *metric,
388 if(unlikely(first_time_s < 0))
389 return false;
390
392 - metric_lock(metric);
393 - metric->first_time_s = first_time_s;
394 - metric_has_retention_unsafe(mrg, metric);
395 - metric_unlock(metric);
391 + __atomic_store_n(&metric->first_time_s, first_time_s, __ATOMIC_RELAXED);
392
393 return true;
394 }
@@ -405,112 +401,56 @@ inline void mrg_metric_expand_retention(MRG *mrg __maybe_unused, METRIC *metric,
401 internal_fatal(last_time_s > max_acceptable_collected_time(),
402 "DBENGINE METRIC: metric last time is in the future");
403
408 - if(unlikely(first_time_s < 0))
409 - first_time_s = 0;
410 -
411 - if(unlikely(last_time_s < 0))
412 - last_time_s = 0;
413 -
414 - if(unlikely(update_every_s < 0))
415 - update_every_s = 0;
416 -
417 - if(unlikely(!first_time_s && !last_time_s && !update_every_s))
418 - return;
404 + if(first_time_s > 0)
405 + set_metric_field_with_condition(metric->first_time_s, first_time_s, _current <= 0 || first_time_s > _current);
406
420 - metric_lock(metric);
421 -
422 - if(unlikely(first_time_s && (!metric->first_time_s || first_time_s < metric->first_time_s)))
423 - metric->first_time_s = first_time_s;
424 -
425 - if(likely(last_time_s && (!metric->latest_time_s_clean || last_time_s > metric->latest_time_s_clean))) {
426 - metric->latest_time_s_clean = last_time_s;
427 -
428 - if(likely(update_every_s))
429 - metric->latest_update_every_s = (uint32_t) update_every_s;
407 + if(last_time_s > 0) {
408 + if(set_metric_field_with_condition(metric->latest_time_s_clean, last_time_s, _current <= 0 || last_time_s > _current) &&
409 + update_every_s > 0)
410 + // set the latest update every too
411 + set_metric_field_with_condition(metric->latest_update_every_s, update_every_s, true);
412 }
431 - else if(unlikely(!metric->latest_update_every_s && update_every_s))
432 - metric->latest_update_every_s = (uint32_t) update_every_s;
433 -
434 - metric_has_retention_unsafe(mrg, metric);
435 - metric_unlock(metric);
413 + else if(update_every_s > 0)
414 + // set it only if it is invalid
415 + set_metric_field_with_condition(metric->latest_update_every_s, update_every_s, _current <= 0);
416 }
417
418 inline bool mrg_metric_set_first_time_s_if_bigger(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s) {
419 internal_fatal(first_time_s < 0, "DBENGINE METRIC: timestamp is negative");
440 -
441 - bool ret = false;
442 -
443 - metric_lock(metric);
444 - if(first_time_s > metric->first_time_s) {
445 - metric->first_time_s = first_time_s;
446 - ret = true;
447 - }
448 - metric_has_retention_unsafe(mrg, metric);
449 - metric_unlock(metric);
450 -
451 - return ret;
420 + return set_metric_field_with_condition(metric->first_time_s, first_time_s, first_time_s > _current);
421 }
422
423 inline time_t mrg_metric_get_first_time_s(MRG *mrg __maybe_unused, METRIC *metric) {
455 - time_t first_time_s;
456 -
457 - metric_lock(metric);
458 -
459 - if(unlikely(!metric->first_time_s)) {
460 - if(metric->latest_time_s_clean)
461 - metric->first_time_s = metric->latest_time_s_clean;
462 -
463 - else if(metric->latest_time_s_hot)
464 - metric->first_time_s = metric->latest_time_s_hot;
465 - }
466 -
467 - first_time_s = metric->first_time_s;
468 -
469 - metric_unlock(metric);
470 -
471 - return first_time_s;
424 + return mrg_metric_get_first_time_s_smart(mrg, metric);
425 }
426
427 inline void mrg_metric_get_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t *first_time_s, time_t *last_time_s, time_t *update_every_s) {
475 - metric_lock(metric);
476 -
477 - if(unlikely(!metric->first_time_s)) {
478 - if(metric->latest_time_s_clean)
479 - metric->first_time_s = metric->latest_time_s_clean;
480 -
481 - else if(metric->latest_time_s_hot)
482 - metric->first_time_s = metric->latest_time_s_hot;
483 - }
484 -
485 - *first_time_s = metric->first_time_s;
486 - *last_time_s = MAX(metric->latest_time_s_clean, metric->latest_time_s_hot);
487 - *update_every_s = metric->latest_update_every_s;
428 + time_t clean = __atomic_load_n(&metric->latest_time_s_clean, __ATOMIC_RELAXED);
429 + time_t hot = __atomic_load_n(&metric->latest_time_s_hot, __ATOMIC_RELAXED);
430
489 - metric_unlock(metric);
431 + *last_time_s = MAX(clean, hot);
432 + *first_time_s = mrg_metric_get_first_time_s_smart(mrg, metric);
433 + *update_every_s = __atomic_load_n(&metric->latest_update_every_s, __ATOMIC_RELAXED);
434 }
435
436 inline bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {
437 internal_fatal(latest_time_s < 0, "DBENGINE METRIC: timestamp is negative");
438
495 - if(unlikely(latest_time_s < 0))
496 - return false;
497 -
498 - metric_lock(metric);
499 -
439 // internal_fatal(latest_time_s > max_acceptable_collected_time(),
440 // "DBENGINE METRIC: metric latest time is in the future");
441
442 // internal_fatal(metric->latest_time_s_clean > latest_time_s,
443 // "DBENGINE METRIC: metric new clean latest time is older than the previous one");
444
506 - metric->latest_time_s_clean = latest_time_s;
445 + if(latest_time_s > 0) {
446 + if(set_metric_field_with_condition(metric->latest_time_s_clean, latest_time_s, true)) {
447 + set_metric_field_with_condition(metric->first_time_s, latest_time_s, _current <= 0 || latest_time_s < _current);
448
508 - if(unlikely(!metric->first_time_s))
509 - metric->first_time_s = latest_time_s;
449 + return true;
450 + }
451 + }
452
511 - metric_has_retention_unsafe(mrg, metric);
512 - metric_unlock(metric);
513 - return true;
453 + return false;
454 }
455
456 // returns true when metric still has retention
@@ -518,7 +458,6 @@ inline bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metr
458 Word_t section = mrg_metric_section(mrg, metric);
459 bool do_again = false;
460 size_t countdown = 5;
521 - bool ret = true;
461
462 do {
463 time_t min_first_time_s = LONG_MAX;
@@ -547,22 +486,20 @@ inline bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metr
486 if (min_first_time_s == LONG_MAX)
487 min_first_time_s = 0;
488
550 - metric_lock(metric);
551 - if (--countdown && !min_first_time_s && metric->latest_time_s_hot)
489 + if (--countdown && !min_first_time_s && __atomic_load_n(&metric->latest_time_s_hot, __ATOMIC_RELAXED))
490 do_again = true;
491 else {
492 internal_error(!countdown, "METRIC: giving up on updating the retention of metric without disk retention");
493
494 do_again = false;
557 - metric->first_time_s = min_first_time_s;
558 - metric->latest_time_s_clean = max_end_time_s;
559 -
560 - ret = metric_has_retention_unsafe(mrg, metric);
495 + set_metric_field_with_condition(metric->first_time_s, min_first_time_s, true);
496 + set_metric_field_with_condition(metric->latest_time_s_clean, max_end_time_s, true);
497 }
562 - metric_unlock(metric);
498 } while(do_again);
499
565 - return ret;
500 + time_t first, last, ue;
501 + mrg_metric_get_retention(mrg, metric, &first, &last, &ue);
502 + return (first && last && first < last);
503 }
504
505 inline bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {
@@ -571,88 +508,80 @@ inline bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *me
508 // internal_fatal(latest_time_s > max_acceptable_collected_time(),
509 // "DBENGINE METRIC: metric latest time is in the future");
510
574 - if(unlikely(latest_time_s < 0))
575 - return false;
576 -
577 - metric_lock(metric);
578 - metric->latest_time_s_hot = latest_time_s;
579 -
580 - if(unlikely(!metric->first_time_s))
581 - metric->first_time_s = latest_time_s;
511 + if(likely(latest_time_s > 0)) {
512 + __atomic_store_n(&metric->latest_time_s_hot, latest_time_s, __ATOMIC_RELAXED);
513 + return true;
514 + }
515
583 - metric_has_retention_unsafe(mrg, metric);
584 - metric_unlock(metric);
585 - return true;
516 + return false;
517 }
518
519 inline time_t mrg_metric_get_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric) {
589 - time_t max;
590 - metric_lock(metric);
591 - max = MAX(metric->latest_time_s_clean, metric->latest_time_s_hot);
592 - metric_unlock(metric);
593 - return max;
520 + time_t clean = __atomic_load_n(&metric->latest_time_s_clean, __ATOMIC_RELAXED);
521 + time_t hot = __atomic_load_n(&metric->latest_time_s_hot, __ATOMIC_RELAXED);
522 +
523 + return MAX(clean, hot);
524 }
525
526 inline bool mrg_metric_set_update_every(MRG *mrg __maybe_unused, METRIC *metric, time_t update_every_s) {
527 internal_fatal(update_every_s < 0, "DBENGINE METRIC: timestamp is negative");
528
599 - if(update_every_s <= 0)
600 - return false;
601 -
602 - metric_lock(metric);
603 - metric->latest_update_every_s = (uint32_t) update_every_s;
604 - metric_unlock(metric);
529 + if(update_every_s > 0)
530 + return set_metric_field_with_condition(metric->latest_update_every_s, update_every_s, true);
531
606 - return true;
532 + return false;
533 }
534
535 inline bool mrg_metric_set_update_every_s_if_zero(MRG *mrg __maybe_unused, METRIC *metric, time_t update_every_s) {
536 internal_fatal(update_every_s < 0, "DBENGINE METRIC: timestamp is negative");
537
612 - if(update_every_s <= 0)
613 - return false;
614 -
615 - metric_lock(metric);
616 - if(!metric->latest_update_every_s)
617 - metric->latest_update_every_s = (uint32_t) update_every_s;
618 - metric_unlock(metric);
538 + if(update_every_s > 0)
539 + return set_metric_field_with_condition(metric->latest_update_every_s, update_every_s, _current <= 0);
540
620 - return true;
541 + return false;
542 }
543
544 inline time_t mrg_metric_get_update_every_s(MRG *mrg __maybe_unused, METRIC *metric) {
624 - time_t update_every_s;
625 -
626 - metric_lock(metric);
627 - update_every_s = metric->latest_update_every_s;
628 - metric_unlock(metric);
629 -
630 - return update_every_s;
545 + return __atomic_load_n(&metric->latest_update_every_s, __ATOMIC_RELAXED);
546 }
547
548 inline bool mrg_metric_set_writer(MRG *mrg, METRIC *metric) {
634 - bool done = false;
635 - metric_lock(metric);
636 - if(!metric->writer) {
637 - metric->writer = gettid();
549 + pid_t expected = __atomic_load_n(&metric->writer, __ATOMIC_RELAXED);
550 + pid_t wanted = gettid();
551 + bool done = true;
552 +
553 + do {
554 + if(expected != 0) {
555 + done = false;
556 + break;
557 + }
558 + } while(!__atomic_compare_exchange_n(&metric->writer, &expected, wanted, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
559 +
560 + if(done)
561 __atomic_add_fetch(&mrg->index[metric->partition].stats.writers, 1, __ATOMIC_RELAXED);
639 - done = true;
640 - }
562 else
563 __atomic_add_fetch(&mrg->index[metric->partition].stats.writers_conflicts, 1, __ATOMIC_RELAXED);
643 - metric_unlock(metric);
564 +
565 return done;
566 }
567
568 inline bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric) {
648 - bool done = false;
649 - metric_lock(metric);
650 - if(metric->writer) {
651 - metric->writer = 0;
569 + // this function can be called from a different thread than the one than the writer
570 +
571 + pid_t expected = __atomic_load_n(&metric->writer, __ATOMIC_RELAXED);
572 + pid_t wanted = 0;
573 + bool done = true;
574 +
575 + do {
576 + if(!expected) {
577 + done = false;
578 + break;
579 + }
580 + } while(!__atomic_compare_exchange_n(&metric->writer, &expected, wanted, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
581 +
582 + if(done)
583 __atomic_sub_fetch(&mrg->index[metric->partition].stats.writers, 1, __ATOMIC_RELAXED);
653 - done = true;
654 - }
655 - metric_unlock(metric);
584 +
585 return done;
586 }
587
@@ -710,7 +639,6 @@ inline void mrg_get_statistics(MRG *mrg, struct mrg_statistics *s) {
639 for(size_t i = 0; i < mrg->partitions ;i++) {
640 s->entries += __atomic_load_n(&mrg->index[i].stats.entries, __ATOMIC_RELAXED);
641 s->entries_referenced += __atomic_load_n(&mrg->index[i].stats.entries_referenced, __ATOMIC_RELAXED);
713 - s->entries_with_retention += __atomic_load_n(&mrg->index[i].stats.entries_with_retention, __ATOMIC_RELAXED);
642 s->size += __atomic_load_n(&mrg->index[i].stats.size, __ATOMIC_RELAXED);
643 s->current_references += __atomic_load_n(&mrg->index[i].stats.current_references, __ATOMIC_RELAXED);
644 s->additions += __atomic_load_n(&mrg->index[i].stats.additions, __ATOMIC_RELAXED);
database/engine/metric.h
-3
@@ -35,9 +35,6 @@ struct mrg_statistics {
35
36 size_t entries_referenced;
37
38 - MRG_CACHE_LINE_PADDING(1);
39 - size_t entries_with_retention;
40 -
38 MRG_CACHE_LINE_PADDING(2);
39 size_t current_references;
40
database/engine/rrdengine.h
+1
@@ -190,6 +190,7 @@ struct rrdeng_collect_handle {
190 RRDENG_COLLECT_HANDLE_OPTIONS options;
191 uint8_t type;
192
193 + struct rrdengine_instance *ctx;
194 struct metric *metric;
195 struct pgc_page *page;
196 void *data;
database/engine/rrdengineapi.c
+6 -5
@@ -201,7 +201,7 @@ static inline bool check_completed_page_consistency(struct rrdeng_collect_handle
201 if (unlikely(!handle->page || !handle->page_entries_max || !handle->page_position || !handle->page_end_time_ut))
202 return false;
203
204 - struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);
204 + struct rrdengine_instance *ctx = handle->ctx;
205
206 uuid_t *uuid = mrg_metric_uuid(main_mrg, handle->metric);
207 time_t start_time_s = pgc_page_start_time_s(handle->page);
@@ -257,6 +257,7 @@ STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metri
257 handle = callocz(1, sizeof(struct rrdeng_collect_handle));
258 handle->common.backend = STORAGE_ENGINE_BACKEND_DBENGINE;
259 handle->metric = metric;
260 + handle->ctx = ctx;
261 handle->page = NULL;
262 handle->data = NULL;
263 handle->data_size = 0;
@@ -312,7 +313,7 @@ static bool page_has_only_empty_metrics(struct rrdeng_collect_handle *handle) {
313 default: {
314 static bool logged = false;
315 if(!logged) {
315 - netdata_log_error("DBENGINE: cannot check page for nulls on unknown page type id %d", (mrg_metric_ctx(handle->metric))->config.page_type);
316 + netdata_log_error("DBENGINE: cannot check page for nulls on unknown page type id %d", handle->ctx->config.page_type);
317 logged = true;
318 }
319 return false;
@@ -442,7 +443,7 @@ static size_t aligned_allocation_entries(size_t max_slots, size_t target_slot, t
443 }
444
445 static void *rrdeng_alloc_new_metric_data(struct rrdeng_collect_handle *handle, size_t *data_size, usec_t point_in_time_ut) {
445 - struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);
446 + struct rrdengine_instance *ctx = handle->ctx;
447
448 size_t max_size = tier_page_size[ctx->config.tier];
449 size_t max_slots = max_size / CTX_POINT_SIZE_BYTES(ctx);
@@ -484,7 +485,7 @@ static void rrdeng_store_metric_append_point(STORAGE_COLLECT_HANDLE *collection_
485 const SN_FLAGS flags)
486 {
487 struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
487 - struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);
488 + struct rrdengine_instance *ctx = handle->ctx;
489
490 if(unlikely(!handle->data))
491 handle->data = rrdeng_alloc_new_metric_data(handle, &handle->data_size, point_in_time_ut);
@@ -656,7 +657,7 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
657 */
658 int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
659 struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
659 - struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric);
660 + struct rrdengine_instance *ctx = handle->ctx;
661
662 handle->page_flags |= RRDENG_PAGE_COLLECT_FINALIZE;
663 rrdeng_store_metric_flush_current_page(collection_handle);
database/rrd.h
+157 -43
@@ -239,7 +239,6 @@ typedef enum __attribute__ ((__packed__)) rrddim_options {
239 RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
240 RRDDIM_OPTION_BACKFILLED_HIGH_TIERS = (1 << 2), // when set, we have backfilled higher tiers
241 RRDDIM_OPTION_UPDATED = (1 << 3), // single-threaded collector updated flag
242 - RRDDIM_OPTION_EXPOSED = (1 << 4), // single-threaded collector exposed flag
242
243 // this is 8-bit
244 } RRDDIM_OPTIONS;
@@ -253,20 +252,22 @@ typedef enum __attribute__ ((__packed__)) rrddim_flags {
252 RRDDIM_FLAG_NONE = 0,
253 RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 0),
254
256 - RRDDIM_FLAG_OBSOLETE = (1 << 2), // this is marked by the collector/module as obsolete
255 + RRDDIM_FLAG_OBSOLETE = (1 << 1), // this is marked by the collector/module as obsolete
256 // No new values have been collected for this dimension since agent start, or it was marked RRDDIM_FLAG_OBSOLETE at
257 // least rrdset_free_obsolete_time seconds ago.
259 - RRDDIM_FLAG_ARCHIVED = (1 << 3),
260 - RRDDIM_FLAG_METADATA_UPDATE = (1 << 4), // Metadata needs to go to the database
258 + RRDDIM_FLAG_ARCHIVED = (1 << 2),
259 + RRDDIM_FLAG_METADATA_UPDATE = (1 << 3), // Metadata needs to go to the database
260 +
261 + RRDDIM_FLAG_META_HIDDEN = (1 << 4), // Status of hidden option in the metadata database
262
262 - RRDDIM_FLAG_META_HIDDEN = (1 << 6), // Status of hidden option in the metadata database
263
264 // this is 8 bit
265 } RRDDIM_FLAGS;
266
267 -#define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & (flag))
268 -#define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
269 -#define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
267 +#define rrddim_flag_get(rd) __atomic_load_n(&((rd)->flags), __ATOMIC_ACQUIRE)
268 +#define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_ACQUIRE) & (flag))
269 +#define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_RELEASE)
270 +#define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_RELEASE)
271
272 // ----------------------------------------------------------------------------
273 // engine-specific iterator state for dimension data collection
@@ -312,7 +313,11 @@ struct rrddim {
313
314 struct rrdset *rrdset;
315 rrd_ml_dimension_t *ml_dimension; // machine learning data about this dimension
315 - RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
316 +
317 + struct {
318 + RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
319 + bool collected;
320 + } rrdcontexts;
321
322 #ifdef NETDATA_LOG_COLLECTION_ERRORS
323 usec_t rrddim_store_metric_last_ut; // the timestamp we last called rrddim_store_metric()
@@ -331,6 +336,16 @@ struct rrddim {
336 storage_number *data; // the array of values
337 } db;
338
339 + // ------------------------------------------------------------------------
340 + // streaming
341 +
342 + struct {
343 + struct {
344 + uint32_t sent_version;
345 + uint32_t dim_slot;
346 + } sender;
347 + } rrdpush;
348 +
349 // ------------------------------------------------------------------------
350 // data collection members
351
@@ -367,10 +382,6 @@ size_t rrddim_size(void);
382 #define rrddim_set_updated(rd) (rd)->collector.options |= RRDDIM_OPTION_UPDATED
383 #define rrddim_clear_updated(rd) (rd)->collector.options &= ~RRDDIM_OPTION_UPDATED
384
370 -#define rrddim_check_exposed(rd) ((rd)->collector.options & RRDDIM_OPTION_EXPOSED)
371 -#define rrddim_set_exposed(rd) (rd)->collector.options |= RRDDIM_OPTION_EXPOSED
372 -#define rrddim_clear_exposed(rd) (rd)->collector.options &= ~RRDDIM_OPTION_EXPOSED
373 -
385 // returns the RRDDIM cache filename, or NULL if it does not exist
386 const char *rrddim_cache_filename(RRDDIM *rd);
387
@@ -685,41 +696,47 @@ typedef enum __attribute__ ((__packed__)) rrdset_flags {
696
697 RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming)
698 RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming)
688 - RRDSET_FLAG_UPSTREAM_EXPOSED = (1 << 8), // if set, we have sent this chart definition to netdata parent (streaming)
699
690 - RRDSET_FLAG_STORE_FIRST = (1 << 9), // if set, do not eliminate the first collection during interpolation
691 - RRDSET_FLAG_HETEROGENEOUS = (1 << 10), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
692 - RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 11), // if set, the chart should be checked to determine if the dimensions are homogeneous
693 - RRDSET_FLAG_HIDDEN = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting
694 - RRDSET_FLAG_SYNC_CLOCK = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
695 - RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions
700 + RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation
701 + RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
702 + RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous
703 + RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting
704 + RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
705 + RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions
706
697 - RRDSET_FLAG_METADATA_UPDATE = (1 << 16), // Mark that metadata needs to be stored
698 - RRDSET_FLAG_ANOMALY_DETECTION = (1 << 18), // flag to identify anomaly detection charts.
699 - RRDSET_FLAG_INDEXED_ID = (1 << 19), // the rrdset is indexed by its id
700 - RRDSET_FLAG_INDEXED_NAME = (1 << 20), // the rrdset is indexed by its name
707 + RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored
708 + RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts.
709 + RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id
710 + RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name
711
702 - RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 21),
712 + RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18),
713
704 - RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 22), // the sending side has replication in progress
705 - RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 23), // the sending side has completed replication
706 - RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 24), // the receiving side has replication in progress
707 - RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 25), // the receiving side has completed replication
714 + RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress
715 + RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication
716 + RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress
717 + RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication
718
709 - RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 26), // a custom variable has been updated and needs to be exposed to parent
719 + RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 23), // a custom variable has been updated and needs to be exposed to parent
720
711 - RRDSET_FLAG_COLLECTION_FINISHED = (1 << 27), // when set, data collection is not available for this chart
721 + RRDSET_FLAG_COLLECTION_FINISHED = (1 << 24), // when set, data collection is not available for this chart
722
713 - RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 28), // this chart has at least one rrdcal linked
723 + RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 25), // this chart has at least one rrdcal linked
724 } RRDSET_FLAGS;
725
716 -#define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_SEQ_CST) & (flag))
717 -#define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_SEQ_CST)
718 -#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~(flag), __ATOMIC_SEQ_CST)
726 +#define rrdset_flag_get(st) __atomic_load_n(&((st)->flags), __ATOMIC_ACQUIRE)
727 +#define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_ACQUIRE) & (flag))
728 +#define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_RELEASE)
729 +#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~(flag), __ATOMIC_RELEASE)
730
731 #define rrdset_is_replicating(st) (rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS|RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS) \
732 && !rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED|RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
733
734 +struct pluginsd_rrddim {
735 + RRDDIM_ACQUIRED *rda;
736 + RRDDIM *rd;
737 + const char *id;
738 +};
739 +
740 struct rrdset {
741 uuid_t chart_uuid; // the global UUID for this chart
742
@@ -749,6 +766,8 @@ struct rrdset {
766 DICTIONARY *rrddimvar_root_index; // dimension variables
767 // we use this dictionary to manage their allocation
768
769 + uint32_t version; // the metadata version (auto-increment)
770 +
771 RRDSET_TYPE chart_type; // line, area, stacked
772
773 // ------------------------------------------------------------------------
@@ -768,8 +787,11 @@ struct rrdset {
787
788 RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to
789
771 - RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart
772 - RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to
790 + struct {
791 + RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart
792 + RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to
793 + bool collected;
794 + } rrdcontexts;
795
796 // ------------------------------------------------------------------------
797 // data collection members
@@ -792,7 +814,15 @@ struct rrdset {
814 // ------------------------------------------------------------------------
815 // data collection - streaming to parents, temp variables
816
795 - time_t upstream_resync_time_s; // the timestamp up to which we should resync clock upstream
817 + struct {
818 + struct {
819 + uint32_t sent_version;
820 + uint32_t chart_slot;
821 + uint32_t dim_last_slot_used;
822 +
823 + time_t resync_time_s; // the timestamp up to which we should resync clock upstream
824 + } sender;
825 + } rrdpush;
826
827 // ------------------------------------------------------------------------
828 // db mode SAVE, MAP specifics
@@ -835,10 +865,12 @@ struct rrdset {
865 struct {
866 SPINLOCK spinlock; // used only for cleanup
867 pid_t collector_tid;
868 + bool with_slots;
869 bool set;
870 uint32_t pos;
871 + int32_t last_slot;
872 uint32_t size;
841 - RRDDIM_ACQUIRED **rda;
873 + struct pluginsd_rrddim *prd_array;
874 } pluginsd;
875
876 #ifdef NETDATA_LOG_REPLICATION_REQUESTS
@@ -861,6 +893,58 @@ struct rrdset {
893 #define rrdset_name(st) string2str((st)->name)
894 #define rrdset_id(st) string2str((st)->id)
895
896 +static inline uint32_t rrdset_metadata_version(RRDSET *st) {
897 + return __atomic_load_n(&st->version, __ATOMIC_RELAXED);
898 +}
899 +
900 +static inline uint32_t rrdset_metadata_upstream_version(RRDSET *st) {
901 + return __atomic_load_n(&st->rrdpush.sender.sent_version, __ATOMIC_RELAXED);
902 +}
903 +
904 +static inline void rrdset_metadata_updated(RRDSET *st) {
905 + __atomic_add_fetch(&st->version, 1, __ATOMIC_RELAXED);
906 +}
907 +
908 +static inline void rrdset_metadata_exposed_upstream(RRDSET *st, uint32_t version) {
909 + __atomic_store_n(&st->rrdpush.sender.sent_version, version, __ATOMIC_RELAXED);
910 +}
911 +
912 +static inline bool rrdset_check_upstream_exposed(RRDSET *st) {
913 + return rrdset_metadata_version(st) == rrdset_metadata_upstream_version(st);
914 +}
915 +
916 +static inline uint32_t rrddim_metadata_version(RRDDIM *rd) {
917 + // the metadata version of the dimension, is the version of the chart
918 + return rrdset_metadata_version(rd->rrdset);
919 +}
920 +
921 +static inline uint32_t rrddim_metadata_upstream_version(RRDDIM *rd) {
922 + return __atomic_load_n(&rd->rrdpush.sender.sent_version, __ATOMIC_RELAXED);
923 +}
924 +
925 +static inline void rrddim_metadata_updated(RRDDIM *rd) {
926 + rrdset_metadata_updated(rd->rrdset);
927 +}
928 +
929 +static inline void rrddim_metadata_exposed_upstream(RRDDIM *rd, uint32_t version) {
930 + __atomic_store_n(&rd->rrdpush.sender.sent_version, version, __ATOMIC_RELAXED);
931 +}
932 +
933 +static inline void rrddim_metadata_exposed_upstream_clear(RRDDIM *rd) {
934 + __atomic_store_n(&rd->rrdpush.sender.sent_version, 0, __ATOMIC_RELAXED);
935 +}
936 +
937 +static inline bool rrddim_check_upstream_exposed(RRDDIM *rd) {
938 + return rrddim_metadata_upstream_version(rd) != 0;
939 +}
940 +
941 +// the collector sets the exposed flag, but anyone can remove it
942 +// still, it can be removed, after the collector has finished
943 +// so, it is safe to check it without atomics
944 +static inline bool rrddim_check_upstream_exposed_collector(RRDDIM *rd) {
945 + return rd->rrdset->version == rd->rrdpush.sender.sent_version;
946 +}
947 +
948 STRING *rrd_string_strdupz(const char *s);
949
950 // ----------------------------------------------------------------------------
@@ -1151,6 +1235,31 @@ struct rrdhost {
1235 // ------------------------------------------------------------------------
1236 // streaming of data to remote hosts - rrdpush sender
1237
1238 + struct {
1239 + struct {
1240 + struct {
1241 + struct {
1242 + SPINLOCK spinlock;
1243 +
1244 + bool ignore; // when set, freeing slots will not put them in the available
1245 + uint32_t used;
1246 + uint32_t size;
1247 + uint32_t *array;
1248 + } available; // keep track of the available chart slots per host
1249 +
1250 + uint32_t last_used; // the last slot we used for a chart (increments only)
1251 + } pluginsd_chart_slots;
1252 + } send;
1253 +
1254 + struct {
1255 + struct {
1256 + SPINLOCK spinlock; // lock for the management of the allocation
1257 + uint32_t size;
1258 + RRDSET **array;
1259 + } pluginsd_chart_slots;
1260 + } receive;
1261 + } rrdpush;
1262 +
1263 char *rrdpush_send_destination; // where to send metrics to
1264 char *rrdpush_send_api_key; // the api key at the receiving netdata
1265 struct rrdpush_destinations *destinations; // a linked list of possible destinations
@@ -1438,8 +1547,8 @@ void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds);
1547 void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next);
1548 void rrdset_done(RRDSET *st);
1549
1441 -void rrdset_is_obsolete(RRDSET *st);
1442 -void rrdset_isnot_obsolete(RRDSET *st);
1550 +void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st);
1551 +void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st);
1552
1553 // checks if the RRDSET should be offered to viewers
1554 #define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
@@ -1488,8 +1597,8 @@ RRDDIM *rrddim_find_active(RRDSET *st, const char *id);
1597 int rrddim_hide(RRDSET *st, const char *id);
1598 int rrddim_unhide(RRDSET *st, const char *id);
1599
1491 -void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd);
1492 -void rrddim_isnot_obsolete(RRDSET *st, RRDDIM *rd);
1600 +void rrddim_is_obsolete___safe_from_collector_thread(RRDSET *st, RRDDIM *rd);
1601 +void rrddim_isnot_obsolete___safe_from_collector_thread(RRDSET *st, RRDDIM *rd);
1602
1603 collected_number rrddim_timed_set_by_pointer(RRDSET *st, RRDDIM *rd, struct timeval collected_time, collected_number value);
1604 collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value);
@@ -1557,6 +1666,11 @@ static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, tim
1666 *to = online ? now : last_time_s;
1667 }
1668
1669 +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host);
1670 +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host);
1671 +void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st);
1672 +void rrdset_pluginsd_receive_unslot(RRDSET *st);
1673 +
1674 // ----------------------------------------------------------------------------
1675 // RRD DB engine declarations
1676
database/rrddim.c
+12 -14
@@ -48,6 +48,8 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
48
49 rd->rrdset = st;
50
51 + rd->rrdpush.sender.dim_slot = __atomic_add_fetch(&st->rrdpush.sender.dim_last_slot_used, 1, __ATOMIC_RELAXED);
52 +
53 if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
54 rd->collector.counter = 1;
55
@@ -155,7 +157,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
157
158 // let the chart resync
159 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
158 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
160 + rrdset_metadata_updated(st);
161
162 ml_dimension_new(rd);
163
@@ -283,7 +285,7 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
285 if(ctr->react_action == RRDDIM_REACT_UPDATED) {
286 // the chart needs to be updated to the parent
287 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
286 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
288 + rrdset_metadata_updated(st);
289 }
290
291 rrdcontext_updated_rrddim(rd);
@@ -369,8 +371,7 @@ inline int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name) {
371
372 rrddimvar_rename_all(rd);
373
372 - rrddim_clear_exposed(rd);
373 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
374 + rrddim_metadata_updated(rd);
375
376 return 1;
377 }
@@ -381,8 +382,7 @@ inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm)
382
383 netdata_log_debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", rrdset_id(st), rrddim_name(rd), rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm));
384 rd->algorithm = algorithm;
384 - rrddim_clear_exposed(rd);
385 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
385 + rrddim_metadata_updated(rd);
386 rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
387 rrdcontext_updated_rrddim_algorithm(rd);
388 return 1;
@@ -395,8 +395,7 @@ inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier) {
395 netdata_log_debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from %d to %d",
396 rrdset_id(st), rrddim_name(rd), rd->multiplier, multiplier);
397 rd->multiplier = multiplier;
398 - rrddim_clear_exposed(rd);
399 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
398 + rrddim_metadata_updated(rd);
399 rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
400 rrdcontext_updated_rrddim_multiplier(rd);
401 return 1;
@@ -409,8 +408,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor) {
408 netdata_log_debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from %d to %d",
409 rrdset_id(st), rrddim_name(rd), rd->divisor, divisor);
410 rd->divisor = divisor;
412 - rrddim_clear_exposed(rd);
413 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
411 + rrddim_metadata_updated(rd);
412 rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
413 rrdcontext_updated_rrddim_divisor(rd);
414 return 1;
@@ -532,8 +530,8 @@ int rrddim_unhide(RRDSET *st, const char *id) {
530 return 0;
531 }
532
535 -inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
536 - netdata_log_debug(D_RRD_CALLS, "rrddim_is_obsolete() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd));
533 +inline void rrddim_is_obsolete___safe_from_collector_thread(RRDSET *st, RRDDIM *rd) {
534 + netdata_log_debug(D_RRD_CALLS, "rrddim_is_obsolete___safe_from_collector_thread() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd));
535
536 if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))) {
537 netdata_log_info("Cannot obsolete already archived dimension %s from chart %s", rrddim_name(rd), rrdset_name(st));
@@ -545,8 +543,8 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
543 rrdcontext_updated_rrddim_flags(rd);
544 }
545
548 -inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
549 - netdata_log_debug(D_RRD_CALLS, "rrddim_isnot_obsolete() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd));
546 +inline void rrddim_isnot_obsolete___safe_from_collector_thread(RRDSET *st __maybe_unused, RRDDIM *rd) {
547 + netdata_log_debug(D_RRD_CALLS, "rrddim_isnot_obsolete___safe_from_collector_thread() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd));
548
549 rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
550 rrdcontext_updated_rrddim_flags(rd);
database/rrdhost.c
+7
@@ -1191,6 +1191,12 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1191 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(localhost, host, prev, next);
1192 }
1193
1194 + // ------------------------------------------------------------------------
1195 + // clean up streaming chart slots
1196 +
1197 + rrdhost_pluginsd_send_chart_slots_free(host);
1198 + rrdhost_pluginsd_receive_chart_slots_free(host);
1199 +
1200 // ------------------------------------------------------------------------
1201 // clean up streaming
1202
@@ -1278,6 +1284,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1284
1285 string_freez(host->hostname);
1286 __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED);
1287 +
1288 freez(host);
1289 }
1290
database/rrdset.c
+143 -13
@@ -5,6 +5,123 @@
5 #include <sched.h>
6 #include "storage_engine.h"
7
8 +// ----------------------------------------------------------------------------
9 +// RRDSET rrdpush send chart_slots
10 +
11 +static void rrdset_rrdpush_send_chart_slot_assign(RRDSET *st) {
12 + RRDHOST *host = st->rrdhost;
13 + spinlock_lock(&host->rrdpush.send.pluginsd_chart_slots.available.spinlock);
14 +
15 + if(host->rrdpush.send.pluginsd_chart_slots.available.used > 0)
16 + st->rrdpush.sender.chart_slot =
17 + host->rrdpush.send.pluginsd_chart_slots.available.array[--host->rrdpush.send.pluginsd_chart_slots.available.used];
18 + else
19 + st->rrdpush.sender.chart_slot = ++host->rrdpush.send.pluginsd_chart_slots.last_used;
20 +
21 + spinlock_unlock(&host->rrdpush.send.pluginsd_chart_slots.available.spinlock);
22 +}
23 +
24 +static void rrdset_rrdpush_send_chart_slot_release(RRDSET *st) {
25 + if(!st->rrdpush.sender.chart_slot || st->rrdhost->rrdpush.send.pluginsd_chart_slots.available.ignore)
26 + return;
27 +
28 + RRDHOST *host = st->rrdhost;
29 + spinlock_lock(&host->rrdpush.send.pluginsd_chart_slots.available.spinlock);
30 +
31 + if(host->rrdpush.send.pluginsd_chart_slots.available.used >= host->rrdpush.send.pluginsd_chart_slots.available.size) {
32 + uint32_t old_size = host->rrdpush.send.pluginsd_chart_slots.available.size;
33 + uint32_t new_size = (old_size > 0) ? (old_size * 2) : 1024;
34 +
35 + host->rrdpush.send.pluginsd_chart_slots.available.array =
36 + reallocz(host->rrdpush.send.pluginsd_chart_slots.available.array, new_size * sizeof(uint32_t));
37 +
38 + host->rrdpush.send.pluginsd_chart_slots.available.size = new_size;
39 + }
40 +
41 + host->rrdpush.send.pluginsd_chart_slots.available.array[host->rrdpush.send.pluginsd_chart_slots.available.used++] =
42 + st->rrdpush.sender.chart_slot;
43 +
44 + st->rrdpush.sender.chart_slot = 0;
45 + spinlock_unlock(&host->rrdpush.send.pluginsd_chart_slots.available.spinlock);
46 +}
47 +
48 +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host) {
49 + spinlock_lock(&host->rrdpush.send.pluginsd_chart_slots.available.spinlock);
50 + host->rrdpush.send.pluginsd_chart_slots.available.ignore = true;
51 + freez(host->rrdpush.send.pluginsd_chart_slots.available.array);
52 + host->rrdpush.send.pluginsd_chart_slots.available.array = NULL;
53 + host->rrdpush.send.pluginsd_chart_slots.available.used = 0;
54 + host->rrdpush.send.pluginsd_chart_slots.available.size = 0;
55 + spinlock_unlock(&host->rrdpush.send.pluginsd_chart_slots.available.spinlock);
56 +
57 + // zero all the slots on all charts, so that they will not attempt to access the array
58 + RRDSET *st;
59 + rrdset_foreach_read(st, host) {
60 + st->rrdpush.sender.chart_slot = 0;
61 + }
62 + rrdset_foreach_done(st);
63 +}
64 +
65 +void rrdset_pluginsd_receive_unslot(RRDSET *st) {
66 + for(size_t i = 0; i < st->pluginsd.size ;i++) {
67 + rrddim_acquired_release(st->pluginsd.prd_array[i].rda); // can be NULL
68 + st->pluginsd.prd_array[i].rda = NULL;
69 + st->pluginsd.prd_array[i].rd = NULL;
70 + st->pluginsd.prd_array[i].id = NULL;
71 + }
72 +
73 + RRDHOST *host = st->rrdhost;
74 +
75 + if(st->pluginsd.last_slot >= 0 &&
76 + (uint32_t)st->pluginsd.last_slot < host->rrdpush.receive.pluginsd_chart_slots.size &&
77 + host->rrdpush.receive.pluginsd_chart_slots.array[st->pluginsd.last_slot] == st) {
78 + host->rrdpush.receive.pluginsd_chart_slots.array[st->pluginsd.last_slot] = NULL;
79 + }
80 +
81 + st->pluginsd.last_slot = -1;
82 + st->pluginsd.with_slots = false;
83 +}
84 +
85 +void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st) {
86 + if(!st)
87 + return;
88 +
89 + spinlock_lock(&st->pluginsd.spinlock);
90 +
91 + rrdset_pluginsd_receive_unslot(st);
92 +
93 + freez(st->pluginsd.prd_array);
94 + st->pluginsd.prd_array = NULL;
95 + st->pluginsd.size = 0;
96 + st->pluginsd.pos = 0;
97 + st->pluginsd.set = false;
98 + st->pluginsd.last_slot = -1;
99 + st->pluginsd.with_slots = false;
100 + st->pluginsd.collector_tid = 0;
101 +
102 + spinlock_unlock(&st->pluginsd.spinlock);
103 +}
104 +
105 +static void rrdset_pluginsd_receive_slots_initialize(RRDSET *st) {
106 + spinlock_init(&st->pluginsd.spinlock);
107 + st->pluginsd.last_slot = -1;
108 +}
109 +
110 +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host) {
111 + spinlock_lock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
112 +
113 + if(host->rrdpush.receive.pluginsd_chart_slots.array) {
114 + for (size_t s = 0; s < host->rrdpush.receive.pluginsd_chart_slots.size; s++)
115 + rrdset_pluginsd_receive_unslot_and_cleanup(host->rrdpush.receive.pluginsd_chart_slots.array[s]);
116 +
117 + freez(host->rrdpush.receive.pluginsd_chart_slots.array);
118 + host->rrdpush.receive.pluginsd_chart_slots.array = NULL;
119 + host->rrdpush.receive.pluginsd_chart_slots.size = 0;
120 + }
121 +
122 + spinlock_unlock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
123 +}
124 +
125 // ----------------------------------------------------------------------------
126 // RRDSET name index
127
@@ -64,7 +181,7 @@ static STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const c
181 i++;
182 } while (rrdset_index_find_name(host, new_name));
183
67 - netdata_log_info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host));
184 +// netdata_log_info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host));
185 }
186 else
187 return NULL;
@@ -135,6 +252,8 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
252 st->chart_type = ctr->chart_type;
253 st->rrdhost = host;
254
255 + rrdset_rrdpush_send_chart_slot_assign(st);
256 +
257 spinlock_init(&st->data_collection_lock);
258
259 st->flags = RRDSET_FLAG_SYNC_CLOCK
@@ -179,13 +298,13 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
298 st->green = NAN;
299 st->red = NAN;
300
301 + rrdset_pluginsd_receive_slots_initialize(st);
302 +
303 ctr->react_action = RRDSET_REACT_NEW;
304
305 ml_chart_new(st);
306 }
307
187 -void pluginsd_rrdset_cleanup(RRDSET *st);
188 -
308 void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
309 RRDHOST *host = st->rrdhost;
310
@@ -208,7 +327,7 @@ void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
327 }
328 }
329
211 - pluginsd_rrdset_cleanup(st);
330 + rrdset_pluginsd_receive_unslot_and_cleanup(st);
331 }
332
333 // the destructor - the dictionary is write locked while this runs
@@ -220,6 +339,8 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
339
340 rrdset_finalize_collection(st, false);
341
342 + rrdset_rrdpush_send_chart_slot_release(st);
343 +
344 // remove it from the name index
345 rrdset_index_del_name(host, st);
346
@@ -288,7 +409,7 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
409 struct rrdset_constructor *ctr = constructor_data;
410 RRDSET *st = rrdset;
411
291 - rrdset_isnot_obsolete(st);
412 + rrdset_isnot_obsolete___safe_from_collector_thread(st);
413
414 ctr->react_action = RRDSET_REACT_NONE;
415
@@ -363,7 +484,7 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
484 rrdset_update_permanent_labels(st);
485
486 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
366 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
487 + rrdset_metadata_updated(st);
488
489 return ctr->react_action != RRDSET_REACT_NONE;
490 }
@@ -542,7 +663,7 @@ int rrdset_reset_name(RRDSET *st, const char *name) {
663 rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);
664 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);
665 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);
545 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
666 + rrdset_metadata_updated(st);
667
668 rrdcontext_updated_rrdset_name(st);
669 return 2;
@@ -651,14 +772,19 @@ void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_
772 *last_time_s = db_last_entry_s;
773 }
774
654 -inline void rrdset_is_obsolete(RRDSET *st) {
775 +inline void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) {
776 + rrdset_pluginsd_receive_unslot(st);
777 +
778 if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
779 +// netdata_log_info("Setting obsolete flag on chart 'host:%s/chart:%s'",
780 +// rrdhost_hostname(st->rrdhost), rrdset_id(st));
781 +
782 rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE);
783 rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS);
784
785 st->last_accessed_time_s = now_realtime_sec();
786
661 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
787 + rrdset_metadata_updated(st);
788
789 // the chart will not get more updates (data collection)
790 // so, we have to push its definition now
@@ -667,12 +793,16 @@ inline void rrdset_is_obsolete(RRDSET *st) {
793 }
794 }
795
670 -inline void rrdset_isnot_obsolete(RRDSET *st) {
796 +inline void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st) {
797 if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
798 +
799 +// netdata_log_info("Clearing obsolete flag on chart 'host:%s/chart:%s'",
800 +// rrdhost_hostname(st->rrdhost), rrdset_id(st));
801 +
802 rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
803 st->last_accessed_time_s = now_realtime_sec();
804
675 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
805 + rrdset_metadata_updated(st);
806
807 // the chart will be pushed upstream automatically
808 // due to data collection
@@ -1527,7 +1657,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1657
1658 if (unlikely(rrdset_flags & RRDSET_FLAG_OBSOLETE)) {
1659 netdata_log_error("Chart '%s' has the OBSOLETE flag set, but it is collected.", rrdset_id(st));
1530 - rrdset_isnot_obsolete(st);
1660 + rrdset_isnot_obsolete___safe_from_collector_thread(st);
1661 }
1662
1663 // check if the chart has a long time to be updated
@@ -1674,7 +1804,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
1804
1805 if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) {
1806 netdata_log_error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
1677 - rrddim_isnot_obsolete(st, rd);
1807 + rrddim_isnot_obsolete___safe_from_collector_thread(st, rd);
1808 }
1809 }
1810 }
database/sqlite/sqlite_functions.c
+1
@@ -85,6 +85,7 @@ sqlite3 *db_meta = NULL;
85
86 #define MAX_PREPARED_STATEMENTS (32)
87 pthread_key_t key_pool[MAX_PREPARED_STATEMENTS];
88 +pthread_key_t plugin_key;
89
90 SQLITE_API int sqlite3_exec_monitored(
91 sqlite3 *db, /* An open database */
exporting/prometheus/prometheus.c
+1 -1
@@ -619,7 +619,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
619 int as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED);
620 int homogeneous = 1;
621 int prometheus_collector = 0;
622 - RRDSET_FLAGS flags = __atomic_load_n(&st->flags, __ATOMIC_RELAXED);
622 + RRDSET_FLAGS flags = rrdset_flag_get(st);
623 if (as_collected) {
624 if (flags & RRDSET_FLAG_HOMOGENEOUS_CHECK)
625 rrdset_update_heterogeneous_flag(st);
libnetdata/functions_evloop/functions_evloop.h
+2
@@ -46,6 +46,8 @@
46
47 #define PLUGINSD_KEYWORD_EXIT "EXIT"
48
49 +#define PLUGINSD_KEYWORD_SLOT "SLOT" // to change the length of this, update pluginsd_extract_chart_slot() too
50 +
51 #define PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT 10 // seconds
52
53 typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, int timeout, bool *cancelled);
libnetdata/string/string.c
-4
@@ -338,10 +338,6 @@ inline const char *string2str(STRING *string) {
338 return string->str;
339 }
340
341 -int string_strcmp(STRING *string, const char *s) {
342 - return strcmp(string2str(string), s);
343 -}
344 -
341 STRING *string_2way_merge(STRING *a, STRING *b) {
342 static STRING *X = NULL;
343
libnetdata/string/string.h
+4 -1
@@ -13,7 +13,6 @@ STRING *string_dup(STRING *string);
13 void string_freez(STRING *string);
14 size_t string_strlen(STRING *string);
15 const char *string2str(STRING *string) NEVERNULL;
16 -int string_strcmp(STRING *string, const char *s);
16
17 // keep common prefix/suffix and replace everything else with [x]
18 STRING *string_2way_merge(STRING *a, STRING *b);
@@ -24,6 +23,10 @@ static inline int string_cmp(STRING *s1, STRING *s2) {
23 return (s1 == s2)?0:strcmp(string2str(s1), string2str(s2));
24 }
25
26 +static inline int string_strcmp(STRING *string, const char *s) {
27 + return strcmp(string2str(string), s);
28 +}
29 +
30 void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases);
31
32 int string_unittest(size_t entries);
streaming/receiver.c
+3
@@ -321,6 +321,9 @@ static inline bool receiver_should_stop(struct receiver_state *rpt) {
321 return false;
322 }
323
324 +extern pthread_key_t plugin_key;
325 +struct plugin_data;
326 +
327 static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, int fd, void *ssl) {
328 size_t result = 0;
329
streaming/replication.c
+58 -30
@@ -168,7 +168,7 @@ static struct replication_query *replication_query_prepare(
168 size_t count = 0;
169 RRDDIM *rd;
170 rrddim_foreach_read(rd, st) {
171 - if (unlikely(!rd || !rd_dfe.item || !rrddim_check_exposed(rd)))
171 + if (unlikely(!rd || !rd_dfe.item || !rrddim_check_upstream_exposed(rd)))
172 continue;
173
174 if (unlikely(rd_dfe.counter >= q->dimensions)) {
@@ -213,31 +213,38 @@ static struct replication_query *replication_query_prepare(
213 }
214
215 static void replication_send_chart_collection_state(BUFFER *wb, RRDSET *st, STREAM_CAPABILITIES capabilities) {
216 - NUMBER_ENCODING encoding = (capabilities & STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
216 + bool with_slots = (capabilities & STREAM_CAP_SLOTS) ? true : false;
217 + NUMBER_ENCODING integer_encoding = (capabilities & STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
218 RRDDIM *rd;
219 rrddim_foreach_read(rd, st){
219 - if (!rrddim_check_exposed(rd)) continue;
220 + if (!rrddim_check_upstream_exposed(rd)) continue;
221
221 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " '",
222 - sizeof(PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE) - 1 + 2);
222 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, sizeof(PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE) - 1);
223 +
224 + if(with_slots) {
225 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
226 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
227 + }
228 +
229 + buffer_fast_strcat(wb, " '", 2);
230 buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
231 buffer_fast_strcat(wb, "' ", 2);
225 - buffer_print_uint64_encoded(wb, encoding, (usec_t) rd->collector.last_collected_time.tv_sec * USEC_PER_SEC +
226 - (usec_t) rd->collector.last_collected_time.tv_usec);
232 + buffer_print_uint64_encoded(wb, integer_encoding, (usec_t) rd->collector.last_collected_time.tv_sec * USEC_PER_SEC +
233 + (usec_t) rd->collector.last_collected_time.tv_usec);
234 buffer_fast_strcat(wb, " ", 1);
228 - buffer_print_int64_encoded(wb, encoding, rd->collector.last_collected_value);
235 + buffer_print_int64_encoded(wb, integer_encoding, rd->collector.last_collected_value);
236 buffer_fast_strcat(wb, " ", 1);
230 - buffer_print_netdata_double_encoded(wb, encoding, rd->collector.last_calculated_value);
237 + buffer_print_netdata_double_encoded(wb, integer_encoding, rd->collector.last_calculated_value);
238 buffer_fast_strcat(wb, " ", 1);
232 - buffer_print_netdata_double_encoded(wb, encoding, rd->collector.last_stored_value);
239 + buffer_print_netdata_double_encoded(wb, integer_encoding, rd->collector.last_stored_value);
240 buffer_fast_strcat(wb, "\n", 1);
241 }
242 rrddim_foreach_done(rd);
243
244 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE " ", sizeof(PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE) - 1 + 1);
238 - buffer_print_uint64_encoded(wb, encoding, (usec_t) st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t) st->last_collected_time.tv_usec);
245 + buffer_print_uint64_encoded(wb, integer_encoding, (usec_t) st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t) st->last_collected_time.tv_usec);
246 buffer_fast_strcat(wb, " ", 1);
240 - buffer_print_uint64_encoded(wb, encoding, (usec_t) st->last_updated.tv_sec * USEC_PER_SEC + (usec_t) st->last_updated.tv_usec);
247 + buffer_print_uint64_encoded(wb, integer_encoding, (usec_t) st->last_updated.tv_sec * USEC_PER_SEC + (usec_t) st->last_updated.tv_usec);
248 buffer_fast_strcat(wb, "\n", 1);
249 }
250
@@ -313,7 +320,8 @@ static void replication_query_align_to_optimal_before(struct replication_query *
320 static bool replication_query_execute(BUFFER *wb, struct replication_query *q, size_t max_msg_size) {
321 replication_query_align_to_optimal_before(q);
322
316 - NUMBER_ENCODING encoding = (q->query.capabilities & STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
323 + bool with_slots = (q->query.capabilities & STREAM_CAP_SLOTS) ? true : false;
324 + NUMBER_ENCODING integer_encoding = (q->query.capabilities & STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
325 time_t after = q->query.after;
326 time_t before = q->query.before;
327 size_t dimensions = q->dimensions;
@@ -444,12 +452,19 @@ static bool replication_query_execute(BUFFER *wb, struct replication_query *q, s
452 }
453 last_end_time_in_buffer = min_end_time;
454
447 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN " '' ", sizeof(PLUGINSD_KEYWORD_REPLAY_BEGIN) - 1 + 4);
448 - buffer_print_uint64_encoded(wb, encoding, min_start_time);
455 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN, sizeof(PLUGINSD_KEYWORD_REPLAY_BEGIN) - 1);
456 +
457 + if(with_slots) {
458 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
459 + buffer_print_uint64_encoded(wb, integer_encoding, q->st->rrdpush.sender.chart_slot);
460 + }
461 +
462 + buffer_fast_strcat(wb, " '' ", 4);
463 + buffer_print_uint64_encoded(wb, integer_encoding, min_start_time);
464 buffer_fast_strcat(wb, " ", 1);
450 - buffer_print_uint64_encoded(wb, encoding, min_end_time);
465 + buffer_print_uint64_encoded(wb, integer_encoding, min_end_time);
466 buffer_fast_strcat(wb, " ", 1);
452 - buffer_print_uint64_encoded(wb, encoding, wall_clock_time);
467 + buffer_print_uint64_encoded(wb, integer_encoding, wall_clock_time);
468 buffer_fast_strcat(wb, "\n", 1);
469
470 // output the replay values for this time
@@ -462,10 +477,17 @@ static bool replication_query_execute(BUFFER *wb, struct replication_query *q, s
477 !storage_point_is_unset(d->sp) &&
478 !storage_point_is_gap(d->sp))) {
479
465 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_SET " \"", sizeof(PLUGINSD_KEYWORD_REPLAY_SET) - 1 + 2);
480 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_SET, sizeof(PLUGINSD_KEYWORD_REPLAY_SET) - 1);
481 +
482 + if(with_slots) {
483 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
484 + buffer_print_uint64_encoded(wb, integer_encoding, d->rd->rrdpush.sender.dim_slot);
485 + }
486 +
487 + buffer_fast_strcat(wb, " \"", 2);
488 buffer_fast_strcat(wb, rrddim_id(d->rd), string_strlen(d->rd->id));
489 buffer_fast_strcat(wb, "\" ", 2);
468 - buffer_print_netdata_double_encoded(wb, encoding, d->sp.sum);
490 + buffer_print_netdata_double_encoded(wb, integer_encoding, d->sp.sum);
491 buffer_fast_strcat(wb, " ", 1);
492 buffer_print_sn_flags(wb, d->sp.flags, q->query.capabilities & STREAM_CAP_INTERPOLATED);
493 buffer_fast_strcat(wb, "\n", 1);
@@ -595,7 +617,8 @@ void replication_response_cancel_and_finalize(struct replication_query *q) {
617 static bool sender_is_still_connected_for_this_request(struct replication_request *rq);
618
619 bool replication_response_execute_and_finalize(struct replication_query *q, size_t max_msg_size) {
598 - NUMBER_ENCODING encoding = (q->query.capabilities & STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
620 + bool with_slots = (q->query.capabilities & STREAM_CAP_SLOTS) ? true : false;
621 + NUMBER_ENCODING integer_encoding = (q->query.capabilities & STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
622 struct replication_request *rq = q->rq;
623 RRDSET *st = q->st;
624 RRDHOST *host = st->rrdhost;
@@ -605,12 +628,17 @@ bool replication_response_execute_and_finalize(struct replication_query *q, size
628 // holding the host's buffer lock for too long
629 BUFFER *wb = sender_start(host->sender);
630
608 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN " '", sizeof(PLUGINSD_KEYWORD_REPLAY_BEGIN) - 1 + 2);
631 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN, sizeof(PLUGINSD_KEYWORD_REPLAY_BEGIN) - 1);
632 +
633 + if(with_slots) {
634 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
635 + buffer_print_uint64_encoded(wb, integer_encoding, q->st->rrdpush.sender.chart_slot);
636 + }
637 +
638 + buffer_fast_strcat(wb, " '", 2);
639 buffer_fast_strcat(wb, rrdset_id(st), string_strlen(st->id));
640 buffer_fast_strcat(wb, "'\n", 2);
641
612 -// buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN " \"%s\"\n", rrdset_id(st));
613 -
642 bool locked_data_collection = q->query.locked_data_collection;
643 q->query.locked_data_collection = false;
644
@@ -634,19 +662,19 @@ bool replication_response_execute_and_finalize(struct replication_query *q, size
662 // last end time of the data we sent
663
664 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_END " ", sizeof(PLUGINSD_KEYWORD_REPLAY_END) - 1 + 1);
637 - buffer_print_int64_encoded(wb, encoding, st->update_every);
665 + buffer_print_int64_encoded(wb, integer_encoding, st->update_every);
666 buffer_fast_strcat(wb, " ", 1);
639 - buffer_print_uint64_encoded(wb, encoding, db_first_entry);
667 + buffer_print_uint64_encoded(wb, integer_encoding, db_first_entry);
668 buffer_fast_strcat(wb, " ", 1);
641 - buffer_print_uint64_encoded(wb, encoding, db_last_entry);
669 + buffer_print_uint64_encoded(wb, integer_encoding, db_last_entry);
670
671 buffer_fast_strcat(wb, enable_streaming ? " true " : " false ", 7);
672
645 - buffer_print_uint64_encoded(wb, encoding, after);
673 + buffer_print_uint64_encoded(wb, integer_encoding, after);
674 buffer_fast_strcat(wb, " ", 1);
647 - buffer_print_uint64_encoded(wb, encoding, before);
675 + buffer_print_uint64_encoded(wb, integer_encoding, before);
676 buffer_fast_strcat(wb, " ", 1);
649 - buffer_print_uint64_encoded(wb, encoding, wall_clock_time);
677 + buffer_print_uint64_encoded(wb, integer_encoding, wall_clock_time);
678 buffer_fast_strcat(wb, "\n", 1);
679
680 worker_is_busy(WORKER_JOB_BUFFER_COMMIT);
@@ -664,7 +692,7 @@ bool replication_response_execute_and_finalize(struct replication_query *q, size
692 rrdhost_sender_replicating_charts_minus_one(st->rrdhost);
693
694 if(!finished_with_gap)
667 - st->upstream_resync_time_s = 0;
695 + st->rrdpush.sender.resync_time_s = 0;
696
697 #ifdef NETDATA_LOG_REPLICATION_REQUESTS
698 internal_error(true, "STREAM_SENDER REPLAY: 'host:%s/chart:%s' streaming starts",
streaming/rrdpush.c
+74 -33
@@ -218,11 +218,13 @@ static void rrdpush_send_clabels(BUFFER *wb, RRDSET *st) {
218 // Send the current chart definition.
219 // Assumes that collector thread has already called sender_start for mutex / buffer state.
220 static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {
221 - bool replication_progress = false;
221 + uint32_t version = rrdset_metadata_version(st);
222
223 RRDHOST *host = st->rrdhost;
224 + NUMBER_ENCODING integer_encoding = stream_has_capability(host->sender, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
225 + bool with_slots = stream_has_capability(host->sender, STREAM_CAP_SLOTS) ? true : false;
226
225 - rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
227 + bool replication_progress = false;
228
229 // properly set the name for the remote end to parse it
230 char *name = "";
@@ -237,10 +239,17 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {
239 }
240 }
241
242 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_CHART, sizeof(PLUGINSD_KEYWORD_CHART) - 1);
243 +
244 + if(with_slots) {
245 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
246 + buffer_print_uint64_encoded(wb, integer_encoding, st->rrdpush.sender.chart_slot);
247 + }
248 +
249 // send the chart
250 buffer_sprintf(
251 wb
243 - , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s %s\" \"%s\" \"%s\"\n"
252 + , " \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s %s\" \"%s\" \"%s\"\n"
253 , rrdset_id(st)
254 , name
255 , rrdset_title(st)
@@ -265,19 +274,25 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {
274 // send the dimensions
275 RRDDIM *rd;
276 rrddim_foreach_read(rd, st) {
277 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_DIMENSION, sizeof(PLUGINSD_KEYWORD_DIMENSION) - 1);
278 +
279 + if(with_slots) {
280 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
281 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
282 + }
283 +
284 buffer_sprintf(
269 - wb
270 - , "DIMENSION \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s\"\n"
271 - , rrddim_id(rd)
272 - , rrddim_name(rd)
273 - , rrd_algorithm_name(rd->algorithm)
274 - , rd->multiplier
275 - , rd->divisor
276 - , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
277 - , rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)?"hidden":""
278 - , rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
285 + wb
286 + , " \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s\"\n"
287 + , rrddim_id(rd)
288 + , rrddim_name(rd)
289 + , rrd_algorithm_name(rd->algorithm)
290 + , rd->multiplier
291 + , rd->divisor
292 + , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
293 + , rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)?"hidden":""
294 + , rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
295 );
280 - rrddim_set_exposed(rd);
296 }
297 rrddim_foreach_done(rd);
298
@@ -312,7 +327,17 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {
327 #endif
328 }
329
315 - st->upstream_resync_time_s = st->last_collected_time.tv_sec + (remote_clock_resync_iterations * st->update_every);
330 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
331 +
332 + // we can set the exposed flag, after we commit the buffer
333 + // because replication may pick it up prematurely
334 + rrddim_foreach_read(rd, st) {
335 + rrddim_metadata_exposed_upstream(rd, version);
336 + }
337 + rrddim_foreach_done(rd);
338 + rrdset_metadata_exposed_upstream(st, version);
339 +
340 + st->rrdpush.sender.resync_time_s = st->last_collected_time.tv_sec + (remote_clock_resync_iterations * st->update_every);
341 return replication_progress;
342 }
343
@@ -322,7 +347,7 @@ static void rrdpush_send_chart_metrics(BUFFER *wb, RRDSET *st, struct sender_sta
347 buffer_fast_strcat(wb, rrdset_id(st), string_strlen(st->id));
348 buffer_fast_strcat(wb, "\" ", 2);
349
325 - if(st->last_collected_time.tv_sec > st->upstream_resync_time_s)
350 + if(st->last_collected_time.tv_sec > st->rrdpush.sender.resync_time_s)
351 buffer_print_uint64(wb, st->usec_since_last_update);
352 else
353 buffer_fast_strcat(wb, "0", 1);
@@ -334,7 +359,7 @@ static void rrdpush_send_chart_metrics(BUFFER *wb, RRDSET *st, struct sender_sta
359 if(unlikely(!rrddim_check_updated(rd)))
360 continue;
361
337 - if(likely(rrddim_check_exposed(rd))) {
362 + if(likely(rrddim_check_upstream_exposed_collector(rd))) {
363 buffer_fast_strcat(wb, "SET \"", 5);
364 buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
365 buffer_fast_strcat(wb, "\" = ", 4);
@@ -345,7 +370,7 @@ static void rrdpush_send_chart_metrics(BUFFER *wb, RRDSET *st, struct sender_sta
370 internal_error(true, "STREAM: 'host:%s/chart:%s/dim:%s' flag 'exposed' is updated but not exposed",
371 rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd));
372 // we will include it in the next iteration
348 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
373 + rrdset_metadata_updated(st);
374 }
375 }
376 rrddim_foreach_done(rd);
@@ -363,12 +388,12 @@ bool rrdset_push_chart_definition_now(RRDSET *st) {
388 RRDHOST *host = st->rrdhost;
389
390 if(unlikely(!rrdhost_can_send_definitions_to_parent(host)
366 - || !should_send_chart_matching(st, __atomic_load_n(&st->flags, __ATOMIC_SEQ_CST))))
391 + || !should_send_chart_matching(st, rrdset_flag_get(st)))) {
392 return false;
393 + }
394
395 BUFFER *wb = sender_start(host->sender);
396 rrdpush_send_chart_definition(wb, st);
371 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
397 sender_thread_buffer_free();
398
399 return true;
@@ -383,6 +408,7 @@ void rrddim_push_metrics_v2(RRDSET_STREAM_BUFFER *rsb, RRDDIM *rd, usec_t point_
408 if(!rsb->wb || !rsb->v2 || !netdata_double_isnumber(n) || !does_storage_number_exist(flags))
409 return;
410
411 + bool with_slots = stream_has_capability(rsb, STREAM_CAP_SLOTS) ? true : false;
412 NUMBER_ENCODING integer_encoding = stream_has_capability(rsb, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
413 NUMBER_ENCODING doubles_encoding = stream_has_capability(rsb, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
414 BUFFER *wb = rsb->wb;
@@ -392,7 +418,14 @@ void rrddim_push_metrics_v2(RRDSET_STREAM_BUFFER *rsb, RRDDIM *rd, usec_t point_
418 if(unlikely(rsb->begin_v2_added))
419 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_END_V2 "\n", sizeof(PLUGINSD_KEYWORD_END_V2) - 1 + 1);
420
395 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2 " '", sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1 + 2);
421 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2, sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1);
422 +
423 + if(with_slots) {
424 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
425 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdset->rrdpush.sender.chart_slot);
426 + }
427 +
428 + buffer_fast_strcat(wb, " '", 2);
429 buffer_fast_strcat(wb, rrdset_id(rd->rrdset), string_strlen(rd->rrdset->id));
430 buffer_fast_strcat(wb, "' ", 2);
431 buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdset->update_every);
@@ -409,7 +442,14 @@ void rrddim_push_metrics_v2(RRDSET_STREAM_BUFFER *rsb, RRDDIM *rd, usec_t point_
442 rsb->begin_v2_added = true;
443 }
444
412 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2 " '", sizeof(PLUGINSD_KEYWORD_SET_V2) - 1 + 2);
445 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2, sizeof(PLUGINSD_KEYWORD_SET_V2) - 1);
446 +
447 + if(with_slots) {
448 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
449 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
450 + }
451 +
452 + buffer_fast_strcat(wb, " '", 2);
453 buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
454 buffer_fast_strcat(wb, "' ", 2);
455 buffer_print_int64_encoded(wb, integer_encoding, rd->collector.last_collected_value);
@@ -465,7 +505,7 @@ void rrdpush_send_job_status_update(RRDHOST *host, const char *plugin_name, cons
505
506 buffer_strcat(wb, "\n");
507
468 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
508 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
509
510 sender_thread_buffer_free();
511
@@ -479,7 +519,7 @@ void rrdpush_send_job_deleted(RRDHOST *host, const char *plugin_name, const char
519
520 buffer_sprintf(wb, PLUGINSD_KEYWORD_DELETE_JOB " %s %s %s\n", plugin_name, module_name, job_name);
521
482 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
522 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
523
524 sender_thread_buffer_free();
525 }
@@ -511,11 +551,11 @@ RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock
551 if(unlikely(host_flags & RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED)) {
552 BUFFER *wb = sender_start(host->sender);
553 rrd_functions_expose_global_rrdpush(host, wb);
514 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
554 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
555 }
556
517 - RRDSET_FLAGS rrdset_flags = __atomic_load_n(&st->flags, __ATOMIC_SEQ_CST);
518 - bool exposed_upstream = (rrdset_flags & RRDSET_FLAG_UPSTREAM_EXPOSED);
557 + bool exposed_upstream = rrdset_check_upstream_exposed(st);
558 + RRDSET_FLAGS rrdset_flags = rrdset_flag_get(st);
559 bool replication_in_progress = !(rrdset_flags & RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
560
561 if(unlikely((exposed_upstream && replication_in_progress) ||
@@ -525,7 +565,6 @@ RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock
565 if(unlikely(!exposed_upstream)) {
566 BUFFER *wb = sender_start(host->sender);
567 replication_in_progress = rrdpush_send_chart_definition(wb, st);
528 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
568 }
569
570 if(replication_in_progress)
@@ -573,7 +612,7 @@ void rrdpush_send_global_functions(RRDHOST *host) {
612
613 rrd_functions_expose_global_rrdpush(host, wb);
614
576 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
615 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
616
617 sender_thread_buffer_free();
618 }
@@ -606,7 +645,7 @@ void rrdpush_send_dyncfg(RRDHOST *host) {
645 }
646 dfe_done(plug);
647
609 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
648 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
649
650 sender_thread_buffer_free();
651 }
@@ -632,7 +671,7 @@ void rrdpush_send_dyncfg_reg_module(RRDHOST *host, const char *plugin_name, cons
671
672 buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE " %s %s %s\n", plugin_name, module_name, module_type2str(type));
673
635 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
674 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
675
676 sender_thread_buffer_free();
677 }
@@ -645,7 +684,7 @@ void rrdpush_send_dyncfg_reg_job(RRDHOST *host, const char *plugin_name, const c
684
685 buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB " %s %s %s %s %"PRIu32"\n", plugin_name, module_name, job_name, job_type2str(type), flags);
686
648 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
687 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
688
689 sender_thread_buffer_free();
690 }
@@ -1397,7 +1436,8 @@ static struct {
1436 {STREAM_CAP_INTERPOLATED, "INTERPOLATED" },
1437 {STREAM_CAP_IEEE754, "IEEE754" },
1438 {STREAM_CAP_DATA_WITH_ML, "ML" },
1400 - {STREAM_CAP_DYNCFG, "DYN_CFG" },
1439 + {STREAM_CAP_DYNCFG, "DYNCFG" },
1440 + {STREAM_CAP_SLOTS, "SLOTS" },
1441 {STREAM_CAP_ZSTD, "ZSTD" },
1442 {STREAM_CAP_GZIP, "GZIP" },
1443 {0 , NULL },
@@ -1475,6 +1515,7 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender) {
1515 STREAM_CAP_REPLICATION |
1516 STREAM_CAP_BINARY |
1517 STREAM_CAP_INTERPOLATED |
1518 + STREAM_CAP_SLOTS |
1519 STREAM_CAP_COMPRESSIONS_AVAILABLE |
1520 #ifdef NETDATA_TEST_DYNCFG
1521 STREAM_CAP_DYNCFG |
streaming/rrdpush.h
+2
@@ -48,6 +48,7 @@ typedef enum {
48 STREAM_CAP_IEEE754 = (1 << 15), // streaming supports binary/hex transfer of double values
49 STREAM_CAP_DATA_WITH_ML = (1 << 16), // streaming supports transferring anomaly bit
50 STREAM_CAP_DYNCFG = (1 << 17), // dynamic configuration of plugins trough streaming
51 + STREAM_CAP_SLOTS = (1 << 18), // the sender can appoint a unique slot for each chart
52 STREAM_CAP_ZSTD = (1 << 19), // ZSTD compression supported
53 STREAM_CAP_GZIP = (1 << 20), // GZIP compression supported
54
@@ -153,6 +154,7 @@ typedef enum __attribute__((packed)) {
154 STREAM_TRAFFIC_TYPE_FUNCTIONS,
155 STREAM_TRAFFIC_TYPE_METADATA,
156 STREAM_TRAFFIC_TYPE_DATA,
157 + STREAM_TRAFFIC_TYPE_DYNCFG,
158
159 // terminator
160 STREAM_TRAFFIC_TYPE_MAX,
streaming/sender.c
+6 -5
@@ -224,7 +224,7 @@ void sender_commit(struct sender_state *s, BUFFER *wb, STREAM_TRAFFIC_TYPE type)
224
225 sender_unlock(s);
226
227 - if(signal_sender)
227 + if(signal_sender && (!stream_has_capability(s, STREAM_CAP_INTERPOLATED) || type != STREAM_TRAFFIC_TYPE_DATA))
228 rrdpush_signal_sender_to_wake_up(s);
229 }
230
@@ -284,14 +284,15 @@ static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) {
284 static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {
285 RRDSET *st;
286 rrdset_foreach_read(st, host) {
287 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED | RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS);
287 + rrdset_flag_clear(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS);
288 rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
289
290 - st->upstream_resync_time_s = 0;
290 + st->rrdpush.sender.resync_time_s = 0;
291 + rrdset_metadata_updated(st);
292
293 RRDDIM *rd;
294 rrddim_foreach_read(rd, st)
294 - rrddim_clear_exposed(rd);
295 + rrddim_metadata_exposed_upstream_clear(rd);
296 rrddim_foreach_done(rd);
297 }
298 rrdset_foreach_done(st);
@@ -1493,7 +1494,7 @@ void *rrdpush_sender_thread(void *ptr) {
1494 }
1495 };
1496
1496 - int poll_rc = poll(fds, 2, 1000);
1497 + int poll_rc = poll(fds, 2, 50); // timeout in milliseconds
1498
1499 netdata_log_debug(D_STREAM, "STREAM: poll() finished collector=%d socket=%d (current chunk %zu bytes)...",
1500 fds[Collector].revents, fds[Socket].revents, outstanding);