master
c 521 lines 18 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "rrdset-index-id.h"
4 #include "rrdset-index-name.h"
5 #include "rrdset-slots.h"
6
7 // --------------------------------------------------------------------------------------------------------------------
8 // tier1/2 spread over time
9
10 static size_t global_rrdset_counter = 0;
11 static uint16_t rrdset_collection_modulo_init(void) {
12 return __atomic_fetch_add(&global_rrdset_counter, 1, __ATOMIC_RELAXED) % 65535;
13 }
14
15 uint16_t rrddim_collection_modulo(RRDSET *st, uint32_t spread) {
16 if(!spread) spread = 65535;
17 spread = MIN(spread, 65535);
18 return 1 + (st->collection_modulo % spread);
19 }
20
21 // --------------------------------------------------------------------------------------------------------------------
22
23 static inline void rrdset_update_permanent_labels(RRDSET *st) {
24 if(!st->rrdlabels) return;
25
26 rrdlabels_add(st->rrdlabels, "_collect_plugin", rrdset_plugin_name(st), RRDLABEL_SRC_AUTO | RRDLABEL_FLAG_DONT_DELETE);
27 rrdlabels_add(st->rrdlabels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO | RRDLABEL_FLAG_DONT_DELETE);
28 }
29
30 // --------------------------------------------------------------------------------------------------------------------
31 // RRDSET index
32
33 struct rrdset_constructor {
34 RRDHOST *host;
35 const char *type;
36 const char *id;
37 const char *name;
38 const char *family;
39 const char *context;
40 const char *title;
41 const char *units;
42 const char *plugin;
43 const char *module;
44 long priority;
45 int update_every;
46 RRDSET_TYPE chart_type;
47 RRD_DB_MODE memory_mode;
48 long history_entries;
49
50 enum {
51 RRDSET_REACT_NONE = 0,
52 RRDSET_REACT_NEW = (1 << 0),
53 RRDSET_REACT_UPDATED = (1 << 1),
54 RRDSET_REACT_PLUGIN_UPDATED = (1 << 2),
55 RRDSET_REACT_MODULE_UPDATED = (1 << 3),
56 RRDSET_REACT_CHART_ACTIVATED = (1 << 4),
57 } react_action;
58 };
59
60 // the constructor - the dictionary is write locked while this runs
61 static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) {
62 struct rrdset_constructor *ctr = constructor_data;
63 RRDHOST *host = ctr->host;
64 RRDSET *st = rrdset;
65
66 spinlock_init(&st->destroy_lock);
67
68 const char *chart_full_id = dictionary_acquired_item_name(item);
69
70 st->id = string_strdupz(chart_full_id);
71
72 st->collection_modulo = rrdset_collection_modulo_init();
73
74 st->parts.id = string_strdupz(ctr->id);
75 st->parts.type = string_strdupz(ctr->type);
76 st->parts.name = string_strdupz(ctr->name);
77
78 st->family = (ctr->family && *ctr->family) ? rrd_string_strdupz(ctr->family) : rrd_string_strdupz(ctr->type);
79 st->context = (ctr->context && *ctr->context) ? rrd_string_strdupz(ctr->context) : rrd_string_strdupz(chart_full_id);
80
81 st->units = rrd_string_strdupz(ctr->units);
82 st->title = rrd_string_strdupz(ctr->title);
83 st->plugin_name = rrd_string_strdupz(ctr->plugin);
84 st->module_name = rrd_string_strdupz(ctr->module);
85 st->priority = ctr->priority;
86
87 st->db.entries = (ctr->memory_mode != RRD_DB_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;
88 st->update_every = ctr->update_every;
89 st->rrd_memory_mode = ctr->memory_mode;
90
91 st->chart_type = ctr->chart_type;
92 st->rrdhost = host;
93
94 rrdset_stream_send_chart_slot_assign(st);
95
96 spinlock_init(&st->data_collection_lock);
97
98 st->flags = RRDSET_FLAG_SYNC_CLOCK
99 | RRDSET_FLAG_INDEXED_ID
100 | RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED
101 | RRDSET_FLAG_SENDER_REPLICATION_FINISHED
102 ;
103
104 rw_spinlock_init(&st->alerts.spinlock);
105
106 // Initialize replication stuck detection counter
107 st->replication_empty_response_count = 0;
108
109 // initialize the db tiers
110 {
111 for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
112 STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
113 if(!eng) continue;
114
115 st->smg[tier] = storage_engine_metrics_group_get(eng->seb, host->db[tier].si, &st->chart_uuid);
116 }
117 }
118
119 rrddim_index_init(st);
120
121 st->rrdvars = rrdvariables_create();
122 st->rrdlabels = rrdlabels_create();
123 rrdset_update_permanent_labels(st);
124
125 st->green = NAN;
126 st->red = NAN;
127
128 rrdset_pluginsd_receive_slots_initialize(st);
129
130 rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
131 rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
132
133 ctr->react_action = RRDSET_REACT_NEW;
134
135 ml_chart_new(st);
136 }
137
138 // the destructor - the dictionary is write locked while this runs
139 static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost) {
140 RRDHOST *host = rrdhost; (void)host;
141 RRDSET *st = rrdset;
142
143 rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID);
144
145 rrdset_finalize_collection(st, false);
146
147 rrdset_stream_send_chart_slot_release(st);
148
149 dictionary_destroy(st->functions_view);
150
151 rrdcalc_unlink_and_delete_all_rrdset_alerts(st);
152
153 // ------------------------------------------------------------------------
154 // the order of destruction is important here
155
156 // 1. delete RRDVAR index after the above, to avoid triggering its garbage collector (they have references on this)
157 rrdvariables_destroy(st->rrdvars); // free all variables and destroy the rrdvar dictionary
158
159 // 2. delete RRDDIMs, now their variables are not existing, so this is fast
160 rrddim_index_destroy(st); // free all the dimensions and destroy the dimensions index
161
162 // 3. this has to be after the dimensions are freed, but before labels are freed (contexts need the labels)
163 rrdcontext_removed_rrdset(st); // let contexts know
164
165 // 4. destroy the chart labels
166 rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know
167
168 // 5. destroy the ml handle
169 ml_chart_delete(st);
170
171 // ------------------------------------------------------------------------
172 // free it
173
174 string_freez(st->id);
175 string_freez(st->name);
176 string_freez(st->parts.id);
177 string_freez(st->parts.type);
178 string_freez(st->parts.name);
179 string_freez(st->family);
180 string_freez(st->title);
181 string_freez(st->units);
182 string_freez(st->context);
183 string_freez(st->plugin_name);
184 string_freez(st->module_name);
185
186 freez(st->exporting_flags);
187
188 if(st->destroy_lock.locked)
189 spinlock_unlock(&st->destroy_lock);
190
191 memset(st, 0, sizeof(RRDSET));
192 }
193
194 // the item to be inserted, is already in the dictionary
195 // this callback deals with the situation, migrating the existing object to the new values
196 // the dictionary is write locked while this runs
197 static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *new_rrdset, void *constructor_data) {
198 (void)new_rrdset; // it is NULL
199
200 struct rrdset_constructor *ctr = constructor_data;
201 RRDSET *st = rrdset;
202
203 ctr->react_action = RRDSET_REACT_NONE;
204
205 if (unlikely(st->priority != ctr->priority)) {
206 st->priority = ctr->priority;
207 ctr->react_action |= RRDSET_REACT_UPDATED;
208 }
209
210 if (unlikely(st->update_every != ctr->update_every)) {
211 rrdset_set_update_every_s(st, ctr->update_every);
212 ctr->react_action |= RRDSET_REACT_UPDATED;
213 }
214
215 if(ctr->plugin && *ctr->plugin) {
216 STRING *old_plugin = st->plugin_name;
217 st->plugin_name = rrd_string_strdupz(ctr->plugin);
218 if (old_plugin != st->plugin_name)
219 ctr->react_action |= RRDSET_REACT_PLUGIN_UPDATED;
220 string_freez(old_plugin);
221 }
222
223 if(ctr->module && *ctr->module) {
224 STRING *old_module = st->module_name;
225 st->module_name = rrd_string_strdupz(ctr->module);
226 if (old_module != st->module_name)
227 ctr->react_action |= RRDSET_REACT_MODULE_UPDATED;
228 string_freez(old_module);
229 }
230
231 if(ctr->title && *ctr->title) {
232 STRING *old_title = st->title;
233 st->title = rrd_string_strdupz(ctr->title);
234 if(old_title != st->title)
235 ctr->react_action |= RRDSET_REACT_UPDATED;
236 string_freez(old_title);
237 }
238
239 if(ctr->units && *ctr->units) {
240 STRING *old_units = st->units;
241 st->units = rrd_string_strdupz(ctr->units);
242 if(old_units != st->units)
243 ctr->react_action |= RRDSET_REACT_UPDATED;
244 string_freez(old_units);
245 }
246
247 if(ctr->family && *ctr->family) {
248 STRING *old_family = st->family;
249 st->family = rrd_string_strdupz(ctr->family);
250 if(old_family != st->family)
251 ctr->react_action |= RRDSET_REACT_UPDATED;
252 string_freez(old_family);
253 }
254
255 if(ctr->context && *ctr->context) {
256 STRING *old_context = st->context;
257 st->context = rrd_string_strdupz(ctr->context);
258 if(old_context != st->context)
259 ctr->react_action |= RRDSET_REACT_UPDATED;
260 string_freez(old_context);
261 }
262
263 if(st->chart_type != ctr->chart_type) {
264 st->chart_type = ctr->chart_type;
265 ctr->react_action |= RRDSET_REACT_UPDATED;
266 }
267
268 rrdset_update_permanent_labels(st);
269
270 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
271 rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
272 rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
273
274 return ctr->react_action != RRDSET_REACT_NONE;
275 }
276
277 // this is called after all insertions/conflicts, with the dictionary unlocked, with a reference to RRDSET
278 // so, any actions requiring locks on other objects, should be placed here
279 static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) {
280 struct rrdset_constructor *ctr = constructor_data;
281 RRDSET *st = rrdset;
282 RRDHOST *host = st->rrdhost;
283
284 st->collector_tid = gettid_cached();
285 st->last_accessed_time_s = now_realtime_sec();
286
287 if(ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
288 if (ctr->react_action & RRDSET_REACT_NEW) {
289 if(unlikely(rrdcontext_find_chart_uuid(st, &st->chart_uuid)))
290 uuid_generate(st->chart_uuid);
291 }
292 rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
293 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_UPDATE);
294 }
295
296 rrdset_metadata_updated(st);
297 }
298
299 void rrdset_index_init(RRDHOST *host) {
300 if(!host->rrdset_root_index) {
301 host->rrdset_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
302 &dictionary_stats_category_rrdset, sizeof(RRDSET));
303
304 dictionary_register_insert_callback(host->rrdset_root_index, rrdset_insert_callback, NULL);
305 dictionary_register_conflict_callback(host->rrdset_root_index, rrdset_conflict_callback, NULL);
306 dictionary_register_react_callback(host->rrdset_root_index, rrdset_react_callback, NULL);
307 dictionary_register_delete_callback(host->rrdset_root_index, rrdset_delete_callback, host);
308 }
309
310 rrdset_index_byname_init(host);
311 }
312
313 void rrdset_index_destroy(RRDHOST *host) {
314 // destroy the name index first
315 dictionary_destroy(host->rrdset_root_index_name);
316 host->rrdset_root_index_name = NULL;
317
318 // destroy the id index last
319 dictionary_destroy(host->rrdset_root_index);
320 host->rrdset_root_index = NULL;
321 }
322
323 static inline RRDSET *rrdset_index_add(RRDHOST *host, const char *id, struct rrdset_constructor *st_ctr) {
324 return dictionary_set_advanced(host->rrdset_root_index, id, -1, NULL, sizeof(RRDSET), st_ctr);
325 }
326
327 static inline void rrdset_index_del(RRDHOST *host, RRDSET *st) {
328 if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID))
329 dictionary_del(host->rrdset_root_index, rrdset_id(st));
330 }
331
332 static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) {
333 // TODO - the name index should have an acquired dictionary item, not just a pointer to RRDSET
334 if (unlikely(!host->rrdset_root_index))
335 return NULL;
336 return dictionary_get(host->rrdset_root_index, id);
337 }
338
339 RRDSET *rrdset_find(RRDHOST *host, const char *id, bool include_obsolete) {
340 netdata_log_debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, rrdhost_hostname(host));
341 RRDSET *st = rrdset_index_find(host, id);
342
343 if(st) {
344 if(!include_obsolete && !rrdset_is_discoverable(st))
345 return NULL;
346
347 st->last_accessed_time_s = now_realtime_sec();
348 }
349
350 return(st);
351 }
352
353 RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id, bool include_obsolete) {
354 netdata_log_debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, rrdhost_hostname(host));
355
356 char buf[RRD_ID_LENGTH_MAX + 1];
357 strncpyz(buf, type, RRD_ID_LENGTH_MAX - 1);
358 strcat(buf, ".");
359 int len = (int) strlen(buf);
360 strncpyz(&buf[len], id, (size_t) (RRD_ID_LENGTH_MAX - len));
361
362 return rrdset_find(host, buf, include_obsolete);
363 }
364
365 RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id, bool include_obsolete) {
366 netdata_log_debug(D_RRD_CALLS, "rrdset_find_and_acquire() for host %s, chart %s", rrdhost_hostname(host), id);
367
368 RRDSET_ACQUIRED *sta = (RRDSET_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdset_root_index, id);
369 if(sta) {
370 RRDSET *st = dictionary_acquired_item_value((const DICTIONARY_ITEM *)sta);
371 if(st) {
372 if(!include_obsolete && !rrdset_is_discoverable(st)) {
373 dictionary_acquired_item_release(host->rrdset_root_index, (const DICTIONARY_ITEM *)sta);
374 return NULL;
375 }
376
377 st->last_accessed_time_s = now_realtime_sec();
378 }
379 }
380
381 return sta;
382 }
383
384 RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa) {
385 if(unlikely(!rsa))
386 return NULL;
387
388 return (RRDSET *) dictionary_acquired_item_value((const DICTIONARY_ITEM *)rsa);
389 }
390
391 void rrdset_acquired_release(RRDSET_ACQUIRED *rsa) {
392 if(unlikely(!rsa))
393 return;
394
395 RRDSET *rs = rrdset_acquired_to_rrdset(rsa);
396 dictionary_acquired_item_release(rs->rrdhost->rrdset_root_index, (const DICTIONARY_ITEM *)rsa);
397 }
398
399 RRDSET *rrdset_create_custom(
400 RRDHOST *host
401 , const char *type
402 , const char *id
403 , const char *name
404 , const char *family
405 , const char *context
406 , const char *title
407 , const char *units
408 , const char *plugin
409 , const char *module
410 , long priority
411 , int update_every
412 , RRDSET_TYPE chart_type
413 ,
414 RRD_DB_MODE memory_mode
415 , long history_entries
416 ) {
417 if(!type || !type[0])
418 fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
419 , (id && *id)?id:"<unset>"
420 , (name && *name)?name:"<unset>"
421 , (family && *family)?family:"<unset>"
422 , (context && *context)?context:"<unset>"
423 , (title && *title)?title:"<unset>"
424 , (units && *units)?units:"<unset>"
425 , (plugin && *plugin)?plugin:"<unset>"
426 , (module && *module)?module:"<unset>"
427 );
428
429 if(!id || !id[0])
430 fatal("Cannot create rrd stats without an id: type '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
431 , type
432 , (name && *name)?name:"<unset>"
433 , (family && *family)?family:"<unset>"
434 , (context && *context)?context:"<unset>"
435 , (title && *title)?title:"<unset>"
436 , (units && *units)?units:"<unset>"
437 , (plugin && *plugin)?plugin:"<unset>"
438 , (module && *module)?module:"<unset>"
439 );
440
441 // ------------------------------------------------------------------------
442 // check if it already exists
443
444 char chart_full_id[RRD_ID_LENGTH_MAX + 1];
445 snprintfz(chart_full_id, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
446
447 // ------------------------------------------------------------------------
448 // allocate it
449
450 netdata_log_debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
451
452 struct rrdset_constructor ctr;
453
454 RRDSET *st = NULL;
455 while(!st) {
456 st = rrdset_index_find(host, chart_full_id);
457 if(st) {
458 if(spinlock_trylock(&st->destroy_lock)) {
459 rrdset_isnot_obsolete___safe_from_collector_thread(st);
460 spinlock_unlock(&st->destroy_lock);
461 }
462 else {
463 #ifdef FSANITIZE_ADDRESS
464 fprintf(stderr, "rrdset_create_custom() - chart '%s' of host '%s' is being deleted but we need it. Retrying...\n",
465 chart_full_id, rrdhost_hostname(host));
466 #endif
467 st = NULL;
468 microsleep(1 * USEC_PER_MS);
469 continue;
470 }
471 }
472
473 ctr = (struct rrdset_constructor){
474 .host = host,
475 .type = type,
476 .id = id,
477 .name = name,
478 .family = family,
479 .context = context,
480 .title = title,
481 .units = units,
482 .plugin = plugin,
483 .module = module,
484 .priority = priority,
485 .update_every = update_every,
486 .chart_type = chart_type,
487 .memory_mode = memory_mode,
488 .history_entries = history_entries,
489 };
490
491 st = rrdset_index_add(host, chart_full_id, &ctr);
492 }
493
494 bool name_updated = false;
495 if(!st->name) {
496 st->name = rrdset_fix_name(host, chart_full_id, ctr.type, NULL, ctr.name);
497 if(!st->name)
498 st->name = rrdset_fix_name(host, chart_full_id, ctr.type, NULL, ctr.id);
499
500 if(st->name) {
501 name_updated = true;
502 rrdset_index_add_name(host, st);
503 }
504 }
505 else if(rrdset_reset_name(st, (name && *name) ? name : id) == 2)
506 name_updated = true;
507
508 if(name_updated) {
509 rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
510 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_UPDATE);
511 rrdset_metadata_updated(st);
512 }
513
514 return st;
515 }
516
517 void rrdset_free(RRDSET *st) {
518 if(unlikely(!st)) return;
519 rrdset_index_del_name(st->rrdhost, st);
520 rrdset_index_del(st->rrdhost, st);
521 }