master
h 578 lines 19.8 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #ifndef NETDATA_DICTIONARY_ITEM_H
4 #define NETDATA_DICTIONARY_ITEM_H
5
6 #include "dictionary-internals.h"
7
8 // ----------------------------------------------------------------------------
9 // ITEM initialization and updates
10
11 static inline size_t item_set_name(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *name, size_t name_len) {
12 if(likely(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE)) {
13 item->caller_name = (char *)name;
14 item->key_len = name_len;
15 }
16 else {
17 item->string_name = string_strdupz(name);
18 item->key_len = string_strlen(item->string_name);
19 item->options |= ITEM_OPTION_ALLOCATED_NAME;
20 }
21
22 return item->key_len;
23 }
24
25 static inline size_t item_free_name(DICTIONARY *dict, DICTIONARY_ITEM *item) {
26 if(likely(!(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE)))
27 string_freez(item->string_name);
28
29 return item->key_len;
30 }
31
32 static inline const char *item_get_name(const DICTIONARY_ITEM *item) {
33 if(item->options & ITEM_OPTION_ALLOCATED_NAME)
34 return string2str(item->string_name);
35 else
36 return item->caller_name;
37 }
38
39 static inline size_t item_get_name_len(const DICTIONARY_ITEM *item) {
40 if(item->options & ITEM_OPTION_ALLOCATED_NAME)
41 return string_strlen(item->string_name);
42 else
43 return strlen(item->caller_name);
44 }
45
46 // ----------------------------------------------------------------------------
47
48 static inline DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) {
49 DICTIONARY_ITEM *item;
50
51 size_t size = sizeof(DICTIONARY_ITEM);
52 item = aral_mallocz(dict_items_aral);
53 memset(item, 0, sizeof(DICTIONARY_ITEM));
54
55 #ifdef NETDATA_INTERNAL_CHECKS
56 item->creator_pid = gettid_cached();
57 #endif
58 #ifdef FSANITIZE_ADDRESS
59 // Initialize stacktrace tracking
60 stacktrace_array_init(&item->stacktraces);
61
62 // Add the first stack trace at creation time
63 stacktrace_array_add(&item->stacktraces, 1);
64 #endif
65
66 item->refcount = 1;
67 item->flags = ITEM_FLAG_BEING_CREATED;
68
69 *allocated_bytes += size;
70
71 if(master_item) {
72 item->shared = master_item->shared;
73
74 if(unlikely(__atomic_add_fetch(&item->shared->links, 1, __ATOMIC_ACQUIRE) <= 1))
75 fatal("DICTIONARY: attempted to link to a shared item structure that had zero references");
76 }
77 else {
78 size = sizeof(DICTIONARY_ITEM_SHARED);
79 item->shared = aral_mallocz(dict_shared_items_aral);
80 memset(item->shared, 0, sizeof(DICTIONARY_ITEM_SHARED));
81
82 item->shared->links = 1;
83 *allocated_bytes += size;
84 }
85
86 #if defined(FSANITIZE_ADDRESS) || defined(NETDATA_INTERNAL_CHECKS)
87 item->dict = dict;
88 #endif
89 return item;
90 }
91
92 static inline void *dict_item_value_mallocz(DICTIONARY *dict, size_t value_len) {
93 if(dict->value_aral) {
94 internal_fatal(aral_requested_element_size(dict->value_aral) != value_len,
95 "DICTIONARY: item value size %zu does not match the configured fixed one %zu",
96 value_len, aral_requested_element_size(dict->value_aral));
97 return aral_mallocz(dict->value_aral);
98 }
99 else
100 return mallocz(value_len);
101 }
102
103 static inline void dict_item_value_freez(DICTIONARY *dict, void *ptr) {
104 if(dict->value_aral)
105 aral_freez(dict->value_aral, ptr);
106 else
107 freez(ptr);
108 }
109
110 static inline void *dict_item_value_create(DICTIONARY *dict, void *value, size_t value_len) {
111 void *ptr = NULL;
112
113 if(likely(value_len)) {
114 if (likely(value)) {
115 // a value has been supplied
116 // copy it
117 ptr = dict_item_value_mallocz(dict, value_len);
118 memcpy(ptr, value, value_len);
119 }
120 else {
121 // no value has been supplied
122 // allocate a clear memory block
123 ptr = dict_item_value_mallocz(dict, value_len);
124 memset(ptr, 0, value_len);
125 }
126 }
127 // else
128 // the caller wants an item without any value
129
130 return ptr;
131 }
132
133 static inline DICTIONARY_ITEM *dict_item_create_with_hooks(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
134 #ifdef NETDATA_INTERNAL_CHECKS
135 if(unlikely(name_len > KEY_LEN_MAX))
136 fatal("DICTIONARY: tried to index a key of size %zu, but the maximum acceptable is %zu", name_len, (size_t)KEY_LEN_MAX);
137
138 if(unlikely(value_len > VALUE_LEN_MAX))
139 fatal("DICTIONARY: tried to add an item of size %zu, but the maximum acceptable is %zu", value_len, (size_t)VALUE_LEN_MAX);
140 #endif
141
142 size_t item_size = 0, key_size = 0, value_size = 0;
143
144 DICTIONARY_ITEM *item = dict_item_create(dict, &item_size, master_item);
145 key_size += item_set_name(dict, item, name, name_len);
146
147 if(unlikely(is_view_dictionary(dict))) {
148 // we are on a view dictionary
149 // do not touch the value
150 ;
151
152 #ifdef NETDATA_INTERNAL_CHECKS
153 if(unlikely(!master_item))
154 fatal("DICTIONARY: cannot add an item to a view without a master item.");
155 #endif
156 }
157 else {
158 // we are on the master dictionary
159
160 if(unlikely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))
161 item->shared->value = value;
162 else
163 item->shared->value = dict_item_value_create(dict, value, value_len);
164
165 item->shared->value_len = value_len;
166 value_size += value_len;
167
168 dictionary_execute_insert_callback(dict, item, constructor_data);
169 }
170
171 DICTIONARY_ENTRIES_PLUS1(dict);
172 DICTIONARY_STATS_PLUS_MEMORY(dict, key_size, item_size, value_size);
173
174 return item;
175 }
176
177 static inline void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item, void *value, size_t value_len, void *constructor_data) {
178 if(unlikely(is_view_dictionary(dict)))
179 fatal("DICTIONARY: %s() should never be called on views.", __FUNCTION__ );
180
181 netdata_log_debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", item_get_name(item));
182
183 DICTIONARY_VALUE_RESETS_PLUS1(dict);
184
185 if(item->shared->value_len != value_len) {
186 DICTIONARY_STATS_PLUS_MEMORY(dict, 0, 0, value_len);
187 DICTIONARY_STATS_MINUS_MEMORY(dict, 0, 0, item->shared->value_len);
188 }
189
190 dictionary_execute_delete_callback(dict, item);
191
192 if(likely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE)) {
193 netdata_log_debug(D_DICTIONARY, "Dictionary: linking value to '%s'", item_get_name(item));
194 item->shared->value = value;
195 item->shared->value_len = value_len;
196 }
197 else {
198 netdata_log_debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", item_get_name(item));
199
200 void *old_value = item->shared->value;
201 void *new_value = NULL;
202 if(value_len) {
203 new_value = dict_item_value_mallocz(dict, value_len);
204 if(value) memcpy(new_value, value, value_len);
205 else memset(new_value, 0, value_len);
206 }
207 item->shared->value = new_value;
208 item->shared->value_len = value_len;
209
210 netdata_log_debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", item_get_name(item));
211 dict_item_value_freez(dict, old_value);
212 }
213
214 dictionary_execute_insert_callback(dict, item, constructor_data);
215 }
216
217 static inline size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) {
218 netdata_log_debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", item_get_name(item));
219
220 if(!item_flag_check(item, ITEM_FLAG_DELETED))
221 DICTIONARY_ENTRIES_MINUS1(dict);
222
223 size_t item_size = 0, key_size = 0, value_size = 0;
224
225 key_size += item->key_len;
226
227 if(item_shared_release_and_check_if_it_can_be_freed(dict, item)) {
228 dictionary_execute_delete_callback(dict, item);
229
230 if(unlikely(!(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))) {
231 netdata_log_debug(D_DICTIONARY, "Dictionary freeing value of '%s'", item_get_name(item));
232 dict_item_value_freez(dict, item->shared->value);
233 item->shared->value = NULL;
234 }
235 value_size += item->shared->value_len;
236
237 aral_freez(dict_shared_items_aral, item->shared);
238 item->shared = NULL;
239 item_size += sizeof(DICTIONARY_ITEM_SHARED);
240 }
241
242 // free the name after calling the delete callback
243 if(unlikely(!(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE)))
244 item_free_name(dict, item);
245
246 aral_freez(dict_items_aral, item);
247
248 item_size += sizeof(DICTIONARY_ITEM);
249
250 DICTIONARY_STATS_MINUS_MEMORY(dict, key_size, item_size, value_size);
251
252 // we return the memory we actually freed
253 return item_size + ((dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE) ? 0 : value_size);
254 }
255
256 // ----------------------------------------------------------------------------
257 // linked list management
258
259 static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) {
260 ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
261
262 if(dict->options & DICT_OPTION_ADD_IN_FRONT)
263 DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(dict->items.list, item, prev, next);
264 else
265 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(dict->items.list, item, prev, next);
266
267 #ifdef NETDATA_INTERNAL_CHECKS
268 item->ll_adder_pid = gettid_cached();
269 #endif
270
271 // clear the BEING created flag,
272 // after it has been inserted into the linked list
273 item_flag_clear(item, ITEM_FLAG_BEING_CREATED);
274
275 garbage_collect_pending_deletes(dict);
276 ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
277 }
278
279 static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item) {
280 ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
281
282 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next);
283
284 #ifdef NETDATA_INTERNAL_CHECKS
285 item->ll_remover_pid = gettid_cached();
286 #endif
287
288 garbage_collect_pending_deletes(dict);
289 ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
290 }
291
292 // ----------------------------------------------------------------------------
293 // item operations
294
295 static inline void dict_item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
296 if(is_master_dictionary(dict)) {
297 item_shared_flag_set(item, ITEM_FLAG_DELETED);
298
299 if(dict->hooks)
300 __atomic_store_n(&dict->hooks->last_master_deletion_us, now_realtime_usec(), __ATOMIC_RELAXED);
301 }
302 }
303
304 // returns true if we set the deleted flag on this item
305 static inline bool dict_item_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
306 ITEM_FLAGS expected, desired;
307
308 expected = __atomic_load_n(&item->flags, __ATOMIC_RELAXED);
309
310 do {
311
312 if (expected & ITEM_FLAG_DELETED)
313 return false;
314
315 desired = expected | ITEM_FLAG_DELETED;
316
317 } while(!__atomic_compare_exchange_n(&item->flags, &expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
318
319 DICTIONARY_ENTRIES_MINUS1(dict);
320 return true;
321 }
322
323 static inline void dict_item_free_or_mark_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
324 int rc = item_is_not_referenced_and_can_be_removed_advanced(dict, item);
325 switch(rc) {
326 case RC_ITEM_OK:
327 // the item is ours, refcount set to -100
328 dict_item_shared_set_deleted(dict, item);
329 item_linked_list_remove(dict, item);
330 dict_item_free_with_hooks(dict, item);
331 break;
332
333 case RC_ITEM_IS_REFERENCED:
334 case RC_ITEM_IS_CURRENTLY_BEING_CREATED:
335 // the item is currently referenced by others
336 dict_item_shared_set_deleted(dict, item);
337 dict_item_set_deleted(dict, item);
338 // after this point do not touch the item
339 break;
340
341 case RC_ITEM_IS_CURRENTLY_BEING_DELETED:
342 // an item that is currently being deleted by someone else - don't touch it
343 break;
344
345 default:
346 internal_error(true, "Hey dev! You forgot to add the new condition here!");
347 break;
348 }
349 }
350
351 // this is used by traversal functions to remove the current item
352 // if it is deleted, and it has zero references. This will eliminate
353 // the need for the garbage collector to kick-in later.
354 // Most deletions happen during traversal, so this is a nice hack
355 // to speed up everything!
356 static inline void dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(DICTIONARY *dict, DICTIONARY_ITEM *item, char rw) {
357 if(rw == DICTIONARY_LOCK_WRITE) {
358 bool should_be_deleted = item_flag_check(item, ITEM_FLAG_DELETED);
359
360 item_release(dict, item);
361
362 if(should_be_deleted && item_is_not_referenced_and_can_be_removed(dict, item)) {
363 // this has to be before removing from the linked list,
364 // otherwise the garbage collector will also kick in!
365 DICTIONARY_PENDING_DELETES_MINUS1(dict);
366
367 item_linked_list_remove(dict, item);
368 dict_item_free_with_hooks(dict, item);
369 }
370 }
371 else {
372 // we can't do anything under this mode
373 item_release(dict, item);
374 }
375 }
376
377 static inline bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len) {
378 if(name_len == -1)
379 name_len = (ssize_t)strlen(name);
380
381 netdata_log_debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name);
382
383 // Unfortunately, the JudyHSDel() does not return the value of the
384 // item that was deleted, so we have to find it before we delete it,
385 // since we need to release our structures too.
386
387 dictionary_index_lock_wrlock(dict);
388
389 if(unlikely(is_dictionary_destroyed(dict))) {
390 dictionary_index_wrlock_unlock(dict);
391 return false;
392 }
393
394 int ret;
395 DICTIONARY_ITEM *item = hashtable_get_unsafe(dict, name, name_len);
396 if(unlikely(!item)) {
397 dictionary_index_wrlock_unlock(dict);
398 ret = false;
399 }
400 else {
401 if(hashtable_delete_unsafe(dict, name, name_len, item) == 0)
402 netdata_log_error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s', "
403 "name_len %zd that is not in the index",
404 name, name_len);
405 else
406 pointer_del(dict, item);
407
408 dictionary_index_wrlock_unlock(dict);
409
410 dict_item_free_or_mark_deleted(dict, item);
411 ret = true;
412 }
413
414 return ret;
415 }
416
417 static inline DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
418 if(unlikely(!name || !*name)) {
419 dictionary_internal_error(true, dict,
420 "DICTIONARY: attempted to %s() without a name on a dictionary.",
421 __FUNCTION__);
422 return NULL;
423 }
424
425 if(unlikely(is_dictionary_destroyed(dict))) {
426 internal_error(true, "DICTIONARY: attempted to dictionary_set() on a destroyed dictionary");
427 return NULL;
428 }
429
430 if(name_len == -1)
431 name_len = (ssize_t)strlen(name);
432
433 netdata_log_debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name);
434
435 // DISCUSSION:
436 // Is it better to gain a read-lock and do a hashtable_get_unsafe()
437 // before we write lock to do hashtable_insert_unsafe()?
438 //
439 // Probably this depends on the use case.
440 // For statsd for example that does dictionary_set() to update received values,
441 // it could be beneficial to do a get() before we insert().
442 //
443 // But the caller has the option to do this on his/her own.
444 // So, let's do the fastest here and let the caller decide the flow of calls.
445
446 dictionary_index_lock_wrlock(dict);
447
448 // Re-check under the index lock. This synchronizes with
449 // dictionary_destroy(), which sets the destroyed flag and then takes
450 // this lock before tearing down the index.
451 if(unlikely(is_dictionary_destroyed(dict))) {
452 dictionary_index_wrlock_unlock(dict);
453 return NULL;
454 }
455
456 bool added_or_updated = false;
457 size_t spins = 0;
458 DICTIONARY_ITEM *item = NULL;
459 do {
460 void *handle = hashtable_insert_unsafe(dict, name, name_len);
461 item = hashtable_insert_handle_to_item_unsafe(dict, handle);
462 if (likely(item == NULL)) {
463 // a new item added to the index
464
465 // create the dictionary item
466 item = dict_item_create_with_hooks(dict, name, name_len, value, value_len, constructor_data, master_item);
467
468 pointer_add(dict, item);
469
470 hashtable_set_item_unsafe(dict, handle, item);
471
472 // unlock the index lock, before we add it to the linked list
473 // DON'T DO IT THE OTHER WAY AROUND - DO NOT CROSS THE LOCKS!
474 dictionary_index_wrlock_unlock(dict);
475
476 item_linked_list_add(dict, item);
477
478 added_or_updated = true;
479 }
480 else {
481 pointer_check(dict, item);
482
483 if(item_check_and_acquire_advanced(dict, item, true) != RC_ITEM_OK) {
484 spins++;
485 item = NULL;
486 continue;
487 }
488
489 // the item is already in the index
490 // so, either we will return the old one
491 // or overwrite the value, depending on dictionary flags
492
493 // We should not compare the values here!
494 // even if they are the same, we have to do the whole job
495 // so that the callbacks will be called.
496
497 if(is_view_dictionary(dict)) {
498 // view dictionary
499 // the item is already there and can be used
500 if(item->shared != master_item->shared)
501 netdata_log_error("DICTIONARY: changing the master item on a view is not supported. The previous item will remain. To change the key of an item in a view, delete it and add it again.");
502 }
503 else {
504 // master dictionary
505 // the user wants to reset its value
506
507 if (!(dict->options & DICT_OPTION_DONT_OVERWRITE_VALUE)) {
508 dict_item_reset_value_with_hooks(dict, item, value, value_len, constructor_data);
509 added_or_updated = true;
510 }
511
512 else if (dictionary_execute_conflict_callback(dict, item, value, constructor_data)) {
513 dictionary_version_increment(dict);
514 added_or_updated = true;
515 }
516
517 else {
518 // conflict callback returned false
519 // we did really nothing!
520 ;
521 }
522 }
523
524 dictionary_index_wrlock_unlock(dict);
525 }
526 } while(!item);
527
528
529 if(unlikely(spins > 0))
530 DICTIONARY_STATS_INSERT_SPINS_PLUS(dict, spins);
531
532 if(is_master_dictionary(dict) && added_or_updated)
533 dictionary_execute_react_callback(dict, item, constructor_data);
534
535 return item;
536 }
537
538 static inline DICTIONARY_ITEM *dict_item_find_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len) {
539 if(unlikely(!name || !*name)) {
540 dictionary_internal_error(true, dict,
541 "DICTIONARY: attempted to %s() without a name on a dictionary.",
542 __FUNCTION__);
543 return NULL;
544 }
545
546 if(unlikely(is_dictionary_destroyed(dict))) {
547 internal_error(true, "DICTIONARY: attempted to dictionary_get() on a destroyed dictionary");
548 return NULL;
549 }
550
551 if(name_len == -1)
552 name_len = (ssize_t)strlen(name);
553
554 netdata_log_debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name);
555
556 dictionary_index_lock_rdlock(dict);
557
558 // Re-check under the index lock. This synchronizes with
559 // dictionary_destroy(), which sets the destroyed flag and then takes
560 // this lock before tearing down the index.
561 if(unlikely(is_dictionary_destroyed(dict))) {
562 dictionary_index_rdlock_unlock(dict);
563 return NULL;
564 }
565
566 DICTIONARY_ITEM *item = hashtable_get_unsafe(dict, name, name_len);
567 if(unlikely(item && !item_check_and_acquire(dict, item))) {
568 item = NULL;
569 DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(dict);
570 }
571
572 dictionary_index_rdlock_unlock(dict);
573
574 return item;
575 }
576
577
578 #endif //NETDATA_DICTIONARY_ITEM_H