Dictionary with JudyHS and double linked list (#13032)
* dictionary internals isolation * more dictionary cleanups * added unit test * we should use DICT internally * disable cups in cmake * implement DICTIONARY with Judy arrays * operational JUDY implementation * JUDY cleanup * JUDY summary added * JudyHS implementation with double linked list * test negative searches too * optimize destruction * optimize set to insert first without lookup * updated stats * code cleanup; better organization; updated info * more code cleanup and commenting * more cleanup, renames and comments * fix rename * more cleanups * use Judy.h from system paths * added foreach traversal; added flag to add item in front; isolated locks to their own functions; destruction returns the number of bytes freed * more comments; flags are now 16-bit * completed unittesting * addressed comments and added reference counters maintainance * added unittest in main; tested removal of items in front, back and middle * added read/write walkthrough and foreach; allowed walkthrough and foreach in write mode to delete the current element (used by cups.plugin); referenced counters removed from the API * DICTFE.name should be const too * added API calls for exposing all statistics * dictionary flags as enum and reference counters as atomic operations * more comments; improved error handling at unit tests * added functions to allow unsafe access while traversing the dictionary with locks in place * check for libcups in cmake * added delete callback; implemented statsd with this dictionary * added missing dfe_done() * added alternative implementation with AVL * added documentation * added comments and warning about AVL * dictionary walktrhough on new code * simplified foreach; updated docs * updated docs * AVL is much faster without hashes * AVL should follow DBENGINE
Costa Tsaousis committed
Jun 1, 2022 at 20:01 UTC
7784a16cc7af8260bb8877873a60d7dc6d2c9e73
21 files changed
+1772
-463
CMakeLists.txt
+33
@@ -230,6 +230,15 @@ pkg_check_modules(CURL libcurl)
230
# ${CURL_CFLAGS_OTHER}
231
# ${CURL_INCLUDE_DIRS}
232
233
+# -----------------------------------------------------------------------------
234
+# Detect libcups
235
+
236
+pkg_check_modules(CURL libcups)
237
+# later we use:
238
+# ${CUPS_LIBRARIES}
239
+# ${CUPS_CFLAGS_OTHER}
240
+# ${CUPS_INCLUDE_DIRS}
241
+
242
# -----------------------------------------------------------------------------
243
# Detect libaws-c-common
244
@@ -491,6 +500,10 @@ set(SLABINFO_PLUGIN_FILES
500
collectors/slabinfo.plugin/slabinfo.c
501
)
502
503
+set(CUPS_PLUGIN_FILES
504
+ collectors/cups.plugin/cups_plugin.c
505
+ )
506
+
507
set(EBPF_PROCESS_PLUGIN_FILES
508
collectors/ebpf.plugin/ebpf.c
509
collectors/ebpf.plugin/ebpf.h
@@ -1229,6 +1242,12 @@ ELSE()
1242
SET(ENABLE_PLUGIN_XENSTAT False)
1243
ENDIF()
1244
1245
+IF(CUPS_LIBRARIES)
1246
+ SET(ENABLE_PLUGIN_CUPS True)
1247
+ELSE()
1248
+ SET(ENABLE_PLUGIN_CUPS False)
1249
+ENDIF()
1250
+
1251
1252
# -----------------------------------------------------------------------------
1253
# netdatacli
@@ -1337,6 +1356,20 @@ ELSE()
1356
ENDIF()
1357
1358
1359
+# -----------------------------------------------------------------------------
1360
+# cups.plugin
1361
+
1362
+IF(ENABLE_PLUGIN_CUPS)
1363
+ message(STATUS "cups.plugin: enabled")
1364
+ add_executable(cups.plugin config.h ${CUPS_PLUGIN_FILES})
1365
+ target_link_libraries (cups.plugin libnetdata ${NETDATA_COMMON_LIBRARIES} ${CUPS_LIBRARIES})
1366
+ target_include_directories(cups.plugin PUBLIC ${NETDATA_COMMON_INCLUDE_DIRS} ${CUPS_INCLUDE_DIRS})
1367
+ target_compile_options(cups.plugin PUBLIC ${NETDATA_COMMON_CFLAGS} ${CUPS_CFLAGS_OTHER})
1368
+ELSE()
1369
+ message(STATUS "cups.plugin: disabled")
1370
+ENDIF()
1371
+
1372
+
1373
# -----------------------------------------------------------------------------
1374
# cgroup-network
1375
collectors/cups.plugin/cups_plugin.c
+8
-7
@@ -137,7 +137,8 @@ getIntegerOption(
137
return ((int)intvalue);
138
}
139
140
-int reset_job_metrics(void *entry, void *data) {
140
+static int reset_job_metrics(const char *name, void *entry, void *data) {
141
+ (void)name;
142
(void)data;
143
144
struct job_metrics *jm = (struct job_metrics *)entry;
@@ -158,7 +159,7 @@ struct job_metrics *get_job_metrics(char *dest) {
159
160
if (unlikely(!jm)) {
161
struct job_metrics new_job_metrics;
161
- reset_job_metrics(&new_job_metrics, NULL);
162
+ reset_job_metrics(NULL, &new_job_metrics, NULL);
163
jm = dictionary_set(dict_dest_job_metrics, dest, &new_job_metrics, sizeof(struct job_metrics));
164
165
printf("CHART cups.job_num_%s '' 'Active job number of destination %s' jobs '%s' cups.job_num stacked %i %i\n", dest, dest, dest, netdata_priority++, netdata_update_every);
@@ -174,7 +175,7 @@ struct job_metrics *get_job_metrics(char *dest) {
175
return jm;
176
}
177
177
-int collect_job_metrics(char *name, void *entry, void *data) {
178
+int collect_job_metrics(const char *name, void *entry, void *data) {
179
(void)data;
180
181
struct job_metrics *jm = (struct job_metrics *)entry;
@@ -204,7 +205,7 @@ int collect_job_metrics(char *name, void *entry, void *data) {
205
printf("DIMENSION pending '' absolute 1 1\n");
206
printf("DIMENSION held '' absolute 1 1\n");
207
printf("DIMENSION processing '' absolute 1 1\n");
207
- dictionary_del(dict_dest_job_metrics, name);
208
+ dictionary_del_having_write_lock(dict_dest_job_metrics, name);
209
}
210
211
return 0;
@@ -219,8 +220,8 @@ void reset_metrics() {
220
num_dest_printing = 0;
221
num_dest_stopped = 0;
222
222
- reset_job_metrics(&global_job_metrics, NULL);
223
- dictionary_get_all(dict_dest_job_metrics, reset_job_metrics, NULL);
223
+ reset_job_metrics(NULL, &global_job_metrics, NULL);
224
+ dictionary_walkthrough_write(dict_dest_job_metrics, reset_job_metrics, NULL);
225
}
226
227
int main(int argc, char **argv) {
@@ -370,7 +371,7 @@ int main(int argc, char **argv) {
371
}
372
cupsFreeJobs(num_jobs, jobs);
373
373
- dictionary_get_all_name_value(dict_dest_job_metrics, collect_job_metrics, NULL);
374
+ dictionary_walkthrough_write(dict_dest_job_metrics, collect_job_metrics, NULL);
375
376
static int cups_printer_by_option_created = 0;
377
if (unlikely(!cups_printer_by_option_created))
collectors/diskspace.plugin/plugin_diskspace.c
+3
-2
@@ -52,7 +52,8 @@ static DICTIONARY *dict_mountpoints = NULL;
52
53
#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
54
55
-int mount_point_cleanup(void *entry, void *data) {
55
+int mount_point_cleanup(const char *name, void *entry, void *data) {
56
+ (void)name;
57
(void)data;
58
59
struct mount_point_metadata *mp = (struct mount_point_metadata *)entry;
@@ -439,7 +440,7 @@ void *diskspace_main(void *ptr) {
440
441
if(dict_mountpoints) {
442
worker_is_busy(WORKER_JOB_CLEANUP);
442
- dictionary_get_all(dict_mountpoints, mount_point_cleanup, NULL);
443
+ dictionary_walkthrough_read(dict_mountpoints, mount_point_cleanup, NULL);
444
}
445
446
}
collectors/proc.plugin/proc_spl_kstat_zfs.c
+2
-2
@@ -252,7 +252,7 @@ void disable_zfs_pool_state(struct zfs_pool *pool)
252
pool->disabled = 1;
253
}
254
255
-int update_zfs_pool_state_chart(char *name, void *pool_p, void *update_every_p)
255
+int update_zfs_pool_state_chart(const char *name, void *pool_p, void *update_every_p)
256
{
257
struct zfs_pool *pool = (struct zfs_pool *)pool_p;
258
int update_every = *(int *)update_every_p;
@@ -408,7 +408,7 @@ int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt)
408
}
409
410
if (do_zfs_pool_state)
411
- dictionary_get_all_name_value(zfs_pools, update_zfs_pool_state_chart, &update_every);
411
+ dictionary_walkthrough_read(zfs_pools, update_zfs_pool_state_chart, &update_every);
412
413
while (deleted_zfs_pools) {
414
struct deleted_zfs_pool *current_pool = deleted_zfs_pools;
collectors/proc.plugin/sys_block_zram.c
+7
-7
@@ -165,7 +165,7 @@ static int init_devices(DICTIONARY *devices, unsigned int zram_id, int update_ev
165
return count;
166
}
167
168
-static void free_device(DICTIONARY *dict, char *name)
168
+static void free_device(DICTIONARY *dict, const char *name)
169
{
170
ZRAM_DEVICE *d = (ZRAM_DEVICE*)dictionary_get(dict, name);
171
info("ZRAM : Disabling monitoring of device %s", name);
@@ -173,7 +173,7 @@ static void free_device(DICTIONARY *dict, char *name)
173
rrdset_obsolete_and_pointer_null(d->st_savings);
174
rrdset_obsolete_and_pointer_null(d->st_alloc_efficiency);
175
rrdset_obsolete_and_pointer_null(d->st_comp_ratio);
176
- dictionary_del(dict, name);
176
+ dictionary_del_having_write_lock(dict, name);
177
}
178
// --------------------------------------------------------------------
179
@@ -200,7 +200,7 @@ static inline int read_mm_stat(procfile *ff, MM_STAT *stats) {
200
return 0;
201
}
202
203
-static inline int _collect_zram_metrics(char* name, ZRAM_DEVICE *d, int advance, DICTIONARY* dict) {
203
+static inline int _collect_zram_metrics(const char* name, ZRAM_DEVICE *d, int advance, DICTIONARY* dict) {
204
MM_STAT mm;
205
int value;
206
if (unlikely(read_mm_stat(d->file, &mm) < 0))
@@ -235,12 +235,12 @@ static inline int _collect_zram_metrics(char* name, ZRAM_DEVICE *d, int advance,
235
return 0;
236
}
237
238
-static int collect_first_zram_metrics(char *name, void *entry, void *data) {
238
+static int collect_first_zram_metrics(const char *name, void *entry, void *data) {
239
// collect without calling rrdset_next (init only)
240
return _collect_zram_metrics(name, (ZRAM_DEVICE *)entry, 0, (DICTIONARY *)data);
241
}
242
243
-static int collect_zram_metrics(char *name, void *entry, void *data) {
243
+static int collect_zram_metrics(const char *name, void *entry, void *data) {
244
(void)name;
245
// collect with calling rrdset_next
246
return _collect_zram_metrics(name, (ZRAM_DEVICE *)entry, 1, (DICTIONARY *)data);
@@ -280,13 +280,13 @@ int do_sys_block_zram(int update_every, usec_t dt) {
280
device_count = init_devices(devices, (unsigned int)zram_id, update_every);
281
if (device_count < 1)
282
return 1;
283
- dictionary_get_all_name_value(devices, collect_first_zram_metrics, devices);
283
+ dictionary_walkthrough_write(devices, collect_first_zram_metrics, devices);
284
}
285
else
286
{
287
if (unlikely(device_count < 1))
288
return 1;
289
- dictionary_get_all_name_value(devices, collect_zram_metrics, devices);
289
+ dictionary_walkthrough_write(devices, collect_zram_metrics, devices);
290
}
291
return 0;
292
}
\ No newline at end of file
collectors/statsd.plugin/statsd.c
+163
-171
@@ -18,35 +18,15 @@
18
#error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least 4
19
#endif
20
21
-#define STATSD_MAX_UNITS_LENGTH 20
22
-#define STATSD_MAX_DIMNAME_LENGTH 20
23
-#define STATSD_MAX_FAMILY_LENGTH 20
24
-
21
// --------------------------------------------------------------------------------------
22
23
// #define STATSD_MULTITHREADED 1
24
25
#ifdef STATSD_MULTITHREADED
26
// DO NOT ENABLE MULTITHREADING - IT IS NOT WELL TESTED
31
-#define STATSD_AVL_TREE avl_tree_lock
32
-#define STATSD_AVL_INSERT avl_insert_lock
33
-#define STATSD_AVL_SEARCH avl_search_lock
34
-#define STATSD_AVL_INDEX_INIT { .avl_tree = { NULL, statsd_metric_compare }, .rwlock = AVL_LOCK_INITIALIZER }
35
-#define STATSD_FIRST_PTR_MUTEX netdata_mutex_t first_mutex
36
-#define STATSD_FIRST_PTR_MUTEX_INIT .first_mutex = NETDATA_MUTEX_INITIALIZER
37
-#define STATSD_FIRST_PTR_MUTEX_LOCK(index) netdata_mutex_lock(&((index)->first_mutex))
38
-#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index) netdata_mutex_unlock(&((index)->first_mutex))
39
-#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_NONE
27
+#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DONT_OVERWRITE_VALUE|DICTIONARY_FLAG_ADD_IN_FRONT
28
#else
41
-#define STATSD_AVL_TREE avl_tree_type
42
-#define STATSD_AVL_INSERT avl_insert
43
-#define STATSD_AVL_SEARCH avl_search
44
-#define STATSD_AVL_INDEX_INIT { .root = NULL, .compar = statsd_metric_compare }
45
-#define STATSD_FIRST_PTR_MUTEX
46
-#define STATSD_FIRST_PTR_MUTEX_INIT
47
-#define STATSD_FIRST_PTR_MUTEX_LOCK(index)
48
-#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index)
49
-#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_SINGLE_THREADED
29
+#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DONT_OVERWRITE_VALUE|DICTIONARY_FLAG_ADD_IN_FRONT|DICTIONARY_FLAG_SINGLE_THREADED
30
#endif
31
32
#define STATSD_DECIMAL_DETAIL 1000 // floating point values get multiplied by this, with the same divisor
@@ -96,21 +76,14 @@ typedef struct statsd_metric_set {
76
size_t unique;
77
} STATSD_METRIC_SET;
78
99
-#define STATSD_METRIC_DICTIONARY_FLAGS_DICTFULL_LOGGED 0x000001
100
-
79
typedef struct statsd_metric_dictionary_item {
102
- char *name;
80
size_t count;
81
RRDDIM *rd;
105
- struct statsd_metric_dictionary_item *next;
82
} STATSD_METRIC_DICTIONARY_ITEM;
83
84
typedef struct statsd_metric_dictionary {
109
- STATSD_METRIC_DICTIONARY_ITEM *other;
85
DICTIONARY *dict;
86
size_t unique;
112
- uint32_t flags;
113
- STATSD_METRIC_DICTIONARY_ITEM *base;
87
} STATSD_METRIC_DICTIONARY;
88
89
@@ -127,6 +100,7 @@ typedef enum statsd_metric_options {
100
STATSD_METRIC_OPTION_USED_IN_APPS = 0x00000020, // set when this metric is used in apps
101
STATSD_METRIC_OPTION_CHECKED = 0x00000040, // set when the charting thread checks this metric for use in charts (its usefulness)
102
STATSD_METRIC_OPTION_USEFUL = 0x00000080, // set when the charting thread finds the metric useful (i.e. used in a chart)
103
+ STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED = 0x00000100, // set when the collection is full for this metric
104
} STATS_METRIC_OPTIONS;
105
106
typedef enum statsd_metric_type {
@@ -141,9 +115,7 @@ typedef enum statsd_metric_type {
115
116
117
typedef struct statsd_metric {
144
- avl_t avl; // indexing - has to be first
145
-
146
- const char *name; // the name of the metric
118
+ const char *name; // the name of the metric - linked to dictionary name
119
uint32_t hash; // hash of the name
120
121
STATSD_METRIC_TYPE type;
@@ -161,9 +133,9 @@ typedef struct statsd_metric {
133
STATSD_METRIC_DICTIONARY dictionary;
134
};
135
164
- char units[STATSD_MAX_UNITS_LENGTH+1];
165
- char dimname[STATSD_MAX_DIMNAME_LENGTH+1];
166
- char family[STATSD_MAX_FAMILY_LENGTH+1];
136
+ char *units;
137
+ char *dimname;
138
+ char *family;
139
140
// chart related members
141
STATS_METRIC_OPTIONS options; // STATSD_METRIC_OPTION_* (bitfield)
@@ -174,7 +146,6 @@ typedef struct statsd_metric {
146
RRDDIM *rd_count; // the dimension for the number of events received
147
148
// linking, used for walking through all metrics
177
- struct statsd_metric *next;
149
struct statsd_metric *next_useful;
150
} STATSD_METRIC;
151
@@ -188,17 +159,14 @@ typedef struct statsd_index {
159
size_t metrics; // the number of metrics in this index
160
size_t useful; // the number of useful metrics in this index
161
191
- STATSD_AVL_TREE index; // the AVL tree
162
+ STATSD_METRIC_TYPE type; // the type of index
163
+ DICTIONARY *dict;
164
193
- STATSD_METRIC *first; // the linked list of metrics (new metrics are added in front)
165
STATSD_METRIC *first_useful; // the linked list of useful metrics (new metrics are added in front)
195
- STATSD_FIRST_PTR_MUTEX; // when multi-threading is enabled, a lock to protect the linked list
166
167
STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index
168
} STATSD_INDEX;
169
200
-static int statsd_metric_compare(void* a, void* b);
201
-
170
// --------------------------------------------------------------------------------------------------------------------
171
// synthetic charts
172
@@ -332,64 +300,57 @@ static struct statsd {
300
.name = "gauge",
301
.events = 0,
302
.metrics = 0,
335
- .index = STATSD_AVL_INDEX_INIT,
336
- .default_options = STATSD_METRIC_OPTION_NONE,
337
- .first = NULL,
338
- STATSD_FIRST_PTR_MUTEX_INIT
303
+ .dict = NULL,
304
+ .type = STATSD_METRIC_TYPE_GAUGE,
305
+ .default_options = STATSD_METRIC_OPTION_NONE
306
},
307
.counters = {
308
.name = "counter",
309
.events = 0,
310
.metrics = 0,
344
- .index = STATSD_AVL_INDEX_INIT,
345
- .default_options = STATSD_METRIC_OPTION_NONE,
346
- .first = NULL,
347
- STATSD_FIRST_PTR_MUTEX_INIT
311
+ .dict = NULL,
312
+ .type = STATSD_METRIC_TYPE_COUNTER,
313
+ .default_options = STATSD_METRIC_OPTION_NONE
314
},
315
.timers = {
316
.name = "timer",
317
.events = 0,
318
.metrics = 0,
353
- .index = STATSD_AVL_INDEX_INIT,
354
- .default_options = STATSD_METRIC_OPTION_NONE,
355
- .first = NULL,
356
- STATSD_FIRST_PTR_MUTEX_INIT
319
+ .dict = NULL,
320
+ .type = STATSD_METRIC_TYPE_TIMER,
321
+ .default_options = STATSD_METRIC_OPTION_NONE
322
},
323
.histograms = {
324
.name = "histogram",
325
.events = 0,
326
.metrics = 0,
362
- .index = STATSD_AVL_INDEX_INIT,
363
- .default_options = STATSD_METRIC_OPTION_NONE,
364
- .first = NULL,
365
- STATSD_FIRST_PTR_MUTEX_INIT
327
+ .dict = NULL,
328
+ .type = STATSD_METRIC_TYPE_HISTOGRAM,
329
+ .default_options = STATSD_METRIC_OPTION_NONE
330
},
331
.meters = {
332
.name = "meter",
333
.events = 0,
334
.metrics = 0,
371
- .index = STATSD_AVL_INDEX_INIT,
372
- .default_options = STATSD_METRIC_OPTION_NONE,
373
- .first = NULL,
374
- STATSD_FIRST_PTR_MUTEX_INIT
335
+ .dict = NULL,
336
+ .type = STATSD_METRIC_TYPE_METER,
337
+ .default_options = STATSD_METRIC_OPTION_NONE
338
},
339
.sets = {
340
.name = "set",
341
.events = 0,
342
.metrics = 0,
380
- .index = STATSD_AVL_INDEX_INIT,
381
- .default_options = STATSD_METRIC_OPTION_NONE,
382
- .first = NULL,
383
- STATSD_FIRST_PTR_MUTEX_INIT
343
+ .dict = NULL,
344
+ .type = STATSD_METRIC_TYPE_SET,
345
+ .default_options = STATSD_METRIC_OPTION_NONE
346
},
347
.dictionaries = {
348
.name = "dictionary",
349
.events = 0,
350
.metrics = 0,
389
- .index = STATSD_AVL_INDEX_INIT,
390
- .default_options = STATSD_METRIC_OPTION_NONE,
391
- .first = NULL,
392
- STATSD_FIRST_PTR_MUTEX_INIT
351
+ .dict = NULL,
352
+ .type = STATSD_METRIC_TYPE_DICTIONARY,
353
+ .default_options = STATSD_METRIC_OPTION_NONE
354
},
355
356
.tcp_idle_timeout = 600,
@@ -413,54 +374,54 @@ static struct statsd {
374
// --------------------------------------------------------------------------------------------------------------------
375
// statsd index management - add/find metrics
376
416
-static int statsd_metric_compare(void* a, void* b) {
417
- if(((STATSD_METRIC *)a)->hash < ((STATSD_METRIC *)b)->hash) return -1;
418
- else if(((STATSD_METRIC *)a)->hash > ((STATSD_METRIC *)b)->hash) return 1;
419
- else return strcmp(((STATSD_METRIC *)a)->name, ((STATSD_METRIC *)b)->name);
420
-}
377
+static void dictionary_metric_insert_callback(const char *name, void *value, void *data) {
378
+ STATSD_INDEX *index = (STATSD_INDEX *)data;
379
+ STATSD_METRIC *m = (STATSD_METRIC *)value;
380
+
381
+ debug(D_STATSD, "Creating new %s metric '%s'", index->name, name);
382
422
-static inline STATSD_METRIC *statsd_metric_index_find(STATSD_INDEX *index, const char *name, uint32_t hash) {
423
- STATSD_METRIC tmp;
424
- tmp.name = name;
425
- tmp.hash = (hash)?hash:simple_hash(tmp.name);
383
+ m->name = name;
384
+ m->hash = simple_hash(name);
385
+ m->type = index->type;
386
+ m->options = index->default_options;
387
+
388
+ if (m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
389
+ m->histogram.ext = callocz(1,sizeof(STATSD_METRIC_HISTOGRAM_EXTENSIONS));
390
+ netdata_mutex_init(&m->histogram.ext->mutex);
391
+ }
392
427
- return (STATSD_METRIC *)STATSD_AVL_SEARCH(&index->index, (avl_t *)&tmp);
393
+ __atomic_fetch_add(&index->metrics, 1, __ATOMIC_SEQ_CST);
394
}
395
430
-static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, const char *name, STATSD_METRIC_TYPE type) {
431
- debug(D_STATSD, "searching for metric '%s' under '%s'", name, index->name);
396
+static void dictionary_metric_delete_callback(const char *name, void *value, void *data) {
397
+ (void)data; // STATSD_INDEX *index = (STATSD_INDEX *)data;
398
+ (void)name;
399
+ STATSD_METRIC *m = (STATSD_METRIC *)value;
400
433
- uint32_t hash = simple_hash(name);
401
+ if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
402
+ freez(m->histogram.ext);
403
+ m->histogram.ext = NULL;
404
+ }
405
435
- STATSD_METRIC *m = statsd_metric_index_find(index, name, hash);
436
- if(unlikely(!m)) {
437
- debug(D_STATSD, "Creating new %s metric '%s'", index->name, name);
406
+ freez(m->units);
407
+ freez(m->family);
408
+ freez(m->dimname);
409
+}
410
439
- m = (STATSD_METRIC *)callocz(sizeof(STATSD_METRIC), 1);
440
- m->name = strdupz(name);
441
- m->hash = hash;
442
- m->type = type;
443
- m->options = index->default_options;
411
+static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, const char *name) {
412
+ debug(D_STATSD, "searching for metric '%s' under '%s'", name, index->name);
413
445
- if(type == STATSD_METRIC_TYPE_HISTOGRAM || type == STATSD_METRIC_TYPE_TIMER) {
446
- m->histogram.ext = callocz(sizeof(STATSD_METRIC_HISTOGRAM_EXTENSIONS), 1);
447
- netdata_mutex_init(&m->histogram.ext->mutex);
448
- }
449
- STATSD_METRIC *n = (STATSD_METRIC *)STATSD_AVL_INSERT(&index->index, (avl_t *)m);
450
- if(unlikely(n != m)) {
451
- freez((void *)m->histogram.ext);
452
- freez((void *)m->name);
453
- freez((void *)m);
454
- m = n;
455
- }
456
- else {
457
- STATSD_FIRST_PTR_MUTEX_LOCK(index);
458
- index->metrics++;
459
- m->next = index->first;
460
- index->first = m;
461
- STATSD_FIRST_PTR_MUTEX_UNLOCK(index);
462
- }
463
- }
414
+#ifdef STATSD_MULTITHREADED
415
+ // avoid the write lock of dictionary_set() for existing metrics
416
+ STATSD_METRIC *m = dictionary_get(index->dict, name);
417
+ if(!m) m = dictionary_set(index->dict, name, NULL, sizeof(STATSD_METRIC));
418
+#else
419
+ // no locks here, go faster
420
+ // this will call the dictionary_metric_insert_callback() if an item
421
+ // is inserted, otherwise it will return the existing one.
422
+ // We used the flag DICTIONARY_FLAG_DONT_OVERWRITE_VALUE to support this.
423
+ STATSD_METRIC *m = dictionary_set(index->dict, name, NULL, sizeof(STATSD_METRIC));
424
+#endif
425
426
index->events++;
427
return m;
@@ -614,6 +575,13 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha
575
#define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer")
576
#define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram")
577
578
+static void dictionary_metric_set_value_insert_callback(const char *name, void *value, void *data) {
579
+ (void)name;
580
+ (void)value;
581
+ STATSD_METRIC *m = (STATSD_METRIC *)data;
582
+ m->set.unique++;
583
+}
584
+
585
static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
586
if(!is_metric_useful_for_collection(m)) return;
587
@@ -625,13 +593,14 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
593
if(unlikely(m->reset)) {
594
if(likely(m->set.dict)) {
595
dictionary_destroy(m->set.dict);
596
+ dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m);
597
m->set.dict = NULL;
598
}
599
statsd_reset_metric(m);
600
}
601
602
if (unlikely(!m->set.dict)) {
634
- m->set.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
603
+ m->set.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
604
m->set.unique = 0;
605
}
606
@@ -639,24 +608,25 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
608
// magic loading of metric, without affecting anything
609
}
610
else {
642
- char c = 'N'; // new
643
- char *cptr = (char *)dictionary_set(m->set.dict, value, &c, sizeof(char));
644
-
645
- // since we pass DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
646
- // the dictionary will return an existing value, if the key is already there
647
- // and based on the returned value, we can know if it is New or Old.
648
-
649
- if(*cptr == 'N') {
650
- // it is a new item
651
- *cptr = 'O'; // mark it as old
652
- m->set.unique++;
653
- }
654
-
611
+#ifdef STATSD_MULTITHREADED
612
+ // avoid the write lock to check if something is already there
613
+ if(!dictionary_get(m->set.dict, value))
614
+ dictionary_set(m->set.dict, value, NULL, 0);
615
+#else
616
+ dictionary_set(m->set.dict, value, NULL, 0);
617
+#endif
618
m->events++;
619
m->count++;
620
}
621
}
622
623
+static void dictionary_metric_dict_value_insert_callback(const char *name, void *value, void *data) {
624
+ (void)name;
625
+ (void)value;
626
+ STATSD_METRIC *m = (STATSD_METRIC *)data;
627
+ m->dictionary.unique++;
628
+}
629
+
630
static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value) {
631
if(!is_metric_useful_for_collection(m)) return;
632
@@ -669,9 +639,9 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value
639
statsd_reset_metric(m);
640
641
if (unlikely(!m->dictionary.dict)) {
672
- m->dictionary.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
642
+ m->dictionary.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
643
+ dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m);
644
m->dictionary.unique = 0;
674
- m->dictionary.base = NULL;
645
}
646
647
if(unlikely(value_is_zinit(value))) {
@@ -684,21 +654,7 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value
654
if(!t && m->dictionary.unique >= statsd.dictionary_max_unique)
655
value = "other";
656
687
- STATSD_METRIC_DICTIONARY_ITEM tmp = {
688
- .name = NULL,
689
- .count = 0,
690
- .rd = NULL,
691
- .next = NULL
692
- };
693
- char *name_ptr = NULL;
694
- t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_set_with_name_ptr(m->dictionary.dict, value, &tmp, sizeof(STATSD_METRIC_DICTIONARY_ITEM), &name_ptr);
695
- if(!t->name) {
696
- // we just added this
697
- t->name = name_ptr;
698
- t->next = m->dictionary.base;
699
- m->dictionary.base = t;
700
- m->dictionary.unique++;
701
- }
657
+ t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_set(m->dictionary.dict, value, NULL, sizeof(STATSD_METRIC_DICTIONARY_ITEM));
658
}
659
660
t->count++;
@@ -755,39 +711,39 @@ static void statsd_process_metric(const char *name, const char *value, const cha
711
char t0 = type[0], t1 = type[1];
712
if(unlikely(t0 == 'g' && t1 == '\0')) {
713
statsd_process_gauge(
758
- m = statsd_find_or_add_metric(&statsd.gauges, name, STATSD_METRIC_TYPE_GAUGE),
714
+ m = statsd_find_or_add_metric(&statsd.gauges, name),
715
value, sampling);
716
}
717
else if(unlikely((t0 == 'c' || t0 == 'C') && t1 == '\0')) {
718
// etsy/statsd uses 'c'
719
// brubeck uses 'C'
720
statsd_process_counter(
765
- m = statsd_find_or_add_metric(&statsd.counters, name, STATSD_METRIC_TYPE_COUNTER),
721
+ m = statsd_find_or_add_metric(&statsd.counters, name),
722
value, sampling);
723
}
724
else if(unlikely(t0 == 'm' && t1 == '\0')) {
725
statsd_process_meter(
770
- m = statsd_find_or_add_metric(&statsd.meters, name, STATSD_METRIC_TYPE_METER),
726
+ m = statsd_find_or_add_metric(&statsd.meters, name),
727
value, sampling);
728
}
729
else if(unlikely(t0 == 'h' && t1 == '\0')) {
730
statsd_process_histogram(
775
- m = statsd_find_or_add_metric(&statsd.histograms, name, STATSD_METRIC_TYPE_HISTOGRAM),
731
+ m = statsd_find_or_add_metric(&statsd.histograms, name),
732
value, sampling);
733
}
734
else if(unlikely(t0 == 's' && t1 == '\0')) {
735
statsd_process_set(
780
- m = statsd_find_or_add_metric(&statsd.sets, name, STATSD_METRIC_TYPE_SET),
736
+ m = statsd_find_or_add_metric(&statsd.sets, name),
737
value);
738
}
739
else if(unlikely(t0 == 'd' && t1 == '\0')) {
740
statsd_process_dictionary(
785
- m = statsd_find_or_add_metric(&statsd.dictionaries, name, STATSD_METRIC_TYPE_DICTIONARY),
741
+ m = statsd_find_or_add_metric(&statsd.dictionaries, name),
742
value);
743
}
744
else if(unlikely(t0 == 'm' && t1 == 's' && type[2] == '\0')) {
745
statsd_process_timer(
790
- m = statsd_find_or_add_metric(&statsd.timers, name, STATSD_METRIC_TYPE_TIMER),
746
+ m = statsd_find_or_add_metric(&statsd.timers, name),
747
value, sampling);
748
}
749
else {
@@ -819,14 +775,15 @@ static void statsd_process_metric(const char *name, const char *value, const cha
775
statsd_parse_field_trim(tagvalue, tagvalue_end);
776
777
if(tagkey && tagkey && tagvalue && *tagvalue) {
822
- if (!m->units[0] && strcmp(tagkey, "units") == 0)
823
- strncpyz(m->units, tagvalue, STATSD_MAX_UNITS_LENGTH);
778
+ if (!m->units && strcmp(tagkey, "units") == 0) {
779
+ m->units = strdupz(tagvalue);
780
+ }
781
825
- if (!m->dimname[0] && strcmp(tagkey, "name") == 0)
826
- strncpyz(m->dimname, tagvalue, STATSD_MAX_DIMNAME_LENGTH);
782
+ if (!m->dimname && strcmp(tagkey, "name") == 0)
783
+ m->dimname = strdupz(tagvalue);
784
828
- if (!m->family[0] && strcmp(tagkey, "family") == 0)
829
- strncpyz(m->family, tagvalue, STATSD_MAX_FAMILY_LENGTH);
785
+ if (!m->family && strcmp(tagkey, "family") == 0)
786
+ m->family = strdupz(tagvalue);
787
}
788
}
789
}
@@ -1676,16 +1633,16 @@ static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
1633
, type
1634
, id
1635
, NULL // name
1679
- , m->family[0]?m->family:"gauges" // family (submenu)
1636
+ , m->family?m->family:"gauges" // family (submenu)
1637
, context // context
1638
, title // title
1682
- , m->units[0]?m->units:"value" // units
1639
+ , m->units?m->units:"value" // units
1640
, NETDATA_CHART_PRIO_STATSD_PRIVATE
1641
, statsd.update_every
1642
, RRDSET_TYPE_LINE
1643
);
1644
1688
- m->rd_value = rrddim_add(m->st, "gauge", m->dimname[0]?m->dimname:NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
1645
+ m->rd_value = rrddim_add(m->st, "gauge", m->dimname?m->dimname:NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
1646
1647
if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
1648
m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1715,16 +1672,16 @@ static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const
1672
, type
1673
, id
1674
, NULL // name
1718
- , m->family[0]?m->family:family // family (submenu)
1675
+ , m->family?m->family:family // family (submenu)
1676
, context // context
1677
, title // title
1721
- , m->units[0]?m->units:"events/s" // units
1678
+ , m->units?m->units:"events/s" // units
1679
, NETDATA_CHART_PRIO_STATSD_PRIVATE
1680
, statsd.update_every
1681
, RRDSET_TYPE_AREA
1682
);
1683
1727
- m->rd_value = rrddim_add(m->st, dim, m->dimname[0]?m->dimname:NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1684
+ m->rd_value = rrddim_add(m->st, dim, m->dimname?m->dimname:NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1685
1686
if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
1687
m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1754,16 +1711,16 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) {
1711
, type
1712
, id
1713
, NULL // name
1757
- , m->family[0]?m->family:"sets" // family (submenu)
1714
+ , m->family?m->family:"sets" // family (submenu)
1715
, context // context
1716
, title // title
1760
- , m->units[0]?m->units:"entries" // units
1717
+ , m->units?m->units:"entries" // units
1718
, NETDATA_CHART_PRIO_STATSD_PRIVATE
1719
, statsd.update_every
1720
, RRDSET_TYPE_LINE
1721
);
1722
1766
- m->rd_value = rrddim_add(m->st, "set", m->dimname[0]?m->dimname:"unique", 1, 1, RRD_ALGORITHM_ABSOLUTE);
1723
+ m->rd_value = rrddim_add(m->st, "set", m->dimname?m->dimname:"unique", 1, 1, RRD_ALGORITHM_ABSOLUTE);
1724
1725
if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
1726
m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1793,10 +1750,10 @@ static inline void statsd_private_chart_dictionary(STATSD_METRIC *m) {
1750
, type
1751
, id
1752
, NULL // name
1796
- , m->family[0]?m->family:"dictionaries" // family (submenu)
1753
+ , m->family?m->family:"dictionaries" // family (submenu)
1754
, context // context
1755
, title // title
1799
- , m->units[0]?m->units:"events/s" // units
1756
+ , m->units?m->units:"events/s" // units
1757
, NETDATA_CHART_PRIO_STATSD_PRIVATE
1758
, statsd.update_every
1759
, RRDSET_TYPE_STACKED
@@ -1808,10 +1765,11 @@ static inline void statsd_private_chart_dictionary(STATSD_METRIC *m) {
1765
else rrdset_next(m->st);
1766
1767
STATSD_METRIC_DICTIONARY_ITEM *t;
1811
- for(t = m->dictionary.base; t ;t = t->next) {
1812
- if(!t->rd) t->rd = rrddim_add(m->st, t->name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1768
+ dfe_start_read(m->dictionary.dict, t) {
1769
+ if (!t->rd) t->rd = rrddim_add(m->st, t_name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1770
rrddim_set_by_pointer(m->st, t->rd, (collected_number)t->count);
1771
}
1772
+ dfe_done(t);
1773
1774
if(m->rd_count)
1775
rrddim_set_by_pointer(m->st, m->rd_count, m->events);
@@ -1834,10 +1792,10 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con
1792
, type
1793
, id
1794
, NULL // name
1837
- , m->family[0]?m->family:family // family (submenu)
1795
+ , m->family?m->family:family // family (submenu)
1796
, context // context
1797
, title // title
1840
- , m->units[0]?m->units:units // units
1798
+ , m->units?m->units:units // units
1799
, NETDATA_CHART_PRIO_STATSD_PRIVATE
1800
, statsd.update_every
1801
, RRDSET_TYPE_AREA
@@ -1947,8 +1905,8 @@ static inline void statsd_flush_dictionary(STATSD_METRIC *m) {
1905
statsd_private_chart_dictionary(m);
1906
1907
if(m->dictionary.unique >= statsd.dictionary_max_unique) {
1950
- if(!(m->dictionary.flags & STATSD_METRIC_DICTIONARY_FLAGS_DICTFULL_LOGGED)) {
1951
- m->dictionary.flags |= STATSD_METRIC_DICTIONARY_FLAGS_DICTFULL_LOGGED;
1908
+ if(!(m->options & STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED)) {
1909
+ m->options |= STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED;
1910
info(
1911
"STATSD dictionary '%s' reach max of %zu items - try increasing 'dictionaries max unique dimensions' in netdata.conf",
1912
m->name,
@@ -2306,7 +2264,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
2264
STATSD_METRIC *m;
2265
2266
// find the useful metrics (incremental = each time we are called, we check the new metrics only)
2309
- for(m = index->first; m ; m = m->next) {
2267
+ dfe_start_read(index->dict, m) {
2268
// since we add new metrics at the beginning
2269
// check for useful charts, until the point we last checked
2270
if(unlikely(is_metric_checked(m))) break;
@@ -2347,6 +2305,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
2305
index->first_useful = m;
2306
}
2307
}
2308
+ dfe_done(m);
2309
2310
// flush all the useful metrics
2311
for(m = index->first_useful; m ; m = m->next_useful) {
@@ -2383,6 +2342,15 @@ static void statsd_main_cleanup(void *data) {
2342
info("STATSD: closing sockets...");
2343
listen_sockets_close(&statsd.sockets);
2344
2345
+ // destroy the dictionaries
2346
+ dictionary_destroy(statsd.gauges.dict);
2347
+ dictionary_destroy(statsd.meters.dict);
2348
+ dictionary_destroy(statsd.counters.dict);
2349
+ dictionary_destroy(statsd.histograms.dict);
2350
+ dictionary_destroy(statsd.dictionaries.dict);
2351
+ dictionary_destroy(statsd.sets.dict);
2352
+ dictionary_destroy(statsd.timers.dict);
2353
+
2354
info("STATSD: cleanup completed.");
2355
static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
2356
@@ -2415,6 +2383,30 @@ void *statsd_main(void *ptr) {
2383
2384
netdata_thread_cleanup_push(statsd_main_cleanup, ptr);
2385
2386
+ statsd.gauges.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
2387
+ statsd.meters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
2388
+ statsd.counters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
2389
+ statsd.histograms.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
2390
+ statsd.dictionaries.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
2391
+ statsd.sets.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
2392
+ statsd.timers.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
2393
+
2394
+ dictionary_register_insert_callback(statsd.gauges.dict, dictionary_metric_insert_callback, &statsd.gauges);
2395
+ dictionary_register_insert_callback(statsd.meters.dict, dictionary_metric_insert_callback, &statsd.meters);
2396
+ dictionary_register_insert_callback(statsd.counters.dict, dictionary_metric_insert_callback, &statsd.counters);
2397
+ dictionary_register_insert_callback(statsd.histograms.dict, dictionary_metric_insert_callback, &statsd.histograms);
2398
+ dictionary_register_insert_callback(statsd.dictionaries.dict, dictionary_metric_insert_callback, &statsd.dictionaries);
2399
+ dictionary_register_insert_callback(statsd.sets.dict, dictionary_metric_insert_callback, &statsd.sets);
2400
+ dictionary_register_insert_callback(statsd.timers.dict, dictionary_metric_insert_callback, &statsd.timers);
2401
+
2402
+ dictionary_register_delete_callback(statsd.gauges.dict, dictionary_metric_delete_callback, &statsd.gauges);
2403
+ dictionary_register_delete_callback(statsd.meters.dict, dictionary_metric_delete_callback, &statsd.meters);
2404
+ dictionary_register_delete_callback(statsd.counters.dict, dictionary_metric_delete_callback, &statsd.counters);
2405
+ dictionary_register_delete_callback(statsd.histograms.dict, dictionary_metric_delete_callback, &statsd.histograms);
2406
+ dictionary_register_delete_callback(statsd.dictionaries.dict, dictionary_metric_delete_callback, &statsd.dictionaries);
2407
+ dictionary_register_delete_callback(statsd.sets.dict, dictionary_metric_delete_callback, &statsd.sets);
2408
+ dictionary_register_delete_callback(statsd.timers.dict, dictionary_metric_delete_callback, &statsd.timers);
2409
+
2410
// ----------------------------------------------------------------------------------------------------------------
2411
// statsd configuration
2412
daemon/analytics.c
+4
-3
@@ -249,8 +249,9 @@ void analytics_exporters(void)
249
buffer_free(bi);
250
}
251
252
-int collector_counter_callb(void *entry, void *data)
253
-{
252
+int collector_counter_callb(const char *name, void *entry, void *data) {
253
+ (void)name;
254
+
255
struct array_printer *ap = (struct array_printer *)data;
256
struct collector *col = (struct collector *)entry;
257
@@ -296,7 +297,7 @@ void analytics_collectors(void)
297
ap.c = 0;
298
ap.both = bt;
299
299
- dictionary_get_all(dict, collector_counter_callb, &ap);
300
+ dictionary_walkthrough_read(dict, collector_counter_callb, &ap);
301
dictionary_destroy(dict);
302
303
analytics_set_data(&analytics_data.netdata_collectors, (char *)buffer_tostring(ap.both));
daemon/main.c
+3
@@ -895,6 +895,9 @@ int main(int argc, char **argv) {
895
}
896
#endif
897
#ifdef ENABLE_DBENGINE
898
+ else if(strcmp(optarg, "dicttest") == 0) {
899
+ return dictionary_unittest(10000);
900
+ }
901
else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
902
optarg += strlen(createdataset_string);
903
unsigned history_seconds = strtoul(optarg, NULL, 0);
libnetdata/dictionary/README.md
+201
@@ -2,4 +2,205 @@
2
custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/dictionary/README.md
3
-->
4
5
+# Dictionaries
6
7
+Netdata dictionaries associate a `name` with a `value`:
8
+
9
+- A `name` can be any string.
10
+- A `value` can be anything.
11
+
12
+Such a pair of a `name` and a `value` consists of an `item` or an `entry` in the dictionary.
13
+
14
+Dictionaries provide an interface to:
15
+
16
+- **Add** an item to the dictionary
17
+- **Get** an item from the dictionary (provided its `name`)
18
+- **Delete** an item from the dictionary (provided its `name`)
19
+- **Traverse** the list of items in the dictionary
20
+
21
+Dictionaries are **ordered**, meaning that the order they have been added is preserved while traversing them. The caller may reverse this order by passing the flag `DICTIONARY_FLAG_ADD_IN_FRONT` when creating the dictionary.
22
+
23
+Dictionaries guarantee **uniqueness** of all items added to them, meaning that only one item with a given name can exist in the dictionary at any given time.
24
+
25
+Dictionaries are extremely fast in all operations. They are indexing the keys with `JudyHS` and they utilize a double-linked-list for the traversal operations. Deletion is the most expensive operation, usually somewhat slower than insertion.
26
+
27
+## Memory management
28
+
29
+Dictionaries come with 2 memory management options:
30
+
31
+- **Clone** (copy) the name and/or the value to memory allocated by the dictionary.
32
+- **Link** the name and/or the value, without allocating any memory about them.
33
+
34
+In **clone** mode, the dictionary guarantees that all operations on the dictionary items will automatically take care of the memory used by the name and/or the value. In case the value is an object needs to have user allocated memory, two callback functions can be registered:
35
+
36
+ 1.`dictionary_register_insert_callback()` that will be called just after the insertion of an item to the dictionary (but while the dictionary is write-locked - if locking is enabled).
37
+ 2. `dictionary_register_delete_callback()` that will be called just prior to the deletion of an item from the dictionary (but while the dictionary is write-locked - if locking is enabled).
38
+
39
+In **link** mode, the name and/or the value are just linked to the dictionary item, and it is the user's responsibility to free the memory used after an item is deleted from the dictionary.
40
+
41
+By default, **clone** mode is used for both the name and the value.
42
+
43
+To use **link** mode for names, add `DICTIONARY_FLAG_NAME_LINK_DONT_CLONE` to the flags when creating the dictionary.
44
+
45
+To use **link** mode for values, add `DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE` to the flags when creating the dictionary.
46
+
47
+## Locks
48
+
49
+The dictionary allows both **single-threaded** operation (no locks - faster) and **multi-threaded** operation utilizing a read-write lock.
50
+
51
+The default is **multi-threaded**. To enable **single-threaded** add `DICTIONARY_FLAG_SINGLE_THREADED` to the flags when creating the dictionary.
52
+
53
+## Hash table operations
54
+
55
+The dictionary supports the following operations supported by the hash table:
56
+
57
+- `dictionary_set()` to add an item to the dictionary, or change its value.
58
+- `dictionary_get()` to get an item from the dictionary.
59
+- `dictionary_del()` to delete an item from the dictionary.
60
+
61
+## Creation and destruction
62
+
63
+Use `dictionary_create()` to create a dictionary.
64
+
65
+Use `dictionary_destroy()` to destroy a dictionary. When destroyed, a dictionary frees all the memory it has allocated on its own. The exception is the registration of a deletion callback function that can be called on deletion of an item, which may free additional resources.
66
+
67
+### dictionary_set()
68
+
69
+This call is used to:
70
+
71
+- **add** an item to the dictionary.
72
+- **reset** the value of an existing item in the dictionary.
73
+
74
+If **resetting** is not desired, add `DICTIONARY_FLAG_DONT_OVERWRITE_VALUE` to the flags when creating the dictionary. In this case, `dictionary_set()` will return the value of the original item found in the dictionary instead of resetting it and the value passed to the call will be ignored.
75
+
76
+For **multi-threaded** operation, the `dictionary_set()` calls get an exclusive write lock on the dictionary.
77
+
78
+The format is:
79
+
80
+```c
81
+value = dictionary_set(dict, name, value, value_len);
82
+```
83
+
84
+Where:
85
+
86
+* `dict` is a pointer to the dictionary previously created.
87
+* `name` is a pointer to a string to be used as the key of this item. The name must not be `NULL` and must not be an empty string `""`.
88
+* `value` is a pointer to the value associated with this item. In **clone** mode, if `value` is `NULL`, a new memory allocation will be made of `value_len` size and will be initialized to zero.
89
+* `value_len` is the size of the `value` data. If `value_len` is zero, no allocation will be done and the dictionary item will permanently have the `NULL` value.
90
+
91
+> **IMPORTANT**<br/>There is also an **unsafe** version (without locks) of this call. This is to be used when traversing the dictionary. It should never be called without an active lock on the dictionary, which can only be acquired while traversing.
92
+
93
+### dictionary_get()
94
+
95
+This call is used to get the value of an item, given its name. It utilizes the JudyHS hash table for making the lookup.
96
+
97
+For **multi-threaded** operation, the `dictionary_get()` call gets a shared read lock on the dictionary.
98
+
99
+The format is:
100
+
101
+```c
102
+value = dictionary_get(dict, name);
103
+```
104
+
105
+Where:
106
+
107
+* `dict` is a pointer to the dictionary previously created.
108
+* `name` is a pointer to a string to be used as the key of this item. The name must not be `NULL` and must not be an empty string `""`.
109
+
110
+> **IMPORTANT**<br/>There is also an **unsafe** version (without locks) of this call. This is to be used when traversing the dictionary. It should never be called without an active lock on the dictionary, which can only be acquired while traversing.
111
+
112
+### dictionary_del()
113
+
114
+This call is used to delete an item from the dictionary, given its name.
115
+
116
+If there is a delete callback registered to the dictionary (`dictionary_register_delete_callback()`), it is called prior to the actual deletion of the item.
117
+
118
+For **multi-threaded** operation, the `dictionary_del()` calls get an exclusive write lock on the dictionary.
119
+
120
+The format is:
121
+
122
+```c
123
+value = dictionary_del(dict, name);
124
+```
125
+
126
+Where:
127
+
128
+* `dict` is a pointer to the dictionary previously created.
129
+* `name` is a pointer to a string to be used as the key of this item. The name must not be `NULL` and must not be an empty string `""`.
130
+
131
+> **IMPORTANT**<br/>There is also an **unsafe** version (without locks) of this call. This is to be used when traversing the dictionary, to delete the current item. It should never be called without an active lock on the dictionary, which can only be acquired while traversing.
132
+
133
+## Traversal
134
+
135
+Dictionaries offer 2 ways to traverse the entire dictionary:
136
+
137
+- **walkthrough**, implemented by setting a callback function to be called for every item.
138
+- **foreach**, a way to traverse the dictionary with a for-next loop.
139
+
140
+Both of these methods are available in **read** or **write** mode. In **read** mode only lookups are allowed to the dictionary. In **write** both lookups but also deletion of the currently working item is also allowed.
141
+
142
+While traversing the dictionary with any of these methods, all calls to the dictionary have to use the `_unsafe` versions of the function calls, otherwise deadlock may arise.
143
+
144
+> **IMPORTANT**<br/>The dictionary itself does not check to ensure that a user is actually using the right lock mode (read or write) while traversing the dictionary for each of the unsafe calls.
145
+
146
+### walkthrough (callback)
147
+
148
+There are 2 calls:
149
+
150
+- `dictionary_walkthrough_read()` that acquires a shared read lock, and it calls a callback function for every item of the dictionary. The callback function may use the unsafe versions of the `dictionary_get()` calls to lookup other items in the dictionary, but it should not add or remove item from the dictionary.
151
+- `dictionary_walkthrough_write()` that acquires an exclusive write lock, and it calls a callback function for every item of the dictionary. This is to be used when items need to be added to the dictionary, or when the current item may need to be deleted. If the callback function deletes any other items, the behavior may be undefined (actually, the item next to the one currently working should not be deleted - a pointer to it is held by the traversal function to move on traversing the dictionary).
152
+
153
+The items are traversed in the same order they have been added to the dictionary (or the reverse order if the flag `DICTIONARY_FLAG_ADD_IN_FRONT` is set during dictionary creation).
154
+
155
+The callback function returns an `int`. If this value is negative, traversal of the dictionary is stopped immediately and the negative value is returned to the caller. If the returned value of all callbacks is zero or positive, the walkthrough functions return the sum of the return values of all callbacks. So, if you are just interested to know how many items fall into some condition, write a callback function that returns 1 when the item satisfies that condition and 0 when it does not and the walkthrough function will return how many tested positive.
156
+
157
+### foreach (for-next loop)
158
+
159
+The following is a snippet of such a loop:
160
+
161
+```c
162
+MY_ITEM *item;
163
+dfe_start_read(dict, item) {
164
+ printf("hey, I got an item named '%s' with value ptr %08X", item_name, item);
165
+}
166
+dfe_done(item);
167
+```
168
+
169
+The `item` parameter gives the name of the pointer to be used while iterating the items. Any name is accepted.
170
+
171
+The `item_name` is a variable that is automatically created, by concatenating whatever is given as `item` and `_name`. So, if you call `dfe_start_read(dict, myvar)`, the name will be `myvar_name`.
172
+
173
+Both `dfe_start_read(dict, item)` and `dfe_done(item)` are together inside a `do { ... } while(0)` loop, so that the following will work:
174
+
175
+```c
176
+MY_ITEM *item;
177
+
178
+if(x = 1)
179
+ // do {
180
+ dfe_start_read(dict, item)
181
+ printf("hey, I got an item named '%s' with value ptr %08X", item_name, item);
182
+ dfe_done(item);
183
+ // } while(0);
184
+else
185
+ something else;
186
+```
187
+
188
+In the above, the `if(x)` condition will work as expected. It will do the foreach loop when x is 1, otherwise it will run `something else`.
189
+
190
+There are 2 versions of `dfe_start`:
191
+
192
+- `dfe_start_read()` that acquires a shared read lock to the dictionary.
193
+- `dfe_start_write()` that acquires an exclusive write lock to the dictionary.
194
+
195
+While in the loop, depending on the read or write versions of `dfe_start`, the caller may lookup or manipulate the dictionary. The rules are the same with the walkthrough callback functions.
196
+
197
+PS: DFE is Dictionary For Each.
198
+
199
+## special multi-threaded lockless case
200
+
201
+Since the dictionary uses a hash table and a double linked list, if the contract between 2 threads is for one to use the hash table functions only (`set`, `get` - but no `del`) and the other to use the traversal ones only, the dictionary allows concurrent use without locks.
202
+
203
+This is currently used in statsd:
204
+
205
+- the data collection thread uses only `get` and `set`. It never uses `del`. New items are added at the front of the linked list (`DICTIONARY_FLAG_ADD_IN_FRONT`).
206
+- the flushing thread is only traversing the dictionary up to the point it last traversed it (it uses a flag for that to know where it stopped last time). It never uses `get`, `set` or `del`.
libnetdata/dictionary/dictionary.c
+1093
-160
@@ -1,226 +1,754 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
+// NOT TO BE USED BY USERS YET
4
+#define DICTIONARY_FLAG_REFERENCE_COUNTERS (1 << 6) // maintain reference counter in walkthrough and foreach
5
+
6
+typedef struct dictionary DICTIONARY;
7
+#define DICTIONARY_INTERNALS
8
+
9
#include "../libnetdata.h"
10
11
+#ifndef ENABLE_DBENGINE
12
+#define DICTIONARY_WITH_AVL
13
+#warning Compiling DICTIONARY with an AVL index
14
+#else
15
+#define DICTIONARY_WITH_JUDYHS
16
+#endif
17
+
18
+#ifdef DICTIONARY_WITH_JUDYHS
19
+#include <Judy.h>
20
+#endif
21
+
22
+/*
23
+ * This version uses JudyHS arrays to index the dictionary
24
+ *
25
+ * The following output is from the unit test, at the end of this file:
26
+ *
27
+ * This is the JudyHS version:
28
+ *
29
+ * 1000000 x dictionary_set() (dictionary size 0 entries, 0 KB)...
30
+ * 1000000 x dictionary_get(existing) (dictionary size 1000000 entries, 74001 KB)...
31
+ * 1000000 x dictionary_get(non-existing) (dictionary size 1000000 entries, 74001 KB)...
32
+ * Walking through the dictionary (dictionary size 1000000 entries, 74001 KB)...
33
+ * 1000000 x dictionary_del(existing) (dictionary size 1000000 entries, 74001 KB)...
34
+ * 1000000 x dictionary_set() (dictionary size 0 entries, 0 KB)...
35
+ * Destroying dictionary (dictionary size 1000000 entries, 74001 KB)...
36
+ *
37
+ * TIMINGS:
38
+ * adding 316027 usec, positive search 156740 usec, negative search 84524, walk through 15036 usec, deleting 361444, destroy 107394 usec
39
+ *
40
+ * This is from the JudySL version:
41
+ *
42
+ * Creating dictionary of 1000000 entries...
43
+ * Checking index of 1000000 entries...
44
+ * Walking 1000000 entries and checking name-value pairs...
45
+ * Created and checked 1000000 entries, found 0 errors - used 58376 KB of memory
46
+ * Destroying dictionary of 1000000 entries...
47
+ * Deleted 1000000 entries
48
+ * create 338975 usec, check 156080 usec, walk 80764 usec, destroy 444569 usec
49
+ *
50
+ * This is the AVL version:
51
+ *
52
+ * Creating dictionary of 1000000 entries...
53
+ * Checking index of 1000000 entries...
54
+ * Walking 1000000 entries and checking name-value pairs...
55
+ * Created and checked 1000000 entries, found 0 errors - used 89626 KB of memory
56
+ * Destroying dictionary of 1000000 entries...
57
+ * create 413892 usec, check 220006 usec, walk 34247 usec, destroy 98062 usec
58
+ *
59
+ * So, the JudySL is a lot slower to WALK and DESTROY (DESTROY does a WALK)
60
+ * It is slower, because for every item, JudySL copies the KEY/NAME to a
61
+ * caller supplied buffer (Index). So, by just walking over 1 million items,
62
+ * JudySL does 1 million strcpy() !!!
63
+ *
64
+ * It also seems that somehow JudySLDel() is unbelievably slow too!
65
+ *
66
+ */
67
+
68
+
69
+/*
70
+ * Every item in the dictionary has the following structure.
71
+ */
72
+typedef struct name_value {
73
+#ifdef DICTIONARY_WITH_AVL
74
+ avl_t avl_node;
75
+#endif
76
+
77
+ struct name_value *next; // a double linked list to allow fast insertions and deletions
78
+ struct name_value *prev;
79
+
80
+ char *name; // the name of the dictionary item
81
+ void *value; // the value of the dictionary item
82
+} NAME_VALUE;
83
+
84
+/*
85
+ * When DICTIONARY_FLAG_WITH_STATISTICS is set, we need to keep track of all the memory
86
+ * we allocate and free. So, we need to keep track of the sizes of all names and values.
87
+ * We do this by overloading NAME_VALUE with the following additional fields.
88
+ */
89
+
90
+typedef enum name_value_flags {
91
+ NAME_VALUE_FLAG_NONE = 0,
92
+ NAME_VALUE_FLAG_DELETED = (1 << 0), // this item is deleted
93
+} NAME_VALUE_FLAGS;
94
+
95
+typedef struct name_value_with_stats {
96
+ NAME_VALUE name_value_data_here; // never used - just to put the lengths at the right position
97
+
98
+ size_t name_len; // the size of the name, including the terminating zero
99
+ size_t value_len; // the size of the value (assumed binary)
100
+
101
+ size_t refcount; // the reference counter
102
+ NAME_VALUE_FLAGS flags; // the flags for this item
103
+} NAME_VALUE_WITH_STATS;
104
+
105
+struct dictionary_stats {
106
+ size_t inserts;
107
+ size_t deletes;
108
+ size_t searches;
109
+ size_t resets;
110
+ size_t entries;
111
+ size_t memory;
112
+};
113
+
114
+struct dictionary {
115
+ DICTIONARY_FLAGS flags; // the flags of the dictionary
116
+
117
+ NAME_VALUE *first_item; // the double linked list base pointers
118
+ NAME_VALUE *last_item;
119
+
120
+#ifdef DICTIONARY_WITH_AVL
121
+ avl_tree_type values_index;
122
+ NAME_VALUE *hash_base;
123
+#endif
124
+
125
+#ifdef DICTIONARY_WITH_JUDYHS
126
+ Pvoid_t JudyHSArray; // the hash table
127
+#endif
128
+
129
+ netdata_rwlock_t *rwlock; // the r/w lock when DICTIONARY_FLAG_SINGLE_THREADED is not set
130
+
131
+ void (*ins_callback)(const char *name, void *value, void *data);
132
+ void *ins_callback_data;
133
+
134
+ void (*del_callback)(const char *name, void *value, void *data);
135
+ void *del_callback_data;
136
+
137
+ struct dictionary_stats *stats; // the statistics when DICTIONARY_FLAG_WITH_STATISTICS is set
138
+};
139
+
140
+void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const char *name, void *value, void *data), void *data) {
141
+ dict->ins_callback = ins_callback;
142
+ dict->ins_callback_data = data;
143
+}
144
+
145
+void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const char *name, void *value, void *data), void *data) {
146
+ dict->del_callback = del_callback;
147
+ dict->del_callback_data = data;
148
+}
149
+
150
// ----------------------------------------------------------------------------
6
-// dictionary statistics
151
+// dictionary statistics maintenance
152
8
-static inline void NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(DICTIONARY *dict) {
9
- if(likely(dict->stats))
10
- dict->stats->inserts++;
153
+size_t dictionary_stats_allocated_memory(DICTIONARY *dict) {
154
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
155
+ return dict->stats->memory;
156
+ return 0;
157
}
12
-static inline void NETDATA_DICTIONARY_STATS_DELETES_PLUS1(DICTIONARY *dict) {
13
- if(likely(dict->stats))
14
- dict->stats->deletes++;
158
+size_t dictionary_stats_entries(DICTIONARY *dict) {
159
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
160
+ return dict->stats->entries;
161
+ return 0;
162
+}
163
+size_t dictionary_stats_searches(DICTIONARY *dict) {
164
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
165
+ return dict->stats->searches;
166
+ return 0;
167
+}
168
+size_t dictionary_stats_inserts(DICTIONARY *dict) {
169
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
170
+ return dict->stats->inserts;
171
+ return 0;
172
}
16
-static inline void NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) {
17
- if(likely(dict->stats))
173
+size_t dictionary_stats_deletes(DICTIONARY *dict) {
174
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
175
+ return dict->stats->deletes;
176
+ return 0;
177
+}
178
+size_t dictionary_stats_resets(DICTIONARY *dict) {
179
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
180
+ return dict->stats->resets;
181
+ return 0;
182
+}
183
+
184
+static inline void DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) {
185
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
186
dict->stats->searches++;
187
}
20
-static inline void NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict) {
21
- if(likely(dict->stats))
188
+static inline void DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict, size_t size) {
189
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
190
+ dict->stats->inserts++;
191
dict->stats->entries++;
192
+ dict->stats->memory += size;
193
+ }
194
}
24
-static inline void NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict) {
25
- if(likely(dict->stats))
195
+static inline void DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict, size_t size) {
196
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
197
+ dict->stats->deletes++;
198
dict->stats->entries--;
199
+ dict->stats->memory -= size;
200
+ }
201
+}
202
+static inline void DICTIONARY_STATS_VALUE_RESETS_PLUS1(DICTIONARY *dict, size_t oldsize, size_t newsize) {
203
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
204
+ dict->stats->resets++;
205
+ dict->stats->memory += newsize;
206
+ dict->stats->memory -= oldsize;
207
+ }
208
}
28
-
209
210
// ----------------------------------------------------------------------------
211
// dictionary locks
212
33
-static inline void dictionary_read_lock(DICTIONARY *dict) {
34
- if(likely(dict->rwlock)) {
213
+static inline size_t dictionary_lock_init(DICTIONARY *dict) {
214
+ if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
215
+ dict->rwlock = mallocz(sizeof(netdata_rwlock_t));
216
+ netdata_rwlock_init(dict->rwlock);
217
+ return sizeof(netdata_rwlock_t);
218
+ }
219
+ dict->rwlock = NULL;
220
+ return 0;
221
+}
222
+
223
+static inline size_t dictionary_lock_free(DICTIONARY *dict) {
224
+ if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
225
+ netdata_rwlock_destroy(dict->rwlock);
226
+ freez(dict->rwlock);
227
+ return sizeof(netdata_rwlock_t);
228
+ }
229
+ return 0;
230
+}
231
+
232
+static inline void dictionary_lock_rlock(DICTIONARY *dict) {
233
+ if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
234
// debug(D_DICTIONARY, "Dictionary READ lock");
235
netdata_rwlock_rdlock(dict->rwlock);
236
}
237
}
238
40
-static inline void dictionary_write_lock(DICTIONARY *dict) {
41
- if(likely(dict->rwlock)) {
239
+static inline void dictionary_lock_wrlock(DICTIONARY *dict) {
240
+ if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
241
// debug(D_DICTIONARY, "Dictionary WRITE lock");
242
netdata_rwlock_wrlock(dict->rwlock);
243
}
244
}
245
246
static inline void dictionary_unlock(DICTIONARY *dict) {
48
- if(likely(dict->rwlock)) {
247
+ if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
248
// debug(D_DICTIONARY, "Dictionary UNLOCK lock");
249
netdata_rwlock_unlock(dict->rwlock);
250
}
251
}
252
253
+// ----------------------------------------------------------------------------
254
+// reference counters
255
+
256
+static inline size_t reference_counter_init(DICTIONARY *dict) {
257
+ (void)dict;
258
+
259
+ // allocate memory required for reference counters
260
+ // return number of bytes
261
+ return 0;
262
+}
263
+
264
+static inline size_t reference_counter_free(DICTIONARY *dict) {
265
+ (void)dict;
266
+
267
+ // free memory required for reference counters
268
+ // return number of bytes
269
+ return 0;
270
+}
271
+
272
+static void reference_counter_acquire(DICTIONARY *dict, NAME_VALUE *nv) {
273
+ if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
274
+ NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
275
+ __atomic_fetch_add(&nvs->refcount, 1, __ATOMIC_SEQ_CST);
276
+ }
277
+}
278
+
279
+static void reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv) {
280
+ if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
281
+ NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
282
+ __atomic_fetch_sub(&nvs->refcount, 1, __ATOMIC_SEQ_CST);
283
+ }
284
+}
285
+
286
+static int reference_counter_mark_deleted(DICTIONARY *dict, NAME_VALUE *nv) {
287
+ if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
288
+ NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
289
+ nvs->flags |= NAME_VALUE_FLAG_DELETED;
290
+ return 1;
291
+ }
292
+ return 0;
293
+}
294
295
// ----------------------------------------------------------------------------
56
-// avl index
296
+// hash table
297
298
+#ifdef DICTIONARY_WITH_AVL
299
static int name_value_compare(void* a, void* b) {
59
- if(((NAME_VALUE *)a)->hash < ((NAME_VALUE *)b)->hash) return -1;
60
- else if(((NAME_VALUE *)a)->hash > ((NAME_VALUE *)b)->hash) return 1;
61
- else return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name);
300
+ return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name);
301
}
302
64
-static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *dict, const char *name, uint32_t hash) {
303
+static void hashtable_init_unsafe(DICTIONARY *dict) {
304
+ avl_init(&dict->values_index, name_value_compare);
305
+}
306
+
307
+static size_t hashtable_destroy_unsafe(DICTIONARY *dict) {
308
+ (void)dict;
309
+ return 0;
310
+}
311
+
312
+static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) {
313
+ (void)name;
314
+ (void)name_len;
315
+
316
+ if(unlikely(avl_remove(&(dict->values_index), (avl_t *)(nv)) != (avl_t *)nv))
317
+ return 0;
318
+
319
+ return 1;
320
+}
321
+
322
+static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
323
+ (void)name_len;
324
+
325
NAME_VALUE tmp;
66
- tmp.hash = (hash)?hash:simple_hash(name);
326
tmp.name = (char *)name;
68
-
69
- NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(dict);
327
return (NAME_VALUE *)avl_search(&(dict->values_index), (avl_t *) &tmp);
328
}
329
330
+static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
331
+ // AVL needs a NAME_VALUE to insert into the dictionary but we don't have it yet.
332
+ // So, the only thing we can do, is return an existing one if it is already there.
333
+ // Returning NULL will make the caller thing we added it, will allocate one
334
+ // and will call hashtable_inserted_name_value_unsafe(), at which we will do
335
+ // the actual indexing.
336
+
337
+ dict->hash_base = hashtable_get_unsafe(dict, name, name_len);
338
+ return &dict->hash_base;
339
+}
340
+
341
+static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) {
342
+ // we have our new NAME_VALUE object.
343
+ // Let's index it.
344
+
345
+ (void)name;
346
+ (void)name_len;
347
+
348
+ if(unlikely(avl_insert(&((dict)->values_index), (avl_t *)(nv)) != (avl_t *)nv))
349
+ error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary.");
350
+}
351
+#endif
352
+
353
+#ifdef DICTIONARY_WITH_JUDYHS
354
+static void hashtable_init_unsafe(DICTIONARY *dict) {
355
+ dict->JudyHSArray = NULL;
356
+}
357
+
358
+static size_t hashtable_destroy_unsafe(DICTIONARY *dict) {
359
+ if(unlikely(!dict->JudyHSArray)) return 0;
360
+
361
+ JError_t J_Error;
362
+ Word_t ret = JudyHSFreeArray(&dict->JudyHSArray, &J_Error);
363
+ if(unlikely(ret == (Word_t) JERR)) {
364
+ error("DICTIONARY: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d",
365
+ JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
366
+ }
367
+
368
+ debug(D_DICTIONARY, "Dictionary: hash table freed %lu bytes", ret);
369
+
370
+ dict->JudyHSArray = NULL;
371
+ return (size_t)ret;
372
+}
373
+
374
+static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
375
+ JError_t J_Error;
376
+ Pvoid_t *Rc = JudyHSIns(&dict->JudyHSArray, (void *)name, name_len, &J_Error);
377
+ if (unlikely(Rc == PJERR)) {
378
+ fatal("DICTIONARY: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d",
379
+ name, JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
380
+ }
381
+
382
+ // if *Rc == 0, new item added to the array
383
+ // otherwise the existing item value is returned in *Rc
384
+
385
+ // we return a pointer to a pointer, so that the caller can
386
+ // put anything needed at the value of the index.
387
+ // The pointer to pointer we return has to be used before
388
+ // any other operation that may change the index (insert/delete).
389
+ return (NAME_VALUE **)Rc;
390
+}
391
+
392
+static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) {
393
+ (void)nv;
394
+
395
+ if(unlikely(!dict->JudyHSArray)) return 0;
396
+
397
+ JError_t J_Error;
398
+ int ret = JudyHSDel(&dict->JudyHSArray, (void *)name, name_len, &J_Error);
399
+ if(unlikely(ret == JERR)) {
400
+ error("DICTIONARY: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", name,
401
+ JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
402
+ return 0;
403
+ }
404
+
405
+ // Hey, this is problematic! We need the value back, not just an int with a status!
406
+ // https://sourceforge.net/p/judy/feature-requests/23/
407
+
408
+ if(unlikely(ret == 0)) {
409
+ // not found in the dictionary
410
+ return 0;
411
+ }
412
+ else {
413
+ // found and deleted from the dictionary
414
+ return 1;
415
+ }
416
+}
417
+
418
+static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
419
+ if(unlikely(!dict->JudyHSArray)) return NULL;
420
+
421
+ DICTIONARY_STATS_SEARCHES_PLUS1(dict);
422
+
423
+ Pvoid_t *Rc;
424
+ Rc = JudyHSGet(dict->JudyHSArray, (void *)name, name_len);
425
+ if(likely(Rc)) {
426
+ // found in the hash table
427
+ return (NAME_VALUE *)*Rc;
428
+ }
429
+ else {
430
+ // not found in the hash table
431
+ return NULL;
432
+ }
433
+}
434
+
435
+static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) {
436
+ (void)dict;
437
+ (void)name;
438
+ (void)name_len;
439
+ (void)nv;
440
+ ;
441
+}
442
+
443
+#endif // DICTIONARY_WITH_JUDYHS
444
+
445
// ----------------------------------------------------------------------------
74
-// internal methods
446
+// linked list management
447
+
448
+static inline void linkedlist_namevalue_link_unsafe(DICTIONARY *dict, NAME_VALUE *nv) {
449
+ if (unlikely(!dict->first_item)) {
450
+ // we are the only ones here
451
+ nv->next = NULL;
452
+ nv->prev = NULL;
453
+ dict->first_item = dict->last_item = nv;
454
+ return;
455
+ }
456
76
-static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const char *name, void *value, size_t value_len, uint32_t hash) {
457
+ if(dict->flags & DICTIONARY_FLAG_ADD_IN_FRONT) {
458
+ // add it at the beginning
459
+ nv->prev = NULL;
460
+ nv->next = dict->first_item;
461
+
462
+ if (likely(nv->next)) nv->next->prev = nv;
463
+ dict->first_item = nv;
464
+ }
465
+ else {
466
+ // add it at the end
467
+ nv->next = NULL;
468
+ nv->prev = dict->last_item;
469
+
470
+ if (likely(nv->prev)) nv->prev->next = nv;
471
+ dict->last_item = nv;
472
+ }
473
+}
474
+
475
+static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VALUE *nv) {
476
+ if(nv->next) nv->next->prev = nv->prev;
477
+ if(nv->prev) nv->prev->next = nv->next;
478
+ if(dict->first_item == nv) dict->first_item = nv->next;
479
+ if(dict->last_item == nv) dict->last_item = nv->prev;
480
+}
481
+
482
+// ----------------------------------------------------------------------------
483
+// NAME_VALUE methods
484
+
485
+static inline size_t namevalue_alloc_size(DICTIONARY *dict) {
486
+ return (dict->flags & DICTIONARY_FLAG_WITH_STATISTICS) ? sizeof(NAME_VALUE_WITH_STATS) : sizeof(NAME_VALUE);
487
+}
488
+
489
+static inline size_t namevalue_get_namelen(DICTIONARY *dict, NAME_VALUE *nv) {
490
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
491
+ NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
492
+ return nvs->name_len;
493
+ }
494
+ return 0;
495
+}
496
+static inline size_t namevalue_get_valuelen(DICTIONARY *dict, NAME_VALUE *nv) {
497
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
498
+ NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
499
+ return nvs->value_len;
500
+ }
501
+ return 0;
502
+}
503
+static inline void namevalue_set_valuelen(DICTIONARY *dict, NAME_VALUE *nv, size_t value_len) {
504
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
505
+ NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
506
+ nvs->value_len = value_len;
507
+ }
508
+}
509
+static inline void namevalue_set_namevaluelen(DICTIONARY *dict, NAME_VALUE *nv, size_t name_len, size_t value_len) {
510
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
511
+ NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
512
+ nvs->name_len = name_len;
513
+ nvs->value_len = value_len;
514
+ }
515
+}
516
+
517
+static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len) {
518
debug(D_DICTIONARY, "Creating name value entry for name '%s'.", name);
519
79
- NAME_VALUE *nv = callocz(1, sizeof(NAME_VALUE));
520
+ size_t size = namevalue_alloc_size(dict);
521
+ NAME_VALUE *nv = mallocz(size);
522
+ size_t allocated = size;
523
81
- if(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)
524
+ namevalue_set_namevaluelen(dict, nv, name_len, value_len);
525
+
526
+ if(likely(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))
527
nv->name = (char *)name;
528
else {
84
- nv->name = strdupz(name);
529
+ nv->name = mallocz(name_len);
530
+ memcpy(nv->name, name, name_len);
531
+ allocated += name_len;
532
}
533
87
- nv->hash = (hash)?hash:simple_hash(nv->name);
88
-
89
- if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)
534
+ if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))
535
nv->value = value;
536
else {
92
- nv->value = mallocz(value_len);
93
- memcpy(nv->value, value, value_len);
94
- }
537
+ if(likely(value_len)) {
538
+ if (value) {
539
+ // a value has been supplied
540
+ // copy it
541
+ nv->value = mallocz(value_len);
542
+ memcpy(nv->value, value, value_len);
543
+ }
544
+ else {
545
+ // no value has been supplied
546
+ // allocate a clear memory block
547
+ nv->value = callocz(1, value_len);
548
+ }
549
+ }
550
+ else {
551
+ // the caller want an item without any value
552
+ nv->value = NULL;
553
+ }
554
96
- // index it
97
- NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(dict);
98
- if(unlikely(avl_insert(&((dict)->values_index), (avl_t *)(nv)) != (avl_t *)nv))
99
- error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary.");
555
+ allocated += value_len;
556
+ }
557
101
- NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(dict);
558
+ DICTIONARY_STATS_ENTRIES_PLUS1(dict, allocated);
559
560
return nv;
561
}
562
106
-static void dictionary_name_value_destroy_nolock(DICTIONARY *dict, NAME_VALUE *nv) {
107
- debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name);
563
+static void namevalue_reset_unsafe(DICTIONARY *dict, NAME_VALUE *nv, void *value, size_t value_len) {
564
+ debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", nv->name);
565
109
- NETDATA_DICTIONARY_STATS_DELETES_PLUS1(dict);
110
- if(unlikely(avl_remove(&(dict->values_index), (avl_t *)(nv)) != (avl_t *)nv))
111
- error("dictionary: INTERNAL ERROR: dictionary invalid removal of node.");
566
+ if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) {
567
+ debug(D_DICTIONARY, "Dictionary: linking value to '%s'", nv->name);
568
+ nv->value = value;
569
+ namevalue_set_valuelen(dict, nv, value_len);
570
+ }
571
+ else {
572
+ debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", nv->name);
573
+ DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, namevalue_get_valuelen(dict, nv), value_len);
574
+
575
+ void *old = nv->value;
576
+ void *new = mallocz(value_len);
577
+ memcpy(new, value, value_len);
578
+ nv->value = new;
579
+ namevalue_set_valuelen(dict, nv, value_len);
580
+
581
+ debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", nv->name);
582
+ freez(old);
583
+ }
584
+}
585
+
586
+static size_t namevalue_destroy_unsafe(DICTIONARY *dict, NAME_VALUE *nv) {
587
+ debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name);
588
113
- NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(dict);
589
+ size_t freed = 0;
590
115
- if(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) {
116
- debug(D_REGISTRY, "Dictionary freeing value of '%s'", nv->name);
591
+ if(unlikely(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))) {
592
+ debug(D_DICTIONARY, "Dictionary freeing value of '%s'", nv->name);
593
freez(nv->value);
594
+ freed += namevalue_get_valuelen(dict, nv);
595
}
596
120
- if(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)) {
121
- debug(D_REGISTRY, "Dictionary freeing name '%s'", nv->name);
597
+ if(unlikely(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))) {
598
+ debug(D_DICTIONARY, "Dictionary freeing name '%s'", nv->name);
599
freez(nv->name);
600
+ freed += namevalue_get_namelen(dict, nv);
601
}
602
603
freez(nv);
604
+ freed += namevalue_alloc_size(dict);
605
+
606
+ DICTIONARY_STATS_ENTRIES_MINUS1(dict, freed);
607
+
608
+ return freed;
609
}
610
611
// ----------------------------------------------------------------------------
129
-// API - basic methods
612
+// API - dictionary management
613
131
-DICTIONARY *dictionary_create(uint8_t flags) {
614
+DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags) {
615
debug(D_DICTIONARY, "Creating dictionary.");
616
134
- DICTIONARY *dict = callocz(1, sizeof(DICTIONARY));
135
-
136
- if(flags & DICTIONARY_FLAG_WITH_STATISTICS)
137
- dict->stats = callocz(1, sizeof(struct dictionary_stats));
617
+ if((flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) && (flags & DICTIONARY_FLAG_SINGLE_THREADED)) {
618
+ error("DICTIONARY: requested reference counters on single threaded dictionary. Not adding reference counters.");
619
+ flags &= ~DICTIONARY_FLAG_REFERENCE_COUNTERS;
620
+ }
621
139
- if(!(flags & DICTIONARY_FLAG_SINGLE_THREADED)) {
140
- dict->rwlock = callocz(1, sizeof(netdata_rwlock_t));
141
- netdata_rwlock_init(dict->rwlock);
622
+ if(flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) {
623
+ // we need statistics to allocate the extra NAME_VALUE attributes
624
+ flags |= DICTIONARY_FLAG_WITH_STATISTICS;
625
}
626
144
- avl_init(&dict->values_index, name_value_compare);
627
+ DICTIONARY *dict = callocz(1, sizeof(DICTIONARY));
628
+ size_t allocated = sizeof(DICTIONARY);
629
+
630
dict->flags = flags;
631
+ dict->first_item = dict->last_item = NULL;
632
147
- return dict;
633
+ allocated += dictionary_lock_init(dict);
634
+ allocated += reference_counter_init(dict);
635
+
636
+ if(flags & DICTIONARY_FLAG_WITH_STATISTICS) {
637
+ dict->stats = callocz(1, sizeof(struct dictionary_stats));
638
+ allocated += sizeof(struct dictionary_stats);
639
+ dict->stats->memory = allocated;
640
+ }
641
+ else
642
+ dict->stats = NULL;
643
+
644
+ hashtable_init_unsafe(dict);
645
+ return (DICTIONARY *)dict;
646
}
647
150
-void dictionary_destroy(DICTIONARY *dict) {
648
+size_t dictionary_destroy(DICTIONARY *dict) {
649
debug(D_DICTIONARY, "Destroying dictionary.");
650
153
- dictionary_write_lock(dict);
651
+ dictionary_lock_wrlock(dict);
652
+
653
+ size_t freed = 0;
654
+ NAME_VALUE *nv = dict->first_item;
655
+ while (nv) {
656
+ // cache nv->next
657
+ // because we are going to free nv
658
+ NAME_VALUE *nvnext = nv->next;
659
+ freed += namevalue_destroy_unsafe(dict, nv);
660
+ nv = nvnext;
661
+ // to speed up destruction, we don't
662
+ // unlink nv from the linked-list here
663
+ }
664
+
665
+ dict->first_item = NULL;
666
+ dict->last_item = NULL;
667
155
- while(dict->values_index.root)
156
- dictionary_name_value_destroy_nolock(dict, (NAME_VALUE *)dict->values_index.root);
668
+ // destroy the dictionary
669
+ freed += hashtable_destroy_unsafe(dict);
670
671
dictionary_unlock(dict);
672
+ freed += dictionary_lock_free(dict);
673
+ freed += reference_counter_free(dict);
674
160
- if(dict->stats)
675
+ if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
676
freez(dict->stats);
162
-
163
- if(dict->rwlock) {
164
- netdata_rwlock_destroy(dict->rwlock);
165
- freez(dict->rwlock);
677
+ dict->stats = NULL;
678
+ freed += sizeof(struct dictionary_stats);
679
}
680
681
freez(dict);
682
+ freed += sizeof(DICTIONARY);
683
+
684
+ return freed;
685
}
686
687
// ----------------------------------------------------------------------------
688
+// API - items management
689
173
-void *dictionary_set_with_name_ptr(DICTIONARY *dict, const char *name, void *value, size_t value_len, char **name_ptr) {
174
- debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name);
175
-
176
- uint32_t hash = simple_hash(name);
690
+void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) {
691
+ if(unlikely(!name || !*name)) {
692
+ error("Attempted to dictionary_set() a dictionary item without a name");
693
+ return NULL;
694
+ }
695
178
- dictionary_write_lock(dict);
696
+ size_t name_len = strlen(name) + 1; // we need the terminating null too
697
180
- NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, hash);
181
- if(unlikely(!nv)) {
182
- debug(D_DICTIONARY, "Dictionary entry with name '%s' not found. Creating a new one.", name);
698
+ debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name);
699
184
- nv = dictionary_name_value_create_nolock(dict, name, value, value_len, hash);
185
- if(unlikely(!nv))
186
- fatal("Cannot create name_value.");
700
+ // DISCUSSION:
701
+ // Is it better to gain a read-lock and do a hashtable_get_unsafe()
702
+ // before we write lock to do hashtable_insert_unsafe()?
703
+ //
704
+ // Probably this depends on the use case.
705
+ // For statsd for example that does dictionary_set() to update received values,
706
+ // it could be beneficial to do a get() before we insert().
707
+ //
708
+ // But the caller has the option to do this on his/her own.
709
+ // So, let's do the fastest here and let the caller decide the flow of calls.
710
+
711
+ NAME_VALUE *nv, **pnv = hashtable_insert_unsafe(dict, name, name_len);
712
+ if(likely(*pnv == 0)) {
713
+ // a new item added to the index
714
+ nv = *pnv = namevalue_create_unsafe(dict, name, name_len, value, value_len);
715
+ hashtable_inserted_name_value_unsafe(dict, name, name_len, nv);
716
+ linkedlist_namevalue_link_unsafe(dict, nv);
717
+
718
+ if(dict->ins_callback)
719
+ dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
720
}
188
- else if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE)) {
189
- debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", name);
721
+ else {
722
+ // the item is already in the index
723
+ // so, either we will return the old one
724
+ // or overwrite the value, depending on dictionary flags
725
191
- if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE) {
192
- debug(D_REGISTRY, "Dictionary: linking value to '%s'", name);
193
- nv->value = value;
194
- }
195
- else {
196
- debug(D_REGISTRY, "Dictionary: cloning value to '%s'", name);
726
+ nv = *pnv;
727
+ if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE))
728
+ namevalue_reset_unsafe(dict, nv, value, value_len);
729
+ }
730
198
- // copy the new value without breaking
199
- // any other thread accessing the same entry
200
- void *new = mallocz(value_len),
201
- *old = nv->value;
731
+ return nv->value;
732
+}
733
203
- memcpy(new, value, value_len);
204
- nv->value = new;
734
+void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) {
735
+ dictionary_lock_wrlock(dict);
736
+ void *ret = dictionary_set_unsafe(dict, name, value, value_len);
737
+ dictionary_unlock(dict);
738
+ return ret;
739
+}
740
206
- debug(D_REGISTRY, "Dictionary: freeing old value of '%s'", name);
207
- freez(old);
208
- }
741
+void *dictionary_get_unsafe(DICTIONARY *dict, const char *name) {
742
+ if(unlikely(!name || !*name)) {
743
+ error("Attempted to dictionary_get() without a name");
744
+ return NULL;
745
}
746
211
- dictionary_unlock(dict);
747
+ size_t name_len = strlen(name) + 1; // we need the terminating null too
748
213
- if(name_ptr) *name_ptr = nv->name;
214
- return nv->value;
215
-}
216
-
217
-void *dictionary_get(DICTIONARY *dict, const char *name) {
749
debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name);
750
220
- dictionary_read_lock(dict);
221
- NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0);
222
- dictionary_unlock(dict);
223
-
751
+ NAME_VALUE *nv = hashtable_get_unsafe(dict, name, name_len);
752
if(unlikely(!nv)) {
753
debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name);
754
return NULL;
@@ -230,101 +758,506 @@ void *dictionary_get(DICTIONARY *dict, const char *name) {
758
return nv->value;
759
}
760
233
-int dictionary_del(DICTIONARY *dict, const char *name) {
234
- int ret;
761
+void *dictionary_get(DICTIONARY *dict, const char *name) {
762
+ dictionary_lock_rlock(dict);
763
+ void *ret = dictionary_get_unsafe(dict, name);
764
+ dictionary_unlock(dict);
765
+ return ret;
766
+}
767
+
768
+int dictionary_del_unsafe(DICTIONARY *dict, const char *name) {
769
+ if(unlikely(!name || !*name)) {
770
+ error("Attempted to dictionary_det() without a name");
771
+ return -1;
772
+ }
773
+
774
+ size_t name_len = strlen(name) + 1; // we need the terminating null too
775
776
debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name);
777
238
- dictionary_write_lock(dict);
778
+ // Unfortunately, the JudyHSDel() does not return the value of the
779
+ // item that was deleted, so we have to find it before we delete it,
780
+ // since we need to release our structures too.
781
240
- NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0);
782
+ int ret;
783
+ NAME_VALUE *nv = hashtable_get_unsafe(dict, name, name_len);
784
if(unlikely(!nv)) {
785
debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name);
786
ret = -1;
787
}
788
else {
789
debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name);
247
- dictionary_name_value_destroy_nolock(dict, nv);
790
+
791
+ if(hashtable_delete_unsafe(dict, name, name_len, nv) == 0)
792
+ error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s' that is not in the index", name);
793
+
794
+ if(!reference_counter_mark_deleted(dict, nv)) {
795
+ linkedlist_namevalue_unlink_unsafe(dict, nv);
796
+
797
+ if(dict->del_callback)
798
+ dict->del_callback(nv->name, nv->value, dict->del_callback_data);
799
+
800
+ namevalue_destroy_unsafe(dict, nv);
801
+ }
802
ret = 0;
803
}
804
+ return ret;
805
+}
806
807
+int dictionary_del(DICTIONARY *dict, const char *name) {
808
+ dictionary_lock_wrlock(dict);
809
+ int ret = dictionary_del_unsafe(dict, name);
810
dictionary_unlock(dict);
252
-
811
return ret;
812
}
813
256
-
814
// ----------------------------------------------------------------------------
258
-// API - walk through the dictionary
259
-// the dictionary is locked for reading while this happens
260
-// do not user other dictionary calls while walking the dictionary - deadlock!
815
+// traversal with loop
816
+
817
+void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
818
+ if(unlikely(!dfe || !dict)) return NULL;
819
262
-static int dictionary_walker(avl_t *a, int (*callback)(void *entry, void *data), void *data) {
263
- int total = 0, ret = 0;
820
+ dfe->dict = dict;
821
+ dfe->started_ut = now_realtime_usec();
822
265
- if(a->avl_link[0]) {
266
- ret = dictionary_walker(a->avl_link[0], callback, data);
267
- if(ret < 0) return ret;
268
- total += ret;
823
+ if(rw == 'r' || rw == 'R')
824
+ dictionary_lock_rlock(dict);
825
+ else
826
+ dictionary_lock_wrlock(dict);
827
+
828
+ NAME_VALUE *nv = dict->first_item;
829
+ dfe->last_position_index = (void *)nv;
830
+
831
+ if(likely(nv)) {
832
+ dfe->next_position_index = (void *)nv->next;
833
+ dfe->name = nv->name;
834
+ dfe->value = (void *)nv->value;
835
+ reference_counter_acquire(dict, nv);
836
}
837
+ else {
838
+ dfe->next_position_index = NULL;
839
+ dfe->name = NULL;
840
+ dfe->value = NULL;
841
+ }
842
+
843
+ return dfe->value;
844
+}
845
+
846
+void *dictionary_foreach_next(DICTFE *dfe) {
847
+ if(unlikely(!dfe || !dfe->dict)) return NULL;
848
+
849
+ NAME_VALUE *nv = (NAME_VALUE *)dfe->last_position_index;
850
+ if(likely(nv))
851
+ reference_counter_release(dfe->dict, nv);
852
+
853
+ nv = dfe->last_position_index = dfe->next_position_index;
854
271
- ret = callback(((NAME_VALUE *)a)->value, data);
272
- if(ret < 0) return ret;
273
- total += ret;
855
+ if(likely(nv)) {
856
+ dfe->next_position_index = (void *)nv->next;
857
+ dfe->name = nv->name;
858
+ dfe->value = (void *)nv->value;
859
275
- if(a->avl_link[1]) {
276
- ret = dictionary_walker(a->avl_link[1], callback, data);
277
- if (ret < 0) return ret;
278
- total += ret;
860
+ reference_counter_acquire(dfe->dict, nv);
861
+ }
862
+ else {
863
+ dfe->next_position_index = NULL;
864
+ dfe->name = NULL;
865
+ dfe->value = NULL;
866
}
867
281
- return total;
868
+ return dfe->value;
869
}
870
284
-int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) {
871
+usec_t dictionary_foreach_done(DICTFE *dfe) {
872
+ if(unlikely(!dfe || !dfe->dict)) return 0;
873
+
874
+ NAME_VALUE *nv = (NAME_VALUE *)dfe->last_position_index;
875
+ if(nv)
876
+ reference_counter_release(dfe->dict, nv);
877
+
878
+ dictionary_unlock((DICTIONARY *)dfe->dict);
879
+ dfe->dict = NULL;
880
+ dfe->last_position_index = NULL;
881
+ dfe->next_position_index = NULL;
882
+ dfe->name = NULL;
883
+ dfe->value = NULL;
884
+
885
+ usec_t usec = now_realtime_usec() - dfe->started_ut;
886
+ dfe->started_ut = 0;
887
+
888
+ return usec;
889
+}
890
+
891
+// ----------------------------------------------------------------------------
892
+// API - walk through the dictionary
893
+// the dictionary is locked for reading while this happens
894
+// do not use other dictionary calls while walking the dictionary - deadlock!
895
+
896
+int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) {
897
+ if(rw == 'r' || rw == 'R')
898
+ dictionary_lock_rlock(dict);
899
+ else
900
+ dictionary_lock_wrlock(dict);
901
+
902
+ // written in such a way, that the callback can delete the active element
903
+
904
int ret = 0;
905
+ NAME_VALUE *nv = dict->first_item, *nv_next = nv->next;
906
+ while(nv) {
907
+ nv_next = nv->next;
908
+
909
+ reference_counter_acquire(dict, nv);
910
+ int r = callback(nv->name, nv->value, data);
911
+ reference_counter_release(dict, nv);
912
+ if(unlikely(r < 0)) {
913
+ ret = r;
914
+ break;
915
+ }
916
287
- dictionary_read_lock(dict);
917
+ ret += r;
918
289
- if(likely(dict->values_index.root))
290
- ret = dictionary_walker(dict->values_index.root, callback, data);
919
+ nv = nv_next;
920
+ }
921
922
dictionary_unlock(dict);
923
924
return ret;
925
}
926
297
-static int dictionary_walker_name_value(avl_t *a, int (*callback)(char *name, void *entry, void *data), void *data) {
298
- int total = 0, ret = 0;
927
+// ----------------------------------------------------------------------------
928
+// unit test
929
300
- if(a->avl_link[0]) {
301
- ret = dictionary_walker_name_value(a->avl_link[0], callback, data);
302
- if(ret < 0) return ret;
303
- total += ret;
930
+static void dictionary_unittest_free_char_pp(char **pp, size_t entries) {
931
+ for(size_t i = 0; i < entries ;i++)
932
+ freez(pp[i]);
933
+
934
+ freez(pp);
935
+}
936
+
937
+static char **dictionary_unittest_generate_names(size_t entries) {
938
+ char **names = mallocz(sizeof(char *) * entries);
939
+ for(size_t i = 0; i < entries ;i++) {
940
+ char buf[25 + 1] = "";
941
+ snprintfz(buf, 25, "name.%zu.0123456789.%zu \t !@#$%%^&*(),./[]{}\\|~`", i, entries / 2 + i);
942
+ names[i] = strdupz(buf);
943
}
944
+ return names;
945
+}
946
306
- ret = callback(((NAME_VALUE *)a)->name, ((NAME_VALUE *)a)->value, data);
307
- if(ret < 0) return ret;
308
- total += ret;
947
+static char **dictionary_unittest_generate_values(size_t entries) {
948
+ char **values = mallocz(sizeof(char *) * entries);
949
+ for(size_t i = 0; i < entries ;i++) {
950
+ char buf[25 + 1] = "";
951
+ snprintfz(buf, 25, "value-%zu-0987654321.%zu%%^&*(),. \t !@#$/[]{}\\|~`", i, entries / 2 + i);
952
+ values[i] = strdupz(buf);
953
+ }
954
+ return values;
955
+}
956
310
- if(a->avl_link[1]) {
311
- ret = dictionary_walker_name_value(a->avl_link[1], callback, data);
312
- if (ret < 0) return ret;
313
- total += ret;
957
+static size_t dictionary_unittest_set_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
958
+ size_t errors = 0;
959
+ for(size_t i = 0; i < entries ;i++) {
960
+ size_t vallen = strlen(values[i]) + 1;
961
+ char *val = (char *)dictionary_set(dict, names[i], values[i], vallen);
962
+ if(val == values[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
963
+ if(!val || memcmp(val, values[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
964
}
965
+ return errors;
966
+}
967
316
- return total;
968
+static size_t dictionary_unittest_set_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
969
+ size_t errors = 0;
970
+ for(size_t i = 0; i < entries ;i++) {
971
+ size_t vallen = strlen(values[i]) + 1;
972
+ char *val = (char *)dictionary_set(dict, names[i], values[i], vallen);
973
+ if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
974
+ }
975
+ return errors;
976
}
977
319
-int dictionary_get_all_name_value(DICTIONARY *dict, int (*callback)(char *name, void *entry, void *data), void *data) {
320
- int ret = 0;
978
+static size_t dictionary_unittest_get_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
979
+ size_t errors = 0;
980
+ for(size_t i = 0; i < entries ;i++) {
981
+ size_t vallen = strlen(values[i]) + 1;
982
+ char *val = (char *)dictionary_get(dict, names[i]);
983
+ if(val == values[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
984
+ if(!val || memcmp(val, values[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
985
+ }
986
+ return errors;
987
+}
988
322
- dictionary_read_lock(dict);
989
+static size_t dictionary_unittest_get_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
990
+ size_t errors = 0;
991
+ for(size_t i = 0; i < entries ;i++) {
992
+ char *val = (char *)dictionary_get(dict, names[i]);
993
+ if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
994
+ }
995
+ return errors;
996
+}
997
324
- if(likely(dict->values_index.root))
325
- ret = dictionary_walker_name_value(dict->values_index.root, callback, data);
998
+static size_t dictionary_unittest_get_nonexisting(DICTIONARY *dict, char **names, char **values, size_t entries) {
999
+ (void)names;
1000
+ size_t errors = 0;
1001
+ for(size_t i = 0; i < entries ;i++) {
1002
+ char *val = (char *)dictionary_get(dict, values[i]);
1003
+ if(val) { fprintf(stderr, ">>> %s() returns non-existing item\n", __FUNCTION__); errors++; }
1004
+ }
1005
+ return errors;
1006
+}
1007
327
- dictionary_unlock(dict);
1008
+static size_t dictionary_unittest_del_nonexisting(DICTIONARY *dict, char **names, char **values, size_t entries) {
1009
+ (void)names;
1010
+ size_t errors = 0;
1011
+ for(size_t i = 0; i < entries ;i++) {
1012
+ int ret = dictionary_del(dict, values[i]);
1013
+ if(ret != -1) { fprintf(stderr, ">>> %s() deleted non-existing item\n", __FUNCTION__); errors++; }
1014
+ }
1015
+ return errors;
1016
+}
1017
329
- return ret;
1018
+static size_t dictionary_unittest_del_existing(DICTIONARY *dict, char **names, char **values, size_t entries) {
1019
+ (void)values;
1020
+ size_t errors = 0;
1021
+
1022
+ size_t forward_from = 0, forward_to = entries / 3;
1023
+ size_t middle_from = forward_to, middle_to = entries * 2 / 3;
1024
+ size_t backward_from = middle_to, backward_to = entries;
1025
+
1026
+ for(size_t i = forward_from; i < forward_to ;i++) {
1027
+ int ret = dictionary_del(dict, names[i]);
1028
+ if(ret == -1) { fprintf(stderr, ">>> %s() didn't delete (forward) existing item\n", __FUNCTION__); errors++; }
1029
+ }
1030
+
1031
+ for(size_t i = middle_to - 1; i >= middle_from ;i--) {
1032
+ int ret = dictionary_del(dict, names[i]);
1033
+ if(ret == -1) { fprintf(stderr, ">>> %s() didn't delete (middle) existing item\n", __FUNCTION__); errors++; }
1034
+ }
1035
+
1036
+ for(size_t i = backward_to - 1; i >= backward_from ;i--) {
1037
+ int ret = dictionary_del(dict, names[i]);
1038
+ if(ret == -1) { fprintf(stderr, ">>> %s() didn't delete (backward) existing item\n", __FUNCTION__); errors++; }
1039
+ }
1040
+
1041
+ return errors;
1042
+}
1043
+
1044
+static size_t dictionary_unittest_reset_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
1045
+ (void)values;
1046
+ // set the name as value too
1047
+ size_t errors = 0;
1048
+ for(size_t i = 0; i < entries ;i++) {
1049
+ size_t vallen = strlen(names[i]) + 1;
1050
+ char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
1051
+ if(val == names[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
1052
+ if(!val || memcmp(val, names[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
1053
+ }
1054
+ return errors;
1055
+}
1056
+
1057
+static size_t dictionary_unittest_reset_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
1058
+ (void)values;
1059
+ // set the name as value too
1060
+ size_t errors = 0;
1061
+ for(size_t i = 0; i < entries ;i++) {
1062
+ size_t vallen = strlen(names[i]) + 1;
1063
+ char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
1064
+ if(val != names[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
1065
+ if(!val) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
1066
+ }
1067
+ return errors;
1068
+}
1069
+
1070
+static size_t dictionary_unittest_reset_dont_overwrite_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
1071
+ // set the name as value too
1072
+ size_t errors = 0;
1073
+ for(size_t i = 0; i < entries ;i++) {
1074
+ size_t vallen = strlen(names[i]) + 1;
1075
+ char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
1076
+ if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
1077
+ }
1078
+ return errors;
1079
+}
1080
+
1081
+static int dictionary_unittest_walkthrough_callback(const char *name, void *value, void *data) {
1082
+ (void)name;
1083
+ (void)value;
1084
+ (void)data;
1085
+ return 1;
1086
+}
1087
+
1088
+static size_t dictionary_unittest_walkthrough(DICTIONARY *dict, char **names, char **values, size_t entries) {
1089
+ (void)names;
1090
+ (void)values;
1091
+ int sum = dictionary_walkthrough_read(dict, dictionary_unittest_walkthrough_callback, NULL);
1092
+ if(sum < (int)entries) return entries - sum;
1093
+ else return sum - entries;
1094
+}
1095
+
1096
+static int dictionary_unittest_walkthrough_delete_this_callback(const char *name, void *value, void *data) {
1097
+ (void)value;
1098
+
1099
+ if(dictionary_del_having_write_lock((DICTIONARY *)data, name) == -1)
1100
+ return 0;
1101
+
1102
+ return 1;
1103
+}
1104
+
1105
+static size_t dictionary_unittest_walkthrough_delete_this(DICTIONARY *dict, char **names, char **values, size_t entries) {
1106
+ (void)names;
1107
+ (void)values;
1108
+ int sum = dictionary_walkthrough_write(dict, dictionary_unittest_walkthrough_delete_this_callback, dict);
1109
+ if(sum < (int)entries) return entries - sum;
1110
+ else return sum - entries;
1111
+}
1112
+
1113
+static int dictionary_unittest_walkthrough_stop_callback(const char *name, void *value, void *data) {
1114
+ (void)name;
1115
+ (void)value;
1116
+ (void)data;
1117
+ return -1;
1118
+}
1119
+
1120
+static size_t dictionary_unittest_walkthrough_stop(DICTIONARY *dict, char **names, char **values, size_t entries) {
1121
+ (void)names;
1122
+ (void)values;
1123
+ (void)entries;
1124
+ int sum = dictionary_walkthrough_read(dict, dictionary_unittest_walkthrough_stop_callback, NULL);
1125
+ if(sum != -1) return 1;
1126
+ return 0;
1127
+}
1128
+
1129
+static size_t dictionary_unittest_foreach(DICTIONARY *dict, char **names, char **values, size_t entries) {
1130
+ (void)names;
1131
+ (void)values;
1132
+ (void)entries;
1133
+ size_t count = 0;
1134
+ char *item;
1135
+ dfe_start_read(dict, item)
1136
+ count++;
1137
+ dfe_done(item);
1138
+
1139
+ if(count > entries) return count - entries;
1140
+ return entries - count;
1141
+}
1142
+
1143
+static size_t dictionary_unittest_foreach_delete_this(DICTIONARY *dict, char **names, char **values, size_t entries) {
1144
+ (void)names;
1145
+ (void)values;
1146
+ (void)entries;
1147
+ size_t count = 0;
1148
+ char *item;
1149
+ dfe_start_write(dict, item)
1150
+ if(dictionary_del_having_write_lock(dict, item_name) != -1) count++;
1151
+ dfe_done(item);
1152
+
1153
+ if(count > entries) return count - entries;
1154
+ return entries - count;
1155
+}
1156
+
1157
+static size_t dictionary_unittest_destroy(DICTIONARY *dict, char **names, char **values, size_t entries) {
1158
+ (void)names;
1159
+ (void)values;
1160
+ (void)entries;
1161
+ size_t bytes = dictionary_destroy(dict);
1162
+ fprintf(stderr, " %s() freed %zu bytes,", __FUNCTION__, bytes);
1163
+ return 0;
1164
+}
1165
+
1166
+static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *message, char **names, char **values, size_t entries, size_t *errors, size_t (*callback)(DICTIONARY *dict, char **names, char **values, size_t entries)) {
1167
+ fprintf(stderr, "%-40s... ", message);
1168
+
1169
+ usec_t started = now_realtime_usec();
1170
+ size_t errs = callback(dict, names, values, entries);
1171
+ usec_t ended = now_realtime_usec();
1172
+ usec_t dt = ended - started;
1173
+
1174
+ if(callback == dictionary_unittest_destroy) dict = NULL;
1175
+
1176
+ fprintf(stderr, " %zu errors, %zu items in dictionary, %llu usec \n", errs, dict? dictionary_stats_entries(dict):0, dt);
1177
+ *errors += errs;
1178
+ return dt;
1179
+}
1180
+
1181
+void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
1182
+ dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone);
1183
+ dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_clone);
1184
+ dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
1185
+ dictionary_unittest_run_and_measure_time(dict, "resetting entries", names, values, entries, errors, dictionary_unittest_reset_clone);
1186
+ dictionary_unittest_run_and_measure_time(dict, "deleting non-existing entries", names, values, entries, errors, dictionary_unittest_del_nonexisting);
1187
+ dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, errors, dictionary_unittest_foreach);
1188
+ dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback", names, values, entries, errors, dictionary_unittest_walkthrough);
1189
+ dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback stop", names, values, entries, errors, dictionary_unittest_walkthrough_stop);
1190
+ dictionary_unittest_run_and_measure_time(dict, "deleting existing entries", names, values, entries, errors, dictionary_unittest_del_existing);
1191
+ dictionary_unittest_run_and_measure_time(dict, "walking through empty", names, values, 0, errors, dictionary_unittest_walkthrough);
1192
+ dictionary_unittest_run_and_measure_time(dict, "traverse foreach empty", names, values, 0, errors, dictionary_unittest_foreach);
1193
+ dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
1194
+}
1195
+
1196
+void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
1197
+ dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_nonclone);
1198
+ dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_nonclone);
1199
+ dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
1200
+ dictionary_unittest_run_and_measure_time(dict, "resetting entries", names, values, entries, errors, dictionary_unittest_reset_nonclone);
1201
+ dictionary_unittest_run_and_measure_time(dict, "deleting non-existing entries", names, values, entries, errors, dictionary_unittest_del_nonexisting);
1202
+ dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, errors, dictionary_unittest_foreach);
1203
+ dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback", names, values, entries, errors, dictionary_unittest_walkthrough);
1204
+ dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback stop", names, values, entries, errors, dictionary_unittest_walkthrough_stop);
1205
+ dictionary_unittest_run_and_measure_time(dict, "deleting existing entries", names, values, entries, errors, dictionary_unittest_del_existing);
1206
+ dictionary_unittest_run_and_measure_time(dict, "walking through empty", names, values, 0, errors, dictionary_unittest_walkthrough);
1207
+ dictionary_unittest_run_and_measure_time(dict, "traverse foreach empty", names, values, 0, errors, dictionary_unittest_foreach);
1208
+ dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
1209
+}
1210
+
1211
+int dictionary_unittest(size_t entries) {
1212
+ if(entries < 10) entries = 10;
1213
+
1214
+ DICTIONARY *dict;
1215
+ size_t errors = 0;
1216
+
1217
+ fprintf(stderr, "Generating %zu names and values...\n", entries);
1218
+ char **names = dictionary_unittest_generate_names(entries);
1219
+ char **values = dictionary_unittest_generate_values(entries);
1220
+
1221
+ fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
1222
+ dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS);
1223
+ dictionary_unittest_clone(dict, names, values, entries, &errors);
1224
+
1225
+ fprintf(stderr, "\nCreating dictionary multi threaded, clone, %zu items\n", entries);
1226
+ dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS);
1227
+ dictionary_unittest_clone(dict, names, values, entries, &errors);
1228
+
1229
+ fprintf(stderr, "\nCreating dictionary single threaded, non-clone, add-in-front options, %zu items\n", entries);
1230
+ dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
1231
+ dictionary_unittest_nonclone(dict, names, values, entries, &errors);
1232
+
1233
+ fprintf(stderr, "\nCreating dictionary multi threaded, non-clone, add-in-front options, %zu items\n", entries);
1234
+ dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
1235
+ dictionary_unittest_nonclone(dict, names, values, entries, &errors);
1236
+
1237
+ fprintf(stderr, "\nCreating dictionary single-threaded, non-clone, don't overwrite options, %zu items\n", entries);
1238
+ dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1239
+ dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
1240
+ dictionary_unittest_run_and_measure_time(dict, "resetting non-overwrite entries", names, values, entries, &errors, dictionary_unittest_reset_dont_overwrite_nonclone);
1241
+ dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, &errors, dictionary_unittest_foreach);
1242
+ dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback", names, values, entries, &errors, dictionary_unittest_walkthrough);
1243
+ dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback stop", names, values, entries, &errors, dictionary_unittest_walkthrough_stop);
1244
+ dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
1245
+
1246
+ fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
1247
+ dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1248
+ dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
1249
+ dictionary_unittest_run_and_measure_time(dict, "walkthrough write delete this", names, values, entries, &errors, dictionary_unittest_walkthrough_delete_this);
1250
+ dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
1251
+
1252
+ fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
1253
+ dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1254
+ dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
1255
+ dictionary_unittest_run_and_measure_time(dict, "foreach write delete this", names, values, entries, &errors, dictionary_unittest_foreach_delete_this);
1256
+ dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
1257
+
1258
+ dictionary_unittest_free_char_pp(names, entries);
1259
+ dictionary_unittest_free_char_pp(values, entries);
1260
+
1261
+ fprintf(stderr, "\n%zu errors found\n", errors);
1262
+ return (int)errors;
1263
}
libnetdata/dictionary/dictionary.h
+173
-39
@@ -5,47 +5,181 @@
5
6
#include "../libnetdata.h"
7
8
-struct dictionary_stats {
9
- unsigned long long inserts;
10
- unsigned long long deletes;
11
- unsigned long long searches;
12
- unsigned long long entries;
13
-};
14
-
15
-typedef struct name_value {
16
- avl_t avl_node; // the index - this has to be first!
17
-
18
- uint32_t hash; // a simple hash to speed up searching
19
- // we first compare hashes, and only if the hashes are equal we do string comparisons
20
-
21
- char *name;
22
- void *value;
23
-} NAME_VALUE;
24
-
25
-typedef struct dictionary {
26
- avl_tree_type values_index;
27
-
28
- uint8_t flags;
29
-
30
- struct dictionary_stats *stats;
31
- netdata_rwlock_t *rwlock;
32
-} DICTIONARY;
33
-
34
-#define DICTIONARY_FLAG_NONE 0x00000000
35
-#define DICTIONARY_FLAG_SINGLE_THREADED 0x00000001
36
-#define DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE 0x00000002
37
-#define DICTIONARY_FLAG_NAME_LINK_DONT_CLONE 0x00000004
38
-#define DICTIONARY_FLAG_WITH_STATISTICS 0x00000008
39
-#define DICTIONARY_FLAG_DONT_OVERWRITE_VALUE 0x00000010
40
-
41
-extern DICTIONARY *dictionary_create(uint8_t flags);
42
-extern void dictionary_destroy(DICTIONARY *dict);
43
-extern void *dictionary_set_with_name_ptr(DICTIONARY *dict, const char *name, void *value, size_t value_len, char **name_ptr) NEVERNULL;
44
-#define dictionary_set(dict, name, value, value_len) dictionary_set_with_name_ptr(dict, name, value, value_len, NULL)
8
+
9
+/*
10
+ * Netdata DICTIONARY features:
11
+ *
12
+ * CLONE or LINK
13
+ * Names and Values in the dictionary can be cloned or linked.
14
+ * In clone mode, the dictionary does all the memory management.
15
+ * The default is clone for both names and values.
16
+ * Set DICTIONARY_FLAG_NAME_LINK_DONT_CLONE to link names.
17
+ * Set DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE to link names.
18
+ *
19
+ * ORDERED
20
+ * Items are ordered in the order they are added (new items are appended at the end).
21
+ * You may reverse the order by setting the flag DICTIONARY_FLAG_ADD_IN_FRONT.
22
+ *
23
+ * LOOKUP
24
+ * The dictionary uses JudyHS to maintain a very fast randomly accessible hash table.
25
+ *
26
+ * MULTI-THREADED and SINGLE-THREADED
27
+ * Each dictionary may be single threaded (no locks), or multi-threaded (multiple readers or one writer).
28
+ * The default is multi-threaded. Add the flag DICTIONARY_FLAG_SINGLE_THREADED for single-threaded.
29
+ *
30
+ * WALK-THROUGH and FOREACH traversal
31
+ * The dictionary can be traversed on read or write mode, either with a callback (walkthrough) or with
32
+ * a loop (foreach).
33
+ *
34
+ * In write mode traversal, the caller may delete only the current item, but may add as many items as needed.
35
+ *
36
+ */
37
+
38
+#ifndef DICTIONARY_INTERNALS
39
+typedef void DICTIONARY;
40
+#endif
41
+
42
+typedef enum dictionary_flags {
43
+ DICTIONARY_FLAG_NONE = 0, // the default is the opposite of all below
44
+ DICTIONARY_FLAG_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks)
45
+ DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy)
46
+ DICTIONARY_FLAG_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy)
47
+ DICTIONARY_FLAG_WITH_STATISTICS = (1 << 3), // maintain statistics about dictionary operations (default: disabled)
48
+ DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 4), // don't overwrite values of dictionary items (default: overwrite)
49
+ DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 5), // add dictionary items at the front of the linked list (default: at the end)
50
+ DICTIONARY_FLAG_RESERVED1 = (1 << 6), // this is reserved for DICTIONARY_FLAG_REFERENCE_COUNTERS
51
+} DICTIONARY_FLAGS;
52
+
53
+// Create a dictionary
54
+extern DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags);
55
+
56
+// an insert callback to be called just after an item is added to the dictionary
57
+// this callback is called while the dictionary is write locked!
58
+extern void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const char *name, void *value, void *data), void *data);
59
+
60
+// a delete callback to be called just before an item is deleted forever
61
+// this callback is called while the dictionary is write locked!
62
+extern void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const char *name, void *value, void *data), void *data);
63
+
64
+// Destroy a dictionary
65
+// returns the number of bytes freed
66
+// the returned value will not include name and value sizes if DICTIONARY_FLAG_WITH_STATISTICS is not set
67
+extern size_t dictionary_destroy(DICTIONARY *dict);
68
+
69
+// Set an item in the dictionary
70
+// - if an item with the same name does not exist, create one
71
+// - if an item with the same name exists, then:
72
+// a) if DICTIONARY_FLAG_DONT_OVERWRITE_VALUE is set, just return the existing value (ignore the new value)
73
+// else b) reset the value to the new value passed at the call
74
+//
75
+// When DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE is set, the value is linked, otherwise it is copied
76
+// When DICTIONARY_FLAG_NAME_LINK_DONT_CLONE is set, the name is linked, otherwise it is copied
77
+//
78
+// When neither DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE nor DICTIONARY_FLAG_NAME_LINK_DONT_CLONE are set, all the
79
+// memory management for names and values is done by the dictionary.
80
+//
81
+// Passing NULL as value, the dictionary will callocz() the newly allocated value, otherwise it will copy it.
82
+// Passing 0 as value_len, the dictionary will set the value to NULL (no allocations for value will be made).
83
+extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) NEVERNULL;
84
+
85
+// Get an item from the dictionary
86
+// If it returns NULL, the item is not found
87
extern void *dictionary_get(DICTIONARY *dict, const char *name);
88
+
89
+// Delete an item from the dictionary
90
+// returns 0 if the item was found and has been deleted
91
+// returns -1 if the item was not found in the index
92
extern int dictionary_del(DICTIONARY *dict, const char *name);
93
48
-extern int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *d), void *data);
49
-extern int dictionary_get_all_name_value(DICTIONARY *dict, int (*callback)(char *name, void *entry, void *d), void *data);
94
+// UNSAFE functions, without locks
95
+// to be used when the user is traversing with the right lock type
96
+// Read lock is acquired by dictionary_walktrhough_read() and dfe_start_read()
97
+// Write lock is acquired by dictionary_walktrhough_write() and dfe_start_write()
98
+// For code readability, please use these macros:
99
+#define dictionary_get_having_read_lock(dict, name) dictionary_get_unsafe(dict, name)
100
+#define dictionary_get_having_write_lock(dict, name) dictionary_get_unsafe(dict, name)
101
+#define dictionary_set_having_write_lock(dict, name, value, value_len) dictionary_set_unsafe(dict, name, value, value_len)
102
+#define dictionary_del_having_write_lock(dict, name) dictionary_del_unsafe(dict, name)
103
+
104
+extern void *dictionary_get_unsafe(DICTIONARY *dict, const char *name);
105
+extern void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len);
106
+extern int dictionary_del_unsafe(DICTIONARY *dict, const char *name);
107
+
108
+// Traverse (walk through) the items of the dictionary.
109
+// The order of traversal is currently the order of insertion.
110
+//
111
+// The callback function may return a negative number to stop the traversal,
112
+// in which case that negative value is returned to the caller.
113
+//
114
+// If all callback calls return zero or positive numbers, the sum of all of
115
+// them is returned to the caller.
116
+//
117
+// You cannot alter the dictionary from inside a dictionary_walkthrough_read() - deadlock!
118
+// You can only delete the current item from inside a dictionary_walkthrough_write() - you can add as many as you want.
119
+//
120
+#define dictionary_walkthrough_read(dict, callback, data) dictionary_walkthrough_rw(dict, 'r', callback, data)
121
+#define dictionary_walkthrough_write(dict, callback, data) dictionary_walkthrough_rw(dict, 'w', callback, data)
122
+extern int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *value, void *data), void *data);
123
+
124
+// Traverse with foreach
125
+//
126
+// Use like this:
127
+//
128
+// DICTFE dfe = {};
129
+// for(MY_ITEM *item = dfe_start_read(&dfe, dict); item ; item = dfe_next(&dfe)) {
130
+// // do things with the item and its dfe.name
131
+// }
132
+// dfe_done(&dfe);
133
+//
134
+// You cannot alter the dictionary from within a dfe_read_start() - deadlock!
135
+// You can only delete the current item from inside a dfe_start_write() - you can add as many as you want.
136
+//
137
+
138
+#ifdef DICTIONARY_INTERNALS
139
+#define DICTFE_CONST
140
+#else
141
+#define DICTFE_CONST const
142
+#endif
143
+
144
+typedef DICTFE_CONST struct dictionary_foreach {
145
+ DICTFE_CONST char *name; // the dictionary name of the last item used
146
+ void *value; // the dictionary value of the last item used
147
+ // same as the return value of dictfe_start() and dictfe_next()
148
+
149
+ // the following are for internal use only - to keep track of the point we are
150
+ usec_t started_ut; // the time the caller started iterating (now_realtime_usec())
151
+ DICTIONARY *dict; // the dictionary upon we work
152
+ void *last_position_index; // the internal position index, to remember the position we are at
153
+ void *next_position_index; // the internal position index, of the next item
154
+} DICTFE;
155
+
156
+#define dfe_start_read(dict, value) dfe_start_rw(dict, value, 'r')
157
+#define dfe_start_write(dict, value) dfe_start_rw(dict, value, 'r')
158
+#define dfe_start_rw(dict, value, mode) \
159
+ do { \
160
+ DICTFE dfe_ ## value = {}; \
161
+ const char *value ## _name; (void)(value ## _name); \
162
+ for((value) = dictionary_foreach_start_rw(&dfe_ ## value, (dict), (mode)), ( value ## _name ) = dfe_ ## value.name; \
163
+ (value) ;\
164
+ (value) = dictionary_foreach_next(&dfe_ ## value), ( value ## _name ) = dfe_ ## value.name)
165
+
166
+#define dfe_done(value) \
167
+ dictionary_foreach_done(&dfe_ ## value); \
168
+ } while(0)
169
+
170
+extern void * dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw);
171
+extern void * dictionary_foreach_next(DICTFE *dfe);
172
+extern usec_t dictionary_foreach_done(DICTFE *dfe);
173
+
174
+// Get statistics about the dictionary
175
+// If DICTIONARY_FLAG_WITH_STATISTICS is not set, these return zero
176
+extern size_t dictionary_stats_allocated_memory(DICTIONARY *dict);
177
+extern size_t dictionary_stats_entries(DICTIONARY *dict);
178
+extern size_t dictionary_stats_inserts(DICTIONARY *dict);
179
+extern size_t dictionary_stats_searches(DICTIONARY *dict);
180
+extern size_t dictionary_stats_deletes(DICTIONARY *dict);
181
+extern size_t dictionary_stats_resets(DICTIONARY *dict);
182
+
183
+extern int dictionary_unittest(size_t entries);
184
185
#endif /* NETDATA_DICTIONARY_H */
registry/registry.c
+7
-5
@@ -108,7 +108,9 @@ static int registry_json_person_url_callback(void *entry, void *data) {
108
}
109
110
// callback for rendering MACHINE_URLs
111
-static int registry_json_machine_url_callback(void *entry, void *data) {
111
+static int registry_json_machine_url_callback(const char *name, void *entry, void *data) {
112
+ (void)name;
113
+
114
REGISTRY_MACHINE_URL *mu = (REGISTRY_MACHINE_URL *)entry;
115
struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;
116
struct web_client *w = c->w;
@@ -272,7 +274,7 @@ int registry_request_search_json(RRDHOST *host, struct web_client *w, char *pers
274
275
buffer_strcat(w->response.data, ",\n\t\"urls\": [");
276
struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 };
275
- dictionary_get_all(m->machine_urls, registry_json_machine_url_callback, &c);
277
+ dictionary_walkthrough_read(m->machine_urls, registry_json_machine_url_callback, &c);
278
buffer_strcat(w->response.data, "\n\t]\n");
279
280
registry_json_footer(w);
@@ -441,10 +443,10 @@ void registry_statistics(void) {
443
}
444
else rrdset_next(stm);
445
444
- rrddim_set(stm, "persons", registry.persons_memory + registry.persons_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
445
- rrddim_set(stm, "machines", registry.machines_memory + registry.machines_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
446
+ rrddim_set(stm, "persons", registry.persons_memory + dictionary_stats_allocated_memory(registry.persons));
447
+ rrddim_set(stm, "machines", registry.machines_memory + dictionary_stats_allocated_memory(registry.machines));
448
rrddim_set(stm, "urls", registry.urls_memory);
449
rrddim_set(stm, "persons_urls", registry.persons_urls_memory);
448
- rrddim_set(stm, "machines_urls", registry.machines_urls_memory + registry.machines_count * sizeof(DICTIONARY) + registry.machines_urls_count * sizeof(NAME_VALUE));
450
+ rrddim_set(stm, "machines_urls", registry.machines_urls_memory);
451
rrdset_done(stm);
452
}
registry/registry_db.c
+14
-8
@@ -11,7 +11,9 @@ int registry_db_should_be_saved(void) {
11
// ----------------------------------------------------------------------------
12
// INTERNAL FUNCTIONS FOR SAVING REGISTRY OBJECTS
13
14
-static int registry_machine_save_url(void *entry, void *file) {
14
+static int registry_machine_save_url(const char *name, void *entry, void *file) {
15
+ (void)name;
16
+
17
REGISTRY_MACHINE_URL *mu = entry;
18
FILE *fp = file;
19
@@ -30,7 +32,9 @@ static int registry_machine_save_url(void *entry, void *file) {
32
return ret;
33
}
34
33
-static int registry_machine_save(void *entry, void *file) {
35
+static int registry_machine_save(const char *name, void *entry, void *file) {
36
+ (void)name;
37
+
38
REGISTRY_MACHINE *m = entry;
39
FILE *fp = file;
40
@@ -44,7 +48,7 @@ static int registry_machine_save(void *entry, void *file) {
48
);
49
50
if(ret >= 0) {
47
- int ret2 = dictionary_get_all(m->machine_urls, registry_machine_save_url, fp);
51
+ int ret2 = dictionary_walkthrough_read(m->machine_urls, registry_machine_save_url, fp);
52
if(ret2 < 0) return ret2;
53
ret += ret2;
54
}
@@ -75,7 +79,9 @@ static inline int registry_person_save_url(void *entry, void *file) {
79
return ret;
80
}
81
78
-static inline int registry_person_save(void *entry, void *file) {
82
+static inline int registry_person_save(const char *name, void *entry, void *file) {
83
+ (void)name;
84
+
85
REGISTRY_PERSON *p = entry;
86
FILE *fp = file;
87
@@ -89,7 +95,7 @@ static inline int registry_person_save(void *entry, void *file) {
95
);
96
97
if(ret >= 0) {
92
- //int ret2 = dictionary_get_all(p->person_urls, registry_person_save_url, fp);
98
+ //int ret2 = dictionary_walkthrough_read(p->person_urls, registry_person_save_url, fp);
99
int ret2 = avl_traverse(&p->person_urls, registry_person_save_url, fp);
100
if (ret2 < 0) return ret2;
101
ret += ret2;
@@ -126,10 +132,10 @@ int registry_db_save(void) {
132
return -1;
133
}
134
129
- // dictionary_get_all() has its own locking, so this is safe to do
135
+ // dictionary_walkthrough_read() has its own locking, so this is safe to do
136
137
debug(D_REGISTRY, "Saving all machines");
132
- int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);
138
+ int bytes1 = dictionary_walkthrough_read(registry.machines, registry_machine_save, fp);
139
if(bytes1 < 0) {
140
error("Registry: Cannot save registry machines - return value %d", bytes1);
141
fclose(fp);
@@ -139,7 +145,7 @@ int registry_db_save(void) {
145
debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);
146
147
debug(D_REGISTRY, "Saving all persons");
142
- int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp);
148
+ int bytes2 = dictionary_walkthrough_read(registry.persons, registry_person_save, fp);
149
if(bytes2 < 0) {
150
error("Registry: Cannot save registry persons - return value %d", bytes2);
151
fclose(fp);
registry/registry_init.c
+41
-34
@@ -76,8 +76,8 @@ int registry_init(void) {
76
netdata_mutex_init(®istry.lock);
77
78
// create dictionaries
79
- registry.persons = dictionary_create(DICTIONARY_FLAGS);
80
- registry.machines = dictionary_create(DICTIONARY_FLAGS);
79
+ registry.persons = dictionary_create(REGISTRY_DICTIONARY_FLAGS);
80
+ registry.machines = dictionary_create(REGISTRY_DICTIONARY_FLAGS);
81
avl_init(®istry.registry_urls_root_index, registry_url_compare);
82
83
// load the registry database
@@ -93,56 +93,63 @@ int registry_init(void) {
93
return 0;
94
}
95
96
-void registry_free(void) {
97
- if(!registry.enabled) return;
96
+static int machine_urls_delete_callback(const char *name, void *entry, void *data) {
97
+ (void)name;
98
99
- // we need to destroy the dictionaries ourselves
100
- // since the dictionaries use memory we allocated
99
+ REGISTRY_MACHINE *m = (REGISTRY_MACHINE *)data;
100
+ (void)m;
101
102
- while(registry.persons->values_index.root) {
103
- REGISTRY_PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;
104
- registry_person_del(p);
105
- }
102
+ REGISTRY_MACHINE_URL *mu = (REGISTRY_MACHINE_URL *)entry;
103
107
- while(registry.machines->values_index.root) {
108
- REGISTRY_MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value;
104
+ debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);
105
+ registry_url_unlink(mu->url);
106
110
- // fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);
107
+ debug(D_REGISTRY, "Registry: freeing machine url");
108
+ freez(mu);
109
112
- while(m->machine_urls->values_index.root) {
113
- REGISTRY_MACHINE_URL *mu = ((NAME_VALUE *)m->machine_urls->values_index.root)->value;
110
+ return 1;
111
+}
112
115
- // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);
113
+static int machine_delete_callback(const char *name, void *entry, void *data) {
114
+ (void)name;
115
+ (void)data;
116
117
- //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url);
118
- //dictionary_destroy(mu->persons);
117
+ REGISTRY_MACHINE *m = (REGISTRY_MACHINE *)entry;
118
+ int ret = dictionary_walkthrough_read(m->machine_urls, machine_urls_delete_callback, m);
119
120
- debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid);
121
- dictionary_del(m->machine_urls, mu->url->url);
120
+ dictionary_destroy(m->machine_urls);
121
+ freez(m);
122
123
- debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);
124
- registry_url_unlink(mu->url);
123
+ return ret + 1;
124
+}
125
+static int registry_person_del_callback(const char *name, void *entry, void *d) {
126
+ (void)name;
127
+ (void)d;
128
126
- debug(D_REGISTRY, "Registry: freeing machine url");
127
- freez(mu);
128
- }
129
+ REGISTRY_PERSON *p = (REGISTRY_PERSON *)entry;
130
130
- debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid);
131
- dictionary_del(registry.machines, m->guid);
131
+ debug(D_REGISTRY, "Registry: registry_person_del('%s'): deleting person", p->guid);
132
133
- debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid);
134
- dictionary_destroy(m->machine_urls);
133
+ while(p->person_urls.root)
134
+ registry_person_unlink_from_url(p, (REGISTRY_PERSON_URL *)p->person_urls.root);
135
136
- debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid);
137
- freez(m);
138
- }
136
+ //debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);
137
+ //dictionary_del(registry.persons, p->guid);
138
140
- // and free the memory of remaining dictionary structures
139
+ debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid);
140
+ freez(p);
141
+
142
+ return 1;
143
+}
144
+
145
+void registry_free(void) {
146
+ if(!registry.enabled) return;
147
148
debug(D_REGISTRY, "Registry: destroying persons dictionary");
149
+ dictionary_walkthrough_read(registry.persons, registry_person_del_callback, NULL);
150
dictionary_destroy(registry.persons);
151
152
debug(D_REGISTRY, "Registry: destroying machines dictionary");
153
+ dictionary_walkthrough_read(registry.machines, machine_delete_callback, NULL);
154
dictionary_destroy(registry.machines);
155
}
148
-
registry/registry_internals.c
+2
-2
@@ -198,7 +198,7 @@ REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid,
198
}
199
200
201
-// a structure to pass to the dictionary_get_all() callback handler
201
+// a structure to pass to the dictionary_walkthrough_read() callback handler
202
struct machine_request_callback_data {
203
REGISTRY_MACHINE *find_this_machine;
204
REGISTRY_PERSON_URL *result;
@@ -246,7 +246,7 @@ REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid
246
// We will walk through the PERSON_URLs to find the machine
247
// linking to our machine
248
249
- // a structure to pass to the dictionary_get_all() callback handler
249
+ // a structure to pass to the dictionary_walkthrough_read() callback handler
250
struct machine_request_callback_data rdata = { m, NULL };
251
252
// request a walk through on the dictionary
registry/registry_internals.h
+1
-1
@@ -8,7 +8,7 @@
8
#define REGISTRY_URL_FLAGS_DEFAULT 0x00
9
#define REGISTRY_URL_FLAGS_EXPIRED 0x01
10
11
-#define DICTIONARY_FLAGS (DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED)
11
+#define REGISTRY_DICTIONARY_FLAGS (DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED | DICTIONARY_FLAG_WITH_STATISTICS)
12
13
// ----------------------------------------------------------------------------
14
// COMMON structures
registry/registry_machine.c
+7
-2
@@ -24,7 +24,10 @@ REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTR
24
registry.machines_urls_memory += sizeof(REGISTRY_MACHINE_URL);
25
26
debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): indexing URL in machine", m->guid, u->url);
27
+
28
+ registry.machines_urls_memory -= dictionary_stats_allocated_memory(m->machine_urls);
29
dictionary_set(m->machine_urls, u->url, mu, sizeof(REGISTRY_MACHINE_URL));
30
+ registry.machines_urls_memory += dictionary_stats_allocated_memory(m->machine_urls);
31
32
registry_url_link(u);
33
@@ -39,15 +42,17 @@ REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t whe
42
strncpyz(m->guid, machine_guid, GUID_LEN);
43
44
debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);
42
- m->machine_urls = dictionary_create(DICTIONARY_FLAGS);
45
+ m->machine_urls = dictionary_create(REGISTRY_DICTIONARY_FLAGS);
46
47
m->first_t = m->last_t = (uint32_t)when;
48
m->usages = 0;
49
50
registry.machines_memory += sizeof(REGISTRY_MACHINE);
48
-
51
registry.machines_count++;
52
+
53
+ registry.machines_urls_memory -= dictionary_stats_allocated_memory(m->machine_urls);
54
dictionary_set(registry.machines, m->guid, m, sizeof(REGISTRY_MACHINE));
55
+ registry.machines_urls_memory += dictionary_stats_allocated_memory(m->machine_urls);
56
57
return m;
58
}
registry/registry_person.c
-13
@@ -199,19 +199,6 @@ REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when) {
199
return p;
200
}
201
202
-void registry_person_del(REGISTRY_PERSON *p) {
203
- debug(D_REGISTRY, "Registry: registry_person_del('%s'): creating dictionary of urls", p->guid);
204
-
205
- while(p->person_urls.root)
206
- registry_person_unlink_from_url(p, (REGISTRY_PERSON_URL *)p->person_urls.root);
207
-
208
- debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);
209
- dictionary_del(registry.persons, p->guid);
210
-
211
- debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid);
212
- freez(p);
213
-}
214
-
202
// ----------------------------------------------------------------------------
203
// LINKING OF OBJECTS
204
registry/registry_person.h
-1
@@ -53,7 +53,6 @@ extern REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, R
53
extern REGISTRY_PERSON *registry_person_find(const char *person_guid);
54
extern REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when);
55
extern REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when);
56
-extern void registry_person_del(REGISTRY_PERSON *p);
56
57
// LINKING PERSON -> PERSON_URL
58
extern REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when);
web/api/formatters/charts2json.c
+4
-2
@@ -150,7 +150,9 @@ struct array_printer {
150
BUFFER *wb;
151
};
152
153
-int print_collector(void *entry, void *data) {
153
+static int print_collector_callback(const char *name, void *entry, void *data) {
154
+ (void)name;
155
+
156
struct array_printer *ap = (struct array_printer *)data;
157
BUFFER *wb = ap->wb;
158
struct collector *col=(struct collector *) entry;
@@ -187,6 +189,6 @@ void chartcollectors2json(RRDHOST *host, BUFFER *wb) {
189
.c = 0,
190
.wb = wb
191
};
190
- dictionary_get_all(dict, print_collector, &ap);
192
+ dictionary_walkthrough_read(dict, print_collector_callback, &ap);
193
dictionary_destroy(dict);
194
}
web/api/formatters/json_wrapper.c
+6
-4
@@ -7,7 +7,9 @@ struct value_output {
7
BUFFER *wb;
8
};
9
10
-static int value_list_output(void *entry, void *data) {
10
+static int value_list_output(const char *name, void *entry, void *data) {
11
+ (void)name;
12
+
13
struct value_output *ap = (struct value_output *)data;
14
BUFFER *wb = ap->wb;
15
char *output = (char *) entry;
@@ -130,7 +132,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
132
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name);
133
dictionary_set(dict, name, output, len+1);
134
}
133
- dictionary_get_all(dict, value_list_output, &co);
135
+ dictionary_walkthrough_read(dict, value_list_output, &co);
136
dictionary_destroy(dict);
137
138
co.c = 0;
@@ -142,7 +144,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
144
dictionary_set(dict, name, output, len + 1);
145
}
146
145
- dictionary_get_all(dict, value_list_output, &co);
147
+ dictionary_walkthrough_read(dict, value_list_output, &co);
148
dictionary_destroy(dict);
149
150
RRDSET *st;
@@ -165,7 +167,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
167
}
168
}
169
}
168
- dictionary_get_all(dict, value_list_output, &co);
170
+ dictionary_walkthrough_read(dict, value_list_output, &co);
171
dictionary_destroy(dict);
172
buffer_strcat(wb, "],\n");
173
}