@cryptotaxi247 / netdata-1 / commits / d4a462d10

finalized dictionary.c

Costa Tsaousis committed May 6, 2016 at 01:41 UTC d4a462d10af143d17a574e8fe0a1280875e8ca41
5 files changed +266 -68
profile/benchmark-dictionary.c new
+132
@@ -0,0 +1,132 @@
1 +
2 +/*
3 + * 1. build netdata (as normally)
4 + * 2. cd profile/
5 + * 3. compile with:
6 + * gcc -O3 -Wall -Wextra -I ../src/ -I ../ -o benchmark-dictionary benchmark-dictionary.c ../src/dictionary.o ../src/log.o ../src/avl.o ../src/common.o -pthread
7 + *
8 + */
9 +
10 +#include <stdio.h>
11 +#include <inttypes.h>
12 +
13 +#include "dictionary.h"
14 +#include "main.h"
15 +#include "log.h"
16 +#include "common.h"
17 +
18 +struct myvalue {
19 + int i;
20 +};
21 +
22 +int main(int argc, char **argv) {
23 + if(argc || argv) {;}
24 +
25 + DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
26 + if(!dict) fatal("Cannot create dictionary.");
27 +
28 + struct rusage start, end;
29 + unsigned long long dt;
30 + char buf[100 + 1];
31 + struct myvalue value, *v;
32 + int i, max = 1000000, max2;
33 +
34 + // ------------------------------------------------------------------------
35 +
36 + getrusage(RUSAGE_SELF, &start);
37 + dict->inserts = dict->deletes = dict->searches = 0ULL;
38 + fprintf(stderr, "Inserting %d entries in the dictionary\n", max);
39 + for(i = 0; i < max; i++) {
40 + value.i = i;
41 + snprintf(buf, 100, "%d", i);
42 +
43 + dictionary_set(dict, buf, &value, sizeof(struct myvalue));
44 + }
45 + getrusage(RUSAGE_SELF, &end);
46 + dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
47 + fprintf(stderr, "Added %d entries in %llu nanoseconds: %llu inserts per second\n", max, dt, max * 1000000ULL / dt);
48 + fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
49 +
50 + // ------------------------------------------------------------------------
51 +
52 + getrusage(RUSAGE_SELF, &start);
53 + dict->inserts = dict->deletes = dict->searches = 0ULL;
54 + fprintf(stderr, "Retrieving %d entries from the dictionary\n", max);
55 + for(i = 0; i < max; i++) {
56 + value.i = i;
57 + snprintf(buf, 100, "%d", i);
58 +
59 + v = dictionary_get(dict, buf);
60 + if(!v)
61 + fprintf(stderr, "ERROR: cannot get value %d from the dictionary\n", i);
62 + else if(v->i != i)
63 + fprintf(stderr, "ERROR: expected %d but got %d\n", i, v->i);
64 + }
65 + getrusage(RUSAGE_SELF, &end);
66 + dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
67 + fprintf(stderr, "Read %d entries in %llu nanoseconds: %llu searches per second\n", max, dt, max * 1000000ULL / dt);
68 + fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
69 +
70 + // ------------------------------------------------------------------------
71 +
72 + getrusage(RUSAGE_SELF, &start);
73 + dict->inserts = dict->deletes = dict->searches = 0ULL;
74 + fprintf(stderr, "Resetting %d entries in the dictionary\n", max);
75 + for(i = 0; i < max; i++) {
76 + value.i = i;
77 + snprintf(buf, 100, "%d", i);
78 +
79 + dictionary_set(dict, buf, &value, sizeof(struct myvalue));
80 + }
81 + getrusage(RUSAGE_SELF, &end);
82 + dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
83 + fprintf(stderr, "Reset %d entries in %llu nanoseconds: %llu inserts per second\n", max, dt, max * 1000000ULL / dt);
84 + fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
85 +
86 + // ------------------------------------------------------------------------
87 +
88 + getrusage(RUSAGE_SELF, &start);
89 + dict->inserts = dict->deletes = dict->searches = 0ULL;
90 + fprintf(stderr, "Searching %d non-existing entries in the dictionary\n", max);
91 + max2 = max * 2;
92 + for(i = max; i < max2; i++) {
93 + value.i = i;
94 + snprintf(buf, 100, "%d", i);
95 +
96 + v = dictionary_get(dict, buf);
97 + if(v)
98 + fprintf(stderr, "ERROR: cannot got non-existing value %d from the dictionary\n", i);
99 + }
100 + getrusage(RUSAGE_SELF, &end);
101 + dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
102 + fprintf(stderr, "Searched %d non-existing entries in %llu nanoseconds: %llu searches per second\n", max, dt, max * 1000000ULL / dt);
103 + fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
104 +
105 + // ------------------------------------------------------------------------
106 +
107 + getrusage(RUSAGE_SELF, &start);
108 + dict->inserts = dict->deletes = dict->searches = 0ULL;
109 + fprintf(stderr, "Deleting %d entries from the dictionary\n", max);
110 + for(i = 0; i < max; i++) {
111 + value.i = i;
112 + snprintf(buf, 100, "%d", i);
113 +
114 + dictionary_del(dict, buf);
115 + }
116 + getrusage(RUSAGE_SELF, &end);
117 + dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
118 + fprintf(stderr, "Deleted %d entries in %llu nanoseconds: %llu deletes per second\n", max, dt, max * 1000000ULL / dt);
119 + fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
120 +
121 + // ------------------------------------------------------------------------
122 +
123 + getrusage(RUSAGE_SELF, &start);
124 + dict->inserts = dict->deletes = dict->searches = 0ULL;
125 + fprintf(stderr, "Destroying dictionary\n");
126 + dictionary_destroy(dict);
127 + getrusage(RUSAGE_SELF, &end);
128 + dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
129 + fprintf(stderr, "Destroyed in %llu nanoseconds\n", dt);
130 +
131 + return 0;
132 +}
src/appconfig.c
+2
@@ -90,11 +90,13 @@ static int config_compare(void* a, void* b) {
90 avl_tree config_root_index = {
91 NULL,
92 config_compare,
93 +#ifndef AVL_WITHOUT_PTHREADS
94 #ifdef AVL_LOCK_WITH_MUTEX
95 PTHREAD_MUTEX_INITIALIZER
96 #else
97 PTHREAD_RWLOCK_INITIALIZER
98 #endif
99 +#endif
100 };
101
102 #define config_index_add(cfg) avl_insert(&config_root_index, (avl *)(cfg))
src/dictionary.c
+111 -62
@@ -22,93 +22,110 @@ static int name_value_compare(void* a, void* b) {
22 else return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name);
23 }
24
25 -#define name_value_index_add(dict, cv) avl_insert(&((dict)->values_index), (avl *)(cv))
26 -#define name_value_index_del(dict, cv) avl_remove(&((dict)->values_index), (avl *)(cv))
25 +#define dictionary_name_value_index_add_nolock(dict, nv) do { (dict)->inserts++; avl_insert(&((dict)->values_index), (avl *)(nv)); } while(0)
26 +#define dictionary_name_value_index_del_nolock(dict, nv) do { (dict)->deletes++; avl_remove(&(dict->values_index), (avl *)(nv)); } while(0)
27
28 -static NAME_VALUE *dictionary_name_value_index_find(DICTIONARY *dict, const char *name, uint32_t hash) {
28 +static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *dict, const char *name, uint32_t hash) {
29 NAME_VALUE *result = NULL, tmp;
30 tmp.hash = (hash)?hash:simple_hash(name);
31 tmp.name = (char *)name;
32
33 + dict->searches++;
34 avl_search(&(dict->values_index), (avl *)&tmp, name_value_iterator, (avl **)&result);
35 return result;
36 }
37
38 +static void dictionary_read_lock(DICTIONARY *dict) {
39 + if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)))
40 + pthread_rwlock_rdlock(&dict->rwlock);
41 +}
42 +
43 +static void dictionary_write_lock(DICTIONARY *dict) {
44 + if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)))
45 + pthread_rwlock_wrlock(&dict->rwlock);
46 +}
47 +
48 +static void dictionary_unlock(DICTIONARY *dict) {
49 + if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)))
50 + pthread_rwlock_unlock(&dict->rwlock);
51 +}
52 +
53 // ----------------------------------------------------------------------------
54
39 -static NAME_VALUE *dictionary_name_value_create(DICTIONARY *dict, const char *name, void *value, size_t value_len) {
55 +static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const char *name, void *value, size_t value_len, uint32_t hash) {
56 debug(D_DICTIONARY, "Creating name value entry for name '%s', value '%s'.", name, value);
57
58 NAME_VALUE *nv = calloc(1, sizeof(NAME_VALUE));
43 - if(!nv) {
44 - fatal("Cannot allocate name_value of size %z", sizeof(NAME_VALUE));
45 - exit(1);
46 - }
59 + if(unlikely(!nv)) fatal("Cannot allocate name_value of size %z", sizeof(NAME_VALUE));
60
61 nv->name = strdup(name);
49 - if(!nv->name) fatal("Cannot allocate name_value.name of size %z", strlen(name));
50 - nv->hash = simple_hash(nv->name);
62 + if(unlikely(!nv->name))
63 + fatal("Cannot allocate name_value.name of size %z", strlen(name));
64
52 - nv->value = malloc(value_len);
53 - if(!nv->value) fatal("Cannot allocate name_value.value of size %z", value_len);
54 - memcpy(nv->value, value, value_len);
65 + nv->hash = (hash)?hash:simple_hash(nv->name);
66
56 - // link it
57 - pthread_rwlock_wrlock(&dict->rwlock);
58 - nv->next = dict->values;
59 - dict->values = nv;
60 - pthread_rwlock_unlock(&dict->rwlock);
67 + if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)
68 + nv->value = value;
69 + else {
70 + nv->value = malloc(value_len);
71 + if (unlikely(!nv->value))
72 + fatal("Cannot allocate name_value.value of size %z", value_len);
73 +
74 + memcpy(nv->value, value, value_len);
75 + }
76 +
77 + dictionary_write_lock(dict);
78
79 // index it
63 - name_value_index_add(dict, nv);
80 + dictionary_name_value_index_add_nolock(dict, nv);
81 +
82 + dict->entries++;
83 +
84 + dictionary_unlock(dict);
85
86 return nv;
87 }
88
68 -static void dictionary_name_value_destroy(DICTIONARY *dict, NAME_VALUE *nv) {
89 +static void dictionary_name_value_destroy_nolock(DICTIONARY *dict, NAME_VALUE *nv) {
90 debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name);
91
71 - pthread_rwlock_wrlock(&dict->rwlock);
72 - if(dict->values == nv) dict->values = nv->next;
73 - else {
74 - NAME_VALUE *n = dict->values;
75 - while(n && n->next && n->next != nv) nv = nv->next;
76 - if(!n || n->next != nv) {
77 - fatal("Cannot find name_value with name '%s' in dictionary.", nv->name);
78 - exit(1);
79 - }
80 - n->next = nv->next;
81 - nv->next = NULL;
82 - }
83 - pthread_rwlock_unlock(&dict->rwlock);
92 + dictionary_name_value_index_del_nolock(dict, nv);
93 +
94 + dict->entries--;
95 +
96 + free(nv->name);
97 +
98 + if(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))
99 + free(nv->value);
100
85 - free(nv->value);
101 free(nv);
102 }
103
104 // ----------------------------------------------------------------------------
105
91 -DICTIONARY *dictionary_create(void) {
106 +DICTIONARY *dictionary_create(uint32_t flags) {
107 debug(D_DICTIONARY, "Creating dictionary.");
108
109 DICTIONARY *dict = calloc(1, sizeof(DICTIONARY));
95 - if(!dict) {
96 - fatal("Cannot allocate DICTIONARY");
97 - exit(1);
98 - }
110 + if(unlikely(!dict)) fatal("Cannot allocate DICTIONARY");
111
112 avl_init(&dict->values_index, name_value_compare);
113 pthread_rwlock_init(&dict->rwlock, NULL);
114
115 + dict->flags = flags;
116 +
117 return dict;
118 }
119
120 void dictionary_destroy(DICTIONARY *dict) {
121 debug(D_DICTIONARY, "Destroying dictionary.");
122
109 - pthread_rwlock_wrlock(&dict->rwlock);
110 - while(dict->values) dictionary_name_value_destroy(dict, dict->values);
111 - pthread_rwlock_unlock(&dict->rwlock);
123 + dictionary_write_lock(dict);
124 +
125 + while(dict->values_index.root)
126 + dictionary_name_value_destroy_nolock(dict, (NAME_VALUE *)dict->values_index.root);
127 +
128 + dictionary_unlock(dict);
129
130 free(dict);
131 }
@@ -118,27 +135,40 @@ void dictionary_destroy(DICTIONARY *dict) {
135 void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) {
136 debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name);
137
121 - pthread_rwlock_rdlock(&dict->rwlock);
122 - NAME_VALUE *nv = dictionary_name_value_index_find(dict, name, 0);
123 - pthread_rwlock_unlock(&dict->rwlock);
124 - if(!nv) {
138 + uint32_t hash = simple_hash(name);
139 +
140 + dictionary_read_lock(dict);
141 + NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, hash);
142 + dictionary_unlock(dict);
143 +
144 + if(unlikely(!nv)) {
145 debug(D_DICTIONARY, "Dictionary entry with name '%s' not found. Creating a new one.", name);
126 - nv = dictionary_name_value_create(dict, name, value, value_len);
127 - if(!nv) {
146 +
147 + pthread_rwlock_wrlock(&dict->rwlock);
148 + nv = dictionary_name_value_create_nolock(dict, name, value, value_len, hash);
149 +
150 + if(unlikely(!nv))
151 fatal("Cannot create name_value.");
129 - exit(1);
130 - }
131 - return nv->value;
152 +
153 + dictionary_unlock(dict);
154 }
155 else {
156 debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", name);
135 - pthread_rwlock_wrlock(&dict->rwlock);
136 - void *old = nv->value;
137 - nv->value = malloc(value_len);
138 - if(!nv->value) fatal("Cannot allocate value of size %z", value_len);
139 - memcpy(nv->value, value, value_len);
140 - pthread_rwlock_unlock(&dict->rwlock);
141 - free(old);
157 +
158 + if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)
159 + nv->value = value;
160 + else {
161 + void *value = malloc(value_len),
162 + *old = nv->value;
163 +
164 + if(unlikely(!nv->value))
165 + fatal("Cannot allocate value of size %z", value_len);
166 +
167 + memcpy(value, value, value_len);
168 + nv->value = value;
169 +
170 + free(old);
171 + }
172 }
173
174 return nv->value;
@@ -147,10 +177,11 @@ void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t val
177 void *dictionary_get(DICTIONARY *dict, const char *name) {
178 debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name);
179
150 - pthread_rwlock_rdlock(&dict->rwlock);
151 - NAME_VALUE *nv = dictionary_name_value_index_find(dict, name, 0);
152 - pthread_rwlock_unlock(&dict->rwlock);
153 - if(!nv) {
180 + dictionary_read_lock(dict);
181 + NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0);
182 + dictionary_unlock(dict);
183 +
184 + if(unlikely(!nv)) {
185 debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name);
186 return NULL;
187 }
@@ -158,3 +189,21 @@ void *dictionary_get(DICTIONARY *dict, const char *name) {
189 debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name);
190 return nv->value;
191 }
192 +
193 +void dictionary_del(DICTIONARY *dict, const char *name) {
194 + debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name);
195 +
196 + dictionary_read_lock(dict);
197 + NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0);
198 + dictionary_unlock(dict);
199 +
200 + if(unlikely(!nv)) {
201 + debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name);
202 + return;
203 + }
204 +
205 + debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name);
206 + pthread_rwlock_wrlock(&dict->rwlock);
207 + dictionary_name_value_destroy_nolock(dict, nv);
208 + dictionary_unlock(dict);
209 +}
src/dictionary.h
+17 -6
@@ -1,4 +1,7 @@
1 +#include <pthread.h>
2 +
3 #include "web_buffer.h"
4 +#include "avl.h"
5
6 #ifndef NETDATA_DICTIONARY_H
7 #define NETDATA_DICTIONARY_H 1
@@ -8,22 +11,30 @@ typedef struct name_value {
11
12 uint32_t hash; // a simple hash to speed up searching
13 // we first compare hashes, and only if the hashes are equal we do string comparisons
11 -
14 char *name;
13 - char *value;
14 -
15 - struct name_value *next;
15 + void *value;
16 } NAME_VALUE;
17
18 typedef struct dictionary {
19 - NAME_VALUE *values;
19 + uint32_t flags;
20 +
21 + unsigned long long inserts;
22 + unsigned long long deletes;
23 + unsigned long long searches;
24 + unsigned long long entries;
25 +
26 avl_tree values_index;
27 pthread_rwlock_t rwlock;
28 } DICTIONARY;
29
24 -extern DICTIONARY *dictionary_create(void);
30 +#define DICTIONARY_FLAG_DEFAULT 0x00000000
31 +#define DICTIONARY_FLAG_SINGLE_THREADED 0x00000001
32 +#define DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE 0x00000002
33 +
34 +extern DICTIONARY *dictionary_create(uint32_t flags);
35 extern void dictionary_destroy(DICTIONARY *dict);
36 extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len);
37 extern void *dictionary_get(DICTIONARY *dict, const char *name);
38 +extern void dictionary_del(DICTIONARY *dict, const char *name);
39
40 #endif /* NETDATA_DICTIONARY_H */
src/plugin_tc.c
+4
@@ -94,11 +94,13 @@ static int tc_device_compare(void* a, void* b) {
94 avl_tree tc_device_root_index = {
95 NULL,
96 tc_device_compare,
97 +#ifndef AVL_WITHOUT_PTHREADS
98 #ifdef AVL_LOCK_WITH_MUTEX
99 PTHREAD_MUTEX_INITIALIZER
100 #else
101 PTHREAD_RWLOCK_INITIALIZER
102 #endif
103 +#endif
104 };
105
106 #define tc_device_index_add(st) avl_insert(&tc_device_root_index, (avl *)(st))
@@ -349,10 +351,12 @@ static struct tc_device *tc_device_create(char *id)
351 d->classes_index.compar = tc_class_compare;
352
353 int lock;
354 +#ifndef AVL_WITHOUT_PTHREADS
355 #ifdef AVL_LOCK_WITH_MUTEX
356 lock = pthread_mutex_init(&d->classes_index.mutex, NULL);
357 #else
358 lock = pthread_rwlock_init(&d->classes_index.rwlock, NULL);
359 +#endif
360 #endif
361 if(lock != 0)
362 fatal("Failed to initialize plugin_tc mutex/rwlock, return code %d.", lock);