lower dictionary memory requirements by keeping pointers to optional features
Costa Tsaousis committed
Jun 2, 2016 at 14:21 UTC
e26fedfc7a43f30e5fdfdf624c01171d28ce807a
3 files changed
+74
-45
profile/benchmark-dictionary.c
+16
-15
@@ -22,19 +22,20 @@ struct myvalue {
22
int main(int argc, char **argv) {
23
if(argc || argv) {;}
24
25
- DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
25
+// DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS);
26
+ DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS);
27
if(!dict) fatal("Cannot create dictionary.");
28
29
struct rusage start, end;
30
unsigned long long dt;
31
char buf[100 + 1];
32
struct myvalue value, *v;
32
- int i, max = 10000000, max2;
33
+ int i, max = 100000, max2;
34
35
// ------------------------------------------------------------------------
36
37
getrusage(RUSAGE_SELF, &start);
37
- dict->inserts = dict->deletes = dict->searches = 0ULL;
38
+ dict->stats->inserts = dict->stats->deletes = dict->stats->searches = 0ULL;
39
fprintf(stderr, "Inserting %d entries in the dictionary\n", max);
40
for(i = 0; i < max; i++) {
41
value.i = i;
@@ -45,12 +46,12 @@ int main(int argc, char **argv) {
46
getrusage(RUSAGE_SELF, &end);
47
dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
48
fprintf(stderr, "Added %d entries in %llu nanoseconds: %llu inserts per second\n", max, dt, max * 1000000ULL / dt);
48
- fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
49
+ fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->stats->inserts, dict->stats->deletes, dict->stats->searches);
50
51
// ------------------------------------------------------------------------
52
53
getrusage(RUSAGE_SELF, &start);
53
- dict->inserts = dict->deletes = dict->searches = 0ULL;
54
+ dict->stats->inserts = dict->stats->deletes = dict->stats->searches = 0ULL;
55
fprintf(stderr, "Retrieving %d entries from the dictionary\n", max);
56
for(i = 0; i < max; i++) {
57
value.i = i;
@@ -65,12 +66,12 @@ int main(int argc, char **argv) {
66
getrusage(RUSAGE_SELF, &end);
67
dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
68
fprintf(stderr, "Read %d entries in %llu nanoseconds: %llu searches per second\n", max, dt, max * 1000000ULL / dt);
68
- fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
69
+ fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->stats->inserts, dict->stats->deletes, dict->stats->searches);
70
71
// ------------------------------------------------------------------------
72
73
getrusage(RUSAGE_SELF, &start);
73
- dict->inserts = dict->deletes = dict->searches = 0ULL;
74
+ dict->stats->inserts = dict->stats->deletes = dict->stats->searches = 0ULL;
75
fprintf(stderr, "Resetting %d entries in the dictionary\n", max);
76
for(i = 0; i < max; i++) {
77
value.i = i;
@@ -80,13 +81,13 @@ int main(int argc, char **argv) {
81
}
82
getrusage(RUSAGE_SELF, &end);
83
dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
83
- fprintf(stderr, "Reset %d entries in %llu nanoseconds: %llu inserts per second\n", max, dt, max * 1000000ULL / dt);
84
- fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
84
+ fprintf(stderr, "Reset %d entries in %llu nanoseconds: %llu resets per second\n", max, dt, max * 1000000ULL / dt);
85
+ fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->stats->inserts, dict->stats->deletes, dict->stats->searches);
86
87
// ------------------------------------------------------------------------
88
89
getrusage(RUSAGE_SELF, &start);
89
- dict->inserts = dict->deletes = dict->searches = 0ULL;
90
+ dict->stats->inserts = dict->stats->deletes = dict->stats->searches = 0ULL;
91
fprintf(stderr, "Searching %d non-existing entries in the dictionary\n", max);
92
max2 = max * 2;
93
for(i = max; i < max2; i++) {
@@ -99,13 +100,13 @@ int main(int argc, char **argv) {
100
}
101
getrusage(RUSAGE_SELF, &end);
102
dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
102
- fprintf(stderr, "Searched %d non-existing entries in %llu nanoseconds: %llu searches per second\n", max, dt, max * 1000000ULL / dt);
103
- fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
103
+ fprintf(stderr, "Searched %d non-existing entries in %llu nanoseconds: %llu not found searches per second\n", max, dt, max * 1000000ULL / dt);
104
+ fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->stats->inserts, dict->stats->deletes, dict->stats->searches);
105
106
// ------------------------------------------------------------------------
107
108
getrusage(RUSAGE_SELF, &start);
108
- dict->inserts = dict->deletes = dict->searches = 0ULL;
109
+ dict->stats->inserts = dict->stats->deletes = dict->stats->searches = 0ULL;
110
fprintf(stderr, "Deleting %d entries from the dictionary\n", max);
111
for(i = 0; i < max; i++) {
112
value.i = i;
@@ -116,12 +117,12 @@ int main(int argc, char **argv) {
117
getrusage(RUSAGE_SELF, &end);
118
dt = (end.ru_utime.tv_sec * 1000000ULL + end.ru_utime.tv_usec) - (start.ru_utime.tv_sec * 1000000ULL + start.ru_utime.tv_usec);
119
fprintf(stderr, "Deleted %d entries in %llu nanoseconds: %llu deletes per second\n", max, dt, max * 1000000ULL / dt);
119
- fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->inserts, dict->deletes, dict->searches);
120
+ fprintf(stderr, " > Dictionary: %llu inserts, %llu deletes, %llu searches\n\n", dict->stats->inserts, dict->stats->deletes, dict->stats->searches);
121
122
// ------------------------------------------------------------------------
123
124
getrusage(RUSAGE_SELF, &start);
124
- dict->inserts = dict->deletes = dict->searches = 0ULL;
125
+ dict->stats->inserts = dict->stats->deletes = dict->stats->searches = 0ULL;
126
fprintf(stderr, "Destroying dictionary\n");
127
dictionary_destroy(dict);
128
getrusage(RUSAGE_SELF, &end);
src/dictionary.c
+48
-8
@@ -12,27 +12,52 @@
12
13
#include "dictionary.h"
14
15
+// ----------------------------------------------------------------------------
16
+// dictionary statistics
17
+
18
+static inline void NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(DICTIONARY *dict) {
19
+ if(likely(dict->stats))
20
+ dict->stats->inserts++;
21
+}
22
+static inline void NETDATA_DICTIONARY_STATS_DELETES_PLUS1(DICTIONARY *dict) {
23
+ if(likely(dict->stats))
24
+ dict->stats->deletes++;
25
+}
26
+static inline void NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) {
27
+ if(likely(dict->stats))
28
+ dict->stats->searches++;
29
+}
30
+static inline void NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict) {
31
+ if(likely(dict->stats))
32
+ dict->stats->entries++;
33
+}
34
+static inline void NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict) {
35
+ if(likely(dict->stats))
36
+ dict->stats->entries--;
37
+}
38
+
39
+
40
// ----------------------------------------------------------------------------
41
// dictionary locks
42
43
static inline void dictionary_read_lock(DICTIONARY *dict) {
19
- if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
44
+ if(likely(dict->rwlock)) {
45
// debug(D_DICTIONARY, "Dictionary READ lock");
21
- pthread_rwlock_rdlock(&dict->rwlock);
46
+ pthread_rwlock_rdlock(dict->rwlock);
47
}
48
}
49
50
static inline void dictionary_write_lock(DICTIONARY *dict) {
26
- if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
51
+ if(likely(dict->rwlock)) {
52
// debug(D_DICTIONARY, "Dictionary WRITE lock");
28
- pthread_rwlock_wrlock(&dict->rwlock);
53
+ pthread_rwlock_wrlock(dict->rwlock);
54
}
55
}
56
57
static inline void dictionary_unlock(DICTIONARY *dict) {
33
- if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
58
+ if(likely(dict->rwlock)) {
59
// debug(D_DICTIONARY, "Dictionary UNLOCK lock");
35
- pthread_rwlock_unlock(&dict->rwlock);
60
+ pthread_rwlock_unlock(dict->rwlock);
61
}
62
}
63
@@ -123,9 +148,18 @@ DICTIONARY *dictionary_create(uint32_t flags) {
148
DICTIONARY *dict = calloc(1, sizeof(DICTIONARY));
149
if(unlikely(!dict)) fatal("Cannot allocate DICTIONARY");
150
126
- avl_init(&dict->values_index, name_value_compare);
127
- pthread_rwlock_init(&dict->rwlock, NULL);
151
+ if(flags & DICTIONARY_FLAG_WITH_STATISTICS) {
152
+ dict->stats = calloc(1, sizeof(struct dictionary_stats));
153
+ if(!dict->stats) fatal("Cannot allocate statistics for DICTIONARY");
154
+ }
155
+
156
+ if(!(flags & DICTIONARY_FLAG_SINGLE_THREADED)) {
157
+ dict->rwlock = calloc(1, sizeof(pthread_rwlock_t));
158
+ if(!dict->rwlock) fatal("Cannot allocate pthread_rwlock_t for DICTIONARY");
159
+ pthread_rwlock_init(dict->rwlock, NULL);
160
+ }
161
162
+ avl_init(&dict->values_index, name_value_compare);
163
dict->flags = flags;
164
165
return dict;
@@ -141,6 +175,12 @@ void dictionary_destroy(DICTIONARY *dict) {
175
176
dictionary_unlock(dict);
177
178
+ if(dict->stats)
179
+ free(dict->stats);
180
+
181
+ if(dict->rwlock)
182
+ free(dict->rwlock);
183
+
184
free(dict);
185
}
186
src/dictionary.h
+10
-22
@@ -6,6 +6,13 @@
6
#ifndef NETDATA_DICTIONARY_H
7
#define NETDATA_DICTIONARY_H 1
8
9
+struct dictionary_stats {
10
+ unsigned long long inserts;
11
+ unsigned long long deletes;
12
+ unsigned long long searches;
13
+ unsigned long long entries;
14
+};
15
+
16
typedef struct name_value {
17
avl avl; // the index - this has to be first!
18
@@ -21,34 +28,15 @@ typedef struct dictionary {
28
29
uint8_t flags;
30
24
-#ifdef NETDATA_DICTIONARY_WITH_STATISTICS
25
- unsigned long long inserts;
26
- unsigned long long deletes;
27
- unsigned long long searches;
28
- unsigned long long entries;
29
-#endif /* NETDATA_DICTIONARY_WITH_STATISTICS */
30
-
31
- pthread_rwlock_t rwlock;
31
+ struct dictionary_stats *stats;
32
+ pthread_rwlock_t *rwlock;
33
} DICTIONARY;
34
34
-#ifdef NETDATA_DICTIONARY_WITH_STATISTICS
35
-#define NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(dict) (dict)->inserts++
36
-#define NETDATA_DICTIONARY_STATS_DELETES_PLUS1(dict) (dict)->deletes++
37
-#define NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(dict) (dict)->searches++
38
-#define NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(dict) (dict)->entries++
39
-#define NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(dict) (dict)->entries--
40
-#else /* NETDATA_DICTIONARY_WITH_STATISTICS */
41
-#define NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(dict)
42
-#define NETDATA_DICTIONARY_STATS_DELETES_PLUS1(dict)
43
-#define NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(dict)
44
-#define NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(dict)
45
-#define NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(dict)
46
-#endif /* NETDATA_DICTIONARY_WITH_STATISTICS */
47
-
35
#define DICTIONARY_FLAG_DEFAULT 0x00000000
36
#define DICTIONARY_FLAG_SINGLE_THREADED 0x00000001
37
#define DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE 0x00000002
38
#define DICTIONARY_FLAG_NAME_LINK_DONT_CLONE 0x00000004
39
+#define DICTIONARY_FLAG_WITH_STATISTICS 0x00000008
40
41
extern DICTIONARY *dictionary_create(uint32_t flags);
42
extern void dictionary_destroy(DICTIONARY *dict);