@cryptotaxi247 / netdata-1 / commits / e44821197

registry progress: working internal prototype - no load/save and network yet

Costa Tsaousis committed May 8, 2016 at 00:50 UTC e44821197ea9ffe92cb723b1a3994876c8670a15
3 files changed +673 -127
src/dictionary.c
+50 -19
@@ -37,24 +37,30 @@ static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *di
37 }
38
39 static void dictionary_read_lock(DICTIONARY *dict) {
40 - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)))
40 + if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
41 + // debug(D_DICTIONARY, "Dictionary READ lock");
42 pthread_rwlock_rdlock(&dict->rwlock);
43 + }
44 }
45
46 static void dictionary_write_lock(DICTIONARY *dict) {
45 - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)))
47 + if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
48 + // debug(D_DICTIONARY, "Dictionary WRITE lock");
49 pthread_rwlock_wrlock(&dict->rwlock);
50 + }
51 }
52
53 static void dictionary_unlock(DICTIONARY *dict) {
50 - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)))
54 + if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) {
55 + // debug(D_DICTIONARY, "Dictionary UNLOCK lock");
56 pthread_rwlock_unlock(&dict->rwlock);
57 + }
58 }
59
60 // ----------------------------------------------------------------------------
61
62 static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const char *name, void *value, size_t value_len, uint32_t hash) {
57 - debug(D_DICTIONARY, "Creating name value entry for name '%s', value '%s'.", name, value);
63 + debug(D_DICTIONARY, "Creating name value entry for name '%s'.", name);
64
65 NAME_VALUE *nv = calloc(1, sizeof(NAME_VALUE));
66 if(unlikely(!nv)) fatal("Cannot allocate name_value of size %z", sizeof(NAME_VALUE));
@@ -79,15 +85,10 @@ static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const c
85 memcpy(nv->value, value, value_len);
86 }
87
82 - dictionary_write_lock(dict);
83 -
88 // index it
89 dictionary_name_value_index_add_nolock(dict, nv);
86 -
90 dict->entries++;
91
89 - dictionary_unlock(dict);
90 -
92 return nv;
93 }
94
@@ -98,10 +99,15 @@ static void dictionary_name_value_destroy_nolock(DICTIONARY *dict, NAME_VALUE *n
99
100 dict->entries--;
101
101 - free(nv->name);
102 -
103 - if(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))
102 + if(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) {
103 + debug(D_REGISTRY, "Dictionary freeing value of '%s'", nv->name);
104 free(nv->value);
105 + }
106 +
107 + if(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)) {
108 + debug(D_REGISTRY, "Dictionary freeing name '%s'", nv->name);
109 + free(nv->name);
110 + }
111
112 free(nv);
113 }
@@ -149,20 +155,23 @@ void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t val
155 if(unlikely(!nv)) {
156 debug(D_DICTIONARY, "Dictionary entry with name '%s' not found. Creating a new one.", name);
157
152 - pthread_rwlock_wrlock(&dict->rwlock);
158 + dictionary_write_lock(dict);
159 nv = dictionary_name_value_create_nolock(dict, name, value, value_len, hash);
160 + dictionary_unlock(dict);
161
162 if(unlikely(!nv))
163 fatal("Cannot create name_value.");
157 -
158 - dictionary_unlock(dict);
164 }
165 else {
166 debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", name);
167
163 - if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)
168 + if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE) {
169 + debug(D_REGISTRY, "Dictionary: linking value to '%s'", name);
170 nv->value = value;
171 + }
172 else {
173 + debug(D_REGISTRY, "Dictionary: cloning value to '%s'", name);
174 +
175 void *value = malloc(value_len),
176 *old = nv->value;
177
@@ -172,6 +181,7 @@ void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t val
181 memcpy(value, value, value_len);
182 nv->value = value;
183
184 + debug(D_REGISTRY, "Dictionary: freeing old value of '%s'", name);
185 free(old);
186 }
187 }
@@ -195,7 +205,7 @@ void *dictionary_get(DICTIONARY *dict, const char *name) {
205 return nv->value;
206 }
207
198 -void dictionary_del(DICTIONARY *dict, const char *name) {
208 +int dictionary_del(DICTIONARY *dict, const char *name) {
209 debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name);
210
211 dictionary_read_lock(dict);
@@ -204,11 +214,32 @@ void dictionary_del(DICTIONARY *dict, const char *name) {
214
215 if(unlikely(!nv)) {
216 debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name);
207 - return;
217 + return -1;
218 }
219
220 debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name);
211 - pthread_rwlock_wrlock(&dict->rwlock);
221 + dictionary_write_lock(dict);
222 dictionary_name_value_destroy_nolock(dict, nv);
223 dictionary_unlock(dict);
224 +
225 + return 0;
226 +}
227 +
228 +
229 +static void dictionary_walker(avl *a, int (*callback)(void *entry, void *data), void *data) {
230 + if(unlikely(!a)) return;
231 +
232 + if(a->right)
233 + dictionary_walker(a->right, callback, data);
234 +
235 + callback(((NAME_VALUE *)a)->value, data);
236 +
237 + if(a->left)
238 + dictionary_walker(a->left, callback, data);
239 +}
240 +
241 +void dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) {
242 + dictionary_read_lock(dict);
243 + dictionary_walker(dict->values_index.root, callback, data);
244 + dictionary_unlock(dict);
245 }
src/dictionary.h
+3 -1
@@ -36,6 +36,8 @@ extern DICTIONARY *dictionary_create(uint32_t flags);
36 extern void dictionary_destroy(DICTIONARY *dict);
37 extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len);
38 extern void *dictionary_get(DICTIONARY *dict, const char *name);
39 -extern void dictionary_del(DICTIONARY *dict, const char *name);
39 +extern int dictionary_del(DICTIONARY *dict, const char *name);
40 +
41 +extern void dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data);
42
43 #endif /* NETDATA_DICTIONARY_H */
src/registry.c
+620 -107
@@ -2,7 +2,7 @@
2 #include <config.h>
3 #endif
4
5 -// gcc -O3 -Wall -Wextra -I ../src/ -I ../ -o registry ../src/registry.c ../src/dictionary.o ../src/log.o ../src/avl.o ../src/common.o -pthread -luuid -DHAVE_CONFIG_H
5 +// gcc -O1 -ggdb -Wall -Wextra -I ../src/ -I ../ -o registry ../src/registry.c ../src/dictionary.o ../src/log.o ../src/avl.o ../src/common.o -pthread -luuid -DHAVE_CONFIG_H
6
7 #include <uuid/uuid.h>
8 #include <inttypes.h>
@@ -13,86 +13,231 @@
13 #include "common.h"
14 #include "dictionary.h"
15
16 +#define REGISTRY_MEMORY_ATTRIBUTE aligned
17 +
18 +#define REGISTRY_URL_FLAGS_DEFAULT 0x00
19 +#define REGISTRY_URL_FLAGS_EXPIRED 0x01
20 +
21 +// ----------------------------------------------------------------------------
22 +// COMMON structures
23 +
24 +struct registry {
25 + unsigned long long persons_count;
26 + unsigned long long machines_count;
27 + unsigned long long usages_count;
28 +
29 + DICTIONARY *persons; // dictionary of PERSON *, with key the PERSON.guid
30 + DICTIONARY *machines; // dictionary of MACHINE *, with key the MACHINE.guid
31 + DICTIONARY *urls; // dictionary of URL *, with key the URL.url
32 +} registry;
33 +
34 +
35 +// ----------------------------------------------------------------------------
36 +// URL structures
37 +// Save memory by de-duplicating URLs
38 +
39 +struct url {
40 + uint32_t links;
41 + uint16_t url_length;
42 + char url[];
43 +} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
44 +typedef struct url URL;
45 +
46 +
47 +// ----------------------------------------------------------------------------
48 +// MACHINE structures
49 +
50 +// For each MACHINE-URL pair we keep this
51 +struct machine_url {
52 + URL *url; // de-duplicated URL
53 +// DICTIONARY *persons; // dictionary of PERSON *
54 +
55 + uint8_t flags;
56 + uint32_t first_t; // the first time we saw this
57 + uint32_t last_t; // the last time we saw this
58 + uint32_t usages; // how many times this has been accessed
59 +} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
60 +typedef struct machine_url MACHINE_URL;
61 +
62 +// A machine
63 struct machine {
64 char guid[36 + 1];
65
19 - time_t first_t;
20 - time_t last_t;
21 - size_t usages;
22 -
23 - DICTIONARY *urls;
24 -};
66 + DICTIONARY *urls; // MACHINE_URL *
67 +
68 + uint32_t first_t; // the first time we saw this
69 + uint32_t last_t; // the last time we saw this
70 + uint32_t usages; // how many times this has been accessed
71 +} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
72 typedef struct machine MACHINE;
73
74
75 +// ----------------------------------------------------------------------------
76 +// PERSON structures
77 +
78 +// for each PERSON-URL pair we keep this
79 +struct person_url {
80 + URL *url; // de-duplicated URL
81 + MACHINE *machine;
82 +
83 + uint8_t flags;
84 + uint32_t first_t; // the first time we saw this
85 + uint32_t last_t; // the last time we saw this
86 + uint32_t usages; // how many times this has been accessed
87 +} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
88 +typedef struct person_url PERSON_URL;
89 +
90 +// A person
91 struct person {
92 char guid[36 + 1];
93
31 - time_t first_t;
32 - time_t last_t;
33 - size_t usages;
94 + DICTIONARY *urls; // PERSON_URL *
95
35 - DICTIONARY *urls;
36 -};
96 + uint32_t first_t; // the first time we saw this
97 + uint32_t last_t; // the last time we saw this
98 + uint32_t usages; // how many times this has been accessed
99 +} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
100 typedef struct person PERSON;
101
102
40 -struct url {
41 - MACHINE *machine;
42 - PERSON *person;
103 +// ----------------------------------------------------------------------------
104 +// URL
105
44 - time_t first_t;
45 - time_t last_t;
46 - size_t usages;
106 +static inline URL *registry_url_get(const char *url) {
107 + debug(D_DICTIONARY, "Registry: registry_url_get('%s')", url);
108 + URL *u = dictionary_get(registry.urls, url);
109 + if(!u) {
110 + size_t len = strlen(url);
111
48 - size_t url_length;
49 - char url[];
50 -};
51 -typedef struct url URL;
112 + debug(D_DICTIONARY, "Registry: registry_url_get('%s'): allocating %zu bytes", url, sizeof(URL) + len + 1);
113 + u = malloc(sizeof(URL) + len + 1);
114 + if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + len + 1, url);
115
116 + strcpy(u->url, url);
117 + u->links = 0;
118 + u->url_length = len;
119
54 -struct registry {
55 - DICTIONARY *persons;
56 - DICTIONARY *machines;
57 -} registry;
120 + debug(D_DICTIONARY, "Registry: registry_url_get('%s'): indexing it", url);
121 + dictionary_set(registry.urls, url, u, sizeof(URL) + len + 1);
122 + }
123 +
124 + return u;
125 +}
126 +
127 +static inline void registry_url_link(URL *u) {
128 + u->links++;
129 +}
130 +
131 +static inline void registry_url_unlink(URL *u) {
132 + u->links--;
133 + if(!u->links) {
134 + dictionary_del(registry.urls, u->url);
135 + free(u);
136 + }
137 +}
138
139
140 // ----------------------------------------------------------------------------
141 // MACHINE
142
143 +int dump_machine_url(void *entry, void *nothing) {
144 + (void)nothing;
145 +
146 + MACHINE_URL *mu = entry;
147 +
148 + fprintf(stderr, "\n\tURL '%s'\n\t\tfirst seen %u, last seen %u, usages %u, flags 0x%02x\n",
149 + mu->url->url,
150 + mu->first_t,
151 + mu->last_t,
152 + mu->usages,
153 + mu->flags
154 + );
155 +
156 + return 0;
157 +}
158 +
159 +int dump_machine(void *entry, void *nothing) {
160 + (void)nothing;
161 +
162 + MACHINE *m = entry;
163 +
164 + fprintf(stderr, "MACHINE '%s'\n\tfirst seen %u, last seen %u, usages %u\n",
165 + m->guid,
166 + (uint32_t)m->first_t,
167 + (uint32_t)m->last_t,
168 + m->usages
169 + );
170 +
171 + dictionary_get_all(m->urls, dump_machine_url, NULL);
172 + return 0;
173 +}
174 +
175 MACHINE *registry_machine_load(const char *machine_guid) {
176 + debug(D_REGISTRY, "Registry: registry_machine_load('%s')", machine_guid);
177 (void)machine_guid;
178
179 + debug(D_REGISTRY, "Registry: registry_machine_load('%s'): not found", machine_guid);
180 return NULL;
181 }
182
183 int registry_machine_save(MACHINE *m) {
70 - (void)m;
184 + debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);
185 +
186 +#ifdef REGISTRY_STDOUT_DUMP
187 + fprintf(stderr, "\nSAVING ");
188 + dump_machine(m, NULL);
189 +#endif /* REGISTRY_STDOUT_DUMP */
190
191 return -1;
192 }
193
194 MACHINE *registry_machine_find(const char *machine_guid) {
195 + debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);
196 +
197 MACHINE *m = dictionary_get(registry.machines, machine_guid);
198 if(!m) m = registry_machine_load(machine_guid);
199 return m;
200 }
201
202
82 -MACHINE *registry_machine_get(const char *machine_guid) {
83 - MACHINE *m = registry_machine_find(machine_guid);
84 - if(!m) {
85 - debug(D_REGISTRY, "Registry: creating new machine '%s'", machine_guid);
203 +MACHINE *registry_machine_allocate(const char *machine_guid) {
204 + debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(MACHINE));
205 +
206 + MACHINE *m = calloc(1, sizeof(MACHINE));
207 + if(!m) fatal("Registry: cannot allocate memory for new machine '%s'", machine_guid);
208
87 - m = calloc(1, sizeof(MACHINE));
88 - if(!m) fatal("Registry: cannot allocate memory for new machine '%s'", machine_guid);
209 + strncpy(m->guid, machine_guid, 36);
210
90 - strncpy(m->guid, machine_guid, 36);
211 + debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);
212 + m->urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
213
92 - dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE));
93 -
94 - m->first_t = time(NULL);
95 - m->usages = 0;
214 + m->first_t = m->last_t = time(NULL);
215 + m->usages = 0;
216 +
217 + dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE));
218 +
219 + return m;
220 +}
221 +
222 +MACHINE *registry_machine_get(const char *machine_guid) {
223 + MACHINE *m = NULL;
224 +
225 + if(likely(machine_guid && *machine_guid)) {
226 + // validate it is a GUID
227 + uuid_t uuid;
228 + if(uuid_parse(machine_guid, uuid) == -1) {
229 + info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);
230 + }
231 + else {
232 + char buf[36 + 1];
233 + uuid_unparse_lower(uuid, buf);
234 + if(strcmp(machine_guid, buf))
235 + info("Registry: machine guid '%s' and re-generated '%s' differ!", machine_guid, buf);
236 +
237 + machine_guid = buf;
238 + m = registry_machine_find(machine_guid);
239 + if(!m) m = registry_machine_allocate(machine_guid);
240 + }
241 }
242
243 return m;
@@ -102,141 +247,509 @@ MACHINE *registry_machine_get(const char *machine_guid) {
247 // ----------------------------------------------------------------------------
248 // PERSON
249
105 -PERSON *registry_person_load(const char *person_guid) {
106 - (void)person_guid;
250 +int dump_person_url(void *entry, void *nothing) {
251 + (void)nothing;
252
108 - return NULL;
253 + PERSON_URL *pu = entry;
254 +
255 + fprintf(stderr, "\n\tURL '%s'\n\t\tfirst seen %u, last seen %u, usages %u, flags 0x%02x\n",
256 + pu->url->url,
257 + pu->first_t,
258 + pu->last_t,
259 + pu->usages,
260 + pu->flags
261 + );
262 +
263 + return 0;
264 +}
265 +
266 +int dump_person(void *entry, void *nothing) {
267 + (void)nothing;
268 +
269 + PERSON *p = entry;
270 +
271 + fprintf(stderr, "PERSON '%s'\n\tfirst seen %u, last seen %u, usages %u\n",
272 + p->guid,
273 + p->first_t,
274 + p->last_t,
275 + p->usages
276 + );
277 +
278 + dictionary_get_all(p->urls, dump_person_url, NULL);
279 + return 0;
280 }
281
282 int registry_person_save(PERSON *p) {
112 - (void)p;
283 + debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);
284 +
285 +#ifdef REGISTRY_STDOUT_DUMP
286 + fprintf(stderr, "\nSAVING ");
287 + dump_person(p, NULL);
288 +#endif /* REGISTRY_STDOUT_DUMP */
289
290 return -1;
291 }
292
293 +PERSON *registry_person_load(const char *person_guid) {
294 + debug(D_REGISTRY, "Registry: registry_person_load('%s')", person_guid);
295 + (void)person_guid;
296 +
297 + debug(D_REGISTRY, "Registry: registry_person_load('%s'): not found", person_guid);
298 + return NULL;
299 +}
300 +
301 PERSON *registry_person_find(const char *person_guid) {
302 + debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid);
303 PERSON *p = dictionary_get(registry.persons, person_guid);
304 if(!p) p = registry_person_load(person_guid);
305 return p;
306 }
307
123 -PERSON *registry_person_get(const char *person_guid) {
308 +PERSON *registry_person_allocate(const char *person_guid) {
309 PERSON *p = NULL;
310
126 - if(person_guid && *person_guid)
127 - p = registry_person_find(person_guid);
311 + debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(PERSON));
312
129 - if(!p) {
130 - if(person_guid && *person_guid)
131 - error("Registry: discarding unknown person guid '%s'. Will give a new PERSONID.", person_guid);
313 + p = calloc(1, sizeof(PERSON));
314 + if(!p) fatal("Registry: cannot allocate memory for new person.");
315
133 - debug(D_REGISTRY, "Registry: creating new person");
316 + if(!person_guid) {
317 + for (; ;) {
318 + uuid_t uuid;
319 + if (uuid_generate_time_safe(uuid) == -1)
320 + info("Registry: uuid_generate_time_safe() reports UUID generation is not safe for uniqueness.");
321
135 - p = calloc(1, sizeof(PERSON));
136 - if(!p) fatal("Registry: cannot allocate memory for new person.");
322 + uuid_unparse_lower(uuid, p->guid);
323
138 - uuid_t uuid;
139 - if(uuid_generate_time_safe(uuid) == -1)
140 - info("Registry: uuid_generate_time_safe() reports UUID generation is not safe for uniqueness.");
324 + debug(D_REGISTRY, "Registry: Checking if the generated person guid '%s' is unique", p->guid);
325 + if (!dictionary_get(registry.persons, p->guid)) {
326 + debug(D_REGISTRY, "Registry: generated person guid '%s' is unique", p->guid);
327 + break;
328 + }
329 + else
330 + info("Registry: generated person guid '%s' found in the registry. Retrying...", p->guid);
331 + }
332 + }
333 + else {
334 + strncpy(p->guid, person_guid, 36);
335 + p->guid[36] = '\0';
336 + }
337 +
338 + debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid);
339 + p->urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
340
142 - uuid_unparse_lower(uuid, p->guid);
341 + p->first_t = p->last_t = time(NULL);
342 + p->usages = 0;
343 +
344 + dictionary_set(registry.persons, p->guid, p, sizeof(PERSON));
345 + return p;
346 +}
347
144 - dictionary_set(registry.persons, p->guid, p, sizeof(PERSON));
348 +PERSON *registry_person_get(const char *person_guid) {
349 + PERSON *p = NULL;
350
146 - p->first_t = time(NULL);
147 - p->usages = 0;
351 + if(person_guid && *person_guid) {
352 + // validate it is a GUID
353 + uuid_t uuid;
354 + if(uuid_parse(person_guid, uuid) == -1) {
355 + info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid);
356 + }
357 + else {
358 + char buf[36 + 1];
359 + uuid_unparse_lower(uuid, buf);
360 + if(strcmp(person_guid, buf))
361 + info("Registry: person guid '%s' and re-generated '%s' differ!", person_guid, buf);
362 +
363 + person_guid = buf;
364 + p = registry_person_find(person_guid);
365 + if(!p) person_guid = NULL;
366 + }
367 }
368
369 + if(!p) p = registry_person_allocate(NULL);
370 +
371 return p;
372 }
373
374
375 // ----------------------------------------------------------------------------
155 -// URL
156 -
157 -URL *registry_url_update(PERSON *p, MACHINE *m, const char *url) {
158 - URL *pu = dictionary_get(p->urls, url);
159 - URL *mu = dictionary_get(m->urls, url);
160 - URL *u = NULL;
161 -
162 - if(pu != mu || pu == NULL || mu == NULL)
163 - error("Registry: person/machine discrepancy on url '%s', for person '%s' and machine '%s'", url, p->guid, m->guid);
164 - else
165 - u = pu;
376 +// LINKING OF OBJECTS
377 +
378 +PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u) {
379 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);
380 +
381 + PERSON_URL *pu = dictionary_get(p->urls, u->url);
382 + if(!pu) {
383 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found, allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL));
384 + pu = malloc(sizeof(PERSON_URL));
385 + pu->machine = m;
386 + pu->first_t = pu->last_t = time(NULL);
387 + pu->usages = 1;
388 + pu->url = u;
389 + pu->flags = REGISTRY_URL_FLAGS_DEFAULT;
390 +
391 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);
392 + dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL));
393 + registry_url_link(u);
394 + }
395 + else {
396 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);
397 + pu->usages++;
398 +
399 + if(pu->machine != m) {
400 + MACHINE_URL *mu = dictionary_get(pu->machine->urls, u->url);
401 + if(mu) {
402 + info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.",
403 + p->guid, m->guid, u->url, pu->machine->guid);
404 + mu->flags |= REGISTRY_URL_FLAGS_EXPIRED;
405 + }
406 +
407 + pu->machine = m;
408 + }
409 + }
410
167 - if(!u) {
168 - size_t len = strlen(url);
411 + p->usages++;
412
170 - URL *u = calloc(1, sizeof(URL) + len);
171 - if(!u) fatal("Registry: cannot allocate memory for person '%s', machine '%s', url '%s'.", p->guid, m->guid, url);
413 + if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED)
414 + info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
415
173 - strcpy(u->url, url);
174 - u->url_length = len;
175 - u->person = p;
176 - u->machine = m;
177 - u->first_t = time(NULL);
178 - u->usages = 1;
416 + return pu;
417 +}
418
180 - dictionary_set(p->urls, url, u, sizeof(URL) + u->url_length);
181 - dictionary_set(m->urls, url, u, sizeof(URL) + u->url_length);
182 - if(pu) free(pu);
183 - if(mu) free(mu);
419 +MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u) {
420 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url);
421 +
422 + MACHINE_URL *mu = dictionary_get(m->urls, u->url);
423 + if(!mu) {
424 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found, allocating %zu bytes", p->guid, m->guid, u->url, sizeof(MACHINE_URL));
425 + mu = malloc(sizeof(MACHINE_URL));
426 + //mu->persons = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
427 + mu->first_t = mu->last_t = time(NULL);
428 + mu->usages = 1;
429 + mu->url = u;
430 + mu->flags = REGISTRY_URL_FLAGS_DEFAULT;
431 +
432 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing URL in machine", p->guid, m->guid, u->url);
433 + dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL));
434 + registry_url_link(u);
435 + }
436 + else {
437 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);
438 + mu->usages++;
439 }
185 - else
186 - u->usages++;
440
188 - p->usages++;
441 + //debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing person in machine", p->guid, m->guid, u->url);
442 + //dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));
443 +
444 m->usages++;
190 - p->last_t = m->last_t = u->last_t = time(NULL);
445
192 - return u;
446 + if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED)
447 + info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
448 +
449 + return mu;
450 }
451
452
453 // ----------------------------------------------------------------------------
197 -// REGISTRY
454 +// REGISTRY REQUESTS
455
456 int registry_save(void) {
457 return -1;
458 }
459
203 -char *registry_request(const char *person_guid, const char *machine_guid, const char *url) {
204 - PERSON *p = NULL;
205 - MACHINE *m = NULL;
460 +PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url) {
461 + debug(D_REGISTRY, "registry_request('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);
462
207 - // --- PERSON ---
208 - p = registry_person_get(person_guid);
209 - person_guid = p->guid;
463 + MACHINE *m = registry_machine_get(machine_guid);
464 + if(!m) return NULL;
465
211 - // --- MACHINE ---
212 - m = registry_machine_get(machine_guid);
213 - machine_guid = m->guid;
466 + URL *u = registry_url_get(url);
467
215 - // --- URL ---
216 - registry_url_update(p, m, url);
468 + PERSON *p = registry_person_get(person_guid);
469 + registry_person_link_to_url(p, m, u);
470 + registry_machine_link_to_url(p, m, u);
471
472 + debug(D_REGISTRY, "Registry: registry_request('%s', '%s', '%s'): saving", person_guid, machine_guid, url);
473 registry_person_save(p);
474 registry_machine_save(m);
475 registry_save();
476
222 - return NULL;
477 + return p;
478 }
479
480 +// ----------------------------------------------------------------------------
481 +// REGISTRY
482 +
483 void registry_init(void) {
226 - registry.persons = dictionary_create(DICTIONARY_FLAG_DEFAULT);
227 - if(!registry.persons)
228 - fatal("Registry: cannot create persons registry");
484 + debug(D_REGISTRY, "Registry: creating global registry dictionary for persons.");
485 + registry.persons = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
486 +
487 + debug(D_REGISTRY, "Registry: creating global registry dictionary for machines.");
488 + registry.machines = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
489
230 - registry.machines = dictionary_create(DICTIONARY_FLAG_DEFAULT);
231 - if(!registry.machines)
232 - fatal("Registry: cannot create machines registry");
490 + debug(D_REGISTRY, "Registry: creating global registry dictionary for urls.");
491 + registry.urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
492 }
493
494 +void registry_free(void) {
495 + while(registry.persons->values_index.root) {
496 + PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;
497 +
498 +#ifdef REGISTRY_STDOUT_DUMP
499 + fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages);
500 +#endif /* REGISTRY_STDOUT_DUMP */
501 +
502 + while(p->urls->values_index.root) {
503 + PERSON_URL *pu = ((NAME_VALUE *)p->urls->values_index.root)->value;
504 +
505 +#ifdef REGISTRY_STDOUT_DUMP
506 + fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags);
507 +#endif /* REGISTRY_STDOUT_DUMP */
508 +
509 + debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid);
510 + dictionary_del(p->urls, pu->url->url);
511 +
512 + debug(D_REGISTRY, "Registry: unlinking url '%s'", pu->url->url);
513 + registry_url_unlink(pu->url);
514 +
515 + debug(D_REGISTRY, "Registry: freeing person url");
516 + free(pu);
517 + }
518 +
519 + debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);
520 + dictionary_del(registry.persons, p->guid);
521 +
522 + debug(D_REGISTRY, "Registry: destroying URL dictionary of person '%s'", p->guid);
523 + dictionary_destroy(p->urls);
524 +
525 + debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid);
526 + free(p);
527 + }
528 +
529 + while(registry.machines->values_index.root) {
530 + MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value;
531 +
532 +#ifdef REGISTRY_STDOUT_DUMP
533 + fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);
534 +#endif /* REGISTRY_STDOUT_DUMP */
535 +
536 + while(m->urls->values_index.root) {
537 + MACHINE_URL *mu = ((NAME_VALUE *)m->urls->values_index.root)->value;
538 +
539 +#ifdef REGISTRY_STDOUT_DUMP
540 + fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);
541 +#endif /* REGISTRY_STDOUT_DUMP */
542 +
543 + //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url);
544 + //dictionary_destroy(mu->persons);
545 +
546 + debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid);
547 + dictionary_del(m->urls, mu->url->url);
548 +
549 + debug(D_REGISTRY, "Registry: unlinking url '%s'", mu->url);
550 + registry_url_unlink(mu->url);
551 +
552 + debug(D_REGISTRY, "Registry: freeing machine url");
553 + free(mu);
554 + }
555 +
556 + debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid);
557 + dictionary_del(registry.machines, m->guid);
558 +
559 + debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid);
560 + dictionary_destroy(m->urls);
561 +
562 + debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid);
563 + free(m);
564 + }
565 +
566 + debug(D_REGISTRY, "Registry: destroying persons dictionary");
567 + dictionary_destroy(registry.persons);
568 +
569 + debug(D_REGISTRY, "Registry: destroying machines dictionary");
570 + dictionary_destroy(registry.machines);
571 +
572 + debug(D_REGISTRY, "Registry: destroying urls dictionary");
573 + dictionary_destroy(registry.urls);
574 +}
575 +
576 +int test1(int argc, char **argv) {
577 + (void) argc;
578 + (void) argv;
579 +
580 + uint32_t u, users = 1000000;
581 + uint32_t m, machines = 50000;
582 + uint32_t machines2 = machines * 2;
583 +
584 + char **users_guids = malloc(users * sizeof(char *));
585 + char **machines_guids = malloc(machines2 * sizeof(char *));
586 + char **machines_urls = malloc(machines2 * sizeof(char *));
587 + unsigned long long start, end;
588 +
589 + registry_init();
590 +
591 + fprintf(stderr, "Generating %u machine guids\n", machines2);
592 + for(m = 0; m < machines2 ;m++) {
593 + uuid_t uuid;
594 + machines_guids[m] = malloc(36+1);
595 + uuid_generate(uuid);
596 + uuid_unparse(uuid, machines_guids[m]);
597 +
598 + char buf[FILENAME_MAX + 1];
599 + snprintf(buf, FILENAME_MAX, "http://%u.netdata.rocks/", m+1);
600 + machines_urls[m] = strdup(buf);
601 +
602 + // fprintf(stderr, "\tmachine %u: '%s', url: '%s'\n", m + 1, machines_guids[m], machines_urls[m]);
603 + }
604 +
605 + start = timems();
606 + fprintf(stderr, "\nGenerating %u users accessing %u machines\n", users, machines);
607 + m = 0;
608 + for(u = 0; u < users ; u++) {
609 + if(++m == machines) m = 0;
610 +
611 + PERSON *p = registry_request(NULL, machines_guids[m], machines_urls[m]);
612 + users_guids[u] = p->guid;
613 + }
614 + end = timems();
615 + fprintf(stderr, "%u accesses served in %0.2f seconds (%llu per second)\n", u, (end-start) / 1000000.0, (unsigned long long)u * 1000000ULL / (end-start));
616 +
617 + start = timems();
618 + fprintf(stderr, "\nAll %u users accessing again the same %u servers\n", users, machines);
619 + m = 0;
620 + for(u = 0; u < users ; u++) {
621 + if(++m == machines) m = 0;
622 +
623 + PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m]);
624 +
625 + if(p->guid != users_guids[u])
626 + fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid);
627 + }
628 + end = timems();
629 + fprintf(stderr, "%u accesses served in %0.2f seconds (%llu per second)\n", u, (end-start) / 1000000.0, (unsigned long long)u * 1000000ULL / (end-start));
630 +
631 + start = timems();
632 + fprintf(stderr, "\nAll %u users accessing a new server, out of the %u servers\n", users, machines);
633 + m = 1;
634 + for(u = 0; u < users ; u++) {
635 + if(++m == machines) m = 0;
636 +
637 + PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m]);
638 +
639 + if(p->guid != users_guids[u])
640 + fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid);
641 + }
642 + end = timems();
643 + fprintf(stderr, "%u accesses served in %0.2f seconds (%llu per second)\n", u, (end-start) / 1000000.0, (unsigned long long)u * 1000000ULL / (end-start));
644 +
645 + start = timems();
646 + fprintf(stderr, "\n%u random users accessing a random server, out of the %u servers\n", users, machines);
647 + for(u = 0; u < users ; u++) {
648 + uint32_t tu = random() * users / RAND_MAX;
649 + uint32_t tm = random() * machines / RAND_MAX;
650 +
651 + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm]);
652 +
653 + if(p->guid != users_guids[tu])
654 + fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
655 + }
656 + end = timems();
657 + fprintf(stderr, "%u accesses served in %0.2f seconds (%llu per second)\n", u, (end-start) / 1000000.0, (unsigned long long)u * 1000000ULL / (end-start));
658 +
659 + start = timems();
660 + fprintf(stderr, "\n%u random users accessing a random server, out of %u servers\n", users, machines2);
661 + for(u = 0; u < users ; u++) {
662 + uint32_t tu = random() * users / RAND_MAX;
663 + uint32_t tm = random() * machines2 / RAND_MAX;
664 +
665 + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm]);
666 +
667 + if(p->guid != users_guids[tu])
668 + fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
669 + }
670 + end = timems();
671 + fprintf(stderr, "%u accesses served in %0.2f seconds (%llu per second)\n", u, (end-start) / 1000000.0, (unsigned long long)u * 1000000ULL / (end-start));
672 +
673 + start = timems();
674 + fprintf(stderr, "\n%u random user accesses to a random server, out of %u servers, using 1 out of 3 a random url\n", users * 2, machines2);
675 + for(u = 0; u < users * 2 ; u++) {
676 + uint32_t tu = random() * users / RAND_MAX;
677 + uint32_t tm = random() * machines2 / RAND_MAX;
678 +
679 + char *url = machines_urls[tm];
680 + char buf[FILENAME_MAX + 1];
681 + if(random() % 3 == 1) {
682 + snprintf(buf, FILENAME_MAX, "http://random.%ld.netdata.rocks/", random());
683 + url = buf;
684 + }
685 +
686 + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], url);
687 +
688 + if(p->guid != users_guids[tu])
689 + fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
690 + }
691 + end = timems();
692 + fprintf(stderr, "%u accesses served in %0.2f seconds (%llu per second)\n", u, (end-start) / 1000000.0, (unsigned long long)u * 1000000ULL / (end-start));
693 +
694 + exit(0);
695 +
696 + registry_free();
697 + return 0;
698 +}
699 +
700 +// ----------------------------------------------------------------------------
701 +// TESTING
702 +
703 int main(int argc, char **argv) {
704 + // debug_flags = 0xFFFFFFFF;
705 + test1(argc, argv);
706 + exit(0);
707 +
708 (void)argc;
709 (void)argv;
710
711 +
712 + PERSON *p1, *p2;
713 +
714 + fprintf(stderr, "\n\nINITIALIZATION\n");
715 +
716 registry_init();
717
718 +
719 + int i = 100000;
720 + while(i--) {
721 +#ifdef REGISTRY_STDOUT_DUMP
722 + fprintf(stderr, "\n\nADDING ENTRY\n");
723 +#endif /* REGISTRY_STDOUT_DUMP */
724 + p1 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/");
725 +
726 +#ifdef REGISTRY_STDOUT_DUMP
727 + fprintf(stderr, "\n\nADDING ANOTHER URL\n");
728 +#endif /* REGISTRY_STDOUT_DUMP */
729 + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://127.0.0.1:19999/");
730 +
731 +#ifdef REGISTRY_STDOUT_DUMP
732 + fprintf(stderr, "\n\nADDING ANOTHER URL\n");
733 +#endif /* REGISTRY_STDOUT_DUMP */
734 + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/");
735 +
736 +#ifdef REGISTRY_STDOUT_DUMP
737 + fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n");
738 +#endif /* REGISTRY_STDOUT_DUMP */
739 + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/");
740 +
741 +#ifdef REGISTRY_STDOUT_DUMP
742 + fprintf(stderr, "\n\nADDING ANOTHER PERSON\n");
743 +#endif /* REGISTRY_STDOUT_DUMP */
744 + p2 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/");
745 +
746 +#ifdef REGISTRY_STDOUT_DUMP
747 + fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n");
748 +#endif /* REGISTRY_STDOUT_DUMP */
749 + p2 = registry_request(p2->guid, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/");
750 + }
751 +
752 + fprintf(stderr, "\n\nCLEANUP\n");
753 + //registry_free();
754 return 0;
755 }