@cryptotaxi247 / netdata-1 / commits / a1abb2f33

registry: working prototype with load/save

Costa Tsaousis committed May 8, 2016 at 22:43 UTC a1abb2f3398730f13a0704133ee1e52b4b6561a9
11 files changed +581 -223
CMakeLists.txt
+1 -1
@@ -91,7 +91,7 @@ set(APPS_PLUGIN_SOURCE_FILES
91
92 include_directories(AFTER .)
93
94 -add_definitions(-DHAVE_CONFIG_H -DNETDATA_INTERNAL_CHECKS=1 -DCACHE_DIR="/tmp" -DCONFIG_DIR="/tmp" -DLOG_DIR="/tmp" -DPLUGINS_DIR="/tmp" -DWEB_DIR="/tmp")
94 +add_definitions(-DHAVE_CONFIG_H -DNETDATA_INTERNAL_CHECKS=1 -DCACHE_DIR="/tmp" -DCONFIG_DIR="/tmp" -DLOG_DIR="/tmp" -DPLUGINS_DIR="/tmp" -DWEB_DIR="/tmp" -DVARLIB_DIR="/tmp")
95
96 add_executable(netdata ${NETDATA_SOURCE_FILES})
97 add_executable(apps.plugin ${APPS_PLUGIN_SOURCE_FILES})
configure.ac
+1
@@ -143,6 +143,7 @@ fi
143
144 AC_DEFINE_UNQUOTED([NETDATA_USER], ["${with_user}"], [use this user to drop privileged])
145
146 +AC_SUBST([varlibdir], ["\$(localstatedir)/lib/netdata"])
147 AC_SUBST([cachedir], ["\$(localstatedir)/cache/netdata"])
148 AC_SUBST([chartsdir], ["\$(libexecdir)/netdata/charts.d"])
149 AC_SUBST([nodedir], ["\$(libexecdir)/netdata/node.d"])
netdata-installer.sh
+4 -1
@@ -120,6 +120,7 @@ It will be installed at these locations:
120 - web files at ${NETDATA_PREFIX}/usr/share/netdata
121 - plugins at ${NETDATA_PREFIX}/usr/libexec/netdata
122 - cache files at ${NETDATA_PREFIX}/var/cache/netdata
123 + - db files at ${NETDATA_PREFIX}/var/lib/netdata
124 - log files at ${NETDATA_PREFIX}/var/log/netdata
125 - pid file at ${NETDATA_PREFIX}/var/run
126
@@ -407,6 +408,7 @@ defport=19999
408 NETDATA_PORT="$( config_option "port" ${defport} )"
409
410 # directories
411 +NETDATA_LIB_DIR="$( config_option "lib directory" "${NETDATA_PREFIX}/var/lib/netdata" )"
412 NETDATA_CACHE_DIR="$( config_option "cache directory" "${NETDATA_PREFIX}/var/cache/netdata" )"
413 NETDATA_WEB_DIR="$( config_option "web files directory" "${NETDATA_PREFIX}/usr/share/netdata/web" )"
414 NETDATA_LOG_DIR="$( config_option "log directory" "${NETDATA_PREFIX}/var/log/netdata" )"
@@ -425,7 +427,7 @@ if [ ! -d "${NETDATA_RUN_DIR}" ]
427 fi
428
429 echo >&2 "Fixing directories (user: ${NETDATA_USER})..."
428 -for x in "${NETDATA_WEB_DIR}" "${NETDATA_CONF_DIR}" "${NETDATA_CACHE_DIR}" "${NETDATA_LOG_DIR}"
430 +for x in "${NETDATA_WEB_DIR}" "${NETDATA_CONF_DIR}" "${NETDATA_CACHE_DIR}" "${NETDATA_LOG_DIR}" "${NETDATA_LIB_DIR}"
431 do
432 if [ ! -d "${x}" ]
433 then
@@ -716,6 +718,7 @@ else
718 deletedir "${NETDATA_PREFIX}/etc/netdata"
719 deletedir "${NETDATA_PREFIX}/usr/share/netdata"
720 deletedir "${NETDATA_PREFIX}/usr/libexec/netdata"
721 + deletedir "${NETDATA_PREFIX}/var/lib/netdata"
722 deletedir "${NETDATA_PREFIX}/var/cache/netdata"
723 deletedir "${NETDATA_PREFIX}/var/log/netdata"
724 fi
profile/benchmark-dictionary.c
+2 -2
@@ -22,14 +22,14 @@ struct myvalue {
22 int main(int argc, char **argv) {
23 if(argc || argv) {;}
24
25 - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
25 + DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_CORRUPTION_CHECK);
26 if(!dict) fatal("Cannot create dictionary.");
27
28 struct rusage start, end;
29 unsigned long long dt;
30 char buf[100 + 1];
31 struct myvalue value, *v;
32 - int i, max = 1000000, max2;
32 + int i, max = 100000, max2;
33
34 // ------------------------------------------------------------------------
35
src/Makefile.am
+1
@@ -4,6 +4,7 @@
4 MAINTAINERCLEANFILES= $(srcdir)/Makefile.in
5
6 AM_CPPFLAGS = \
7 + -DVARLIB_DIR="\"$(varlibdir)\"" \
8 -DCACHE_DIR="\"$(cachedir)\"" \
9 -DCONFIG_DIR="\"$(configdir)\"" \
10 -DLOG_DIR="\"$(logdir)\"" \
src/common.c
+20
@@ -762,3 +762,23 @@ pid_t gettid(void)
762 return syscall(SYS_gettid);
763 }
764
765 +char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {
766 + char *s = fgets(buf, buf_size, fp);
767 + if(!s) return NULL;
768 +
769 + char *t = s;
770 + if(*t != '\0') {
771 + // find the string end
772 + while (*++t != '\0');
773 +
774 + // trim trailing spaces/newlines/tabs
775 + while (--t > s && *t == '\n')
776 + *t = '\0';
777 + }
778 +
779 + if(len)
780 + *len = t - s + 1;
781 +
782 + return s;
783 +}
784 +
src/common.h
+3
@@ -1,5 +1,6 @@
1 #include <sys/time.h>
2 #include <sys/resource.h>
3 +#include <stdio.h>
4
5 #ifndef NETDATA_COMMON_H
6 #define NETDATA_COMMON_H 1
@@ -39,6 +40,8 @@ extern pid_t gettid(void);
40
41 extern unsigned long long timems(void);
42
43 +extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);
44 +
45 /* fix for alpine linux */
46 #ifndef RUSAGE_THREAD
47 #ifdef RUSAGE_CHILDREN
src/dictionary.c
+26 -8
@@ -33,6 +33,7 @@ static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *di
33
34 dict->searches++;
35 avl_search(&(dict->values_index), (avl *)&tmp, name_value_iterator, (avl **)&result);
36 +
37 return result;
38 }
39
@@ -226,20 +227,37 @@ int dictionary_del(DICTIONARY *dict, const char *name) {
227 }
228
229
229 -static void dictionary_walker(avl *a, int (*callback)(void *entry, void *data), void *data) {
230 - if(unlikely(!a)) return;
230 +static int dictionary_walker(avl *a, int (*callback)(void *entry, void *data), void *data) {
231 + int total = 0, ret = 0;
232
232 - if(a->right)
233 - dictionary_walker(a->right, callback, data);
233 + if(a->right) {
234 + ret = dictionary_walker(a->right, callback, data);
235 + if(ret < 0) return ret;
236 + total += ret;
237 + }
238
235 - callback(((NAME_VALUE *)a)->value, data);
239 + ret = callback(((NAME_VALUE *)a)->value, data);
240 + if(ret < 0) return ret;
241 + total += ret;
242
237 - if(a->left)
243 + if(a->left) {
244 dictionary_walker(a->left, callback, data);
245 + if (ret < 0) return ret;
246 + total += ret;
247 + }
248 +
249 + return total;
250 }
251
241 -void dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) {
252 +int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) {
253 + int ret = 0;
254 +
255 dictionary_read_lock(dict);
243 - dictionary_walker(dict->values_index.root, callback, data);
256 +
257 + if(likely(dict->values_index.root))
258 + ret = dictionary_walker(dict->values_index.root, callback, data);
259 +
260 dictionary_unlock(dict);
261 +
262 + return ret;
263 }
src/dictionary.h
+2 -1
@@ -11,6 +11,7 @@ typedef struct name_value {
11
12 uint32_t hash; // a simple hash to speed up searching
13 // we first compare hashes, and only if the hashes are equal we do string comparisons
14 +
15 char *name;
16 void *value;
17 } NAME_VALUE;
@@ -38,6 +39,6 @@ extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, siz
39 extern void *dictionary_get(DICTIONARY *dict, const char *name);
40 extern int dictionary_del(DICTIONARY *dict, const char *name);
41
41 -extern void dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data);
42 +extern int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data);
43
44 #endif /* NETDATA_DICTIONARY_H */
src/main.c
+1
@@ -275,6 +275,7 @@ int main(int argc, char **argv)
275 setenv("NETDATA_PLUGINS_DIR", config_get("global", "plugins directory" , PLUGINS_DIR), 1);
276 setenv("NETDATA_WEB_DIR" , config_get("global", "web files directory", WEB_DIR) , 1);
277 setenv("NETDATA_CACHE_DIR" , config_get("global", "cache directory" , CACHE_DIR) , 1);
278 + setenv("NETDATA_LIB_DIR" , config_get("global", "lib directory" , VARLIB_DIR) , 1);
279 setenv("NETDATA_LOG_DIR" , config_get("global", "log directory" , LOG_DIR) , 1);
280 setenv("NETDATA_HOST_PREFIX", config_get("global", "host access prefix" , "") , 1);
281 setenv("HOME" , config_get("global", "home directory" , CACHE_DIR) , 1);
src/registry.c
+520 -210
@@ -2,22 +2,28 @@
2 #include <config.h>
3 #endif
4
5 -// gcc -O1 -ggdb -Wall -Wextra -I ../src/ -I ../ -o registry ../src/registry.c ../src/dictionary.o ../src/log.o ../src/avl.o ../src/common.o -pthread -luuid -DHAVE_CONFIG_H
5 +// gcc -O1 -ggdb -Wall -Wextra -I ../src/ -I ../ -o registry ../src/registry.c ../src/dictionary.o ../src/log.o ../src/avl.o ../src/common.o ../src/appconfig.o ../src/web_buffer.o ../src/storage_number.o -pthread -luuid -lm -DHAVE_CONFIG_H -DVARLIB_DIR="\"/tmp\""
6
7 #include <uuid/uuid.h>
8 #include <inttypes.h>
9 #include <stdlib.h>
10 #include <string.h>
11 +#include <ctype.h>
12 +#include <unistd.h>
13 +#include <sys/stat.h>
14 +#include <sys/types.h>
15 +#include <errno.h>
16
17 #include "log.h"
18 #include "common.h"
19 #include "dictionary.h"
15 -
16 -#define REGISTRY_MEMORY_ATTRIBUTE aligned
20 +#include "appconfig.h"
21
22 #define REGISTRY_URL_FLAGS_DEFAULT 0x00
23 #define REGISTRY_URL_FLAGS_EXPIRED 0x01
24
25 +#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
26 +
27 // ----------------------------------------------------------------------------
28 // COMMON structures
29
@@ -29,9 +35,15 @@ struct registry {
35 unsigned long long persons_urls_count;
36 unsigned long long machines_urls_count;
37
38 + char *pathname;
39 + char *db_filename;
40 + char *log_filename;
41 + FILE *registry_log_fp;
42 +
43 DICTIONARY *persons; // dictionary of PERSON *, with key the PERSON.guid
44 DICTIONARY *machines; // dictionary of MACHINE *, with key the MACHINE.guid
45 DICTIONARY *urls; // dictionary of URL *, with key the URL.url
46 +
47 } registry;
48
49
@@ -41,9 +53,9 @@ struct registry {
53
54 struct url {
55 uint32_t links;
44 - uint16_t url_length;
45 - char url[];
46 -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
56 + uint16_t len;
57 + char url[1];
58 +};
59 typedef struct url URL;
60
61
@@ -59,7 +71,7 @@ struct machine_url {
71 uint32_t first_t; // the first time we saw this
72 uint32_t last_t; // the last time we saw this
73 uint32_t usages; // how many times this has been accessed
62 -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
74 +};
75 typedef struct machine_url MACHINE_URL;
76
77 // A machine
@@ -71,7 +83,7 @@ struct machine {
83 uint32_t first_t; // the first time we saw this
84 uint32_t last_t; // the last time we saw this
85 uint32_t usages; // how many times this has been accessed
74 -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
86 +};
87 typedef struct machine MACHINE;
88
89
@@ -87,7 +99,7 @@ struct person_url {
99 uint32_t first_t; // the first time we saw this
100 uint32_t last_t; // the last time we saw this
101 uint32_t usages; // how many times this has been accessed
90 -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
102 +};
103 typedef struct person_url PERSON_URL;
104
105 // A person
@@ -99,30 +111,37 @@ struct person {
111 uint32_t first_t; // the first time we saw this
112 uint32_t last_t; // the last time we saw this
113 uint32_t usages; // how many times this has been accessed
102 -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE ));
114 +};
115 typedef struct person PERSON;
116
117 +extern PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url, time_t when);
118
119 // ----------------------------------------------------------------------------
120 // URL
121
109 -static inline URL *registry_url_get(const char *url) {
110 - debug(D_DICTIONARY, "Registry: registry_url_get('%s')", url);
111 - URL *u = dictionary_get(registry.urls, url);
112 - if(!u) {
113 - size_t len = strlen(url);
122 +static inline URL *registry_url_allocate(const char *url) {
123 + size_t len = strlen(url);
124
115 - debug(D_DICTIONARY, "Registry: registry_url_get('%s'): allocating %zu bytes", url, sizeof(URL) + len + 1);
116 - u = malloc(sizeof(URL) + len + 1);
117 - if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + len + 1, url);
125 + debug(D_REGISTRY, "Registry: registry_url_allocate('%s'): allocating %zu bytes", url, sizeof(URL) + len);
126 + URL *u = malloc(sizeof(URL) + len);
127 + if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + len);
128
119 - strcpy(u->url, url);
120 - u->links = 0;
121 - u->url_length = len;
129 + strcpy(u->url, url);
130 + u->len = len;
131 + u->links = 0;
132
123 - debug(D_DICTIONARY, "Registry: registry_url_get('%s'): indexing it", url);
124 - dictionary_set(registry.urls, url, u, sizeof(URL) + len + 1);
133 + debug(D_REGISTRY, "Registry: registry_url_allocate('%s'): indexing it", url);
134 + dictionary_set(registry.urls, u->url, u, sizeof(URL));
135
136 + return u;
137 +}
138 +
139 +static inline URL *registry_url_get(const char *url) {
140 + debug(D_REGISTRY, "Registry: registry_url_get('%s')", url);
141 +
142 + URL *u = dictionary_get(registry.urls, url);
143 + if(!u) {
144 + u = registry_url_allocate(url);
145 registry.urls_count++;
146 }
147
@@ -131,81 +150,51 @@ static inline URL *registry_url_get(const char *url) {
150
151 static inline void registry_url_link(URL *u) {
152 u->links++;
153 + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): This URL has now %u links", u->url, u->links);
154 }
155
156 static inline void registry_url_unlink(URL *u) {
157 u->links--;
158 if(!u->links) {
159 + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): No more links for this URL", u->url);
160 dictionary_del(registry.urls, u->url);
161 free(u);
162 }
163 + else
164 + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): This URL has %u links left", u->url, u->links);
165 }
166
167
168 // ----------------------------------------------------------------------------
169 // MACHINE
170
148 -int dump_machine_url(void *entry, void *nothing) {
149 - (void)nothing;
150 -
151 - MACHINE_URL *mu = entry;
152 -
153 - fprintf(stderr, "\n\tURL '%s'\n\t\tfirst seen %u, last seen %u, usages %u, flags 0x%02x\n",
154 - mu->url->url,
155 - mu->first_t,
156 - mu->last_t,
157 - mu->usages,
158 - mu->flags
159 - );
160 -
161 - return 0;
162 -}
163 -
164 -int dump_machine(void *entry, void *nothing) {
165 - (void)nothing;
166 -
167 - MACHINE *m = entry;
168 -
169 - fprintf(stderr, "MACHINE '%s'\n\tfirst seen %u, last seen %u, usages %u\n",
170 - m->guid,
171 - (uint32_t)m->first_t,
172 - (uint32_t)m->last_t,
173 - m->usages
174 - );
175 -
176 - dictionary_get_all(m->urls, dump_machine_url, NULL);
177 - return 0;
171 +static inline MACHINE *registry_machine_find(const char *machine_guid) {
172 + debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);
173 + return dictionary_get(registry.machines, machine_guid);
174 }
175
180 -MACHINE *registry_machine_load(const char *machine_guid) {
181 - debug(D_REGISTRY, "Registry: registry_machine_load('%s')", machine_guid);
182 - (void)machine_guid;
176 +static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, time_t when) {
177 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(MACHINE_URL));
178
184 - debug(D_REGISTRY, "Registry: registry_machine_load('%s'): not found", machine_guid);
185 - return NULL;
186 -}
179 + MACHINE_URL *mu = malloc(sizeof(MACHINE_URL));
180 + if(!mu) fatal("registry_machine_link_to_url('%s', '%s'): cannot allocate %zu bytes.", m->guid, u->url, sizeof(MACHINE_URL));
181
188 -int registry_machine_save(MACHINE *m) {
189 - debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);
182 + // mu->persons = dictionary_create(DICTIONARY_FLAGS);
183 + // dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));
184
191 -#ifdef REGISTRY_STDOUT_DUMP
192 - fprintf(stderr, "\nSAVING ");
193 - dump_machine(m, NULL);
194 -#endif /* REGISTRY_STDOUT_DUMP */
185 + mu->first_t = mu->last_t = when;
186 + mu->usages = 1;
187 + mu->url = u;
188 + mu->flags = REGISTRY_URL_FLAGS_DEFAULT;
189
196 - return -1;
197 -}
198 -
199 -MACHINE *registry_machine_find(const char *machine_guid) {
200 - debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);
190 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url);
191 + dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL));
192 + registry_url_link(u);
193
202 - MACHINE *m = dictionary_get(registry.machines, machine_guid);
203 - if(!m) m = registry_machine_load(machine_guid);
204 - return m;
194 + return mu;
195 }
196
207 -
208 -MACHINE *registry_machine_allocate(const char *machine_guid) {
197 +static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) {
198 debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(MACHINE));
199
200 MACHINE *m = calloc(1, sizeof(MACHINE));
@@ -214,9 +203,9 @@ MACHINE *registry_machine_allocate(const char *machine_guid) {
203 strncpy(m->guid, machine_guid, 36);
204
205 debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);
217 - m->urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
206 + m->urls = dictionary_create(DICTIONARY_FLAGS);
207
219 - m->first_t = m->last_t = time(NULL);
208 + m->first_t = m->last_t = when;
209 m->usages = 0;
210
211 dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE));
@@ -224,7 +213,10 @@ MACHINE *registry_machine_allocate(const char *machine_guid) {
213 return m;
214 }
215
227 -MACHINE *registry_machine_get(const char *machine_guid) {
216 +// 1. validate machine GUID
217 +// 2. if it is valid, find it or create it and return it
218 +// 3. if it is not valid, return NULL
219 +static inline MACHINE *registry_machine_get(const char *machine_guid, time_t when) {
220 MACHINE *m = NULL;
221
222 if(likely(machine_guid && *machine_guid)) {
@@ -242,7 +234,7 @@ MACHINE *registry_machine_get(const char *machine_guid) {
234 machine_guid = buf;
235 m = registry_machine_find(machine_guid);
236 if(!m) {
245 - m = registry_machine_allocate(machine_guid);
237 + m = registry_machine_allocate(machine_guid, when);
238 registry.machines_count++;
239 }
240 }
@@ -255,65 +247,30 @@ MACHINE *registry_machine_get(const char *machine_guid) {
247 // ----------------------------------------------------------------------------
248 // PERSON
249
258 -int dump_person_url(void *entry, void *nothing) {
259 - (void)nothing;
260 -
261 - PERSON_URL *pu = entry;
262 -
263 - fprintf(stderr, "\n\tURL '%s'\n\t\tfirst seen %u, last seen %u, usages %u, flags 0x%02x\n",
264 - pu->url->url,
265 - pu->first_t,
266 - pu->last_t,
267 - pu->usages,
268 - pu->flags
269 - );
270 -
271 - return 0;
272 -}
273 -
274 -int dump_person(void *entry, void *nothing) {
275 - (void)nothing;
276 -
277 - PERSON *p = entry;
278 -
279 - fprintf(stderr, "PERSON '%s'\n\tfirst seen %u, last seen %u, usages %u\n",
280 - p->guid,
281 - p->first_t,
282 - p->last_t,
283 - p->usages
284 - );
285 -
286 - dictionary_get_all(p->urls, dump_person_url, NULL);
287 - return 0;
250 +static inline PERSON *registry_person_find(const char *person_guid) {
251 + debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid);
252 + return dictionary_get(registry.persons, person_guid);
253 }
254
290 -int registry_person_save(PERSON *p) {
291 - debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);
255 +static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, time_t when) {
256 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL));
257 + PERSON_URL *pu = malloc(sizeof(PERSON_URL));
258 + if(!pu) fatal("registry_person_link_to_url('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL));
259
293 -#ifdef REGISTRY_STDOUT_DUMP
294 - fprintf(stderr, "\nSAVING ");
295 - dump_person(p, NULL);
296 -#endif /* REGISTRY_STDOUT_DUMP */
260 + pu->machine = m;
261 + pu->first_t = pu->last_t = when;
262 + pu->usages = 1;
263 + pu->url = u;
264 + pu->flags = REGISTRY_URL_FLAGS_DEFAULT;
265
298 - return -1;
299 -}
266 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);
267 + dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL));
268 + registry_url_link(u);
269
301 -PERSON *registry_person_load(const char *person_guid) {
302 - debug(D_REGISTRY, "Registry: registry_person_load('%s')", person_guid);
303 - (void)person_guid;
304 -
305 - debug(D_REGISTRY, "Registry: registry_person_load('%s'): not found", person_guid);
306 - return NULL;
307 -}
308 -
309 -PERSON *registry_person_find(const char *person_guid) {
310 - debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid);
311 - PERSON *p = dictionary_get(registry.persons, person_guid);
312 - if(!p) p = registry_person_load(person_guid);
313 - return p;
270 + return pu;
271 }
272
316 -PERSON *registry_person_allocate(const char *person_guid) {
273 +static inline PERSON *registry_person_allocate(const char *person_guid, time_t when) {
274 PERSON *p = NULL;
275
276 debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(PERSON));
@@ -344,16 +301,21 @@ PERSON *registry_person_allocate(const char *person_guid) {
301 }
302
303 debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid);
347 - p->urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
304 + p->urls = dictionary_create(DICTIONARY_FLAGS);
305
349 - p->first_t = p->last_t = time(NULL);
306 + p->first_t = p->last_t = when;
307 p->usages = 0;
308
309 dictionary_set(registry.persons, p->guid, p, sizeof(PERSON));
310 return p;
311 }
312
356 -PERSON *registry_person_get(const char *person_guid) {
313 +
314 +// 1. validate person GUID
315 +// 2. if it is valid, find it
316 +// 3. if it is not valid, create a new one
317 +// 4. return it
318 +static inline PERSON *registry_person_get(const char *person_guid, time_t when) {
319 PERSON *p = NULL;
320
321 if(person_guid && *person_guid) {
@@ -375,33 +337,23 @@ PERSON *registry_person_get(const char *person_guid) {
337 }
338
339 if(!p) {
378 - p = registry_person_allocate(NULL);
340 + p = registry_person_allocate(NULL, when);
341 registry.persons_count++;
342 }
343
344 return p;
345 }
346
385 -
347 // ----------------------------------------------------------------------------
348 // LINKING OF OBJECTS
349
389 -PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u) {
350 +static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) {
351 debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);
352
353 PERSON_URL *pu = dictionary_get(p->urls, u->url);
354 if(!pu) {
394 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found, allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL));
395 - pu = malloc(sizeof(PERSON_URL));
396 - pu->machine = m;
397 - pu->first_t = pu->last_t = time(NULL);
398 - pu->usages = 1;
399 - pu->url = u;
400 - pu->flags = REGISTRY_URL_FLAGS_DEFAULT;
401 -
402 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);
403 - dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL));
404 - registry_url_link(u);
355 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
356 + pu = registry_person_url_allocate(p, m, u, when);
357 registry.persons_urls_count++;
358 }
359 else {
@@ -421,6 +373,7 @@ PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u) {
373 }
374
375 p->usages++;
376 + p->last_t = when;
377
378 if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED)
379 info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
@@ -428,22 +381,13 @@ PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u) {
381 return pu;
382 }
383
431 -MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u) {
384 +static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) {
385 debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url);
386
387 MACHINE_URL *mu = dictionary_get(m->urls, u->url);
388 if(!mu) {
436 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found, allocating %zu bytes", p->guid, m->guid, u->url, sizeof(MACHINE_URL));
437 - mu = malloc(sizeof(MACHINE_URL));
438 - //mu->persons = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
439 - mu->first_t = mu->last_t = time(NULL);
440 - mu->usages = 1;
441 - mu->url = u;
442 - mu->flags = REGISTRY_URL_FLAGS_DEFAULT;
443 -
444 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing URL in machine", p->guid, m->guid, u->url);
445 - dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL));
446 - registry_url_link(u);
389 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
390 + mu = registry_machine_url_allocate(m, u, when);
391 registry.machines_urls_count++;
392 }
393 else {
@@ -455,6 +399,7 @@ MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u) {
399 //dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));
400
401 m->usages++;
402 + m->last_t = when;
403
404 if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED)
405 info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
@@ -462,40 +407,388 @@ MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u) {
407 return mu;
408 }
409
465 -
410 // ----------------------------------------------------------------------------
467 -// REGISTRY REQUESTS
411 +// REGISTRY LOG LOAD/SAVE
412 +
413 +static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u) {
414 + if(likely(registry.registry_log_fp))
415 + fprintf(registry.registry_log_fp, "%c\t%08x\t%s\t%s\t%s\n",
416 + action,
417 + p->last_t,
418 + p->guid,
419 + m->guid,
420 + u->url
421 + );
422 +}
423
469 -int registry_save(void) {
470 - return -1;
424 +void registry_log_open(void) {
425 + registry.registry_log_fp = fopen(registry.log_filename, "a");
426 +
427 + if(registry.registry_log_fp) {
428 + if (setvbuf(registry.registry_log_fp, NULL, _IOLBF, 0) != 0)
429 + error("Cannot set line buffering on registry log file.");
430 + }
431 +}
432 +
433 +void registry_log_close(void) {
434 + if(registry.registry_log_fp)
435 + fclose(registry.registry_log_fp);
436 +
437 + registry.registry_log_fp = NULL;
438 +}
439 +
440 +void registry_log_recreate(void) {
441 + if(registry.registry_log_fp != NULL) {
442 + fclose(registry.registry_log_fp);
443 +
444 + registry.registry_log_fp = fopen(registry.log_filename, "w");
445 + if(registry.registry_log_fp) fclose(registry.registry_log_fp);
446 + registry.registry_log_fp = NULL;
447 +
448 + registry_log_open();
449 + }
450 }
451
473 -PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url) {
452 +int registry_log_load(void) {
453 + char *s, buf[4096 + 1];
454 + size_t line = 0;
455 +
456 + registry_log_close();
457 +
458 + debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename);
459 + FILE *fp = fopen(registry.log_filename, "r");
460 + if(!fp) {
461 + error("Registry: cannot open registry file: %s", registry.log_filename);
462 + return 0;
463 + }
464 +
465 + size_t len = 0;
466 + while((s = fgets_trim_len(buf, 4096, fp, &len))) {
467 + line++;
468 +
469 + switch(s[0]) {
470 + case 'A':
471 + // verify it is valid
472 + if(unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t'))
473 + error("Registry log line %u is wrong (len = %zu).", line, len);
474 +
475 + s[1] = s[10] = s[47] = s[84] = '\0';
476 + registry_request(&s[11], &s[48], &s[85], strtoul(&s[2], NULL, 16));
477 + break;
478 +
479 + default:
480 + error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.log_filename, s);
481 + break;
482 + }
483 + }
484 +
485 + registry_log_open();
486 +
487 + return 0;
488 +}
489 +
490 +
491 +// ----------------------------------------------------------------------------
492 +// REGISTRY REQUESTS
493 +
494 +PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url, time_t when) {
495 debug(D_REGISTRY, "registry_request('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);
496
476 - MACHINE *m = registry_machine_get(machine_guid);
497 + MACHINE *m = registry_machine_get(machine_guid, when);
498 if(!m) return NULL;
499
500 URL *u = registry_url_get(url);
501 + PERSON *p = registry_person_get(person_guid, when);
502
481 - PERSON *p = registry_person_get(person_guid);
482 - registry_person_link_to_url(p, m, u);
483 - registry_machine_link_to_url(p, m, u);
503 + registry_person_link_to_url(p, m, u, when);
504 + registry_machine_link_to_url(p, m, u, when);
505
485 - debug(D_REGISTRY, "Registry: registry_request('%s', '%s', '%s'): saving", person_guid, machine_guid, url);
486 - registry_person_save(p);
487 - registry_machine_save(m);
506 + registry_log('A', p, m, u);
507
508 registry.usages_count++;
490 - registry_save();
491 -
509 return p;
510 }
511
512 +
513 +// ----------------------------------------------------------------------------
514 +// REGISTRY LOAD/SAVE
515 +
516 +int registry_machine_save_url(void *entry, void *file) {
517 + MACHINE_URL *mu = entry;
518 + FILE *fp = file;
519 +
520 + debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url);
521 +
522 + int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n",
523 + mu->first_t,
524 + mu->last_t,
525 + mu->usages,
526 + mu->flags,
527 + mu->url->url
528 + );
529 +
530 + return ret;
531 +}
532 +
533 +int registry_machine_save(void *entry, void *file) {
534 + MACHINE *m = entry;
535 + FILE *fp = file;
536 +
537 + debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);
538 +
539 + int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n",
540 + m->first_t,
541 + m->last_t,
542 + m->usages,
543 + m->guid
544 + );
545 +
546 + if(ret >= 0) {
547 + int ret2 = dictionary_get_all(m->urls, registry_machine_save_url, fp);
548 + if(ret2 < 0) return ret2;
549 + ret += ret2;
550 + }
551 +
552 + return ret;
553 +}
554 +
555 +static inline int registry_person_save_url(void *entry, void *file) {
556 + PERSON_URL *pu = entry;
557 + FILE *fp = file;
558 +
559 + debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);
560 +
561 + int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\n",
562 + pu->first_t,
563 + pu->last_t,
564 + pu->usages,
565 + pu->flags,
566 + pu->machine->guid,
567 + pu->url->url
568 + );
569 +
570 + return ret;
571 +}
572 +
573 +static inline int registry_person_save(void *entry, void *file) {
574 + PERSON *p = entry;
575 + FILE *fp = file;
576 +
577 + debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);
578 +
579 + int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n",
580 + p->first_t,
581 + p->last_t,
582 + p->usages,
583 + p->guid
584 + );
585 +
586 + if(ret >= 0) {
587 + int ret2 = dictionary_get_all(p->urls, registry_person_save_url, fp);
588 + if (ret2 < 0) return ret2;
589 + ret += ret2;
590 + }
591 +
592 + return ret;
593 +}
594 +
595 +static int registry_save(void) {
596 + char tmp_filename[FILENAME_MAX + 1];
597 + char old_filename[FILENAME_MAX + 1];
598 +
599 + snprintf(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);
600 + snprintf(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);
601 +
602 + debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);
603 + FILE *fp = fopen(tmp_filename, "w");
604 + if(!fp) {
605 + error("Registry: Cannot create file: %s", tmp_filename);
606 + return -1;
607 + }
608 +
609 + debug(D_REGISTRY, "Saving all machines");
610 + int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);
611 + if(bytes1 < 0) {
612 + error("Registry: Cannot save registry machines - return value %d", bytes1);
613 + fclose(fp);
614 + return bytes1;
615 + }
616 + debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);
617 +
618 + debug(D_REGISTRY, "Saving all persons");
619 + int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp);
620 + if(bytes2 < 0) {
621 + error("Registry: Cannot save registry persons - return value %d", bytes2);
622 + fclose(fp);
623 + return bytes2;
624 + }
625 + debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);
626 +
627 + fclose(fp);
628 +
629 + errno = 0;
630 +
631 + // remove the .old db
632 + debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename);
633 + if(unlink(old_filename) == -1 && errno != ENOENT)
634 + error("Registry: cannot remove old registry file '%s'", old_filename);
635 +
636 + // rename the db to .old
637 + debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);
638 + if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)
639 + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, registry.db_filename);
640 +
641 + else {
642 + // remove the database (it is saved in .old)
643 + debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename);
644 + if (unlink(registry.db_filename) == -1 && errno != ENOENT)
645 + error("Registry: cannot remove old registry file '%s'", registry.db_filename);
646 +
647 + // move the .tmp to make it active
648 + debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);
649 + if (link(tmp_filename, registry.db_filename) == -1) {
650 + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,
651 + registry.db_filename);
652 +
653 + // move the .old back
654 + debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);
655 + if(link(old_filename, registry.db_filename) == -1)
656 + error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);
657 + }
658 + else {
659 + debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename);
660 + if(unlink(tmp_filename) == -1)
661 + error("Registry: cannot remove tmp registry file '%s'", tmp_filename);
662 +
663 + // it has been moved successfully
664 + // discard the current registry log
665 + registry_log_recreate();
666 + }
667 + }
668 +
669 + return -1;
670 +}
671 +
672 +static inline size_t registry_load(void) {
673 + char *s, buf[4096 + 1];
674 + PERSON *p = NULL;
675 + MACHINE *m = NULL;
676 + URL *u = NULL;
677 + size_t line = 0;
678 +
679 + debug(D_REGISTRY, "Registry: loading active db from: %s", registry.db_filename);
680 + FILE *fp = fopen(registry.db_filename, "r");
681 + if(!fp) {
682 + error("Registry: cannot open registry file: %s", registry.db_filename);
683 + return 0;
684 + }
685 +
686 + size_t len = 0;
687 + while((s = fgets_trim_len(buf, 4096, fp, &len))) {
688 + line++;
689 +
690 + debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);
691 +
692 + switch(*s) {
693 + case 'P': // person
694 + m = NULL;
695 + // verify it is valid
696 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0'))
697 + error("Registry person line %u is wrong (len = %zu).", line, len);
698 +
699 + s[1] = s[10] = s[19] = s[28] = '\0';
700 + p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));
701 + p->last_t = strtoul(&s[11], NULL, 16);
702 + p->usages = strtoul(&s[20], NULL, 16);
703 + debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);
704 + break;
705 +
706 + case 'M': // machine
707 + p = NULL;
708 + // verify it is valid
709 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0'))
710 + error("Registry person line %u is wrong (len = %zu).", line, len);
711 +
712 + s[1] = s[10] = s[19] = s[28] = '\0';
713 + m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));
714 + m->last_t = strtoul(&s[11], NULL, 16);
715 + m->usages = strtoul(&s[20], NULL, 16);
716 + debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);
717 + break;
718 +
719 + case 'U': // person URL
720 + if(unlikely(!p)) {
721 + error("Registry: ignoring line %zu, no person loaded: %s", line, s);
722 + break;
723 + }
724 +
725 + // verify it is valid
726 + if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t')
727 + error("Registry person URL line %u is wrong (len = %zu).", line, len);
728 +
729 + s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0';
730 + u = registry_url_allocate(&s[69]);
731 +
732 + time_t first_t = strtoul(&s[2], NULL, 16);
733 +
734 + m = registry_machine_find(&s[32]);
735 + if(!m) m = registry_machine_allocate(&s[32], first_t);
736 +
737 + PERSON_URL *pu = registry_person_url_allocate(p, m, u, first_t);
738 + pu->last_t = strtoul(&s[11], NULL, 16);
739 + pu->usages = strtoul(&s[20], NULL, 16);
740 + pu->flags = strtoul(&s[29], NULL, 16);
741 + debug(D_REGISTRY, "Registry loaded person URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);
742 + break;
743 +
744 + case 'V': // machine URL
745 + if(unlikely(!m)) {
746 + error("Registry: ignoring line %zu, no machine loaded: %s", line, s);
747 + break;
748 + }
749 +
750 + // verify it is valid
751 + if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t')
752 + error("Registry person URL line %u is wrong (len = %zu).", line, len);
753 +
754 + s[1] = s[10] = s[19] = s[28] = s[31] = '\0';
755 + u = registry_url_allocate(&s[32]);
756 +
757 + MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16));
758 + mu->last_t = strtoul(&s[11], NULL, 16);
759 + mu->usages = strtoul(&s[20], NULL, 16);
760 + mu->flags = strtoul(&s[29], NULL, 16);
761 + debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags);
762 + break;
763 +
764 + default:
765 + error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s);
766 + break;
767 + }
768 + }
769 + fclose(fp);
770 +
771 + registry_log_load();
772 +
773 + return line;
774 +}
775 +
776 // ----------------------------------------------------------------------------
777 // REGISTRY
778
779 void registry_init(void) {
780 + char filename[FILENAME_MAX + 1];
781 +
782 + registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR);
783 + if(mkdir(registry.pathname, 0644) == -1 && errno != EEXIST)
784 + error("Cannot create directory '%s'", registry.pathname);
785 +
786 + snprintf(filename, FILENAME_MAX, "%s/%s", registry.pathname, "registry.db");
787 + registry.db_filename = config_get("registry", "registry db file", filename);
788 +
789 + snprintf(filename, FILENAME_MAX, "%s/%s", registry.pathname, "registry-log.db");
790 + registry.log_filename = config_get("registry", "registry log file", filename);
791 +
792 registry.persons_count = 0;
793 registry.machines_count = 0;
794 registry.usages_count = 0;
@@ -504,34 +797,33 @@ void registry_init(void) {
797 registry.machines_urls_count = 0;
798
799 debug(D_REGISTRY, "Registry: creating global registry dictionary for persons.");
507 - registry.persons = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
800 + registry.persons = dictionary_create(DICTIONARY_FLAGS);
801
802 debug(D_REGISTRY, "Registry: creating global registry dictionary for machines.");
510 - registry.machines = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
803 + registry.machines = dictionary_create(DICTIONARY_FLAGS);
804
805 debug(D_REGISTRY, "Registry: creating global registry dictionary for urls.");
513 - registry.urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED);
806 + registry.urls = dictionary_create(DICTIONARY_FLAGS);
807 +
808 + registry_log_open();
809 + registry_load();
810 }
811
812 void registry_free(void) {
813 while(registry.persons->values_index.root) {
814 PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;
815
520 -#ifdef REGISTRY_STDOUT_DUMP
521 - fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages);
522 -#endif /* REGISTRY_STDOUT_DUMP */
816 + // fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages);
817
818 while(p->urls->values_index.root) {
819 PERSON_URL *pu = ((NAME_VALUE *)p->urls->values_index.root)->value;
820
527 -#ifdef REGISTRY_STDOUT_DUMP
528 - fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags);
529 -#endif /* REGISTRY_STDOUT_DUMP */
821 + // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags);
822
823 debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid);
824 dictionary_del(p->urls, pu->url->url);
825
534 - debug(D_REGISTRY, "Registry: unlinking url '%s'", pu->url->url);
826 + debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url);
827 registry_url_unlink(pu->url);
828
829 debug(D_REGISTRY, "Registry: freeing person url");
@@ -551,16 +843,12 @@ void registry_free(void) {
843 while(registry.machines->values_index.root) {
844 MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value;
845
554 -#ifdef REGISTRY_STDOUT_DUMP
555 - fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);
556 -#endif /* REGISTRY_STDOUT_DUMP */
846 + // fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);
847
848 while(m->urls->values_index.root) {
849 MACHINE_URL *mu = ((NAME_VALUE *)m->urls->values_index.root)->value;
850
561 -#ifdef REGISTRY_STDOUT_DUMP
562 - fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);
563 -#endif /* REGISTRY_STDOUT_DUMP */
851 + // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);
852
853 //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url);
854 //dictionary_destroy(mu->persons);
@@ -568,7 +856,7 @@ void registry_free(void) {
856 debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid);
857 dictionary_del(m->urls, mu->url->url);
858
571 - debug(D_REGISTRY, "Registry: unlinking url '%s'", mu->url);
859 + debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);
860 registry_url_unlink(mu->url);
861
862 debug(D_REGISTRY, "Registry: freeing machine url");
@@ -595,6 +883,10 @@ void registry_free(void) {
883 dictionary_destroy(registry.urls);
884 }
885
886 +
887 +// ----------------------------------------------------------------------------
888 +// TESTS
889 +
890 int test1(int argc, char **argv) {
891
892 void print_stats(uint32_t requests, unsigned long long start, unsigned long long end) {
@@ -609,8 +901,8 @@ int test1(int argc, char **argv) {
901 (void) argc;
902 (void) argv;
903
612 - uint32_t u, users = 500000;
613 - uint32_t m, machines = 50000;
904 + uint32_t u, users = 1000000;
905 + uint32_t m, machines = 200000;
906 uint32_t machines2 = machines * 2;
907
908 char **users_guids = malloc(users * sizeof(char *));
@@ -637,10 +929,11 @@ int test1(int argc, char **argv) {
929 start = timems();
930 fprintf(stderr, "\nGenerating %u users accessing %u machines\n", users, machines);
931 m = 0;
932 + time_t now = time(NULL);
933 for(u = 0; u < users ; u++) {
934 if(++m == machines) m = 0;
935
643 - PERSON *p = registry_request(NULL, machines_guids[m], machines_urls[m]);
936 + PERSON *p = registry_request(NULL, machines_guids[m], machines_urls[m], now);
937 users_guids[u] = p->guid;
938 }
939 print_stats(u, start, timems());
@@ -648,10 +941,11 @@ int test1(int argc, char **argv) {
941 start = timems();
942 fprintf(stderr, "\nAll %u users accessing again the same %u servers\n", users, machines);
943 m = 0;
944 + now = time(NULL);
945 for(u = 0; u < users ; u++) {
946 if(++m == machines) m = 0;
947
654 - PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m]);
948 + PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m], now);
949
950 if(p->guid != users_guids[u])
951 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid);
@@ -661,10 +955,11 @@ int test1(int argc, char **argv) {
955 start = timems();
956 fprintf(stderr, "\nAll %u users accessing a new server, out of the %u servers\n", users, machines);
957 m = 1;
958 + now = time(NULL);
959 for(u = 0; u < users ; u++) {
960 if(++m == machines) m = 0;
961
667 - PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m]);
962 + PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m], now);
963
964 if(p->guid != users_guids[u])
965 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid);
@@ -673,11 +968,12 @@ int test1(int argc, char **argv) {
968
969 start = timems();
970 fprintf(stderr, "\n%u random users accessing a random server, out of the %u servers\n", users, machines);
971 + now = time(NULL);
972 for(u = 0; u < users ; u++) {
973 uint32_t tu = random() * users / RAND_MAX;
974 uint32_t tm = random() * machines / RAND_MAX;
975
680 - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm]);
976 + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm], now);
977
978 if(p->guid != users_guids[tu])
979 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
@@ -686,11 +982,12 @@ int test1(int argc, char **argv) {
982
983 start = timems();
984 fprintf(stderr, "\n%u random users accessing a random server, out of %u servers\n", users, machines2);
985 + now = time(NULL);
986 for(u = 0; u < users ; u++) {
987 uint32_t tu = random() * users / RAND_MAX;
988 uint32_t tm = random() * machines2 / RAND_MAX;
989
693 - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm]);
990 + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm], now);
991
992 if(p->guid != users_guids[tu])
993 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
@@ -702,6 +999,7 @@ int test1(int argc, char **argv) {
999 fprintf(stderr,
1000 "\n%u random user accesses to a random server, out of %u servers,\n > using 1/10000 with a random url, 1/1000 with a mismatched url\n",
1001 users * 2, machines2);
1002 + now = time(NULL);
1003 for (u = 0; u < users * 2; u++) {
1004 uint32_t tu = random() * users / RAND_MAX;
1005 uint32_t tm = random() * machines2 / RAND_MAX;
@@ -715,7 +1013,7 @@ int test1(int argc, char **argv) {
1013 else if (random() % 1000 == 123)
1014 url = machines_urls[random() * machines2 / RAND_MAX];
1015
718 - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], url);
1016 + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], url, now);
1017
1018 if (p->guid != users_guids[tu])
1019 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
@@ -723,6 +1021,11 @@ int test1(int argc, char **argv) {
1021 print_stats(u, start, timems());
1022 }
1023
1024 + fprintf(stderr, "\n\nSAVE\n");
1025 + start = timems();
1026 + registry_save();
1027 + print_stats(registry.persons_count, start, timems());
1028 +
1029 fprintf(stderr, "\n\nCLEANUP\n");
1030 start = timems();
1031 registry_free();
@@ -734,9 +1037,9 @@ int test1(int argc, char **argv) {
1037 // TESTING
1038
1039 int main(int argc, char **argv) {
737 - // debug_flags = 0xFFFFFFFF;
738 - test1(argc, argv);
739 - exit(0);
1040 + debug_flags = 0xFFFFFFFF;
1041 + //test1(argc, argv);
1042 + //exit(0);
1043
1044 (void)argc;
1045 (void)argv;
@@ -748,41 +1051,48 @@ int main(int argc, char **argv) {
1051
1052 registry_init();
1053
1054 + int i = 2;
1055 +
1056 + fprintf(stderr, "\n\nADDING ENTRY\n");
1057 + // p1 = registry_request("2c95abd0-1542-11e6-8c66-00508db7e9c9", "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", time(NULL));
1058
752 - int i = 100000;
1059 + if(0)
1060 while(i--) {
1061 #ifdef REGISTRY_STDOUT_DUMP
1062 fprintf(stderr, "\n\nADDING ENTRY\n");
1063 #endif /* REGISTRY_STDOUT_DUMP */
757 - p1 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/");
1064 + p1 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", time(NULL));
1065
1066 #ifdef REGISTRY_STDOUT_DUMP
1067 fprintf(stderr, "\n\nADDING ANOTHER URL\n");
1068 #endif /* REGISTRY_STDOUT_DUMP */
762 - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://127.0.0.1:19999/");
1069 + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://127.0.0.1:19999/", time(NULL));
1070
1071 #ifdef REGISTRY_STDOUT_DUMP
1072 fprintf(stderr, "\n\nADDING ANOTHER URL\n");
1073 #endif /* REGISTRY_STDOUT_DUMP */
767 - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/");
1074 + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", time(NULL));
1075
1076 #ifdef REGISTRY_STDOUT_DUMP
1077 fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n");
1078 #endif /* REGISTRY_STDOUT_DUMP */
772 - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/");
1079 + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", time(NULL));
1080
1081 #ifdef REGISTRY_STDOUT_DUMP
1082 fprintf(stderr, "\n\nADDING ANOTHER PERSON\n");
1083 #endif /* REGISTRY_STDOUT_DUMP */
777 - p2 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/");
1084 + p2 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", time(NULL));
1085
1086 #ifdef REGISTRY_STDOUT_DUMP
1087 fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n");
1088 #endif /* REGISTRY_STDOUT_DUMP */
782 - p2 = registry_request(p2->guid, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/");
1089 + p2 = registry_request(p2->guid, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", time(NULL));
1090 }
1091
1092 + fprintf(stderr, "\n\nSAVE\n");
1093 + registry_save();
1094 +
1095 fprintf(stderr, "\n\nCLEANUP\n");
786 - //registry_free();
1096 + registry_free();
1097 return 0;
1098 }