@cryptotaxi247 / netdata-1 / commits / 86d20a49c

almost operational registry - still needs testing

Costa Tsaousis committed May 10, 2016 at 04:01 UTC 86d20a49c6bf8b3eab54bbddc227e0a06e8bf267
5 files changed +813 -157
src/Makefile.am
+1
@@ -59,6 +59,7 @@ netdata_SOURCES = \
59 sys_kernel_mm_ksm.c \
60 sys_fs_cgroup.c \
61 procfile.c procfile.h \
62 + registry.c registry.h \
63 rrd.c rrd.h \
64 rrd2json.c rrd2json.h \
65 storage_number.c storage_number.h \
src/registry.c
+637 -123
@@ -19,6 +19,9 @@
19 #include "dictionary.h"
20 #include "appconfig.h"
21
22 +#include "web_client.h"
23 +#include "registry.h"
24 +
25 #define REGISTRY_URL_FLAGS_DEFAULT 0x00
26 #define REGISTRY_URL_FLAGS_EXPIRED 0x01
27
@@ -28,6 +31,9 @@
31 // COMMON structures
32
33 struct registry {
34 + int enabled;
35 +
36 + // counters / statistics
37 unsigned long long persons_count;
38 unsigned long long machines_count;
39 unsigned long long usages_count;
@@ -35,26 +41,38 @@ struct registry {
41 unsigned long long persons_urls_count;
42 unsigned long long machines_urls_count;
43
44 + // files
45 char *pathname;
46 char *db_filename;
47 char *log_filename;
48 FILE *registry_log_fp;
49
50 + // the database
51 DICTIONARY *persons; // dictionary of PERSON *, with key the PERSON.guid
52 DICTIONARY *machines; // dictionary of MACHINE *, with key the MACHINE.guid
53 DICTIONARY *urls; // dictionary of URL *, with key the URL.url
54
55 + // concurrency locking
56 + // we keep different locks for different things
57 + // so that many tasks can be completed in parallel
58 + pthread_mutex_t persons_lock;
59 + pthread_mutex_t machines_lock;
60 + pthread_mutex_t urls_lock;
61 + pthread_mutex_t person_urls_lock;
62 + pthread_mutex_t machine_urls_lock;
63 + pthread_mutex_t log_lock;
64 } registry;
65
49 -
66 // ----------------------------------------------------------------------------
67 // URL structures
68 // Save memory by de-duplicating URLs
69 +// so instead of storing URLs all over the place
70 +// we store them here and we keep pointers elsewhere
71
72 struct url {
55 - uint32_t links;
56 - uint16_t len;
57 - char url[1];
73 + uint32_t links; // the number of links to this URL - when none is left, we free it
74 + uint16_t len; // the length of the URL in bytes
75 + char url[1]; // the URL - dynamically allocated to more size
76 };
77 typedef struct url URL;
78
@@ -76,7 +94,7 @@ typedef struct machine_url MACHINE_URL;
94
95 // A machine
96 struct machine {
79 - char guid[36 + 1];
97 + char guid[36 + 1]; // the GUID
98
99 DICTIONARY *urls; // MACHINE_URL *
100
@@ -92,21 +110,24 @@ typedef struct machine MACHINE;
110
111 // for each PERSON-URL pair we keep this
112 struct person_url {
95 - URL *url; // de-duplicated URL
96 - MACHINE *machine;
113 + URL *url; // de-duplicated URL
114 + MACHINE *machine; // link the MACHINE of this URL
115
116 uint8_t flags;
117 uint32_t first_t; // the first time we saw this
118 uint32_t last_t; // the last time we saw this
119 uint32_t usages; // how many times this has been accessed
120 +
121 + char name[1]; // the name of the URL, as known by the user
122 + // dynamically allocated to fit properly
123 };
124 typedef struct person_url PERSON_URL;
125
126 // A person
127 struct person {
107 - char guid[36 + 1];
128 + char guid[36 + 1]; // the person GUID
129
109 - DICTIONARY *urls; // PERSON_URL *
130 + DICTIONARY *urls; // dictionary of PERSON_URL *
131
132 uint32_t first_t; // the first time we saw this
133 uint32_t last_t; // the last time we saw this
@@ -114,54 +135,191 @@ struct person {
135 };
136 typedef struct person PERSON;
137
117 -extern PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url, time_t when);
138 +
139 +// ----------------------------------------------------------------------------
140 +// REGISTRY concurrency locking
141 +
142 +static inline void registry_persons_lock(void) {
143 + pthread_mutex_lock(&registry.persons_lock);
144 +}
145 +
146 +static inline void registry_persons_unlock(void) {
147 + pthread_mutex_unlock(&registry.persons_lock);
148 +}
149 +
150 +static inline void registry_machines_lock(void) {
151 + pthread_mutex_lock(&registry.machines_lock);
152 +}
153 +
154 +static inline void registry_machines_unlock(void) {
155 + pthread_mutex_unlock(&registry.machines_lock);
156 +}
157 +
158 +static inline void registry_urls_lock(void) {
159 + pthread_mutex_lock(&registry.urls_lock);
160 +}
161 +
162 +static inline void registry_urls_unlock(void) {
163 + pthread_mutex_unlock(&registry.urls_lock);
164 +}
165 +
166 +// ideally, we should not lock the whole registry for
167 +// updating a person's urls.
168 +// however, to save the memory required for keeping a
169 +// mutex (40 bytes) per person, we do...
170 +static inline void registry_person_urls_lock(PERSON *p) {
171 + (void)p;
172 + pthread_mutex_lock(&registry.person_urls_lock);
173 +}
174 +
175 +static inline void registry_person_urls_unlock(PERSON *p) {
176 + (void)p;
177 + pthread_mutex_unlock(&registry.person_urls_lock);
178 +}
179 +
180 +// ideally, we should not lock the whole registry for
181 +// updating a machine's urls.
182 +// however, to save the memory required for keeping a
183 +// mutex (40 bytes) per machine, we do...
184 +static inline void registry_machine_urls_lock(MACHINE *m) {
185 + (void)m;
186 + pthread_mutex_lock(&registry.machine_urls_lock);
187 +}
188 +
189 +static inline void registry_machine_urls_unlock(MACHINE *m) {
190 + (void)m;
191 + pthread_mutex_unlock(&registry.machine_urls_lock);
192 +}
193 +
194 +static inline void registry_log_lock(void) {
195 + pthread_mutex_lock(&registry.log_lock);
196 +}
197 +
198 +static inline void registry_log_unlock(void) {
199 + pthread_mutex_unlock(&registry.log_lock);
200 +}
201 +
202 +
203 +// ----------------------------------------------------------------------------
204 +// common functions
205 +
206 +// parse a GUID and re-generated to be always lower case
207 +// this is used as a protection against the variations of GUIDs
208 +static inline int registry_regenerate_guid(const char *guid, char *result) {
209 + uuid_t uuid;
210 + if(unlikely(uuid_parse(guid, uuid) == -1)) {
211 + info("Registry: GUID '%s' is not a valid GUID.", guid);
212 + return -1;
213 + }
214 + else {
215 + uuid_unparse_lower(uuid, result);
216 +
217 +#ifdef NETDATA_INTERNAL_CHECKS
218 + if(strcmp(guid, result))
219 + info("Registry: source GUID '%s' and re-generated GUID '%s' differ!", guid, result);
220 +#endif /* NETDATA_INTERNAL_CHECKS */
221 + }
222 +
223 + return 0;
224 +}
225 +
226 +// make sure the names of the machines / URLs do not contain any tabs
227 +// (which are used as our separator in the database files)
228 +// and are properly trimmed (before and after)
229 +static inline char *registry_fix_machine_name(char *name, size_t *len) {
230 + char *s = name?name:"";
231 +
232 + // skip leading spaces
233 + while(*s && isspace(*s)) s++;
234 +
235 + // make sure all spaces are a SPACE
236 + char *t = s;
237 + while(*t) {
238 + if(unlikely(isspace(*t)))
239 + *t = ' ';
240 +
241 + t++;
242 + }
243 +
244 + // remove trailing spaces
245 + while(--t >= s) {
246 + if(*t == ' ')
247 + *t = '\0';
248 + else
249 + break;
250 + }
251 + t++;
252 +
253 + if(likely(len))
254 + *len = (t - s);
255 +
256 + return s;
257 +}
258 +
259 +static inline char *registry_fix_url(char *url, size_t *len) {
260 + return registry_fix_machine_name(url, len);
261 +}
262 +
263 +
264 +// ----------------------------------------------------------------------------
265 +// forward definition of functions
266 +
267 +extern PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when);
268 +extern PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);
269 +
270
271 // ----------------------------------------------------------------------------
272 // URL
273
122 -static inline URL *registry_url_allocate(const char *url) {
123 - size_t len = strlen(url);
274 +static inline URL *registry_url_allocate_nolock(const char *url, size_t urllen) {
275 + debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): allocating %zu bytes", url, sizeof(URL) + urllen);
276 + URL *u = malloc(sizeof(URL) + urllen);
277 + if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + urllen);
278
125 - debug(D_REGISTRY, "Registry: registry_url_allocate('%s'): allocating %zu bytes", url, sizeof(URL) + len);
126 - URL *u = malloc(sizeof(URL) + len);
127 - if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + len);
279 + // a simple strcpy() should do the job
280 + // but I prefer to be safe, since the caller specified urllen
281 + strncpy(u->url, url, urllen);
282 + u->url[urllen] = '\0';
283
129 - strcpy(u->url, url);
130 - u->len = len;
284 + u->len = urllen;
285 u->links = 0;
286
133 - debug(D_REGISTRY, "Registry: registry_url_allocate('%s'): indexing it", url);
287 + debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): indexing it", url);
288 dictionary_set(registry.urls, u->url, u, sizeof(URL));
289
290 return u;
291 }
292
139 -static inline URL *registry_url_get(const char *url) {
293 +static inline URL *registry_url_get(const char *url, size_t urllen) {
294 debug(D_REGISTRY, "Registry: registry_url_get('%s')", url);
295
296 + registry_urls_lock();
297 +
298 URL *u = dictionary_get(registry.urls, url);
299 if(!u) {
144 - u = registry_url_allocate(url);
300 + u = registry_url_allocate_nolock(url, urllen);
301 registry.urls_count++;
302 }
303
304 + registry_urls_unlock();
305 +
306 return u;
307 }
308
151 -static inline void registry_url_link(URL *u) {
309 +static inline void registry_url_link_nolock(URL *u) {
310 u->links++;
153 - debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): This URL has now %u links", u->url, u->links);
311 + debug(D_REGISTRY, "Registry: registry_url_link_nolock('%s'): URL has now %u links", u->url, u->links);
312 }
313
156 -static inline void registry_url_unlink(URL *u) {
314 +static inline void registry_url_unlink_nolock(URL *u) {
315 u->links--;
316 if(!u->links) {
159 - debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): No more links for this URL", u->url);
317 + debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): No more links for this URL", u->url);
318 dictionary_del(registry.urls, u->url);
319 free(u);
320 }
321 else
164 - debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): This URL has %u links left", u->url, u->links);
322 + debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): URL has %u links left", u->url, u->links);
323 }
324
325
@@ -189,7 +347,7 @@ static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, tim
347
348 debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url);
349 dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL));
192 - registry_url_link(u);
350 + registry_url_link_nolock(u);
351
352 return mu;
353 }
@@ -219,18 +377,14 @@ static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_
377 static inline MACHINE *registry_machine_get(const char *machine_guid, time_t when) {
378 MACHINE *m = NULL;
379
380 + registry_machines_lock();
381 +
382 if(likely(machine_guid && *machine_guid)) {
383 // validate it is a GUID
224 - uuid_t uuid;
225 - if(uuid_parse(machine_guid, uuid) == -1) {
384 + char buf[36 + 1];
385 + if(unlikely(registry_regenerate_guid(machine_guid, buf) == -1))
386 info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);
227 - }
387 else {
229 - char buf[36 + 1];
230 - uuid_unparse_lower(uuid, buf);
231 - if(strcmp(machine_guid, buf))
232 - info("Registry: machine guid '%s' and re-generated '%s' differ!", machine_guid, buf);
233 -
388 machine_guid = buf;
389 m = registry_machine_find(machine_guid);
390 if(!m) {
@@ -240,6 +394,8 @@ static inline MACHINE *registry_machine_get(const char *machine_guid, time_t whe
394 }
395 }
396
397 + registry_machines_unlock();
398 +
399 return m;
400 }
401
@@ -252,10 +408,15 @@ static inline PERSON *registry_person_find(const char *person_guid) {
408 return dictionary_get(registry.persons, person_guid);
409 }
410
255 -static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, time_t when) {
256 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL));
257 - PERSON_URL *pu = malloc(sizeof(PERSON_URL));
258 - if(!pu) fatal("registry_person_link_to_url('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL));
411 +static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) {
412 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL) + namelen);
413 + PERSON_URL *pu = malloc(sizeof(PERSON_URL) + namelen);
414 + if(!pu) fatal("registry_person_link_to_url('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL) + namelen);
415 +
416 + // a simple strcpy() should do the job
417 + // but I prefer to be safe, since the caller specified urllen
418 + strncpy(pu->name, name, namelen);
419 + pu->name[namelen] = '\0';
420
421 pu->machine = m;
422 pu->first_t = pu->last_t = when;
@@ -265,7 +426,7 @@ static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, UR
426
427 debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);
428 dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL));
268 - registry_url_link(u);
429 + registry_url_link_nolock(u);
430
431 return pu;
432 }
@@ -318,18 +479,14 @@ static inline PERSON *registry_person_allocate(const char *person_guid, time_t w
479 static inline PERSON *registry_person_get(const char *person_guid, time_t when) {
480 PERSON *p = NULL;
481
482 + registry_persons_lock();
483 +
484 if(person_guid && *person_guid) {
485 + char buf[36 + 1];
486 // validate it is a GUID
323 - uuid_t uuid;
324 - if(uuid_parse(person_guid, uuid) == -1) {
487 + if(unlikely(registry_regenerate_guid(person_guid, buf) == -1))
488 info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid);
326 - }
489 else {
328 - char buf[36 + 1];
329 - uuid_unparse_lower(uuid, buf);
330 - if(strcmp(person_guid, buf))
331 - info("Registry: person guid '%s' and re-generated '%s' differ!", person_guid, buf);
332 -
490 person_guid = buf;
491 p = registry_person_find(person_guid);
492 if(!p) person_guid = NULL;
@@ -341,19 +498,23 @@ static inline PERSON *registry_person_get(const char *person_guid, time_t when)
498 registry.persons_count++;
499 }
500
501 + registry_persons_unlock();
502 +
503 return p;
504 }
505
506 // ----------------------------------------------------------------------------
507 // LINKING OF OBJECTS
508
350 -static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) {
509 +static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) {
510 debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);
511
512 + registry_person_urls_lock(p);
513 +
514 PERSON_URL *pu = dictionary_get(p->urls, u->url);
515 if(!pu) {
516 debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
356 - pu = registry_person_url_allocate(p, m, u, when);
517 + pu = registry_person_url_allocate(p, m, u, name, namelen, when);
518 registry.persons_urls_count++;
519 }
520 else {
@@ -375,8 +536,12 @@ static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL
536 p->usages++;
537 p->last_t = when;
538
378 - if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED)
379 - info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
539 + if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) {
540 + info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url);
541 + pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;
542 + }
543 +
544 + registry_person_urls_unlock(p);
545
546 return pu;
547 }
@@ -384,6 +549,8 @@ static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL
549 static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) {
550 debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url);
551
552 + registry_machine_urls_lock(m);
553 +
554 MACHINE_URL *mu = dictionary_get(m->urls, u->url);
555 if(!mu) {
556 debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
@@ -401,8 +568,12 @@ static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, U
568 m->usages++;
569 m->last_t = when;
570
404 - if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED)
405 - info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
571 + if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) {
572 + info("registry_machine_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
573 + mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;
574 + }
575 +
576 + registry_machine_urls_unlock(m);
577
578 return mu;
579 }
@@ -410,18 +581,26 @@ static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, U
581 // ----------------------------------------------------------------------------
582 // REGISTRY LOG LOAD/SAVE
583
413 -static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u) {
414 - if(likely(registry.registry_log_fp))
415 - fprintf(registry.registry_log_fp, "%c\t%08x\t%s\t%s\t%s\n",
584 +static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u, char *name) {
585 + if(likely(registry.registry_log_fp)) {
586 + // we lock only if the file is open
587 + // to allow replaying the log at registry_log_load()
588 + registry_log_lock();
589 +
590 + fprintf(registry.registry_log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",
591 action,
592 p->last_t,
593 p->guid,
594 m->guid,
595 + name,
596 u->url
597 );
598 +
599 + registry_log_unlock();
600 + }
601 }
602
424 -void registry_log_open(void) {
603 +void registry_log_open_nolock(void) {
604 registry.registry_log_fp = fopen(registry.log_filename, "a");
605
606 if(registry.registry_log_fp) {
@@ -430,85 +609,342 @@ void registry_log_open(void) {
609 }
610 }
611
433 -void registry_log_close(void) {
612 +void registry_log_close_nolock(void) {
613 if(registry.registry_log_fp)
614 fclose(registry.registry_log_fp);
615
616 registry.registry_log_fp = NULL;
617 }
618
440 -void registry_log_recreate(void) {
619 +void registry_log_recreate_nolock(void) {
620 if(registry.registry_log_fp != NULL) {
442 - fclose(registry.registry_log_fp);
621 + registry_log_close_nolock();
622
623 registry.registry_log_fp = fopen(registry.log_filename, "w");
624 if(registry.registry_log_fp) fclose(registry.registry_log_fp);
625 registry.registry_log_fp = NULL;
626
448 - registry_log_open();
627 + registry_log_open_nolock();
628 }
629 +
630 }
631
632 int registry_log_load(void) {
633 char *s, buf[4096 + 1];
454 - size_t line = 0;
634 + size_t line = -1;
635
456 - registry_log_close();
636 + // closing the log is required here
637 + // otherwise we will append to it the values we read
638 + registry_log_close_nolock();
639
640 debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename);
641 FILE *fp = fopen(registry.log_filename, "r");
460 - if(!fp) {
642 + if(!fp)
643 error("Registry: cannot open registry file: %s", registry.log_filename);
462 - return 0;
463 - }
464 -
465 - size_t len = 0;
466 - while((s = fgets_trim_len(buf, 4096, fp, &len))) {
467 - line++;
468 -
469 - switch(s[0]) {
470 - case 'A':
471 - // verify it is valid
472 - if(unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t'))
473 - error("Registry log line %u is wrong (len = %zu).", line, len);
644 + else {
645 + line = 0;
646 + size_t len = 0;
647 + while ((s = fgets_trim_len(buf, 4096, fp, &len))) {
648 + line++;
649 +
650 + switch (s[0]) {
651 + case 'A': // accesses
652 + case 'D': // deletes
653 +
654 + // verify it is valid
655 + if (unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) {
656 + error("Registry: log line %u is wrong (len = %zu).", line, len);
657 + continue;
658 + }
659 +
660 + s[1] = s[10] = s[47] = s[84] = '\0';
661 +
662 + // skip the name to find the url
663 + char *url = &s[85];
664 + while(*url && *url != '\t') url++;
665 + if(!*url) {
666 + error("Registry: log line %u does not have a url.", line);
667 + continue;
668 + }
669 + *url++ = '\0';
670 +
671 + if(s[0] == 'A')
672 + registry_request_access(&s[11], &s[48], url, &s[85], strtoul(&s[2], NULL, 16));
673 + else
674 + registry_request_delete(&s[11], &s[48], url, &s[85], strtoul(&s[2], NULL, 16));
675
475 - s[1] = s[10] = s[47] = s[84] = '\0';
476 - registry_request(&s[11], &s[48], &s[85], strtoul(&s[2], NULL, 16));
477 - break;
676 + break;
677
479 - default:
480 - error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.log_filename, s);
481 - break;
678 + default:
679 + error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.log_filename, s);
680 + break;
681 + }
682 }
683 }
684
485 - registry_log_open();
685 + // open the log again
686 + registry_log_open_nolock();
687
487 - return 0;
688 + return line;
689 }
690
691
692 // ----------------------------------------------------------------------------
693 // REGISTRY REQUESTS
694
494 -PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url, time_t when) {
495 - debug(D_REGISTRY, "registry_request('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);
695 +PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) {
696 + debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);
697
698 MACHINE *m = registry_machine_get(machine_guid, when);
699 if(!m) return NULL;
700
500 - URL *u = registry_url_get(url);
701 + // make sure the name is valid
702 + size_t namelen;
703 + name = registry_fix_machine_name(name, &namelen);
704 +
705 + size_t urllen;
706 + url = registry_fix_url(url, &urllen);
707 +
708 + URL *u = registry_url_get(url, urllen);
709 PERSON *p = registry_person_get(person_guid, when);
710
503 - registry_person_link_to_url(p, m, u, when);
711 + registry_person_link_to_url(p, m, u, name, namelen, when);
712 registry_machine_link_to_url(p, m, u, when);
713
506 - registry_log('A', p, m, u);
714 + registry_log('A', p, m, u, name);
715
716 registry.usages_count++;
717 return p;
718 }
719
720 +// verify the person, the machine and the URL exist in our DB
721 +PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, PERSON **pp, MACHINE **mm) {
722 + char pbuf[36 + 1], mbuf[36 + 1];
723 +
724 + if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) {
725 + info("Registry Request Verification: invalid request! person: '%s', machine '%s', url '%s'", person_guid?person_guid:"UNSET", machine_guid?machine_guid:"UNSET", url?url:"UNSET");
726 + return NULL;
727 + }
728 +
729 + // normalize the url
730 + url = registry_fix_url(url, NULL);
731 +
732 + // make sure the person GUID is valid
733 + if(registry_regenerate_guid(person_guid, pbuf) == -1) {
734 + info("Registry Request Verification: invalid person GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
735 + return NULL;
736 + }
737 + person_guid = pbuf;
738 +
739 + // make sure the machine GUID is valid
740 + if(registry_regenerate_guid(machine_guid, mbuf) == -1) {
741 + info("Registry Request Verification: invalid machine GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
742 + return NULL;
743 + }
744 + machine_guid = mbuf;
745 +
746 + // make sure the machine exists
747 + MACHINE *m = registry_machine_find(machine_guid);
748 + if(!m) {
749 + info("Registry Request Verification: machine not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
750 + return NULL;
751 + }
752 + if(mm) *mm = m;
753 +
754 + // make sure the person exist
755 + PERSON *p = registry_person_find(person_guid);
756 + if(!p) {
757 + info("Registry Request Verification: person not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
758 + return NULL;
759 + }
760 + if(pp) *pp = p;
761 +
762 + PERSON_URL *pu = dictionary_get(p->urls, url);
763 + if(!pu) {
764 + info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
765 + return NULL;
766 + }
767 + return pu;
768 +}
769 +
770 +PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {
771 + (void)when;
772 +
773 + PERSON *p = NULL;
774 + MACHINE *m = NULL;
775 + PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);
776 + if(!pu || !p || !m) return NULL;
777 +
778 + // normalize the url
779 + delete_url = registry_fix_url(delete_url, NULL);
780 +
781 + // make sure the user is not deleting the url it uses
782 + if(!strcmp(delete_url, pu->url->url)) {
783 + info("Registry Delete Request: delete URL is the one currently accessing, person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);
784 + return NULL;
785 + }
786 +
787 + registry_person_urls_lock(p);
788 +
789 + PERSON_URL *dpu = dictionary_get(p->urls, delete_url);
790 + if(!dpu) {
791 + info("Registry Delete Request: URL not found for person, person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);
792 + registry_person_urls_unlock(p);
793 + return NULL;
794 + }
795 +
796 + registry_log('D', p, m, pu->url, dpu->url->url);
797 +
798 + dictionary_del(p->urls, dpu->url->url);
799 + registry_url_unlink_nolock(dpu->url);
800 + free(dpu);
801 +
802 + registry_person_urls_unlock(p);
803 + return p;
804 +}
805 +
806 +MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {
807 + (void)when;
808 +
809 + char mbuf[36 + 1];
810 +
811 + PERSON *p = NULL;
812 + MACHINE *m = NULL;
813 + PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);
814 + if(!pu || !p || !m) return NULL;
815 +
816 + // make sure the machine GUID is valid
817 + if(registry_regenerate_guid(request_machine, mbuf) == -1) {
818 + info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);
819 + return NULL;
820 + }
821 + request_machine = mbuf;
822 +
823 + // make sure the machine exists
824 + m = registry_machine_find(request_machine);
825 + if(!m) {
826 + info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);
827 + return NULL;
828 + }
829 +
830 + // Verify the user has in the past accessed this machine
831 + // We will walk through the PERSON_URLs to find the machine
832 + // linking to our machine
833 +
834 + // C magic !
835 +
836 + // a structure to pass to the dictionary_get_all() callback handler
837 + struct machine_request_callback_data {
838 + MACHINE *find_this_machine;
839 + PERSON_URL *result;
840 + } rdata = { m, NULL };
841 +
842 + // the callback function
843 + // this will be run for every PERSON_URL of this PERSON
844 + int machine_request_callback(void *entry, void *data) {
845 + PERSON_URL *mypu = (PERSON_URL *)entry;
846 + struct machine_request_callback_data *myrdata = (struct machine_request_callback_data *)data;
847 +
848 + if(mypu->machine == myrdata->find_this_machine) {
849 + myrdata->result = mypu;
850 + return -1; // this will also stop the walk through
851 + }
852 +
853 + return 0; // continue
854 + }
855 +
856 + // request a walk through on the dictionary
857 + // no need for locking here, the underlying dictionary has its own
858 + dictionary_get_all(p->urls, machine_request_callback, &rdata);
859 +
860 + if(rdata.result)
861 + return m;
862 +
863 + return NULL;
864 +}
865 +
866 +
867 +// ----------------------------------------------------------------------------
868 +// REGISTRY JSON generation
869 +
870 +#ifndef REGISTRY_STANDALONE_TESTS
871 +
872 +static inline void registry_json_header(struct web_client *w, int status) {
873 + buffer_flush(w->response.data);
874 + buffer_sprintf(w->response.data, "{\n\"success\": %s", status?"true":"false");
875 +}
876 +
877 +static inline void registry_json_footer(struct web_client *w) {
878 + buffer_strcat(w->response.data, "\n}\n");
879 +}
880 +
881 +static inline int registry_json_redirect(struct web_client *w) {
882 + registry_json_header(w, 0);
883 + buffer_sprintf(w->response.data, "\"registries\": \"%s\"", config_get("global", "users registries", "reg1.mynetdata.io reg2.mynetdata.io reg3.mynetdata.io"));
884 + registry_json_footer(w);
885 + return 200;
886 +}
887 +
888 +int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) {
889 + if(!registry.enabled)
890 + return registry_json_redirect(w);
891 +
892 + PERSON *p = registry_request_access(person_guid, machine_guid, url, name, when);
893 + if(!p) {
894 + registry_json_header(w, 0);
895 + registry_json_footer(w);
896 + return 400;
897 + }
898 +
899 + // FIXME -- make this a permanent cookie
900 + strncpy(w->cookie, p->guid, COOKIE_MAX);
901 + w->cookie[COOKIE_MAX] = '\0';
902 +
903 + registry_json_header(w, 1);
904 +
905 + // FIXME - print an array of all URLs / Machines / Names
906 +
907 + buffer_strcat(w->response.data, "\n}\n");
908 + return 200;
909 +}
910 +
911 +int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {
912 + if(!registry.enabled)
913 + return registry_json_redirect(w);
914 +
915 + PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when);
916 + if(!p) {
917 + registry_json_header(w, 0);
918 + registry_json_footer(w);
919 + return 400;
920 + }
921 +
922 + registry_json_header(w, 1);
923 + registry_json_footer(w);
924 + return 200;
925 +}
926 +
927 +int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {
928 + if(!registry.enabled)
929 + return registry_json_redirect(w);
930 +
931 + MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when);
932 + if(!m) {
933 + registry_json_header(w, 0);
934 + registry_json_footer(w);
935 + return 400;
936 + }
937 +
938 + registry_json_header(w, 1);
939 +
940 + // FIXME - print an array of all URLs the machine is accessible
941 +
942 + registry_json_footer(w);
943 + return 200;
944 +}
945 +
946 +#endif /* REGISTRY_STANDALONE_TESTS */
947 +
948
949 // ----------------------------------------------------------------------------
950 // REGISTRY LOAD/SAVE
@@ -527,6 +963,8 @@ int registry_machine_save_url(void *entry, void *file) {
963 mu->url->url
964 );
965
966 + // error handling is done at registry_save()
967 +
968 return ret;
969 }
970
@@ -549,6 +987,8 @@ int registry_machine_save(void *entry, void *file) {
987 ret += ret2;
988 }
989
990 + // error handling is done at registry_save()
991 +
992 return ret;
993 }
994
@@ -558,15 +998,18 @@ static inline int registry_person_save_url(void *entry, void *file) {
998
999 debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);
1000
561 - int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\n",
1001 + int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n",
1002 pu->first_t,
1003 pu->last_t,
1004 pu->usages,
1005 pu->flags,
1006 pu->machine->guid,
1007 + pu->name,
1008 pu->url->url
1009 );
1010
1011 + // error handling is done at registry_save()
1012 +
1013 return ret;
1014 }
1015
@@ -589,20 +1032,28 @@ static inline int registry_person_save(void *entry, void *file) {
1032 ret += ret2;
1033 }
1034
1035 + // error handling is done at registry_save()
1036 +
1037 return ret;
1038 }
1039
595 -static int registry_save(void) {
1040 +int registry_save(void) {
1041 + if(!registry.enabled) return -1;
1042 +
1043 char tmp_filename[FILENAME_MAX + 1];
1044 char old_filename[FILENAME_MAX + 1];
1045
1046 snprintf(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);
1047 snprintf(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);
1048
1049 + // make sure the log is not updated
1050 + registry_log_lock();
1051 +
1052 debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);
1053 FILE *fp = fopen(tmp_filename, "w");
1054 if(!fp) {
1055 error("Registry: Cannot create file: %s", tmp_filename);
1056 + registry_log_unlock();
1057 return -1;
1058 }
1059
@@ -611,6 +1062,7 @@ static int registry_save(void) {
1062 if(bytes1 < 0) {
1063 error("Registry: Cannot save registry machines - return value %d", bytes1);
1064 fclose(fp);
1065 + registry_log_unlock();
1066 return bytes1;
1067 }
1068 debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);
@@ -620,10 +1072,21 @@ static int registry_save(void) {
1072 if(bytes2 < 0) {
1073 error("Registry: Cannot save registry persons - return value %d", bytes2);
1074 fclose(fp);
1075 + registry_log_unlock();
1076 return bytes2;
1077 }
1078 debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);
1079
1080 + // save the totals
1081 + fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",
1082 + registry.persons_count,
1083 + registry.machines_count,
1084 + registry.usages_count,
1085 + registry.urls_count,
1086 + registry.persons_urls_count,
1087 + registry.machines_urls_count
1088 + );
1089 +
1090 fclose(fp);
1091
1092 errno = 0;
@@ -662,10 +1125,13 @@ static int registry_save(void) {
1125
1126 // it has been moved successfully
1127 // discard the current registry log
665 - registry_log_recreate();
1128 + registry_log_recreate_nolock();
1129 }
1130 }
1131
1132 + // continue operations
1133 + registry_log_unlock();
1134 +
1135 return -1;
1136 }
1137
@@ -688,13 +1154,27 @@ static inline size_t registry_load(void) {
1154 line++;
1155
1156 debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);
691 -
1157 switch(*s) {
1158 + case 'T': // totals
1159 + if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {
1160 + error("Registry totals line %u is wrong (len = %zu).", line, len);
1161 + continue;
1162 + }
1163 + registry.persons_count = strtoull(&s[2], NULL, 16);
1164 + registry.machines_count = strtoull(&s[19], NULL, 16);
1165 + registry.usages_count = strtoull(&s[36], NULL, 16);
1166 + registry.urls_count = strtoull(&s[53], NULL, 16);
1167 + registry.persons_urls_count = strtoull(&s[70], NULL, 16);
1168 + registry.machines_urls_count = strtoull(&s[87], NULL, 16);
1169 + break;
1170 +
1171 case 'P': // person
1172 m = NULL;
1173 // verify it is valid
696 - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0'))
1174 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
1175 error("Registry person line %u is wrong (len = %zu).", line, len);
1176 + continue;
1177 + }
1178
1179 s[1] = s[10] = s[19] = s[28] = '\0';
1180 p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));
@@ -706,8 +1186,10 @@ static inline size_t registry_load(void) {
1186 case 'M': // machine
1187 p = NULL;
1188 // verify it is valid
709 - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0'))
1189 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
1190 error("Registry person line %u is wrong (len = %zu).", line, len);
1191 + continue;
1192 + }
1193
1194 s[1] = s[10] = s[19] = s[28] = '\0';
1195 m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));
@@ -719,40 +1201,54 @@ static inline size_t registry_load(void) {
1201 case 'U': // person URL
1202 if(unlikely(!p)) {
1203 error("Registry: ignoring line %zu, no person loaded: %s", line, s);
722 - break;
1204 + continue;
1205 }
1206
1207 // verify it is valid
726 - if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t')
1208 + if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') {
1209 error("Registry person URL line %u is wrong (len = %zu).", line, len);
1210 + continue;
1211 + }
1212
1213 s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0';
730 - u = registry_url_allocate(&s[69]);
1214 +
1215 + // skip the name to find the url
1216 + char *url = &s[69];
1217 + while(*url && *url != '\t') url++;
1218 + if(!*url) {
1219 + error("Registry person URL line %u does not have a url.", line);
1220 + continue;
1221 + }
1222 + *url++ = '\0';
1223 +
1224 + u = registry_url_allocate_nolock(url, strlen(url));
1225
1226 time_t first_t = strtoul(&s[2], NULL, 16);
1227
1228 m = registry_machine_find(&s[32]);
1229 if(!m) m = registry_machine_allocate(&s[32], first_t);
1230
737 - PERSON_URL *pu = registry_person_url_allocate(p, m, u, first_t);
1231 + PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t);
1232 pu->last_t = strtoul(&s[11], NULL, 16);
1233 pu->usages = strtoul(&s[20], NULL, 16);
1234 pu->flags = strtoul(&s[29], NULL, 16);
741 - debug(D_REGISTRY, "Registry loaded person URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);
1235 + debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);
1236 break;
1237
1238 case 'V': // machine URL
1239 if(unlikely(!m)) {
1240 error("Registry: ignoring line %zu, no machine loaded: %s", line, s);
747 - break;
1241 + continue;
1242 }
1243
1244 // verify it is valid
751 - if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t')
1245 + if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') {
1246 error("Registry person URL line %u is wrong (len = %zu).", line, len);
1247 + continue;
1248 + }
1249
1250 s[1] = s[10] = s[19] = s[28] = s[31] = '\0';
755 - u = registry_url_allocate(&s[32]);
1251 + u = registry_url_allocate_nolock(&s[32], strlen(&s[32]));
1252
1253 MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16));
1254 mu->last_t = strtoul(&s[11], NULL, 16);
@@ -778,7 +1274,6 @@ static inline size_t registry_load(void) {
1274
1275 void registry_init(void) {
1276 char filename[FILENAME_MAX + 1];
781 -
1277 registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR);
1278 if(mkdir(registry.pathname, 0644) == -1 && errno != EEXIST)
1279 error("Cannot create directory '%s'", registry.pathname);
@@ -805,11 +1300,24 @@ void registry_init(void) {
1300 debug(D_REGISTRY, "Registry: creating global registry dictionary for urls.");
1301 registry.urls = dictionary_create(DICTIONARY_FLAGS);
1302
808 - registry_log_open();
1303 + pthread_mutex_init(&registry.persons_lock, NULL);
1304 + pthread_mutex_init(&registry.machines_lock, NULL);
1305 + pthread_mutex_init(&registry.urls_lock, NULL);
1306 + pthread_mutex_init(&registry.person_urls_lock, NULL);
1307 + pthread_mutex_init(&registry.machine_urls_lock, NULL);
1308 +
1309 + registry_log_open_nolock();
1310 registry_load();
1311 +
1312 + registry.enabled = 1;
1313 }
1314
1315 void registry_free(void) {
1316 + if(!registry.enabled) return;
1317 +
1318 + // we need to destroy the dictionaries ourselves
1319 + // since the dictionaries use memory we allocated
1320 +
1321 while(registry.persons->values_index.root) {
1322 PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;
1323
@@ -824,7 +1332,7 @@ void registry_free(void) {
1332 dictionary_del(p->urls, pu->url->url);
1333
1334 debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url);
827 - registry_url_unlink(pu->url);
1335 + registry_url_unlink_nolock(pu->url);
1336
1337 debug(D_REGISTRY, "Registry: freeing person url");
1338 free(pu);
@@ -857,7 +1365,7 @@ void registry_free(void) {
1365 dictionary_del(m->urls, mu->url->url);
1366
1367 debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);
860 - registry_url_unlink(mu->url);
1368 + registry_url_unlink_nolock(mu->url);
1369
1370 debug(D_REGISTRY, "Registry: freeing machine url");
1371 free(mu);
@@ -873,6 +1381,8 @@ void registry_free(void) {
1381 free(m);
1382 }
1383
1384 + // and free the memory of remaining dictionary structures
1385 +
1386 debug(D_REGISTRY, "Registry: destroying persons dictionary");
1387 dictionary_destroy(registry.persons);
1388
@@ -884,6 +1394,8 @@ void registry_free(void) {
1394 }
1395
1396
1397 +#ifdef REGISTRY_STANDALONE_TESTS
1398 +
1399 // ----------------------------------------------------------------------------
1400 // TESTS
1401
@@ -933,7 +1445,7 @@ int test1(int argc, char **argv) {
1445 for(u = 0; u < users ; u++) {
1446 if(++m == machines) m = 0;
1447
936 - PERSON *p = registry_request(NULL, machines_guids[m], machines_urls[m], now);
1448 + PERSON *p = registry_request_access(NULL, machines_guids[m], machines_urls[m], "test", now);
1449 users_guids[u] = p->guid;
1450 }
1451 print_stats(u, start, timems());
@@ -945,7 +1457,7 @@ int test1(int argc, char **argv) {
1457 for(u = 0; u < users ; u++) {
1458 if(++m == machines) m = 0;
1459
948 - PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m], now);
1460 + PERSON *p = registry_request_access(users_guids[u], machines_guids[m], machines_urls[m], "test", now);
1461
1462 if(p->guid != users_guids[u])
1463 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid);
@@ -959,7 +1471,7 @@ int test1(int argc, char **argv) {
1471 for(u = 0; u < users ; u++) {
1472 if(++m == machines) m = 0;
1473
962 - PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m], now);
1474 + PERSON *p = registry_request_access(users_guids[u], machines_guids[m], machines_urls[m], "test", now);
1475
1476 if(p->guid != users_guids[u])
1477 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid);
@@ -973,7 +1485,7 @@ int test1(int argc, char **argv) {
1485 uint32_t tu = random() * users / RAND_MAX;
1486 uint32_t tm = random() * machines / RAND_MAX;
1487
976 - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm], now);
1488 + PERSON *p = registry_request_access(users_guids[tu], machines_guids[tm], machines_urls[tm], "test", now);
1489
1490 if(p->guid != users_guids[tu])
1491 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
@@ -987,7 +1499,7 @@ int test1(int argc, char **argv) {
1499 uint32_t tu = random() * users / RAND_MAX;
1500 uint32_t tm = random() * machines2 / RAND_MAX;
1501
990 - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm], now);
1502 + PERSON *p = registry_request_access(users_guids[tu], machines_guids[tm], machines_urls[tm], "test", now);
1503
1504 if(p->guid != users_guids[tu])
1505 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
@@ -1013,7 +1525,7 @@ int test1(int argc, char **argv) {
1525 else if (random() % 1000 == 123)
1526 url = machines_urls[random() * machines2 / RAND_MAX];
1527
1016 - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], url, now);
1528 + PERSON *p = registry_request_access(users_guids[tu], machines_guids[tm], url, "test", now);
1529
1530 if (p->guid != users_guids[tu])
1531 fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid);
@@ -1037,9 +1549,9 @@ int test1(int argc, char **argv) {
1549 // TESTING
1550
1551 int main(int argc, char **argv) {
1040 - debug_flags = 0xFFFFFFFF;
1041 - //test1(argc, argv);
1042 - //exit(0);
1552 + // debug_flags = 0xFFFFFFFF;
1553 + // test1(argc, argv);
1554 + // exit(0);
1555
1556 (void)argc;
1557 (void)argv;
@@ -1054,39 +1566,39 @@ int main(int argc, char **argv) {
1566 int i = 2;
1567
1568 fprintf(stderr, "\n\nADDING ENTRY\n");
1057 - // p1 = registry_request("2c95abd0-1542-11e6-8c66-00508db7e9c9", "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", time(NULL));
1569 + p1 = registry_request_access("2c95abd0-1542-11e6-8c66-00508db7e9c9", "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", "test", time(NULL));
1570
1571 if(0)
1572 while(i--) {
1573 #ifdef REGISTRY_STDOUT_DUMP
1574 fprintf(stderr, "\n\nADDING ENTRY\n");
1575 #endif /* REGISTRY_STDOUT_DUMP */
1064 - p1 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", time(NULL));
1576 + p1 = registry_request_access(NULL, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", "test", time(NULL));
1577
1578 #ifdef REGISTRY_STDOUT_DUMP
1579 fprintf(stderr, "\n\nADDING ANOTHER URL\n");
1580 #endif /* REGISTRY_STDOUT_DUMP */
1069 - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://127.0.0.1:19999/", time(NULL));
1581 + p1 = registry_request_access(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://127.0.0.1:19999/", "test", time(NULL));
1582
1583 #ifdef REGISTRY_STDOUT_DUMP
1584 fprintf(stderr, "\n\nADDING ANOTHER URL\n");
1585 #endif /* REGISTRY_STDOUT_DUMP */
1074 - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", time(NULL));
1586 + p1 = registry_request_access(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", "test", time(NULL));
1587
1588 #ifdef REGISTRY_STDOUT_DUMP
1589 fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n");
1590 #endif /* REGISTRY_STDOUT_DUMP */
1079 - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", time(NULL));
1591 + p1 = registry_request_access(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", "test", time(NULL));
1592
1593 #ifdef REGISTRY_STDOUT_DUMP
1594 fprintf(stderr, "\n\nADDING ANOTHER PERSON\n");
1595 #endif /* REGISTRY_STDOUT_DUMP */
1084 - p2 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", time(NULL));
1596 + p2 = registry_request_access(NULL, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", "test", time(NULL));
1597
1598 #ifdef REGISTRY_STDOUT_DUMP
1599 fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n");
1600 #endif /* REGISTRY_STDOUT_DUMP */
1089 - p2 = registry_request(p2->guid, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", time(NULL));
1601 + p2 = registry_request_access(p2->guid, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", "test", time(NULL));
1602 }
1603
1604 fprintf(stderr, "\n\nSAVE\n");
@@ -1096,3 +1608,5 @@ int main(int argc, char **argv) {
1608 registry_free();
1609 return 0;
1610 }
1611 +
1612 +#endif /* REGISTRY_STANDALONE_TESTS */
src/registry.h new
+16
@@ -0,0 +1,16 @@
1 +#include "web_client.h"
2 +
3 +#ifndef NETDATA_REGISTRY_H
4 +#define NETDATA_REGISTRY_H 1
5 +
6 +#define NETDATA_REGISTRY_COOKIE_NAME "netdata_registry_id"
7 +
8 +extern int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when);
9 +extern int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);
10 +extern int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when);
11 +
12 +extern void registry_init(void);
13 +extern void registry_free(void);
14 +extern int registry_save(void);
15 +
16 +#endif /* NETDATA_REGISTRY_H */
src/web_client.c
+155 -34
@@ -26,6 +26,7 @@
26 #include "global_statistics.h"
27 #include "rrd.h"
28 #include "rrd2json.h"
29 +#include "registry.h"
30
31 #include "web_client.h"
32 #include "../config.h"
@@ -173,6 +174,7 @@ void web_client_reset(struct web_client *w)
174 }
175
176 w->last_url[0] = '\0';
177 + w->cookie[0] = '\0';
178
179 w->mode = WEB_CLIENT_MODE_NORMAL;
180
@@ -644,7 +646,7 @@ int web_client_api_request_v1_data(struct web_client *w, char *url)
646 if(!name || !*name) continue;
647 if(!value || !*value) continue;
648
647 - debug(D_WEB_CLIENT, "%llu: API v1 query param '%s' with value '%s'", w->id, name, value);
649 + debug(D_WEB_CLIENT, "%llu: API v1 data query param '%s' with value '%s'", w->id, name, value);
650
651 // name and value are now the parameters
652 // they are not null and not empty
@@ -784,19 +786,111 @@ cleanup:
786 return ret;
787 }
788
789 +int web_client_api_request_v1_registry(struct web_client *w, char *url)
790 +{
791 + char person_guid[36 + 1] = "";
792 +
793 + debug(D_WEB_CLIENT, "%llu: API v1 registry with URL '%s'", w->id, url);
794 +
795 + char *cookie = strstr(w->response.data->buffer, " " NETDATA_REGISTRY_COOKIE_NAME "=");
796 + if(cookie) {
797 + strncpy(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME) + 2], 36);
798 + person_guid[36] = '\0';
799 + }
800 +
801 + char action = '\0';
802 + char *machine_guid = NULL,
803 + *machine_url = NULL,
804 + *url_name = NULL,
805 + *search_machine_guid = NULL,
806 + *delete_url = NULL;
807 +
808 + while(url) {
809 + char *value = mystrsep(&url, "?&[]");
810 + if (!value || !*value) continue;
811 +
812 + char *name = mystrsep(&value, "=");
813 + if (!name || !*name) continue;
814 + if (!value || !*value) continue;
815 +
816 + debug(D_WEB_CLIENT, "%llu: API v1 registry query param '%s' with value '%s'", w->id, name, value);
817 +
818 + if(!strcmp(name, "action")) {
819 + if(!strcmp(value, "access")) action = 'A';
820 + else if(!strcmp(value, "delete")) action = 'D';
821 + else if(!strcmp(value, "search")) action = 'S';
822 + }
823 + else if(!strcmp(name, "machine"))
824 + machine_guid = value;
825 +
826 + else if(!strcmp(name, "url"))
827 + machine_url = value;
828 +
829 + else if(action == 'A') {
830 + if(!strcmp(name, "name"))
831 + url_name = value;
832 + }
833 + else if(action == 'D') {
834 + if(!strcmp(name, "delete_url"))
835 + delete_url = value;
836 + }
837 + else if(action == 'S') {
838 + if(!strcmp(name, "for"))
839 + search_machine_guid = value;
840 + }
841 + }
842 +
843 + if((!action || !machine_guid || !machine_url) || (action == 'A' && !url_name) || (action == 'D' && !delete_url) || (action == 'S' && !search_machine_guid)) {
844 + buffer_flush(w->response.data);
845 + buffer_sprintf(w->response.data, "Invalid registry request - required parameters missing.");
846 + return 400;
847 + }
848 +
849 + switch(action) {
850 + case 'A':
851 + return registry_request_access_json(w, person_guid, machine_guid, machine_url, url_name, time(NULL));
852 +
853 + case 'D':
854 + return registry_request_delete_json(w, person_guid, machine_guid, machine_url, delete_url, time(NULL));
855 +
856 + case 'S':
857 + return registry_request_search_json(w, person_guid, machine_guid, machine_url, search_machine_guid, time(NULL));
858 + }
859 +
860 + buffer_flush(w->response.data);
861 + buffer_sprintf(w->response.data, "Invalid or no registry action.");
862 + return 400;
863 +}
864 +
865 int web_client_api_request_v1(struct web_client *w, char *url)
866 {
867 + static uint32_t data_hash = 0, chart_hash = 0, charts_hash = 0, registry_hash = 0;
868 +
869 + if(unlikely(data_hash == 0)) {
870 + data_hash = simple_hash("data");
871 + chart_hash = simple_hash("chart");
872 + charts_hash = simple_hash("charts");
873 + registry_hash = simple_hash("registry");
874 + }
875 +
876 // get the command
877 char *tok = mystrsep(&url, "/?&");
878 if(tok && *tok) {
879 debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, tok);
880 + uint32_t hash = simple_hash(tok);
881
794 - if(strcmp(tok, "data") == 0)
882 + if(hash == data_hash && !strcmp(tok, "data"))
883 return web_client_api_request_v1_data(w, url);
796 - else if(strcmp(tok, "chart") == 0)
884 +
885 + else if(hash == chart_hash && !strcmp(tok, "chart"))
886 return web_client_api_request_v1_chart(w, url);
798 - else if(strcmp(tok, "charts") == 0)
887 +
888 + else if(hash == charts_hash && !strcmp(tok, "charts"))
889 return web_client_api_request_v1_charts(w, url);
890 +
891 + else if(hash == registry_hash && !strcmp(tok, "registry"))
892 + return web_client_api_request_v1_registry(w, url);
893 +
894 else {
895 buffer_flush(w->response.data);
896 buffer_sprintf(w->response.data, "Unsupported v1 API command: %s", tok);
@@ -1043,6 +1137,32 @@ cleanup:
1137 }
1138 */
1139
1140 +// get the request buffer, just after the GET or OPTIONS
1141 +// and find the url to be decoded, decode it and return
1142 +// a newly allocated buffer with it
1143 +static inline char *find_url_and_decode_it(char *request) {
1144 + char *e = request, *url = NULL;
1145 +
1146 + // find the SPACE + "HTTP/"
1147 + while(*e) {
1148 + // find the space
1149 + while (*e && *e != ' ') e++;
1150 +
1151 + // is it SPACE + "HTTP/" ?
1152 + if(*e && !strncmp(e, " HTTP/", 6))
1153 + break;
1154 + }
1155 +
1156 + if(*e) {
1157 + // we have the end
1158 + *e = '\0';
1159 + url = url_decode(request);
1160 + *e = ' ';
1161 + }
1162 +
1163 + return url;
1164 +}
1165 +
1166 void web_client_process(struct web_client *w) {
1167 int code = 500;
1168 ssize_t bytes;
@@ -1069,33 +1189,24 @@ void web_client_process(struct web_client *w) {
1189 enable_gzip = 1;
1190 #endif // NETDATA_WITH_ZLIB
1191
1072 - int datasource_type = DATASOURCE_DATATABLE_JSONP;
1073 - //if(strstr(w->response.data->buffer, "X-DataSource-Auth"))
1074 - // datasource_type = DATASOURCE_GOOGLE_JSON;
1192 + w->mode = WEB_CLIENT_MODE_NORMAL;
1193
1076 - char *buf = (char *)buffer_tostring(w->response.data);
1077 - char *tok = strsep(&buf, " \r\n");
1078 - char *url = NULL;
1194 + char *tok = (char *)buffer_tostring(w->response.data);
1195 + char *url = NULL, *encoded_url = NULL;
1196 char *pointer_to_free = NULL; // keep url_decode() allocated buffer
1197
1081 - w->mode = WEB_CLIENT_MODE_NORMAL;
1082 -
1083 - if(buf && strcmp(tok, "GET") == 0) {
1084 - tok = strsep(&buf, " \r\n");
1085 - pointer_to_free = url = url_decode(tok);
1086 - debug(D_WEB_CLIENT, "%llu: Processing HTTP GET on url '%s'.", w->id, url);
1087 - }
1088 - else if(buf && strcmp(tok, "OPTIONS") == 0) {
1089 - tok = strsep(&buf, " \r\n");
1090 - pointer_to_free = url = url_decode(tok);
1091 - debug(D_WEB_CLIENT, "%llu: Processing HTTP OPTIONS on url '%s'.", w->id, url);
1198 + if(!strncmp(tok, "GET ", 4))
1199 + encoded_url = &tok[4];
1200 + else if(!strncmp(tok, "OPTIONS ", 8)) {
1201 + encoded_url = &tok[8];
1202 w->mode = WEB_CLIENT_MODE_OPTIONS;
1203 }
1094 - else if (buf && strcmp(tok, "POST") == 0) {
1095 - w->keepalive = 0;
1096 - tok = strsep(&buf, " \r\n");
1097 - pointer_to_free = url = url_decode(tok);
1098 - debug(D_WEB_CLIENT, "%llu: I don't know how to handle POST with form data. Assuming it is a GET on url '%s'.", w->id, url);
1204 +
1205 + if(encoded_url) {
1206 + pointer_to_free = url = find_url_and_decode_it(encoded_url);
1207 +
1208 + if(url) debug(D_WEB_CLIENT, "%llu: Processing url '%s'.", w->id, url);
1209 + else debug(D_WEB_CLIENT, "%llu: Cannot find a valid URL in '%s'", w->id, encoded_url);
1210 }
1211
1212 w->last_url[0] = '\0';
@@ -1124,26 +1235,29 @@ void web_client_process(struct web_client *w) {
1235
1236 if(strcmp(tok, "api") == 0) {
1237 // the client is requesting api access
1127 - datasource_type = DATASOURCE_JSON;
1238 code = web_client_api_request(w, url);
1239 }
1240 #ifdef NETDATA_INTERNAL_CHECKS
1241 else if(strcmp(tok, "exit") == 0) {
1132 - netdata_exit = 1;
1242 code = 200;
1243 w->response.data->contenttype = CT_TEXT_PLAIN;
1244 buffer_flush(w->response.data);
1136 - buffer_strcat(w->response.data, "will do");
1245 +
1246 + if(!netdata_exit)
1247 + buffer_strcat(w->response.data, "ok, will do...");
1248 + else
1249 + buffer_strcat(w->response.data, "I am doing it already");
1250 +
1251 + netdata_exit = 1;
1252 }
1253 #endif
1254 else if(strcmp(tok, WEB_PATH_DATA) == 0) { // "data"
1255 // the client is requesting rrd data
1141 - datasource_type = DATASOURCE_JSON;
1142 - code = web_client_data_request(w, url, datasource_type);
1256 + code = web_client_data_request(w, url, DATASOURCE_JSON);
1257 }
1258 else if(strcmp(tok, WEB_PATH_DATASOURCE) == 0) { // "datasource"
1259 // the client is requesting google datasource
1146 - code = web_client_data_request(w, url, datasource_type);
1260 + code = web_client_data_request(w, url, DATASOURCE_DATATABLE_JSONP);
1261 }
1262 else if(strcmp(tok, WEB_PATH_GRAPH) == 0) { // "graph"
1263 // the client is requesting an rrd graph
@@ -1269,7 +1383,7 @@ void web_client_process(struct web_client *w) {
1383 else {
1384 strcpy(w->last_url, "not a valid response");
1385
1272 - if(buf) debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, buf);
1386 + debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, w->response.data->buffer);
1387
1388 code = 500;
1389 buffer_flush(w->response.data);
@@ -1437,10 +1551,17 @@ void web_client_process(struct web_client *w) {
1551 , date
1552 );
1553
1554 + if(w->cookie[0]) {
1555 + buffer_sprintf(w->response.header_output,
1556 + "Set-Cookie: %s\r\n",
1557 + w->cookie);
1558 + }
1559 +
1560 if(w->mode == WEB_CLIENT_MODE_OPTIONS) {
1561 buffer_strcat(w->response.header_output,
1562 "Access-Control-Allow-Methods: GET, OPTIONS\r\n"
1443 - "Access-Control-Allow-Headers: accept, x-requested-with\r\n"
1563 + "Access-Control-Allow-Credentials: true\r\n"
1564 + "Access-Control-Allow-Headers: Accept, X-Requested-With, Content-Type, Cookie\r\n"
1565 "Access-Control-Max-Age: 1209600\r\n" // 86400 * 14
1566 );
1567 }
src/web_client.h
+4
@@ -11,6 +11,7 @@
11 #include <netdb.h>
12
13 #include "web_buffer.h"
14 +#include "dictionary.h"
15
16 #define DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS 60
17 extern int web_client_timeout;
@@ -26,6 +27,7 @@ extern int web_enable_gzip;
27 #define URL_MAX 8192
28 #define ZLIB_CHUNK 16384
29 #define HTTP_RESPONSE_HEADER_SIZE 4096
30 +#define COOKIE_MAX 200
31
32 struct response {
33 BUFFER *header; // our response header
@@ -58,6 +60,8 @@ struct web_client {
60
61 struct timeval tv_in, tv_ready;
62
63 + char cookie[COOKIE_MAX+1];
64 +
65 int mode;
66 int keepalive;
67