@cryptotaxi247 / netdata-1 / commits / e96b2b05e

registry server side operational

Costa Tsaousis committed May 11, 2016 at 00:56 UTC e96b2b05e17b7358a8ddbafa9c69951208323aee

8 files changed +428 -75

src/appconfig.c

+1 -1

@@ -442,7 +442,7 @@ void generate_config(BUFFER *wb, int only_changed)

442

443 pthread_rwlock_wrlock(&config_rwlock);

444 for(co = config_root; co ; co = co->next) {

445 - if(strcmp(co->name, "global") == 0 || strcmp(co->name, "plugins") == 0) pri = 0;

445 + if(strcmp(co->name, "global") == 0 || strcmp(co->name, "plugins") == 0 || strcmp(co->name, "registry") == 0) pri = 0;

446 else if(strncmp(co->name, "plugin:", 7) == 0) pri = 1;

447 else pri = 2;

448

src/main.c

@@ -34,6 +34,7 @@

34 #include "plugin_checks.h"

35 #include "plugin_proc.h"

36 #include "plugin_nfacct.h"

37 +#include "registry.h"

39 #include "main.h"

@@ -521,6 +522,12 @@ int main(int argc, char **argv)

522 info("Successfully set pthread stacksize to %zu bytes", wanted_stacksize);

523 }

524

525 + // --------------------------------------------------------------------

526 + // initialize the registry

527 +

528 + registry_init();

529 +

530 +

531 // ------------------------------------------------------------------------

532 // spawn the threads

533

src/plugin_proc.c

+9 -4

@@ -14,6 +14,7 @@

14 #include "rrd.h"

15 #include "plugin_proc.h"

16 #include "main.h"

17 +#include "registry.h"

19 void *proc_main(void *ptr)

20 {

@@ -240,7 +241,7 @@ void *proc_main(void *ptr)

241

242 if(!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu");

243 if(!stcpu_thread) {

243 - stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, "NetData Proc Plugin CPU usage", "milliseconds/s", 131000, rrd_update_every, RRDSET_TYPE_STACKED);

244 + stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, RRDSET_TYPE_STACKED);

245

246 rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);

247 rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);

@@ -270,7 +271,7 @@ void *proc_main(void *ptr)

271

272 if(!stclients) stclients = rrdset_find("netdata.clients");

273 if(!stclients) {

273 - stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients", "connected clients", 131000, rrd_update_every, RRDSET_TYPE_LINE);

274 + stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients", "connected clients", 130100, rrd_update_every, RRDSET_TYPE_LINE);

275

276 rrddim_add(stclients, "clients", NULL, 1, 1, RRDDIM_ABSOLUTE);

277 }

@@ -283,7 +284,7 @@ void *proc_main(void *ptr)

284

285 if(!streqs) streqs = rrdset_find("netdata.requests");

286 if(!streqs) {

286 - streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s", 131100, rrd_update_every, RRDSET_TYPE_LINE);

287 + streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s", 130200, rrd_update_every, RRDSET_TYPE_LINE);

288

289 rrddim_add(streqs, "requests", NULL, 1, 1, RRDDIM_INCREMENTAL);

290 }

@@ -296,7 +297,7 @@ void *proc_main(void *ptr)

297

298 if(!stbytes) stbytes = rrdset_find("netdata.net");

299 if(!stbytes) {

299 - stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s", 131200, rrd_update_every, RRDSET_TYPE_AREA);

300 + stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s", 130300, rrd_update_every, RRDSET_TYPE_AREA);

301

302 rrddim_add(stbytes, "in", NULL, 8, 1024, RRDDIM_INCREMENTAL);

303 rrddim_add(stbytes, "out", NULL, -8, 1024, RRDDIM_INCREMENTAL);

@@ -306,6 +307,10 @@ void *proc_main(void *ptr)

307 rrddim_set(stbytes, "in", global_statistics.bytes_received);

308 rrddim_set(stbytes, "out", global_statistics.bytes_sent);

309 rrdset_done(stbytes);

310 +

311 + // ----------------------------------------------------------------

312 +

313 + registry_statistics();

314 }

315 }

316

src/registry.c

+403 -66

@@ -13,6 +13,7 @@

13 #include <sys/stat.h>

14 #include <sys/types.h>

15 #include <errno.h>

16 +#include <fcntl.h>

18 #include "log.h"

19 #include "common.h"

@@ -20,8 +21,31 @@

21 #include "appconfig.h"

23 #include "web_client.h"

24 +#include "rrd.h"

25 #include "registry.h"

27 +

28 +// ----------------------------------------------------------------------------

29 +// TODO

30 +//

31 +// 1. the default tracking cookie expires in 1 year, but the persons are not

32 +// removed from the db - this means the database only grows - ideally the

33 +// database should be cleaned in registry_save() for both on-disk and

34 +// on-memory entries.

35 +//

36 +// 2. add protection to prevent abusing the registry by flooding it with

37 +// requests to fill the memory and crash it.

38 +//

39 +// Possible protections:

40 +// - limit the number of URLs per person

41 +// - limit the number of URLs per machine

42 +// - limit the number of persons

43 +// - limit the number of machines

44 +// - limit the number of requests that add data to the registry,

45 +// per client IP per hour

46 +

47 +

48 +

49 #define REGISTRY_URL_FLAGS_DEFAULT 0x00

50 #define REGISTRY_URL_FLAGS_EXPIRED 0x01

@@ -33,19 +57,38 @@

57 struct registry {

58 int enabled;

36 - // counters / statistics

60 + char machine_guid[36 + 1];

61 +

62 + // entries counters / statistics

63 unsigned long long persons_count;

64 unsigned long long machines_count;

65 unsigned long long usages_count;

66 unsigned long long urls_count;

67 unsigned long long persons_urls_count;

68 unsigned long long machines_urls_count;

43 -

44 - // files

69 + unsigned long long log_count;

70 +

71 + // memory counters / statistics

72 + unsigned long long persons_memory;

73 + unsigned long long machines_memory;

74 + unsigned long long urls_memory;

75 + unsigned long long persons_urls_memory;

76 + unsigned long long machines_urls_memory;

77 +

78 + // configuration

79 + unsigned long long save_registry_every_entries;

80 + char *registry_domain;

81 + char *registry_to_announce;

82 + time_t persons_expiration; // seconds to expire idle persons

83 +

84 + // file/path names

85 char *pathname;

86 char *db_filename;

87 char *log_filename;

48 - FILE *registry_log_fp;

88 + char *machine_guid_filename;

89 +

90 + // open files

91 + FILE *log_fp;

93 // the database

94 DICTIONARY *persons; // dictionary of PERSON *, with key the PERSON.guid

@@ -63,6 +106,7 @@ struct registry {

106 pthread_mutex_t log_lock;

107 } registry;

108

109 +

110 // ----------------------------------------------------------------------------

111 // URL structures

112 // Save memory by de-duplicating URLs

@@ -284,6 +328,8 @@ static inline URL *registry_url_allocate_nolock(const char *url, size_t urllen)

328 u->len = urllen;

329 u->links = 0;

330

331 + registry.urls_memory += sizeof(URL) + urllen;

332 +

333 debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): indexing it", url);

334 dictionary_set(registry.urls, u->url, u, sizeof(URL));

335

@@ -345,6 +391,8 @@ static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, tim

391 mu->url = u;

392 mu->flags = REGISTRY_URL_FLAGS_DEFAULT;

393

394 + registry.machines_urls_memory += sizeof(MACHINE_URL);

395 +

396 debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url);

397 dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL));

398 registry_url_link_nolock(u);

@@ -355,7 +403,7 @@ static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, tim

403 static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) {

404 debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(MACHINE));

405

358 - MACHINE *m = calloc(1, sizeof(MACHINE));

406 + MACHINE *m = malloc(sizeof(MACHINE));

407 if(!m) fatal("Registry: cannot allocate memory for new machine '%s'", machine_guid);

408

409 strncpy(m->guid, machine_guid, 36);

@@ -366,6 +414,8 @@ static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_

414 m->first_t = m->last_t = when;

415 m->usages = 0;

416

417 + registry.machines_memory += sizeof(MACHINE);

418 +

419 dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE));

420

421 return m;

@@ -409,9 +459,11 @@ static inline PERSON *registry_person_find(const char *person_guid) {

459 }

460

461 static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) {

412 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL) + namelen);

462 + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,

463 + sizeof(PERSON_URL) + namelen);

464 +

465 PERSON_URL *pu = malloc(sizeof(PERSON_URL) + namelen);

414 - if(!pu) fatal("registry_person_link_to_url('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL) + namelen);

466 + if(!pu) fatal("registry_person_url_allocate('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL) + namelen);

467

468 // a simple strcpy() should do the job

469 // but I prefer to be safe, since the caller specified urllen

@@ -424,19 +476,42 @@ static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, UR

476 pu->url = u;

477 pu->flags = REGISTRY_URL_FLAGS_DEFAULT;

478

427 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);

479 + registry.persons_urls_memory += sizeof(PERSON_URL) + namelen;

480 +

481 + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);

482 dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL));

483 registry_url_link_nolock(u);

484

485 return pu;

486 }

487

488 +static inline PERSON_URL *registry_person_url_reallocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when, PERSON_URL *pu) {

489 + // this function is needed to change the name of a PERSON_URL

490 +

491 + debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,

492 + sizeof(PERSON_URL) + namelen);

493 +

494 + PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when);

495 + tpu->first_t = pu->first_t;

496 + tpu->last_t = pu->last_t;

497 + tpu->usages = pu->usages;

498 +

499 + // ok, these are a hack - since the registry_person_url_allocate() is

500 + // adding these, we have to subtract them

501 + registry.persons_urls_memory -= sizeof(PERSON_URL) + strlen(pu->name);

502 + registry_url_unlink_nolock(u);

503 +

504 + free(pu);

505 +

506 + return tpu;

507 +}

508 +

509 static inline PERSON *registry_person_allocate(const char *person_guid, time_t when) {

510 PERSON *p = NULL;

511

512 debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(PERSON));

513

439 - p = calloc(1, sizeof(PERSON));

514 + p = malloc(sizeof(PERSON));

515 if(!p) fatal("Registry: cannot allocate memory for new person.");

516

517 if(!person_guid) {

@@ -467,6 +542,8 @@ static inline PERSON *registry_person_allocate(const char *person_guid, time_t w

542 p->first_t = p->last_t = when;

543 p->usages = 0;

544

545 + registry.persons_memory += sizeof(PERSON);

546 +

547 dictionary_set(registry.persons, p->guid, p, sizeof(PERSON));

548 return p;

549 }

@@ -531,6 +608,11 @@ static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL

608

609 pu->machine = m;

610 }

611 +

612 + if(strcmp(pu->name, name)) {

613 + // the name of the PERSON_URL has changed !

614 + pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu);

615 + }

616 }

617

618 p->usages++;

@@ -581,52 +663,76 @@ static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, U

663 // ----------------------------------------------------------------------------

664 // REGISTRY LOG LOAD/SAVE

665

666 +static inline int registry_should_save_db(void) {

667 + debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries);

668 + return registry.log_count > registry.save_registry_every_entries;

669 +}

670 +

671 static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u, char *name) {

585 - if(likely(registry.registry_log_fp)) {

672 + if(likely(registry.log_fp)) {

673 // we lock only if the file is open

674 // to allow replaying the log at registry_log_load()

675 registry_log_lock();

676

590 - fprintf(registry.registry_log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",

677 + if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",

678 action,

679 p->last_t,

680 p->guid,

681 m->guid,

682 name,

596 - u->url

597 - );

683 + u->url) < 0))

684 + error("Registry: failed to save log. Registry data may be lost in case of abnormal restart.");

685 +

686 + // we increase the counter even on failures

687 + // so that the registry will be saved periodically

688 + registry.log_count++;

689

690 registry_log_unlock();

691 +

692 + // this must be outside the log_lock(), or a deadlock will happen.

693 + // registry_save() checks the same inside the log_lock, so only

694 + // one thread will save the db

695 + if(unlikely(registry_should_save_db()))

696 + registry_save();

697 }

698 }

699

603 -void registry_log_open_nolock(void) {

604 - registry.registry_log_fp = fopen(registry.log_filename, "a");

700 +static inline int registry_log_open_nolock(void) {

701 + if(registry.log_fp)

702 + fclose(registry.log_fp);

703 +

704 + registry.log_fp = fopen(registry.log_filename, "a");

705

606 - if(registry.registry_log_fp) {

607 - if (setvbuf(registry.registry_log_fp, NULL, _IOLBF, 0) != 0)

706 + if(registry.log_fp) {

707 + if (setvbuf(registry.log_fp, NULL, _IOLBF, 0) != 0)

708 error("Cannot set line buffering on registry log file.");

709 + return 0;

710 }

610 -}

711

612 -void registry_log_close_nolock(void) {

613 - if(registry.registry_log_fp)

614 - fclose(registry.registry_log_fp);

712 + error("Cannot open registry log file '%s'. Registry data will be lost in case of netdata or server crash.", registry.log_filename);

713 + return -1;

714 +}

715

616 - registry.registry_log_fp = NULL;

716 +static inline void registry_log_close_nolock(void) {

717 + if(registry.log_fp) {

718 + fclose(registry.log_fp);

719 + registry.log_fp = NULL;

720 + }

721 }

722

619 -void registry_log_recreate_nolock(void) {

620 - if(registry.registry_log_fp != NULL) {

723 +static inline void registry_log_recreate_nolock(void) {

724 + if(registry.log_fp != NULL) {

725 registry_log_close_nolock();

726

623 - registry.registry_log_fp = fopen(registry.log_filename, "w");

624 - if(registry.registry_log_fp) fclose(registry.registry_log_fp);

625 - registry.registry_log_fp = NULL;

727 + // open it with truncate

728 + registry.log_fp = fopen(registry.log_filename, "w");

729 + if(registry.log_fp) fclose(registry.log_fp);

730 + else error("Cannot truncate registry log '%s'", registry.log_filename);

731 +

732 + registry.log_fp = NULL;

733

734 registry_log_open_nolock();

735 }

629 -

736 }

737

738 int registry_log_load(void) {

@@ -656,11 +762,16 @@ int registry_log_load(void) {

762 error("Registry: log line %u is wrong (len = %zu).", line, len);

763 continue;

764 }

659 -

765 s[1] = s[10] = s[47] = s[84] = '\0';

766

767 + // get the variables

768 + time_t when = strtoul(&s[2], NULL, 16);

769 + char *person_guid = &s[11];

770 + char *machine_guid = &s[48];

771 + char *name = &s[85];

772 +

773 // skip the name to find the url

663 - char *url = &s[85];

774 + char *url = name;

775 while(*url && *url != '\t') url++;

776 if(!*url) {

777 error("Registry: log line %u does not have a url.", line);

@@ -668,10 +779,15 @@ int registry_log_load(void) {

779 }

780 *url++ = '\0';

781

782 + // make sure the person exists

783 + // without this, a new person guid will be created

784 + PERSON *p = registry_person_find(person_guid);

785 + if(!p) p = registry_person_allocate(person_guid, when);

786 +

787 if(s[0] == 'A')

672 - registry_request_access(&s[11], &s[48], url, &s[85], strtoul(&s[2], NULL, 16));

788 + registry_request_access(p->guid, machine_guid, url, name, when);

789 else

674 - registry_request_delete(&s[11], &s[48], url, &s[85], strtoul(&s[2], NULL, 16));

790 + registry_request_delete(p->guid, machine_guid, url, name, when);

791

792 break;

793

@@ -871,9 +987,27 @@ MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *u

987

988 #ifndef REGISTRY_STANDALONE_TESTS

989

990 +static inline void registry_set_person_cookie(struct web_client *w, PERSON *p) {

991 + char edate[100];

992 + time_t et = time(NULL) + registry.persons_expiration;

993 + struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);

994 + strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm);

995 +

996 + if(registry.registry_domain && registry.registry_domain[0])

997 + snprintf(w->cookie, COOKIE_MAX, NETDATA_REGISTRY_COOKIE_NAME "=%s; Domain=%s; Expires=%s", p->guid, registry.registry_domain, edate);

998 + else

999 + snprintf(w->cookie, COOKIE_MAX, NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s", p->guid, edate);

1000 +

1001 + w->cookie[COOKIE_MAX] = '\0';

1002 +}

1003 +

1004 +

1005 static inline void registry_json_header(struct web_client *w, int status) {

1006 + w->response.data->contenttype = CT_APPLICATION_JSON;

1007 buffer_flush(w->response.data);

876 - buffer_sprintf(w->response.data, "{\n\"success\": %s", status?"true":"false");

1008 + buffer_sprintf(w->response.data, "{\n\t\"success\": %s,\n\t\"machine_guid\": \"%s\"",

1009 + status?"true":"false",

1010 + registry.machine_guid);

1011 }

1012

1013 static inline void registry_json_footer(struct web_client *w) {

@@ -882,11 +1016,55 @@ static inline void registry_json_footer(struct web_client *w) {

1016

1017 static inline int registry_json_redirect(struct web_client *w) {

1018 registry_json_header(w, 0);

885 - buffer_sprintf(w->response.data, "\"registries\": \"%s\"", config_get("global", "users registries", "reg1.mynetdata.io reg2.mynetdata.io reg3.mynetdata.io"));

1019 +

1020 + buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",

1021 + registry.registry_to_announce);

1022 +

1023 registry_json_footer(w);

1024 return 200;

1025 }

1026

1027 +// structure used be the callbacks below

1028 +struct registry_json_walk_person_urls_callback {

1029 + PERSON *p;

1030 + MACHINE *m;

1031 + struct web_client *w;

1032 + int count;

1033 +};

1034 +

1035 +// callback for rendering PERSON_URLs

1036 +static inline int registry_json_person_url_callback(void *entry, void *data) {

1037 + PERSON_URL *pu = (PERSON_URL *)entry;

1038 + struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;

1039 + struct web_client *w = c->w;

1040 +

1041 + if(unlikely(c->count++))

1042 + buffer_strcat(w->response.data, ",");

1043 +

1044 + buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u, \"%s\" ]",

1045 + pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->name);

1046 +

1047 + return 1;

1048 +}

1049 +

1050 +// callback for rendering MACHINE_URLs

1051 +static inline int registry_json_machine_url_callback(void *entry, void *data) {

1052 + MACHINE_URL *mu = (MACHINE_URL *)entry;

1053 + struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;

1054 + struct web_client *w = c->w;

1055 + MACHINE *m = c->m;

1056 +

1057 + if(unlikely(c->count++))

1058 + buffer_strcat(w->response.data, ",");

1059 +

1060 + buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u ]",

1061 + m->guid, mu->url->url, mu->last_t, mu->usages);

1062 +

1063 + return 1;

1064 +}

1065 +

1066 +

1067 +// the main method for registering an access

1068 int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) {

1069 if(!registry.enabled)

1070 return registry_json_redirect(w);

@@ -898,18 +1076,22 @@ int registry_request_access_json(struct web_client *w, char *person_guid, char *

1076 return 400;

1077 }

1078

901 - // FIXME -- make this a permanent cookie

902 - strncpy(w->cookie, p->guid, COOKIE_MAX);

903 - w->cookie[COOKIE_MAX] = '\0';

1079 + // set the cookie

1080 + registry_set_person_cookie(w, p);

1081

1082 + // generate the response

1083 registry_json_header(w, 1);

1084

907 - // FIXME - print an array of all URLs / Machines / Names

1085 + buffer_strcat(w->response.data, ",\n\t\"urls\": [");

1086 + struct registry_json_walk_person_urls_callback c = { p, NULL, w, 0 };

1087 + dictionary_get_all(p->urls, registry_json_person_url_callback, &c);

1088 + buffer_strcat(w->response.data, "\n\t]\n");

1089

909 - buffer_strcat(w->response.data, "\n}\n");

1090 + registry_json_footer(w);

1091 return 200;

1092 }

1093

1094 +// the main method for deleting a URL from a person

1095 int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {

1096 if(!registry.enabled)

1097 return registry_json_redirect(w);

@@ -921,11 +1103,13 @@ int registry_request_delete_json(struct web_client *w, char *person_guid, char *

1103 return 400;

1104 }

1105

1106 + // generate the response

1107 registry_json_header(w, 1);

1108 registry_json_footer(w);

1109 return 200;

1110 }

1111

1112 +// the main method for searching the URLs of a netdata

1113 int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {

1114 if(!registry.enabled)

1115 return registry_json_redirect(w);

@@ -939,13 +1123,69 @@ int registry_request_search_json(struct web_client *w, char *person_guid, char *

1123

1124 registry_json_header(w, 1);

1125

942 - // FIXME - print an array of all URLs the machine is accessible

1126 + buffer_strcat(w->response.data, ",\n\t\"urls\": [");

1127 + struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 };

1128 + dictionary_get_all(m->urls, registry_json_machine_url_callback, &c);

1129 + buffer_strcat(w->response.data, "\n\t]\n");

1130

1131 registry_json_footer(w);

1132 return 200;

1133 }

1134

948 -#endif /* REGISTRY_STANDALONE_TESTS */

1135 +#endif /* ! REGISTRY_STANDALONE_TESTS */

1136 +

1137 +

1138 +// ----------------------------------------------------------------------------

1139 +// REGISTRY THIS MACHINE UNIQUE ID

1140 +

1141 +char *registry_get_this_machine_guid(void) {

1142 + if(likely(registry.machine_guid[0]))

1143 + return registry.machine_guid;

1144 +

1145 + // read it from disk

1146 + int fd = open(registry.machine_guid_filename, O_RDONLY);

1147 + if(fd != -1) {

1148 + char buf[36 + 1];

1149 + if(read(fd, buf, 36) != 36)

1150 + error("Failed to read machine GUID from '%s'", registry.machine_guid_filename);

1151 + else {

1152 + buf[36] = '\0';

1153 + if(registry_regenerate_guid(buf, registry.machine_guid) == -1) {

1154 + error("Failed to validate machine GUID '%s' from '%s'. Ignoring it - this might mean this netdata will appear as duplicate in the registry.",

1155 + buf, registry.machine_guid_filename);

1156 +

1157 + registry.machine_guid[0] = '\0';

1158 + }

1159 + }

1160 + close(fd);

1161 + }

1162 +

1163 + // generate a new one?

1164 + if(!registry.machine_guid[0]) {

1165 + int count = 10, ret = 0;

1166 + uuid_t uuid;

1167 +

1168 + // for some reason it reports unsafe generation the first time

1169 + while(count-- && (ret = uuid_generate_time_safe(uuid)) == -1) ;

1170 + uuid_unparse_lower(uuid, registry.machine_guid);

1171 + registry.machine_guid[36] = '\0';

1172 +

1173 + if(ret == -1)

1174 + info("Warning: generated machine GUID '%s' was not generated using a safe method.", registry.machine_guid);

1175 +

1176 + // save it

1177 + fd = open(registry.machine_guid_filename, O_WRONLY|O_CREAT|O_TRUNC, 444);

1178 + if(fd == -1)

1179 + fatal("Cannot create unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

1180 +

1181 + if(write(fd, registry.machine_guid, 36) != 36)

1182 + fatal("Cannot write the unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

1183 +

1184 + close(fd);

1185 + }

1186 +

1187 + return registry.machine_guid;

1188 +}

1189

1190

1191 // ----------------------------------------------------------------------------

@@ -1042,15 +1282,20 @@ static inline int registry_person_save(void *entry, void *file) {

1282 int registry_save(void) {

1283 if(!registry.enabled) return -1;

1284

1285 + // make sure the log is not updated

1286 + registry_log_lock();

1287 +

1288 + if(unlikely(!registry_should_save_db())) {

1289 + registry_log_unlock();

1290 + return -2;

1291 + }

1292 +

1293 char tmp_filename[FILENAME_MAX + 1];

1294 char old_filename[FILENAME_MAX + 1];

1295

1296 snprintf(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);

1297 snprintf(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);

1298

1051 - // make sure the log is not updated

1052 - registry_log_lock();

1053 -

1299 debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);

1300 FILE *fp = fopen(tmp_filename, "w");

1301 if(!fp) {

@@ -1059,6 +1304,8 @@ int registry_save(void) {

1304 return -1;

1305 }

1306

1307 + // dictionary_get_all() has its own locking, so this is safe to do

1308 +

1309 debug(D_REGISTRY, "Saving all machines");

1310 int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);

1311 if(bytes1 < 0) {

@@ -1083,7 +1330,7 @@ int registry_save(void) {

1330 fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",

1331 registry.persons_count,

1332 registry.machines_count,

1086 - registry.usages_count,

1333 + registry.usages_count + 1, // this is required - it is lost on db rotation

1334 registry.urls_count,

1335 registry.persons_urls_count,

1336 registry.machines_urls_count

@@ -1128,6 +1375,8 @@ int registry_save(void) {

1375 // it has been moved successfully

1376 // discard the current registry log

1377 registry_log_recreate_nolock();

1378 +

1379 + registry.log_count = 0;

1380 }

1381 }

1382

@@ -1144,10 +1393,10 @@ static inline size_t registry_load(void) {

1393 URL *u = NULL;

1394 size_t line = 0;

1395

1147 - debug(D_REGISTRY, "Registry: loading active db from: %s", registry.db_filename);

1396 + debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename);

1397 FILE *fp = fopen(registry.db_filename, "r");

1398 if(!fp) {

1150 - error("Registry: cannot open registry file: %s", registry.db_filename);

1399 + error("Registry: cannot open registry file: '%s'", registry.db_filename);

1400 return 0;

1401 }

1402

@@ -1266,26 +1515,44 @@ static inline size_t registry_load(void) {

1515 }

1516 fclose(fp);

1517

1269 - registry_log_load();

1270 -

1518 return line;

1519 }

1520

1521 // ----------------------------------------------------------------------------

1522 // REGISTRY

1523

1277 -void registry_init(void) {

1524 +int registry_init(void) {

1525 char filename[FILENAME_MAX + 1];

1279 - registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR);

1280 - if(mkdir(registry.pathname, 0644) == -1 && errno != EEXIST)

1281 - error("Cannot create directory '%s'", registry.pathname);

1526

1283 - snprintf(filename, FILENAME_MAX, "%s/%s", registry.pathname, "registry.db");

1527 + // registry enabled?

1528 + registry.enabled = config_get_boolean("registry", "enabled", 0);

1529 +

1530 + // pathnames

1531 + registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR "/registry");

1532 + if(mkdir(registry.pathname, 0755) == -1 && errno != EEXIST) {

1533 + error("Cannot create directory '%s'. Registry disabled.", registry.pathname);

1534 + registry.enabled = 0;

1535 + return -1;

1536 + }

1537 +

1538 + // filenames

1539 + snprintf(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname);

1540 + registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename);

1541 + registry_get_this_machine_guid();

1542 +

1543 + snprintf(filename, FILENAME_MAX, "%s/registry.db", registry.pathname);

1544 registry.db_filename = config_get("registry", "registry db file", filename);

1545

1286 - snprintf(filename, FILENAME_MAX, "%s/%s", registry.pathname, "registry-log.db");

1546 + snprintf(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname);

1547 registry.log_filename = config_get("registry", "registry log file", filename);

1548

1549 + // configuration options

1550 + registry.save_registry_every_entries = config_get_number("registry", "registry save db every new entries", 1000000);

1551 + registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400;

1552 + registry.registry_domain = config_get("registry", "registry domain", "");

1553 + registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.netdata.online");

1554 +

1555 + // initialize entries counters

1556 registry.persons_count = 0;

1557 registry.machines_count = 0;

1558 registry.usages_count = 0;

@@ -1293,25 +1560,33 @@ void registry_init(void) {

1560 registry.persons_urls_count = 0;

1561 registry.machines_urls_count = 0;

1562

1296 - debug(D_REGISTRY, "Registry: creating global registry dictionary for persons.");

1297 - registry.persons = dictionary_create(DICTIONARY_FLAGS);

1298 -

1299 - debug(D_REGISTRY, "Registry: creating global registry dictionary for machines.");

1300 - registry.machines = dictionary_create(DICTIONARY_FLAGS);

1301 -

1302 - debug(D_REGISTRY, "Registry: creating global registry dictionary for urls.");

1303 - registry.urls = dictionary_create(DICTIONARY_FLAGS);

1563 + // initialize memory counters

1564 + registry.persons_memory = 0;

1565 + registry.machines_memory = 0;

1566 + registry.urls_memory = 0;

1567 + registry.persons_urls_memory = 0;

1568 + registry.machines_urls_memory = 0;

1569

1570 + // initialize locks

1571 pthread_mutex_init(&registry.persons_lock, NULL);

1572 pthread_mutex_init(&registry.machines_lock, NULL);

1573 pthread_mutex_init(&registry.urls_lock, NULL);

1574 pthread_mutex_init(&registry.person_urls_lock, NULL);

1575 pthread_mutex_init(&registry.machine_urls_lock, NULL);

1576

1311 - registry_log_open_nolock();

1312 - registry_load();

1577 + // create dictionaries

1578 + registry.persons = dictionary_create(DICTIONARY_FLAGS);

1579 + registry.machines = dictionary_create(DICTIONARY_FLAGS);

1580 + registry.urls = dictionary_create(DICTIONARY_FLAGS);

1581 +

1582 + // load the registry database

1583 + if(registry.enabled) {

1584 + registry_log_open_nolock();

1585 + registry_load();

1586 + registry_log_load();

1587 + }

1588

1314 - registry.enabled = 1;

1589 + return 0;

1590 }

1591

1592 void registry_free(void) {

@@ -1395,6 +1670,68 @@ void registry_free(void) {

1670 dictionary_destroy(registry.urls);

1671 }

1672

1673 +// ----------------------------------------------------------------------------

1674 +// STATISTICS

1675 +

1676 +void registry_statistics(void) {

1677 + if(!registry.enabled) return;

1678 +

1679 + static RRDSET *sts = NULL, *stc = NULL, *stm = NULL;

1680 +

1681 + if(!sts) sts = rrdset_find("netdata.registry_sessions");

1682 + if(!sts) {

1683 + sts = rrdset_create("netdata", "registry_sessions", NULL, "registry", NULL, "NetData Registry Sessions", "session", 131000, rrd_update_every, RRDSET_TYPE_LINE);

1684 +

1685 + rrddim_add(sts, "sessions", NULL, 1, 1, RRDDIM_ABSOLUTE);

1686 + }

1687 + else rrdset_next(sts);

1688 +

1689 + rrddim_set(sts, "sessions", registry.usages_count);

1690 + rrdset_done(sts);

1691 +

1692 + // ------------------------------------------------------------------------

1693 +

1694 + if(!stc) stc = rrdset_find("netdata.registry_entries");

1695 + if(!stc) {

1696 + stc = rrdset_create("netdata", "registry_entries", NULL, "registry", NULL, "NetData Registry Entries", "entries", 131100, rrd_update_every, RRDSET_TYPE_LINE);

1697 +

1698 + rrddim_add(stc, "persons", NULL, 1, 1, RRDDIM_ABSOLUTE);

1699 + rrddim_add(stc, "machines", NULL, 1, 1, RRDDIM_ABSOLUTE);

1700 + rrddim_add(stc, "urls", NULL, 1, 1, RRDDIM_ABSOLUTE);

1701 + rrddim_add(stc, "persons_urls", NULL, 1, 1, RRDDIM_ABSOLUTE);

1702 + rrddim_add(stc, "machines_urls", NULL, 1, 1, RRDDIM_ABSOLUTE);

1703 + }

1704 + else rrdset_next(stc);

1705 +

1706 + rrddim_set(stc, "persons", registry.persons_count);

1707 + rrddim_set(stc, "machines", registry.machines_count);

1708 + rrddim_set(stc, "urls", registry.urls_count);

1709 + rrddim_set(stc, "persons_urls", registry.persons_urls_count);

1710 + rrddim_set(stc, "machines_urls", registry.machines_urls_count);

1711 + rrdset_done(stc);

1712 +

1713 + // ------------------------------------------------------------------------

1714 +

1715 + if(!stm) stm = rrdset_find("netdata.registry_mem");

1716 + if(!stm) {

1717 + stm = rrdset_create("netdata", "registry_mem", NULL, "registry", NULL, "NetData Registry Memory", "KB", 131300, rrd_update_every, RRDSET_TYPE_STACKED);

1718 +

1719 + rrddim_add(stm, "persons", NULL, 1, 1024, RRDDIM_ABSOLUTE);

1720 + rrddim_add(stm, "machines", NULL, 1, 1024, RRDDIM_ABSOLUTE);

1721 + rrddim_add(stm, "urls", NULL, 1, 1024, RRDDIM_ABSOLUTE);

1722 + rrddim_add(stm, "persons_urls", NULL, 1, 1024, RRDDIM_ABSOLUTE);

1723 + rrddim_add(stm, "machines_urls", NULL, 1, 1024, RRDDIM_ABSOLUTE);

1724 + }

1725 + else rrdset_next(stm);

1726 +

1727 + rrddim_set(stm, "persons", registry.persons_memory + registry.persons_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));

1728 + rrddim_set(stm, "machines", registry.machines_memory + registry.machines_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));

1729 + rrddim_set(stm, "urls", registry.urls_memory + registry.urls_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));

1730 + rrddim_set(stm, "persons_urls", registry.persons_urls_memory + registry.persons_count * sizeof(DICTIONARY) + registry.persons_urls_count * sizeof(NAME_VALUE));

1731 + rrddim_set(stm, "machines_urls", registry.machines_urls_memory + registry.machines_count * sizeof(DICTIONARY) + registry.machines_urls_count * sizeof(NAME_VALUE));

1732 + rrdset_done(stm);

1733 +}

1734 +

1735

1736 #ifdef REGISTRY_STANDALONE_TESTS

1737

src/registry.h

+5 -1

@@ -9,8 +9,12 @@ extern int registry_request_access_json(struct web_client *w, char *person_guid,

9 extern int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);

10 extern int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when);

12 -extern void registry_init(void);

12 +extern int registry_init(void);

13 extern void registry_free(void);

14 extern int registry_save(void);

16 +extern char *registry_get_this_machine_guid(void);

17 +

18 +extern void registry_statistics(void);

19 +

20 #endif /* NETDATA_REGISTRY_H */

src/sys_fs_cgroup.c

+1 -1

@@ -1292,7 +1292,7 @@ void *cgroups_main(void *ptr)

1292

1293 if(!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_cgroups_cpu");

1294 if(!stcpu_thread) {

1295 - stcpu_thread = rrdset_create("netdata", "plugin_cgroups_cpu", NULL, "proc.internal", NULL, "NetData CGroups Plugin CPU usage", "milliseconds/s", 131000, rrd_update_every, RRDSET_TYPE_STACKED);

1295 + stcpu_thread = rrdset_create("netdata", "plugin_cgroups_cpu", NULL, "proc.internal", NULL, "NetData CGroups Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, RRDSET_TYPE_STACKED);

1296

1297 rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);

1298 rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);

src/web_client.c

+1 -1

@@ -794,7 +794,7 @@ int web_client_api_request_v1_registry(struct web_client *w, char *url)

794

795 char *cookie = strstr(w->response.data->buffer, " " NETDATA_REGISTRY_COOKIE_NAME "=");

796 if(cookie) {

797 - strncpy(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME) + 2], 36);

797 + strncpy(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME) + 1], 36);

798 person_guid[36] = '\0';

799 }

800

src/web_client.h

+1 -1

@@ -27,7 +27,7 @@ extern int web_enable_gzip;

27 #define URL_MAX 8192

28 #define ZLIB_CHUNK 16384

29 #define HTTP_RESPONSE_HEADER_SIZE 4096

30 -#define COOKIE_MAX 200

30 +#define COOKIE_MAX 1024

32 struct response {

33 BUFFER *header; // our response header