@cryptotaxi247 / netdata-1 / commits / ba97a9f4d

removed over-optimization at the registry to simplify its logic and remove race conditions that could result to crashes; registry is now split into multiple files, to allow easier enhancements

Costa Tsaousis (ktsaou) committed Dec 31, 2016 at 21:08 UTC ba97a9f4d4dd21b74af45e511470bcbbc60ad992

17 files changed +1692 -1680

CMakeLists.txt

+8 -1

@@ -94,7 +94,14 @@ set(NETDATA_SOURCE_FILES

94 src/web_client.h

95 src/web_server.c

96 src/web_server.h

97 - src/socket.c src/socket.h)

97 + src/socket.c src/socket.h

98 + src/registry_url.c

99 + src/registry_url.h

100 + src/registry_internals.h

101 + src/registry_person.c

102 + src/registry_person.h

103 + src/registry_machine.c

104 + src/registry_machine.h src/registry_internals.c src/registry_init.c src/registry_db.c src/registry_log.c)

105

106 set(APPS_PLUGIN_SOURCE_FILES

107 src/appconfig.c

src/Makefile.am

@@ -50,6 +50,13 @@ netdata_SOURCES = \

50 procfile.c procfile.h \

51 proc_self_mountinfo.c proc_self_mountinfo.h \

52 registry.c registry.h \

53 + registry_internals.c registry_internals.h \

54 + registry_url.c registry_url.h \

55 + registry_person.c registry_person.h \

56 + registry_machine.c registry_machine.h \

57 + registry_init.c \

58 + registry_db.c \

59 + registry_log.c \

60 rrd.c rrd.h \

61 rrd2json.c rrd2json.h \

62 storage_number.c storage_number.h \

src/common.h

+6 -4

@@ -170,6 +170,8 @@

170 #endif

171 #define abs(x) ((x < 0)? -x : x)

172

173 +#define GUID_LEN 36

174 +

175 extern void netdata_fix_chart_id(char *s);

176 extern void netdata_fix_chart_name(char *s);

177

@@ -198,10 +200,10 @@ extern void *mallocz_int(const char *file, const char *function, const unsigned

200 extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size);

201 extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr);

202 #else

201 -extern char *strdupz(const char *s);

202 -extern void *callocz(size_t nmemb, size_t size);

203 -extern void *mallocz(size_t size);

204 -extern void *reallocz(void *ptr, size_t size);

203 +extern char *strdupz(const char *s) __attribute__((returns_nonnull));

204 +extern void *callocz(size_t nmemb, size_t size) __attribute__((returns_nonnull));

205 +extern void *mallocz(size_t size) __attribute__((returns_nonnull));

206 +extern void *reallocz(void *ptr, size_t size) __attribute__((returns_nonnull));

207 extern void freez(void *ptr);

208 #endif

209

src/registry.c

+107 -1608

@@ -1,1016 +1,27 @@

1 #include "common.h"

3 -// ----------------------------------------------------------------------------

4 -// TODO

5 -//

6 -// 1. the default tracking cookie expires in 1 year, but the persons are not

7 -// removed from the db - this means the database only grows - ideally the

8 -// database should be cleaned in registry_save() for both on-disk and

9 -// on-memory entries.

10 -//

11 -// Cleanup:

12 -// i. Find all the PERSONs that have expired cookie

13 -// ii. For each of their PERSON_URLs:

14 -// - decrement the linked MACHINE links

15 -// - if the linked MACHINE has no other links, remove the linked MACHINE too

16 -// - remove the PERSON_URL

17 -//

18 -// 2. add protection to prevent abusing the registry by flooding it with

19 -// requests to fill the memory and crash it.

20 -//

21 -// Possible protections:

22 -// - limit the number of URLs per person

23 -// - limit the number of URLs per machine

24 -// - limit the number of persons

25 -// - limit the number of machines

26 -// - [DONE] limit the size of URLs

27 -// - [DONE] limit the size of PERSON_URL names

28 -// - limit the number of requests that add data to the registry,

29 -// per client IP per hour

30 -//

31 -// 3. lower memory requirements

32 -//

33 -// - embed avl structures directly into registry objects, instead of DICTIONARY

34 -// - store GUIDs in memory as UUID instead of char *

35 -// - do not track persons using the demo machines only

36 -// (i.e. start tracking them only when they access a non-demo machine)

37 -// - [DONE] do not track custom dashboards by default

38 -

39 -#define REGISTRY_URL_FLAGS_DEFAULT 0x00

40 -#define REGISTRY_URL_FLAGS_EXPIRED 0x01

41 -

42 -#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE

43 -

44 -#define GUID_LEN 36

45 -

46 -// ----------------------------------------------------------------------------

47 -// COMMON structures

48 -

49 -struct registry {

50 - int enabled;

51 -

52 - char machine_guid[GUID_LEN + 1];

53 -

54 - // entries counters / statistics

55 - unsigned long long persons_count;

56 - unsigned long long machines_count;

57 - unsigned long long usages_count;

58 - unsigned long long urls_count;

59 - unsigned long long persons_urls_count;

60 - unsigned long long machines_urls_count;

61 - unsigned long long log_count;

62 -

63 - // memory counters / statistics

64 - unsigned long long persons_memory;

65 - unsigned long long machines_memory;

66 - unsigned long long urls_memory;

67 - unsigned long long persons_urls_memory;

68 - unsigned long long machines_urls_memory;

69 -

70 - // configuration

71 - unsigned long long save_registry_every_entries;

72 - char *registry_domain;

73 - char *hostname;

74 - char *registry_to_announce;

75 - time_t persons_expiration; // seconds to expire idle persons

76 - int verify_cookies_redirects;

77 -

78 - size_t max_url_length;

79 - size_t max_name_length;

80 -

81 - // file/path names

82 - char *pathname;

83 - char *db_filename;

84 - char *log_filename;

85 - char *machine_guid_filename;

86 -

87 - // open files

88 - FILE *log_fp;

89 -

90 - // the database

91 - DICTIONARY *persons; // dictionary of PERSON *, with key the PERSON.guid

92 - DICTIONARY *machines; // dictionary of MACHINE *, with key the MACHINE.guid

93 - DICTIONARY *urls; // dictionary of URL *, with key the URL.url

94 -

95 - // concurrency locking

96 - // we keep different locks for different things

97 - // so that many tasks can be completed in parallel

98 - pthread_mutex_t persons_lock;

99 - pthread_mutex_t machines_lock;

100 - pthread_mutex_t urls_lock;

101 - pthread_mutex_t person_urls_lock;

102 - pthread_mutex_t machine_urls_lock;

103 - pthread_mutex_t log_lock;

104 -} registry;

105 -

106 -

107 -// ----------------------------------------------------------------------------

108 -// URL structures

109 -// Save memory by de-duplicating URLs

110 -// so instead of storing URLs all over the place

111 -// we store them here and we keep pointers elsewhere

112 -

113 -struct registry_url {

114 - uint32_t links; // the number of links to this URL - when none is left, we free it

115 - uint16_t len; // the length of the URL in bytes

116 - char url[1]; // the URL - dynamically allocated to more size

117 -};

118 -typedef struct registry_url REGISTRY_URL;

119 -

120 -

121 -// ----------------------------------------------------------------------------

122 -// MACHINE structures

123 -

124 -// For each MACHINE-URL pair we keep this

125 -struct registry_machine_url {

126 - REGISTRY_URL *url; // de-duplicated URL

127 -

128 - uint8_t flags;

129 -

130 - uint32_t first_t; // the first time we saw this

131 - uint32_t last_t; // the last time we saw this

132 - uint32_t usages; // how many times this has been accessed

133 -};

134 -typedef struct registry_machine_url REGISTRY_MACHINE_URL;

135 -

136 -// A machine

137 -struct registry_machine {

138 - char guid[GUID_LEN + 1]; // the GUID

139 -

140 - uint32_t links; // the number of REGISTRY_PERSON_URL linked to this machine

141 -

142 - DICTIONARY *machine_urls; // MACHINE_URL *

143 -

144 - uint32_t first_t; // the first time we saw this

145 - uint32_t last_t; // the last time we saw this

146 - uint32_t usages; // how many times this has been accessed

147 -};

148 -typedef struct registry_machine REGISTRY_MACHINE;

149 -

150 -

151 -// ----------------------------------------------------------------------------

152 -// PERSON structures

153 -

154 -// for each PERSON-URL pair we keep this

155 -struct registry_person_url {

156 - REGISTRY_URL *url; // de-duplicated URL

157 - REGISTRY_MACHINE *machine; // link the MACHINE of this URL

158 -

159 - uint8_t flags;

160 -

161 - uint32_t first_t; // the first time we saw this

162 - uint32_t last_t; // the last time we saw this

163 - uint32_t usages; // how many times this has been accessed

164 -

165 - char machine_name[1]; // the name of the machine, as known by the user

166 - // dynamically allocated to fit properly

167 -};

168 -typedef struct registry_person_url REGISTRY_PERSON_URL;

169 -

170 -// A person

171 -struct registry_person {

172 - char guid[GUID_LEN + 1]; // the person GUID

173 -

174 - DICTIONARY *person_urls; // dictionary of PERSON_URL *

175 -

176 - uint32_t first_t; // the first time we saw this

177 - uint32_t last_t; // the last time we saw this

178 - uint32_t usages; // how many times this has been accessed

179 -};

180 -typedef struct registry_person REGISTRY_PERSON;

3 +#include "registry_internals.h"

5 +#define REGISTRY_STATUS_OK "ok"

6 +#define REGISTRY_STATUS_FAILED "failed"

7 +#define REGISTRY_STATUS_DISABLED "disabled"

9 // ----------------------------------------------------------------------------

10 // REGISTRY concurrency locking

186 -static inline void registry_persons_lock(void) {

187 - pthread_mutex_lock(&registry.persons_lock);

188 -}

189 -

190 -static inline void registry_persons_unlock(void) {

191 - pthread_mutex_unlock(&registry.persons_lock);

192 -}

193 -

194 -static inline void registry_machines_lock(void) {

195 - pthread_mutex_lock(&registry.machines_lock);

196 -}

197 -

198 -static inline void registry_machines_unlock(void) {

199 - pthread_mutex_unlock(&registry.machines_lock);

200 -}

201 -

202 -static inline void registry_urls_lock(void) {

203 - pthread_mutex_lock(&registry.urls_lock);

204 -}

205 -

206 -static inline void registry_urls_unlock(void) {

207 - pthread_mutex_unlock(&registry.urls_lock);

208 -}

209 -

210 -// ideally, we should not lock the whole registry for

211 -// updating a person's urls.

212 -// however, to save the memory required for keeping a

213 -// mutex (40 bytes) per person, we do...

214 -static inline void registry_person_urls_lock(REGISTRY_PERSON *p) {

215 - (void)p;

216 - pthread_mutex_lock(&registry.person_urls_lock);

217 -}

218 -

219 -static inline void registry_person_urls_unlock(REGISTRY_PERSON *p) {

220 - (void)p;

221 - pthread_mutex_unlock(&registry.person_urls_lock);

12 +static inline void registry_lock(void) {

13 + pthread_mutex_lock(&registry.lock);

14 }

224 -// ideally, we should not lock the whole registry for

225 -// updating a machine's urls.

226 -// however, to save the memory required for keeping a

227 -// mutex (40 bytes) per machine, we do...

228 -static inline void registry_machine_urls_lock(REGISTRY_MACHINE *m) {

229 - (void)m;

230 - pthread_mutex_lock(&registry.machine_urls_lock);

231 -}

232 -

233 -static inline void registry_machine_urls_unlock(REGISTRY_MACHINE *m) {

234 - (void)m;

235 - pthread_mutex_unlock(&registry.machine_urls_lock);

236 -}

237 -

238 -static inline void registry_log_lock(void) {

239 - pthread_mutex_lock(&registry.log_lock);

240 -}

241 -

242 -static inline void registry_log_unlock(void) {

243 - pthread_mutex_unlock(&registry.log_lock);

16 +static inline void registry_unlock(void) {

17 + pthread_mutex_unlock(&registry.lock);

18 }

21 // ----------------------------------------------------------------------------

248 -// common functions

249 -

250 -// parse a GUID and re-generated to be always lower case

251 -// this is used as a protection against the variations of GUIDs

252 -static inline int registry_regenerate_guid(const char *guid, char *result) {

253 - uuid_t uuid;

254 - if(unlikely(uuid_parse(guid, uuid) == -1)) {

255 - info("Registry: GUID '%s' is not a valid GUID.", guid);

256 - return -1;

257 - }

258 - else {

259 - uuid_unparse_lower(uuid, result);

260 -

261 -#ifdef NETDATA_INTERNAL_CHECKS

262 - if(strcmp(guid, result))

263 - info("Registry: source GUID '%s' and re-generated GUID '%s' differ!", guid, result);

264 -#endif /* NETDATA_INTERNAL_CHECKS */

265 - }

266 -

267 - return 0;

268 -}

269 -

270 -// make sure the names of the machines / URLs do not contain any tabs

271 -// (which are used as our separator in the database files)

272 -// and are properly trimmed (before and after)

273 -static inline char *registry_fix_machine_name(char *name, size_t *len) {

274 - char *s = name?name:"";

275 -

276 - // skip leading spaces

277 - while(*s && isspace(*s)) s++;

278 -

279 - // make sure all spaces are a SPACE

280 - char *t = s;

281 - while(*t) {

282 - if(unlikely(isspace(*t)))

283 - *t = ' ';

284 -

285 - t++;

286 - }

287 -

288 - // remove trailing spaces

289 - while(--t >= s) {

290 - if(*t == ' ')

291 - *t = '\0';

292 - else

293 - break;

294 - }

295 - t++;

296 -

297 - if(likely(len))

298 - *len = (t - s);

299 -

300 - return s;

301 -}

302 -

303 -static inline char *registry_fix_url(char *url, size_t *len) {

304 - return registry_fix_machine_name(url, len);

305 -}

306 -

307 -

308 -// ----------------------------------------------------------------------------

309 -// forward definition of functions

310 -

311 -extern REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when);

312 -extern REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);

313 -

314 -

315 -// ----------------------------------------------------------------------------

316 -// URL

317 -

318 -static inline REGISTRY_URL *registry_url_allocate_nolock(const char *url, size_t urllen) {

319 - // protection from too big URLs

320 - if(urllen > registry.max_url_length)

321 - urllen = registry.max_url_length;

322 -

323 - debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): allocating %zu bytes", url, sizeof(REGISTRY_URL) + urllen);

324 - REGISTRY_URL *u = mallocz(sizeof(REGISTRY_URL) + urllen);

325 -

326 - // a simple strcpy() should do the job

327 - // but I prefer to be safe, since the caller specified urllen

328 - u->len = (uint16_t)urllen;

329 - strncpyz(u->url, url, u->len);

330 - u->links = 0;

331 -

332 - registry.urls_memory += sizeof(REGISTRY_URL) + urllen;

333 -

334 - debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): indexing it", url);

335 - dictionary_set(registry.urls, u->url, u, sizeof(REGISTRY_URL));

336 -

337 - return u;

338 -}

339 -

340 -static inline REGISTRY_URL *registry_url_get_nolock(const char *url, size_t urllen) {

341 - debug(D_REGISTRY, "Registry: registry_url_get_nolock('%s')", url);

342 -

343 - REGISTRY_URL *u = dictionary_get(registry.urls, url);

344 - if(!u) {

345 - u = registry_url_allocate_nolock(url, urllen);

346 - registry.urls_count++;

347 - }

348 -

349 - return u;

350 -}

351 -

352 -static inline REGISTRY_URL *registry_url_get(const char *url, size_t urllen) {

353 - debug(D_REGISTRY, "Registry: registry_url_get('%s')", url);

354 -

355 - registry_urls_lock();

356 -

357 - REGISTRY_URL *u = registry_url_get_nolock(url, urllen);

358 -

359 - registry_urls_unlock();

360 -

361 - return u;

362 -}

363 -

364 -static inline void registry_url_link_nolock(REGISTRY_URL *u) {

365 - u->links++;

366 - debug(D_REGISTRY, "Registry: registry_url_link_nolock('%s'): URL has now %u links", u->url, u->links);

367 -}

368 -

369 -static inline void registry_url_unlink_nolock(REGISTRY_URL *u) {

370 - u->links--;

371 - if(!u->links) {

372 - debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): No more links for this URL", u->url);

373 - dictionary_del(registry.urls, u->url);

374 - freez(u);

375 - }

376 - else

377 - debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): URL has %u links left", u->url, u->links);

378 -}

379 -

380 -

381 -// ----------------------------------------------------------------------------

382 -// MACHINE

383 -

384 -static inline REGISTRY_MACHINE *registry_machine_find(const char *machine_guid) {

385 - debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);

386 - return dictionary_get(registry.machines, machine_guid);

387 -}

388 -

389 -static inline REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) {

390 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(REGISTRY_MACHINE_URL));

391 -

392 - REGISTRY_MACHINE_URL *mu = mallocz(sizeof(REGISTRY_MACHINE_URL));

393 -

394 - // mu->persons = dictionary_create(DICTIONARY_FLAGS);

395 - // dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));

396 -

397 - mu->first_t = mu->last_t = (uint32_t)when;

398 - mu->usages = 1;

399 - mu->url = u;

400 - mu->flags = REGISTRY_URL_FLAGS_DEFAULT;

401 -

402 - registry.machines_urls_memory += sizeof(REGISTRY_MACHINE_URL);

403 -

404 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url);

405 - dictionary_set(m->machine_urls, u->url, mu, sizeof(REGISTRY_MACHINE_URL));

406 - registry_url_link_nolock(u);

407 -

408 - return mu;

409 -}

410 -

411 -static inline REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) {

412 - debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(REGISTRY_MACHINE));

413 -

414 - REGISTRY_MACHINE *m = mallocz(sizeof(REGISTRY_MACHINE));

415 -

416 - strncpyz(m->guid, machine_guid, GUID_LEN);

417 -

418 - debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);

419 - m->machine_urls = dictionary_create(DICTIONARY_FLAGS);

420 -

421 - m->first_t = m->last_t = (uint32_t)when;

422 - m->usages = 0;

423 -

424 - registry.machines_memory += sizeof(REGISTRY_MACHINE);

425 -

426 - registry.machines_count++;

427 - dictionary_set(registry.machines, m->guid, m, sizeof(REGISTRY_MACHINE));

428 -

429 - return m;

430 -}

431 -

432 -// 1. validate machine GUID

433 -// 2. if it is valid, find it or create it and return it

434 -// 3. if it is not valid, return NULL

435 -static inline REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when) {

436 - REGISTRY_MACHINE *m = NULL;

437 -

438 - registry_machines_lock();

439 -

440 - if(likely(machine_guid && *machine_guid)) {

441 - // validate it is a GUID

442 - char buf[GUID_LEN + 1];

443 - if(unlikely(registry_regenerate_guid(machine_guid, buf) == -1))

444 - info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);

445 - else {

446 - machine_guid = buf;

447 - m = registry_machine_find(machine_guid);

448 - if(!m) m = registry_machine_allocate(machine_guid, when);

449 - }

450 - }

451 -

452 - registry_machines_unlock();

453 -

454 - return m;

455 -}

456 -

457 -

458 -// ----------------------------------------------------------------------------

459 -// PERSON

460 -

461 -static inline REGISTRY_PERSON *registry_person_find(const char *person_guid) {

462 - debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid);

463 - return dictionary_get(registry.persons, person_guid);

464 -}

465 -

466 -static inline REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) {

467 - // protection from too big names

468 - if(namelen > registry.max_name_length)

469 - namelen = registry.max_name_length;

470 -

471 - debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,

472 - sizeof(REGISTRY_PERSON_URL) + namelen);

473 -

474 - REGISTRY_PERSON_URL *pu = mallocz(sizeof(REGISTRY_PERSON_URL) + namelen);

475 -

476 - // a simple strcpy() should do the job

477 - // but I prefer to be safe, since the caller specified urllen

478 - strncpyz(pu->machine_name, name, namelen);

479 -

480 - pu->machine = m;

481 - pu->first_t = pu->last_t = when;

482 - pu->usages = 1;

483 - pu->url = u;

484 - pu->flags = REGISTRY_URL_FLAGS_DEFAULT;

485 - m->links++;

486 -

487 - registry.persons_urls_memory += sizeof(REGISTRY_PERSON_URL) + namelen;

488 -

489 - debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);

490 - dictionary_set(p->person_urls, u->url, pu, sizeof(REGISTRY_PERSON_URL));

491 - registry_url_link_nolock(u);

492 -

493 - return pu;

494 -}

495 -

496 -static inline REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when, REGISTRY_PERSON_URL *pu) {

497 - // this function is needed to change the name of a PERSON_URL

498 -

499 - debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,

500 - sizeof(REGISTRY_PERSON_URL) + namelen);

501 -

502 - REGISTRY_PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when);

503 - tpu->first_t = pu->first_t;

504 - tpu->last_t = pu->last_t;

505 - tpu->usages = pu->usages;

506 -

507 - // ok, these are a hack - since the registry_person_url_allocate() is

508 - // adding these, we have to subtract them

509 - tpu->machine->links--;

510 - registry.persons_urls_memory -= sizeof(REGISTRY_PERSON_URL) + strlen(pu->machine_name);

511 - registry_url_unlink_nolock(u);

512 -

513 - freez(pu);

514 -

515 - return tpu;

516 -}

517 -

518 -static inline REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when) {

519 - REGISTRY_PERSON *p = NULL;

520 -

521 - debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(REGISTRY_PERSON));

522 -

523 - p = mallocz(sizeof(REGISTRY_PERSON));

524 -

525 - if(!person_guid) {

526 - for (; ;) {

527 - uuid_t uuid;

528 - uuid_generate(uuid);

529 - uuid_unparse_lower(uuid, p->guid);

530 -

531 - debug(D_REGISTRY, "Registry: Checking if the generated person guid '%s' is unique", p->guid);

532 - if (!dictionary_get(registry.persons, p->guid)) {

533 - debug(D_REGISTRY, "Registry: generated person guid '%s' is unique", p->guid);

534 - break;

535 - }

536 - else

537 - info("Registry: generated person guid '%s' found in the registry. Retrying...", p->guid);

538 - }

539 - }

540 - else

541 - strncpyz(p->guid, person_guid, GUID_LEN);

542 -

543 - debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid);

544 - p->person_urls = dictionary_create(DICTIONARY_FLAGS);

545 -

546 - p->first_t = p->last_t = when;

547 - p->usages = 0;

548 -

549 - registry.persons_memory += sizeof(REGISTRY_PERSON);

550 -

551 - registry.persons_count++;

552 - dictionary_set(registry.persons, p->guid, p, sizeof(REGISTRY_PERSON));

553 -

554 - return p;

555 -}

556 -

557 -

558 -// 1. validate person GUID

559 -// 2. if it is valid, find it

560 -// 3. if it is not valid, create a new one

561 -// 4. return it

562 -static inline REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when) {

563 - REGISTRY_PERSON *p = NULL;

564 -

565 - registry_persons_lock();

566 -

567 - if(person_guid && *person_guid) {

568 - char buf[GUID_LEN + 1];

569 - // validate it is a GUID

570 - if(unlikely(registry_regenerate_guid(person_guid, buf) == -1))

571 - info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid);

572 - else {

573 - person_guid = buf;

574 - p = registry_person_find(person_guid);

575 - }

576 - }

577 -

578 - if(!p) p = registry_person_allocate(NULL, when);

579 -

580 - registry_persons_unlock();

581 -

582 - return p;

583 -}

584 -

585 -// ----------------------------------------------------------------------------

586 -// LINKING OF OBJECTS

587 -

588 -static inline REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) {

589 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);

590 -

591 - registry_person_urls_lock(p);

592 -

593 - REGISTRY_PERSON_URL *pu = dictionary_get(p->person_urls, u->url);

594 - if(!pu) {

595 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);

596 - pu = registry_person_url_allocate(p, m, u, name, namelen, when);

597 - registry.persons_urls_count++;

598 - }

599 - else {

600 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);

601 - pu->usages++;

602 - if(likely(pu->last_t < (uint32_t)when)) pu->last_t = when;

603 -

604 - if(pu->machine != m) {

605 - REGISTRY_MACHINE_URL *mu = dictionary_get(pu->machine->machine_urls, u->url);

606 - if(mu) {

607 - info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.",

608 - p->guid, m->guid, u->url, pu->machine->guid);

609 - mu->flags |= REGISTRY_URL_FLAGS_EXPIRED;

610 - }

611 - else {

612 - info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - but the URL is not linked to the old machine.",

613 - p->guid, m->guid, u->url, pu->machine->guid);

614 - }

615 -

616 - pu->machine->links--;

617 - pu->machine = m;

618 - }

619 -

620 - if(strcmp(pu->machine_name, name)) {

621 - // the name of the PERSON_URL has changed !

622 - pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu);

623 - }

624 - }

625 -

626 - p->usages++;

627 - if(likely(p->last_t < (uint32_t)when)) p->last_t = when;

628 -

629 - if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) {

630 - info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url);

631 - pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;

632 - }

633 -

634 - registry_person_urls_unlock(p);

635 -

636 - return pu;

637 -}

638 -

639 -static inline REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) {

640 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url);

641 -

642 - registry_machine_urls_lock(m);

643 -

644 - REGISTRY_MACHINE_URL *mu = dictionary_get(m->machine_urls, u->url);

645 - if(!mu) {

646 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);

647 - mu = registry_machine_url_allocate(m, u, when);

648 - registry.machines_urls_count++;

649 - }

650 - else {

651 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);

652 - mu->usages++;

653 - if(likely(mu->last_t < (uint32_t)when)) mu->last_t = when;

654 - }

655 -

656 - //debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing person in machine", p->guid, m->guid, u->url);

657 - //dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));

658 -

659 - m->usages++;

660 - if(likely(m->last_t < (uint32_t)when)) m->last_t = when;

661 -

662 - if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) {

663 - info("registry_machine_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);

664 - mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;

665 - }

666 -

667 - registry_machine_urls_unlock(m);

668 -

669 - return mu;

670 -}

671 -

672 -// ----------------------------------------------------------------------------

673 -// REGISTRY LOG LOAD/SAVE

674 -

675 -static inline int registry_should_save_db(void) {

676 - debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries);

677 - return registry.log_count > registry.save_registry_every_entries;

678 -}

679 -

680 -static inline void registry_log(const char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name) {

681 - if(likely(registry.log_fp)) {

682 - // we lock only if the file is open

683 - // to allow replaying the log at registry_log_load()

684 - registry_log_lock();

685 -

686 - if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",

687 - action,

688 - p->last_t,

689 - p->guid,

690 - m->guid,

691 - name,

692 - u->url) < 0))

693 - error("Registry: failed to save log. Registry data may be lost in case of abnormal restart.");

694 -

695 - // we increase the counter even on failures

696 - // so that the registry will be saved periodically

697 - registry.log_count++;

698 -

699 - registry_log_unlock();

700 -

701 - // this must be outside the log_lock(), or a deadlock will happen.

702 - // registry_save() checks the same inside the log_lock, so only

703 - // one thread will save the db

704 - if(unlikely(registry_should_save_db()))

705 - registry_save();

706 - }

707 -}

708 -

709 -static inline int registry_log_open_nolock(void) {

710 - if(registry.log_fp)

711 - fclose(registry.log_fp);

712 -

713 - registry.log_fp = fopen(registry.log_filename, "a");

714 -

715 - if(registry.log_fp) {

716 - if (setvbuf(registry.log_fp, NULL, _IOLBF, 0) != 0)

717 - error("Cannot set line buffering on registry log file.");

718 - return 0;

719 - }

720 -

721 - error("Cannot open registry log file '%s'. Registry data will be lost in case of netdata or server crash.", registry.log_filename);

722 - return -1;

723 -}

724 -

725 -static inline void registry_log_close_nolock(void) {

726 - if(registry.log_fp) {

727 - fclose(registry.log_fp);

728 - registry.log_fp = NULL;

729 - }

730 -}

731 -

732 -static inline void registry_log_recreate_nolock(void) {

733 - if(registry.log_fp != NULL) {

734 - registry_log_close_nolock();

735 -

736 - // open it with truncate

737 - registry.log_fp = fopen(registry.log_filename, "w");

738 - if(registry.log_fp) fclose(registry.log_fp);

739 - else error("Cannot truncate registry log '%s'", registry.log_filename);

740 -

741 - registry.log_fp = NULL;

742 -

743 - registry_log_open_nolock();

744 - }

745 -}

746 -

747 -int registry_log_load(void) {

748 - ssize_t line = -1;

749 -

750 - // closing the log is required here

751 - // otherwise we will append to it the values we read

752 - registry_log_close_nolock();

753 -

754 - debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename);

755 - FILE *fp = fopen(registry.log_filename, "r");

756 - if(!fp)

757 - error("Registry: cannot open registry file: %s", registry.log_filename);

758 - else {

759 - char *s, buf[4096 + 1];

760 - line = 0;

761 - size_t len = 0;

762 -

763 - while ((s = fgets_trim_len(buf, 4096, fp, &len))) {

764 - line++;

765 -

766 - switch (s[0]) {

767 - case 'A': // accesses

768 - case 'D': // deletes

769 -

770 - // verify it is valid

771 - if (unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) {

772 - error("Registry: log line %zd is wrong (len = %zu).", line, len);

773 - continue;

774 - }

775 - s[1] = s[10] = s[47] = s[84] = '\0';

776 -

777 - // get the variables

778 - time_t when = strtoul(&s[2], NULL, 16);

779 - char *person_guid = &s[11];

780 - char *machine_guid = &s[48];

781 - char *name = &s[85];

782 -

783 - // skip the name to find the url

784 - char *url = name;

785 - while(*url && *url != '\t') url++;

786 - if(!*url) {

787 - error("Registry: log line %zd does not have a url.", line);

788 - continue;

789 - }

790 - *url++ = '\0';

791 -

792 - // make sure the person exists

793 - // without this, a new person guid will be created

794 - REGISTRY_PERSON *p = registry_person_find(person_guid);

795 - if(!p) p = registry_person_allocate(person_guid, when);

796 -

797 - if(s[0] == 'A')

798 - registry_request_access(p->guid, machine_guid, url, name, when);

799 - else

800 - registry_request_delete(p->guid, machine_guid, url, name, when);

801 -

802 - registry.log_count++;

803 - break;

804 -

805 - default:

806 - error("Registry: ignoring line %zd of filename '%s': %s.", line, registry.log_filename, s);

807 - break;

808 - }

809 - }

810 -

811 - fclose(fp);

812 - }

813 -

814 - // open the log again

815 - registry_log_open_nolock();

816 -

817 - return line;

818 -}

819 -

820 -

821 -// ----------------------------------------------------------------------------

822 -// REGISTRY REQUESTS

823 -

824 -REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) {

825 - debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);

826 -

827 - REGISTRY_MACHINE *m = registry_machine_get(machine_guid, when);

828 - if(!m) return NULL;

829 -

830 - // make sure the name is valid

831 - size_t namelen;

832 - name = registry_fix_machine_name(name, &namelen);

833 -

834 - size_t urllen;

835 - url = registry_fix_url(url, &urllen);

836 -

837 - REGISTRY_URL *u = registry_url_get(url, urllen);

838 - REGISTRY_PERSON *p = registry_person_get(person_guid, when);

839 -

840 - registry_person_link_to_url(p, m, u, name, namelen, when);

841 - registry_machine_link_to_url(p, m, u, when);

842 -

843 - registry_log('A', p, m, u, name);

844 -

845 - registry.usages_count++;

846 - return p;

847 -}

848 -

849 -// verify the person, the machine and the URL exist in our DB

850 -REGISTRY_PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, REGISTRY_PERSON **pp, REGISTRY_MACHINE **mm) {

851 - char pbuf[GUID_LEN + 1], mbuf[GUID_LEN + 1];

852 -

853 - if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) {

854 - info("Registry Request Verification: invalid request! person: '%s', machine '%s', url '%s'", person_guid?person_guid:"UNSET", machine_guid?machine_guid:"UNSET", url?url:"UNSET");

855 - return NULL;

856 - }

857 -

858 - // normalize the url

859 - url = registry_fix_url(url, NULL);

860 -

861 - // make sure the person GUID is valid

862 - if(registry_regenerate_guid(person_guid, pbuf) == -1) {

863 - info("Registry Request Verification: invalid person GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

864 - return NULL;

865 - }

866 - person_guid = pbuf;

867 -

868 - // make sure the machine GUID is valid

869 - if(registry_regenerate_guid(machine_guid, mbuf) == -1) {

870 - info("Registry Request Verification: invalid machine GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

871 - return NULL;

872 - }

873 - machine_guid = mbuf;

874 -

875 - // make sure the machine exists

876 - REGISTRY_MACHINE *m = registry_machine_find(machine_guid);

877 - if(!m) {

878 - info("Registry Request Verification: machine not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

879 - return NULL;

880 - }

881 - if(mm) *mm = m;

882 -

883 - // make sure the person exist

884 - REGISTRY_PERSON *p = registry_person_find(person_guid);

885 - if(!p) {

886 - info("Registry Request Verification: person not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

887 - return NULL;

888 - }

889 - if(pp) *pp = p;

890 -

891 - REGISTRY_PERSON_URL *pu = dictionary_get(p->person_urls, url);

892 - if(!pu) {

893 - info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

894 - return NULL;

895 - }

896 - return pu;

897 -}

898 -

899 -REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {

900 - (void)when;

22 +// COOKIES

902 - REGISTRY_PERSON *p = NULL;

903 - REGISTRY_MACHINE *m = NULL;

904 - REGISTRY_PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);

905 - if(!pu || !p || !m) return NULL;

906 -

907 - // normalize the url

908 - delete_url = registry_fix_url(delete_url, NULL);

909 -

910 - // make sure the user is not deleting the url it uses

911 - if(!strcmp(delete_url, pu->url->url)) {

912 - info("Registry Delete Request: delete URL is the one currently accessed, person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);

913 - return NULL;

914 - }

915 -

916 - registry_person_urls_lock(p);

917 -

918 - REGISTRY_PERSON_URL *dpu = dictionary_get(p->person_urls, delete_url);

919 - if(!dpu) {

920 - info("Registry Delete Request: URL not found for person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);

921 - registry_person_urls_unlock(p);

922 - return NULL;

923 - }

924 -

925 - registry_log('D', p, m, pu->url, dpu->url->url);

926 -

927 - dictionary_del(p->person_urls, dpu->url->url);

928 - registry_url_unlink_nolock(dpu->url);

929 - freez(dpu);

930 -

931 - registry_person_urls_unlock(p);

932 - return p;

933 -}

934 -

935 -

936 -// a structure to pass to the dictionary_get_all() callback handler

937 -struct machine_request_callback_data {

938 - REGISTRY_MACHINE *find_this_machine;

939 - REGISTRY_PERSON_URL *result;

940 -};

941 -

942 -// the callback function

943 -// this will be run for every PERSON_URL of this PERSON

944 -int machine_request_callback(void *entry, void *data) {

945 - REGISTRY_PERSON_URL *mypu = (REGISTRY_PERSON_URL *)entry;

946 - struct machine_request_callback_data *myrdata = (struct machine_request_callback_data *)data;

947 -

948 - if(mypu->machine == myrdata->find_this_machine) {

949 - myrdata->result = mypu;

950 - return -1; // this will also stop the walk through

951 - }

952 -

953 - return 0; // continue

954 -}

955 -

956 -REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {

957 - (void)when;

958 -

959 - char mbuf[GUID_LEN + 1];

960 -

961 - REGISTRY_PERSON *p = NULL;

962 - REGISTRY_MACHINE *m = NULL;

963 - REGISTRY_PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);

964 - if(!pu || !p || !m) return NULL;

965 -

966 - // make sure the machine GUID is valid

967 - if(registry_regenerate_guid(request_machine, mbuf) == -1) {

968 - info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);

969 - return NULL;

970 - }

971 - request_machine = mbuf;

972 -

973 - // make sure the machine exists

974 - m = registry_machine_find(request_machine);

975 - if(!m) {

976 - info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, machine_guid, pu->url->url, request_machine);

977 - return NULL;

978 - }

979 -

980 - // Verify the user has in the past accessed this machine

981 - // We will walk through the PERSON_URLs to find the machine

982 - // linking to our machine

983 -

984 - // a structure to pass to the dictionary_get_all() callback handler

985 - struct machine_request_callback_data rdata = { m, NULL };

986 -

987 - // request a walk through on the dictionary

988 - // no need for locking here, the underlying dictionary has its own

989 - dictionary_get_all(p->person_urls, machine_request_callback, &rdata);

990 -

991 - if(rdata.result)

992 - return m;

993 -

994 - return NULL;

995 -}

996 -

997 -

998 -// ----------------------------------------------------------------------------

999 -// REGISTRY JSON generation

1000 -

1001 -#define REGISTRY_STATUS_OK "ok"

1002 -#define REGISTRY_STATUS_FAILED "failed"

1003 -#define REGISTRY_STATUS_DISABLED "disabled"

1004 -

1005 -int registry_verify_cookies_redirects(void) {

1006 - return registry.verify_cookies_redirects;

1007 -}

1008 -

1009 -const char *registry_to_announce(void) {

1010 - return registry.registry_to_announce;

1011 -}

1012 -

1013 -void registry_set_cookie(struct web_client *w, const char *guid) {

24 +static void registry_set_cookie(struct web_client *w, const char *guid) {

25 char edate[100];

26 time_t et = now_realtime_sec() + registry.persons_expiration;

27 struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);

@@ -1026,37 +37,35 @@ static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PER

37 registry_set_cookie(w, p->guid);

38 }

40 +

41 +// ----------------------------------------------------------------------------

42 +// JSON GENERATION

43 +

44 static inline void registry_json_header(struct web_client *w, const char *action, const char *status) {

45 buffer_flush(w->response.data);

46 w->response.data->contenttype = CT_APPLICATION_JSON;

47 buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"",

1033 - action, status, registry.hostname, registry.machine_guid);

48 + action, status, registry.hostname, registry.machine_guid);

49 }

51 static inline void registry_json_footer(struct web_client *w) {

52 buffer_strcat(w->response.data, "\n}\n");

53 }

1040 -int registry_request_hello_json(struct web_client *w) {

1041 - registry_json_header(w, "hello", REGISTRY_STATUS_OK);

1042 -

1043 - buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",

1044 - registry.registry_to_announce);

1045 -

1046 - registry_json_footer(w);

1047 - return 200;

1048 -}

1049 -

55 static inline int registry_json_disabled(struct web_client *w, const char *action) {

56 registry_json_header(w, action, REGISTRY_STATUS_DISABLED);

58 buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",

1054 - registry.registry_to_announce);

59 + registry.registry_to_announce);

61 registry_json_footer(w);

62 return 200;

63 }

65 +

66 +// ----------------------------------------------------------------------------

67 +// CALLBACKS FOR WALKING THROUGH REGISTRY OBJECTS

68 +

69 // structure used be the callbacks below

70 struct registry_json_walk_person_urls_callback {

71 REGISTRY_PERSON *p;

@@ -1075,7 +84,7 @@ static inline int registry_json_person_url_callback(void *entry, void *data) {

84 buffer_strcat(w->response.data, ",");

86 buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u, \"%s\" ]",

1078 - pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->machine_name);

87 + pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->machine_name);

89 return 1;

90 }

@@ -1091,21 +100,76 @@ static inline int registry_json_machine_url_callback(void *entry, void *data) {

100 buffer_strcat(w->response.data, ",");

101

102 buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u ]",

1094 - m->guid, mu->url->url, mu->last_t, mu->usages);

103 + m->guid, mu->url->url, mu->last_t, mu->usages);

104

105 return 1;

106 }

107

108 +// ----------------------------------------------------------------------------

109 +

110 +// structure used be the callbacks below

111 +struct registry_person_url_callback_verify_machine_exists_data {

112 + REGISTRY_MACHINE *m;

113 + int count;

114 +};

115 +

116 +int registry_person_url_callback_verify_machine_exists(void *entry, void *data) {

117 + struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data;

118 + REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry;

119 + REGISTRY_MACHINE *m = d->m;

120 +

121 + if(pu->machine == m)

122 + d->count++;

123 +

124 + return 0;

125 +}

126 +

127 +// ----------------------------------------------------------------------------

128 +// public HELLO request

129 +

130 +int registry_request_hello_json(struct web_client *w) {

131 + registry_json_header(w, "hello", REGISTRY_STATUS_OK);

132 +

133 + buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",

134 + registry.registry_to_announce);

135 +

136 + registry_json_footer(w);

137 + return 200;

138 +}

139 +

140 +// ----------------------------------------------------------------------------

141 +//public ACCESS request

142 +

143 +#define REGISTRY_VERIFY_COOKIES_GUID "give-me-back-this-cookie-now--please"

144

145 // the main method for registering an access

146 int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) {

1102 - if(!registry.enabled)

147 + if(unlikely(!registry.enabled))

148 return registry_json_disabled(w, "access");

149

150 + // ------------------------------------------------------------------------

151 + // verify the browser supports cookies

152 +

153 + if(registry.verify_cookies_redirects > 0 && !person_guid[0]) {

154 + buffer_flush(w->response.data);

155 + registry_set_cookie(w, REGISTRY_VERIFY_COOKIES_GUID);

156 + w->response.data->contenttype = CT_APPLICATION_JSON;

157 + buffer_sprintf(w->response.data, "{ \"status\": \"redirect\", \"registry\": \"%s\" }", registry.registry_to_announce);

158 + return 200;

159 + }

160 +

161 + if(unlikely(person_guid[0] && !strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID)))

162 + person_guid[0] = '\0';

163 +

164 + // ------------------------------------------------------------------------

165 +

166 + registry_lock();

167 +

168 REGISTRY_PERSON *p = registry_request_access(person_guid, machine_guid, url, name, when);

169 if(!p) {

170 registry_json_header(w, "access", REGISTRY_STATUS_FAILED);

171 registry_json_footer(w);

172 + registry_unlock();

173 return 412;

174 }

175

@@ -1121,36 +185,50 @@ int registry_request_access_json(struct web_client *w, char *person_guid, char *

185 buffer_strcat(w->response.data, "\n\t]\n");

186

187 registry_json_footer(w);

188 + registry_unlock();

189 return 200;

190 }

191

192 +// ----------------------------------------------------------------------------

193 +// public DELETE request

194 +

195 // the main method for deleting a URL from a person

196 int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {

197 if(!registry.enabled)

198 return registry_json_disabled(w, "delete");

199

200 + registry_lock();

201 +

202 REGISTRY_PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when);

203 if(!p) {

204 registry_json_header(w, "delete", REGISTRY_STATUS_FAILED);

205 registry_json_footer(w);

206 + registry_unlock();

207 return 412;

208 }

209

210 // generate the response

211 registry_json_header(w, "delete", REGISTRY_STATUS_OK);

212 registry_json_footer(w);

213 + registry_unlock();

214 return 200;

215 }

216

217 +// ----------------------------------------------------------------------------

218 +// public SEARCH request

219 +

220 // the main method for searching the URLs of a netdata

221 int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {

222 if(!registry.enabled)

223 return registry_json_disabled(w, "search");

224

225 + registry_lock();

226 +

227 REGISTRY_MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when);

228 if(!m) {

229 registry_json_header(w, "search", REGISTRY_STATUS_FAILED);

230 registry_json_footer(w);

231 + registry_unlock();

232 return 404;

233 }

234

@@ -1162,38 +240,28 @@ int registry_request_search_json(struct web_client *w, char *person_guid, char *

240 buffer_strcat(w->response.data, "\n\t]\n");

241

242 registry_json_footer(w);

243 + registry_unlock();

244 return 200;

245 }

246

1168 -// structure used be the callbacks below

1169 -struct registry_person_url_callback_verify_machine_exists_data {

1170 - REGISTRY_MACHINE *m;

1171 - int count;

1172 -};

1173 -

1174 -int registry_person_url_callback_verify_machine_exists(void *entry, void *data) {

1175 - struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data;

1176 - REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry;

1177 - REGISTRY_MACHINE *m = d->m;

1178 -

1179 - if(pu->machine == m)

1180 - d->count++;

1181 -

1182 - return 0;

1183 -}

247 +// ----------------------------------------------------------------------------

248 +// SWITCH REQUEST

249

250 // the main method for switching user identity

251 int registry_request_switch_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when) {

252 + if(!registry.enabled)

253 + return registry_json_disabled(w, "switch");

254 +

255 (void)url;

256 (void)when;

257

1190 - if(!registry.enabled)

1191 - return registry_json_disabled(w, "switch");

258 + registry_lock();

259

260 REGISTRY_PERSON *op = registry_person_find(person_guid);

261 if(!op) {

262 registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);

263 registry_json_footer(w);

264 + registry_unlock();

265 return 430;

266 }

267

@@ -1201,6 +269,7 @@ int registry_request_switch_json(struct web_client *w, char *person_guid, char *

269 if(!np) {

270 registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);

271 registry_json_footer(w);

272 + registry_unlock();

273 return 431;

274 }

275

@@ -1208,6 +277,7 @@ int registry_request_switch_json(struct web_client *w, char *person_guid, char *

277 if(!m) {

278 registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);

279 registry_json_footer(w);

280 + registry_unlock();

281 return 432;

282 }

283

@@ -1218,6 +288,7 @@ int registry_request_switch_json(struct web_client *w, char *person_guid, char *

288 if(!data.count) {

289 registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);

290 registry_json_footer(w);

291 + registry_unlock();

292 return 433;

293 }

294

@@ -1227,6 +298,7 @@ int registry_request_switch_json(struct web_client *w, char *person_guid, char *

298 if(!data.count) {

299 registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);

300 registry_json_footer(w);

301 + registry_unlock();

302 return 434;

303 }

304

@@ -1238,582 +310,9 @@ int registry_request_switch_json(struct web_client *w, char *person_guid, char *

310 registry_json_header(w, "switch", REGISTRY_STATUS_OK);

311 buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\"", np->guid);

312 registry_json_footer(w);

1241 - return 200;

1242 -}

1243 -

1244 -

1245 -// ----------------------------------------------------------------------------

1246 -// REGISTRY THIS MACHINE UNIQUE ID

1247 -

1248 -static inline int is_machine_guid_blacklisted(const char *guid) {

1249 - // these are machine GUIDs that have been included in distribution packages.

1250 - // we blacklist them here, so that the next version of netdata will generate

1251 - // new ones.

1252 -

1253 - if(!strcmp(guid, "8a795b0c-2311-11e6-8563-000c295076a6")

1254 - || !strcmp(guid, "4aed1458-1c3e-11e6-a53f-000c290fc8f5")

1255 - ) {

1256 - error("Blacklisted machine GUID '%s' found.", guid);

1257 - return 1;

1258 - }

1259 -

1260 - return 0;

1261 -}

1262 -

1263 -char *registry_get_this_machine_guid(void) {

1264 - if(likely(registry.machine_guid[0]))

1265 - return registry.machine_guid;

1266 -

1267 - // read it from disk

1268 - int fd = open(registry.machine_guid_filename, O_RDONLY);

1269 - if(fd != -1) {

1270 - char buf[GUID_LEN + 1];

1271 - if(read(fd, buf, GUID_LEN) != GUID_LEN)

1272 - error("Failed to read machine GUID from '%s'", registry.machine_guid_filename);

1273 - else {

1274 - buf[GUID_LEN] = '\0';

1275 - if(registry_regenerate_guid(buf, registry.machine_guid) == -1) {

1276 - error("Failed to validate machine GUID '%s' from '%s'. Ignoring it - this might mean this netdata will appear as duplicate in the registry.",

1277 - buf, registry.machine_guid_filename);

1278 -

1279 - registry.machine_guid[0] = '\0';

1280 - }

1281 - else if(is_machine_guid_blacklisted(registry.machine_guid))

1282 - registry.machine_guid[0] = '\0';

1283 - }

1284 - close(fd);

1285 - }

1286 -

1287 - // generate a new one?

1288 - if(!registry.machine_guid[0]) {

1289 - uuid_t uuid;

1290 -

1291 - uuid_generate_time(uuid);

1292 - uuid_unparse_lower(uuid, registry.machine_guid);

1293 - registry.machine_guid[GUID_LEN] = '\0';

1294 -

1295 - // save it

1296 - fd = open(registry.machine_guid_filename, O_WRONLY|O_CREAT|O_TRUNC, 444);

1297 - if(fd == -1)

1298 - fatal("Cannot create unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

1299 -

1300 - if(write(fd, registry.machine_guid, GUID_LEN) != GUID_LEN)

1301 - fatal("Cannot write the unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

1302 -

1303 - close(fd);

1304 - }

1305 -

1306 - setenv("NETDATA_REGISTRY_UNIQUE_ID", registry.machine_guid, 1);

1307 -

1308 - return registry.machine_guid;

1309 -}

1310 -

1311 -

1312 -// ----------------------------------------------------------------------------

1313 -// REGISTRY LOAD/SAVE

1314 -

1315 -int registry_machine_save_url(void *entry, void *file) {

1316 - REGISTRY_MACHINE_URL *mu = entry;

1317 - FILE *fp = file;

1318 -

1319 - debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url);

1320 -

1321 - int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n",

1322 - mu->first_t,

1323 - mu->last_t,

1324 - mu->usages,

1325 - mu->flags,

1326 - mu->url->url

1327 - );

1328 -

1329 - // error handling is done at registry_save()

1330 -

1331 - return ret;

1332 -}

1333 -

1334 -int registry_machine_save(void *entry, void *file) {

1335 - REGISTRY_MACHINE *m = entry;

1336 - FILE *fp = file;

1337 -

1338 - debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);

1339 -

1340 - int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n",

1341 - m->first_t,

1342 - m->last_t,

1343 - m->usages,

1344 - m->guid

1345 - );

1346 -

1347 - if(ret >= 0) {

1348 - int ret2 = dictionary_get_all(m->machine_urls, registry_machine_save_url, fp);

1349 - if(ret2 < 0) return ret2;

1350 - ret += ret2;

1351 - }

1352 -

1353 - // error handling is done at registry_save()

1354 -

1355 - return ret;

1356 -}

1357 -

1358 -static inline int registry_person_save_url(void *entry, void *file) {

1359 - REGISTRY_PERSON_URL *pu = entry;

1360 - FILE *fp = file;

1361 -

1362 - debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);

1363 -

1364 - int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n",

1365 - pu->first_t,

1366 - pu->last_t,

1367 - pu->usages,

1368 - pu->flags,

1369 - pu->machine->guid,

1370 - pu->machine_name,

1371 - pu->url->url

1372 - );

1373 -

1374 - // error handling is done at registry_save()

1375 -

1376 - return ret;

1377 -}

1378 -

1379 -static inline int registry_person_save(void *entry, void *file) {

1380 - REGISTRY_PERSON *p = entry;

1381 - FILE *fp = file;

1382 -

1383 - debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);

1384 -

1385 - int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n",

1386 - p->first_t,

1387 - p->last_t,

1388 - p->usages,

1389 - p->guid

1390 - );

1391 -

1392 - if(ret >= 0) {

1393 - int ret2 = dictionary_get_all(p->person_urls, registry_person_save_url, fp);

1394 - if (ret2 < 0) return ret2;

1395 - ret += ret2;

1396 - }

1397 -

1398 - // error handling is done at registry_save()

1399 -

1400 - return ret;

1401 -}

1402 -

1403 -int registry_save(void) {

1404 - if(!registry.enabled) return -1;

1405 -

1406 - // make sure the log is not updated

1407 - registry_log_lock();

1408 -

1409 - if(unlikely(!registry_should_save_db())) {

1410 - registry_log_unlock();

1411 - return -2;

1412 - }

1413 -

1414 - error_log_limit_unlimited();

1415 -

1416 - char tmp_filename[FILENAME_MAX + 1];

1417 - char old_filename[FILENAME_MAX + 1];

1418 -

1419 - snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);

1420 - snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);

1421 -

1422 - debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);

1423 - FILE *fp = fopen(tmp_filename, "w");

1424 - if(!fp) {

1425 - error("Registry: Cannot create file: %s", tmp_filename);

1426 - registry_log_unlock();

1427 - error_log_limit_reset();

1428 - return -1;

1429 - }

313

1431 - // dictionary_get_all() has its own locking, so this is safe to do

1432 -

1433 - debug(D_REGISTRY, "Saving all machines");

1434 - int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);

1435 - if(bytes1 < 0) {

1436 - error("Registry: Cannot save registry machines - return value %d", bytes1);

1437 - fclose(fp);

1438 - registry_log_unlock();

1439 - error_log_limit_reset();

1440 - return bytes1;

1441 - }

1442 - debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);

1443 -

1444 - debug(D_REGISTRY, "Saving all persons");

1445 - int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp);

1446 - if(bytes2 < 0) {

1447 - error("Registry: Cannot save registry persons - return value %d", bytes2);

1448 - fclose(fp);

1449 - registry_log_unlock();

1450 - error_log_limit_reset();

1451 - return bytes2;

1452 - }

1453 - debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);

1454 -

1455 - // save the totals

1456 - fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",

1457 - registry.persons_count,

1458 - registry.machines_count,

1459 - registry.usages_count + 1, // this is required - it is lost on db rotation

1460 - registry.urls_count,

1461 - registry.persons_urls_count,

1462 - registry.machines_urls_count

1463 - );

1464 -

1465 - fclose(fp);

1466 -

1467 - errno = 0;

1468 -

1469 - // remove the .old db

1470 - debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename);

1471 - if(unlink(old_filename) == -1 && errno != ENOENT)

1472 - error("Registry: cannot remove old registry file '%s'", old_filename);

1473 -

1474 - // rename the db to .old

1475 - debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);

1476 - if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)

1477 - error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", registry.db_filename, old_filename);

1478 -

1479 - else {

1480 - // remove the database (it is saved in .old)

1481 - debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename);

1482 - if (unlink(registry.db_filename) == -1 && errno != ENOENT)

1483 - error("Registry: cannot remove old registry file '%s'", registry.db_filename);

1484 -

1485 - // move the .tmp to make it active

1486 - debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);

1487 - if (link(tmp_filename, registry.db_filename) == -1) {

1488 - error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,

1489 - registry.db_filename);

1490 -

1491 - // move the .old back

1492 - debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);

1493 - if(link(old_filename, registry.db_filename) == -1)

1494 - error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);

1495 - }

1496 - else {

1497 - debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename);

1498 - if(unlink(tmp_filename) == -1)

1499 - error("Registry: cannot remove tmp registry file '%s'", tmp_filename);

1500 -

1501 - // it has been moved successfully

1502 - // discard the current registry log

1503 - registry_log_recreate_nolock();

1504 - registry.log_count = 0;

1505 - }

1506 - }

1507 -

1508 - // continue operations

1509 - registry_log_unlock();

1510 - error_log_limit_reset();

1511 -

1512 - return -1;

1513 -}

1514 -

1515 -static inline size_t registry_load(void) {

1516 - char *s, buf[4096 + 1];

1517 - REGISTRY_PERSON *p = NULL;

1518 - REGISTRY_MACHINE *m = NULL;

1519 - REGISTRY_URL *u = NULL;

1520 - size_t line = 0;

1521 -

1522 - debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename);

1523 - FILE *fp = fopen(registry.db_filename, "r");

1524 - if(!fp) {

1525 - error("Registry: cannot open registry file: '%s'", registry.db_filename);

1526 - return 0;

1527 - }

1528 -

1529 - size_t len = 0;

1530 - buf[4096] = '\0';

1531 - while((s = fgets_trim_len(buf, 4096, fp, &len))) {

1532 - line++;

1533 -

1534 - debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);

1535 - switch(*s) {

1536 - case 'T': // totals

1537 - if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {

1538 - error("Registry totals line %zu is wrong (len = %zu).", line, len);

1539 - continue;

1540 - }

1541 - registry.persons_count = strtoull(&s[2], NULL, 16);

1542 - registry.machines_count = strtoull(&s[19], NULL, 16);

1543 - registry.usages_count = strtoull(&s[36], NULL, 16);

1544 - registry.urls_count = strtoull(&s[53], NULL, 16);

1545 - registry.persons_urls_count = strtoull(&s[70], NULL, 16);

1546 - registry.machines_urls_count = strtoull(&s[87], NULL, 16);

1547 - break;

1548 -

1549 - case 'P': // person

1550 - m = NULL;

1551 - // verify it is valid

1552 - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

1553 - error("Registry person line %zu is wrong (len = %zu).", line, len);

1554 - continue;

1555 - }

1556 -

1557 - s[1] = s[10] = s[19] = s[28] = '\0';

1558 - p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));

1559 - p->last_t = strtoul(&s[11], NULL, 16);

1560 - p->usages = strtoul(&s[20], NULL, 16);

1561 - debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);

1562 - break;

1563 -

1564 - case 'M': // machine

1565 - p = NULL;

1566 - // verify it is valid

1567 - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

1568 - error("Registry person line %zu is wrong (len = %zu).", line, len);

1569 - continue;

1570 - }

1571 -

1572 - s[1] = s[10] = s[19] = s[28] = '\0';

1573 - m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));

1574 - m->last_t = strtoul(&s[11], NULL, 16);

1575 - m->usages = strtoul(&s[20], NULL, 16);

1576 - debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);

1577 - break;

1578 -

1579 - case 'U': // person URL

1580 - if(unlikely(!p)) {

1581 - error("Registry: ignoring line %zu, no person loaded: %s", line, s);

1582 - continue;

1583 - }

1584 -

1585 - // verify it is valid

1586 - if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') {

1587 - error("Registry person URL line %zu is wrong (len = %zu).", line, len);

1588 - continue;

1589 - }

1590 -

1591 - s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0';

1592 -

1593 - // skip the name to find the url

1594 - char *url = &s[69];

1595 - while(*url && *url != '\t') url++;

1596 - if(!*url) {

1597 - error("Registry person URL line %zu does not have a url.", line);

1598 - continue;

1599 - }

1600 - *url++ = '\0';

1601 -

1602 - // u = registry_url_allocate_nolock(url, strlen(url));

1603 - u = registry_url_get_nolock(url, strlen(url));

1604 -

1605 - time_t first_t = strtoul(&s[2], NULL, 16);

1606 -

1607 - m = registry_machine_find(&s[32]);

1608 - if(!m) m = registry_machine_allocate(&s[32], first_t);

1609 -

1610 - REGISTRY_PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t);

1611 - pu->last_t = strtoul(&s[11], NULL, 16);

1612 - pu->usages = strtoul(&s[20], NULL, 16);

1613 - pu->flags = strtoul(&s[29], NULL, 16);

1614 - debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->machine_name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);

1615 - break;

1616 -

1617 - case 'V': // machine URL

1618 - if(unlikely(!m)) {

1619 - error("Registry: ignoring line %zu, no machine loaded: %s", line, s);

1620 - continue;

1621 - }

1622 -

1623 - // verify it is valid

1624 - if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') {

1625 - error("Registry person URL line %zu is wrong (len = %zu).", line, len);

1626 - continue;

1627 - }

1628 -

1629 - s[1] = s[10] = s[19] = s[28] = s[31] = '\0';

1630 - // u = registry_url_allocate_nolock(&s[32], strlen(&s[32]));

1631 - u = registry_url_get_nolock(&s[32], strlen(&s[32]));

1632 -

1633 - REGISTRY_MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16));

1634 - mu->last_t = strtoul(&s[11], NULL, 16);

1635 - mu->usages = strtoul(&s[20], NULL, 16);

1636 - mu->flags = strtoul(&s[29], NULL, 16);

1637 - debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags);

1638 - break;

1639 -

1640 - default:

1641 - error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s);

1642 - break;

1643 - }

1644 - }

1645 - fclose(fp);

1646 -

1647 - return line;

1648 -}

1649 -

1650 -// ----------------------------------------------------------------------------

1651 -// REGISTRY

1652 -

1653 -int registry_init(void) {

1654 - char filename[FILENAME_MAX + 1];

1655 -

1656 - // registry enabled?

1657 - registry.enabled = config_get_boolean("registry", "enabled", 0);

1658 -

1659 - // pathnames

1660 - registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR "/registry");

1661 - if(mkdir(registry.pathname, 0770) == -1 && errno != EEXIST)

1662 - fatal("Cannot create directory '%s'.", registry.pathname);

1663 -

1664 - // filenames

1665 - snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname);

1666 - registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename);

1667 - registry_get_this_machine_guid();

1668 -

1669 - snprintfz(filename, FILENAME_MAX, "%s/registry.db", registry.pathname);

1670 - registry.db_filename = config_get("registry", "registry db file", filename);

1671 -

1672 - snprintfz(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname);

1673 - registry.log_filename = config_get("registry", "registry log file", filename);

1674 -

1675 - // configuration options

1676 - registry.save_registry_every_entries = config_get_number("registry", "registry save db every new entries", 1000000);

1677 - registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400;

1678 - registry.registry_domain = config_get("registry", "registry domain", "");

1679 - registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io");

1680 - registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", localhost.hostname));

1681 - registry.verify_cookies_redirects = config_get_boolean("registry", "verify browser cookies support", 1);

1682 -

1683 - setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1);

1684 - setenv("NETDATA_REGISTRY_URL", registry.registry_to_announce, 1);

1685 -

1686 - registry.max_url_length = config_get_number("registry", "max URL length", 1024);

1687 - if(registry.max_url_length < 10) {

1688 - registry.max_url_length = 10;

1689 - config_set_number("registry", "max URL length", registry.max_url_length);

1690 - }

1691 -

1692 - registry.max_name_length = config_get_number("registry", "max URL name length", 50);

1693 - if(registry.max_name_length < 10) {

1694 - registry.max_name_length = 10;

1695 - config_set_number("registry", "max URL name length", registry.max_name_length);

1696 - }

1697 -

1698 - // initialize entries counters

1699 - registry.persons_count = 0;

1700 - registry.machines_count = 0;

1701 - registry.usages_count = 0;

1702 - registry.urls_count = 0;

1703 - registry.persons_urls_count = 0;

1704 - registry.machines_urls_count = 0;

1705 -

1706 - // initialize memory counters

1707 - registry.persons_memory = 0;

1708 - registry.machines_memory = 0;

1709 - registry.urls_memory = 0;

1710 - registry.persons_urls_memory = 0;

1711 - registry.machines_urls_memory = 0;

1712 -

1713 - // initialize locks

1714 - pthread_mutex_init(&registry.persons_lock, NULL);

1715 - pthread_mutex_init(&registry.machines_lock, NULL);

1716 - pthread_mutex_init(&registry.urls_lock, NULL);

1717 - pthread_mutex_init(&registry.person_urls_lock, NULL);

1718 - pthread_mutex_init(&registry.machine_urls_lock, NULL);

1719 -

1720 - // create dictionaries

1721 - registry.persons = dictionary_create(DICTIONARY_FLAGS);

1722 - registry.machines = dictionary_create(DICTIONARY_FLAGS);

1723 - registry.urls = dictionary_create(DICTIONARY_FLAGS);

1724 -

1725 - // load the registry database

1726 - if(registry.enabled) {

1727 - registry_log_open_nolock();

1728 - registry_load();

1729 - registry_log_load();

1730 -

1731 - if(unlikely(registry_should_save_db()))

1732 - registry_save();

1733 - }

1734 -

1735 - return 0;

1736 -}

1737 -

1738 -void registry_free(void) {

1739 - if(!registry.enabled) return;

1740 -

1741 - // we need to destroy the dictionaries ourselves

1742 - // since the dictionaries use memory we allocated

1743 -

1744 - while(registry.persons->values_index.root) {

1745 - REGISTRY_PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;

1746 -

1747 - // fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages);

1748 -

1749 - while(p->person_urls->values_index.root) {

1750 - REGISTRY_PERSON_URL *pu = ((NAME_VALUE *)p->person_urls->values_index.root)->value;

1751 -

1752 - // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags);

1753 -

1754 - debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid);

1755 - dictionary_del(p->person_urls, pu->url->url);

1756 -

1757 - debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url);

1758 - registry_url_unlink_nolock(pu->url);

1759 -

1760 - debug(D_REGISTRY, "Registry: freeing person url");

1761 - freez(pu);

1762 - }

1763 -

1764 - debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);

1765 - dictionary_del(registry.persons, p->guid);

1766 -

1767 - debug(D_REGISTRY, "Registry: destroying URL dictionary of person '%s'", p->guid);

1768 - dictionary_destroy(p->person_urls);

1769 -

1770 - debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid);

1771 - freez(p);

1772 - }

1773 -

1774 - while(registry.machines->values_index.root) {

1775 - REGISTRY_MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value;

1776 -

1777 - // fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);

1778 -

1779 - while(m->machine_urls->values_index.root) {

1780 - REGISTRY_MACHINE_URL *mu = ((NAME_VALUE *)m->machine_urls->values_index.root)->value;

1781 -

1782 - // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);

1783 -

1784 - //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url);

1785 - //dictionary_destroy(mu->persons);

1786 -

1787 - debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid);

1788 - dictionary_del(m->machine_urls, mu->url->url);

1789 -

1790 - debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);

1791 - registry_url_unlink_nolock(mu->url);

1792 -

1793 - debug(D_REGISTRY, "Registry: freeing machine url");

1794 - freez(mu);

1795 - }

1796 -

1797 - debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid);

1798 - dictionary_del(registry.machines, m->guid);

1799 -

1800 - debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid);

1801 - dictionary_destroy(m->machine_urls);

1802 -

1803 - debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid);

1804 - freez(m);

1805 - }

1806 -

1807 - // and free the memory of remaining dictionary structures

1808 -

1809 - debug(D_REGISTRY, "Registry: destroying persons dictionary");

1810 - dictionary_destroy(registry.persons);

1811 -

1812 - debug(D_REGISTRY, "Registry: destroying machines dictionary");

1813 - dictionary_destroy(registry.machines);

1814 -

1815 - debug(D_REGISTRY, "Registry: destroying urls dictionary");

1816 - dictionary_destroy(registry.urls);

314 + registry_unlock();

315 + return 200;

316 }

317

318 // ----------------------------------------------------------------------------

@@ -1872,7 +371,7 @@ void registry_statistics(void) {

371

372 rrddim_set(stm, "persons", registry.persons_memory + registry.persons_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));

373 rrddim_set(stm, "machines", registry.machines_memory + registry.machines_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));

1875 - rrddim_set(stm, "urls", registry.urls_memory + registry.urls_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));

374 + rrddim_set(stm, "urls", registry.urls_memory);

375 rrddim_set(stm, "persons_urls", registry.persons_urls_memory + registry.persons_count * sizeof(DICTIONARY) + registry.persons_urls_count * sizeof(NAME_VALUE));

376 rrddim_set(stm, "machines_urls", registry.machines_urls_memory + registry.machines_count * sizeof(DICTIONARY) + registry.machines_urls_count * sizeof(NAME_VALUE));

377 rrdset_done(stm);

src/registry.h

+56 -10

@@ -1,25 +1,71 @@

1 +/*

2 + * netdata registry

3 + *

4 + * this header file describes the public interface

5 + * to the netdata registry

6 + *

7 + * only these high level functions are exposed

8 + *

9 + */

10 +

11 +// ----------------------------------------------------------------------------

12 +// TODO

13 +//

14 +// 1. the default tracking cookie expires in 1 year, but the persons are not

15 +// removed from the db - this means the database only grows - ideally the

16 +// database should be cleaned in registry_db_save() for both on-disk and

17 +// on-memory entries.

18 +//

19 +// Cleanup:

20 +// i. Find all the PERSONs that have expired cookie

21 +// ii. For each of their PERSON_URLs:

22 +// - decrement the linked MACHINE links

23 +// - if the linked MACHINE has no other links, remove the linked MACHINE too

24 +// - remove the PERSON_URL

25 +//

26 +// 2. add protection to prevent abusing the registry by flooding it with

27 +// requests to fill the memory and crash it.

28 +//

29 +// Possible protections:

30 +// - limit the number of URLs per person

31 +// - limit the number of URLs per machine

32 +// - limit the number of persons

33 +// - limit the number of machines

34 +// - [DONE] limit the size of URLs

35 +// - [DONE] limit the size of PERSON_URL names

36 +// - limit the number of requests that add data to the registry,

37 +// per client IP per hour

38 +//

39 +// 3. lower memory requirements

40 +//

41 +// - embed avl structures directly into registry objects, instead of DICTIONARY

42 +// - store GUIDs in memory as UUID instead of char *

43 +// - do not track persons using the demo machines only

44 +// (i.e. start tracking them only when they access a non-demo machine)

45 +// - [DONE] do not track custom dashboards by default

46 +

47 +

48 #ifndef NETDATA_REGISTRY_H

49 #define NETDATA_REGISTRY_H 1

51 #define NETDATA_REGISTRY_COOKIE_NAME "netdata_registry_id"

6 -extern void registry_set_cookie(struct web_client *w, const char *guid);

7 -extern const char *registry_to_announce(void);

8 -extern int registry_verify_cookies_redirects(void);

53 +// initialize the registry

54 +// should only happen when netdata starts

55 +extern int registry_init(void);

56 +

57 +// free all data held by the registry

58 +// should only happen when netdata exits

59 +extern void registry_free(void);

61 +// HTTP requests handled by the registry

62 extern int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when);

63 extern int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);

64 extern int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when);

65 extern int registry_request_switch_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when);

66 extern int registry_request_hello_json(struct web_client *w);

16 -extern int registry_init(void);

17 -extern void registry_free(void);

18 -extern int registry_save(void);

19 -

20 -extern char *registry_get_this_machine_guid(void);

21 -

68 +// update the registry monitoring charts

69 extern void registry_statistics(void);

24 -

71 #endif /* NETDATA_REGISTRY_H */

src/registry_db.c new

+342

@@ -0,0 +1,342 @@

1 +#include "registry_internals.h"

2 +

3 +int registry_db_should_be_saved(void) {

4 + debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries);

5 + return registry.log_count > registry.save_registry_every_entries;

6 +}

7 +

8 +// ----------------------------------------------------------------------------

9 +// INTERNAL FUNCTIONS FOR SAVING REGISTRY OBJECTS

10 +

11 +static int registry_machine_save_url(void *entry, void *file) {

12 + REGISTRY_MACHINE_URL *mu = entry;

13 + FILE *fp = file;

14 +

15 + debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url);

16 +

17 + int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n",

18 + mu->first_t,

19 + mu->last_t,

20 + mu->usages,

21 + mu->flags,

22 + mu->url->url

23 + );

24 +

25 + // error handling is done at registry_db_save()

26 +

27 + return ret;

28 +}

29 +

30 +static int registry_machine_save(void *entry, void *file) {

31 + REGISTRY_MACHINE *m = entry;

32 + FILE *fp = file;

33 +

34 + debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);

35 +

36 + int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n",

37 + m->first_t,

38 + m->last_t,

39 + m->usages,

40 + m->guid

41 + );

42 +

43 + if(ret >= 0) {

44 + int ret2 = dictionary_get_all(m->machine_urls, registry_machine_save_url, fp);

45 + if(ret2 < 0) return ret2;

46 + ret += ret2;

47 + }

48 +

49 + // error handling is done at registry_db_save()

50 +

51 + return ret;

52 +}

53 +

54 +static inline int registry_person_save_url(void *entry, void *file) {

55 + REGISTRY_PERSON_URL *pu = entry;

56 + FILE *fp = file;

57 +

58 + debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);

59 +

60 + int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n",

61 + pu->first_t,

62 + pu->last_t,

63 + pu->usages,

64 + pu->flags,

65 + pu->machine->guid,

66 + pu->machine_name,

67 + pu->url->url

68 + );

69 +

70 + // error handling is done at registry_db_save()

71 +

72 + return ret;

73 +}

74 +

75 +static inline int registry_person_save(void *entry, void *file) {

76 + REGISTRY_PERSON *p = entry;

77 + FILE *fp = file;

78 +

79 + debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);

80 +

81 + int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n",

82 + p->first_t,

83 + p->last_t,

84 + p->usages,

85 + p->guid

86 + );

87 +

88 + if(ret >= 0) {

89 + int ret2 = dictionary_get_all(p->person_urls, registry_person_save_url, fp);

90 + if (ret2 < 0) return ret2;

91 + ret += ret2;

92 + }

93 +

94 + // error handling is done at registry_db_save()

95 +

96 + return ret;

97 +}

98 +

99 +// ----------------------------------------------------------------------------

100 +// SAVE THE REGISTRY DATABASE

101 +

102 +int registry_db_save(void) {

103 + if(unlikely(!registry.enabled))

104 + return -1;

105 +

106 + if(unlikely(!registry_db_should_be_saved()))

107 + return -2;

108 +

109 + error_log_limit_unlimited();

110 +

111 + char tmp_filename[FILENAME_MAX + 1];

112 + char old_filename[FILENAME_MAX + 1];

113 +

114 + snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);

115 + snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);

116 +

117 + debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);

118 + FILE *fp = fopen(tmp_filename, "w");

119 + if(!fp) {

120 + error("Registry: Cannot create file: %s", tmp_filename);

121 + error_log_limit_reset();

122 + return -1;

123 + }

124 +

125 + // dictionary_get_all() has its own locking, so this is safe to do

126 +

127 + debug(D_REGISTRY, "Saving all machines");

128 + int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);

129 + if(bytes1 < 0) {

130 + error("Registry: Cannot save registry machines - return value %d", bytes1);

131 + fclose(fp);

132 + error_log_limit_reset();

133 + return bytes1;

134 + }

135 + debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);

136 +

137 + debug(D_REGISTRY, "Saving all persons");

138 + int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp);

139 + if(bytes2 < 0) {

140 + error("Registry: Cannot save registry persons - return value %d", bytes2);

141 + fclose(fp);

142 + error_log_limit_reset();

143 + return bytes2;

144 + }

145 + debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);

146 +

147 + // save the totals

148 + fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",

149 + registry.persons_count,

150 + registry.machines_count,

151 + registry.usages_count + 1, // this is required - it is lost on db rotation

152 + registry.urls_count,

153 + registry.persons_urls_count,

154 + registry.machines_urls_count

155 + );

156 +

157 + fclose(fp);

158 +

159 + errno = 0;

160 +

161 + // remove the .old db

162 + debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename);

163 + if(unlink(old_filename) == -1 && errno != ENOENT)

164 + error("Registry: cannot remove old registry file '%s'", old_filename);

165 +

166 + // rename the db to .old

167 + debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);

168 + if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)

169 + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", registry.db_filename, old_filename);

170 +

171 + else {

172 + // remove the database (it is saved in .old)

173 + debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename);

174 + if (unlink(registry.db_filename) == -1 && errno != ENOENT)

175 + error("Registry: cannot remove old registry file '%s'", registry.db_filename);

176 +

177 + // move the .tmp to make it active

178 + debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);

179 + if (link(tmp_filename, registry.db_filename) == -1) {

180 + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,

181 + registry.db_filename);

182 +

183 + // move the .old back

184 + debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);

185 + if(link(old_filename, registry.db_filename) == -1)

186 + error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);

187 + }

188 + else {

189 + debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename);

190 + if(unlink(tmp_filename) == -1)

191 + error("Registry: cannot remove tmp registry file '%s'", tmp_filename);

192 +

193 + // it has been moved successfully

194 + // discard the current registry log

195 + registry_log_recreate();

196 + registry.log_count = 0;

197 + }

198 + }

199 +

200 + // continue operations

201 + error_log_limit_reset();

202 +

203 + return -1;

204 +}

205 +

206 +// ----------------------------------------------------------------------------

207 +// LOAD THE REGISTRY DATABASE

208 +

209 +size_t registry_db_load(void) {

210 + char *s, buf[4096 + 1];

211 + REGISTRY_PERSON *p = NULL;

212 + REGISTRY_MACHINE *m = NULL;

213 + REGISTRY_URL *u = NULL;

214 + size_t line = 0;

215 +

216 + debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename);

217 + FILE *fp = fopen(registry.db_filename, "r");

218 + if(!fp) {

219 + error("Registry: cannot open registry file: '%s'", registry.db_filename);

220 + return 0;

221 + }

222 +

223 + size_t len = 0;

224 + buf[4096] = '\0';

225 + while((s = fgets_trim_len(buf, 4096, fp, &len))) {

226 + line++;

227 +

228 + debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);

229 + switch(*s) {

230 + case 'T': // totals

231 + if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {

232 + error("Registry totals line %zu is wrong (len = %zu).", line, len);

233 + continue;

234 + }

235 + registry.persons_count = strtoull(&s[2], NULL, 16);

236 + registry.machines_count = strtoull(&s[19], NULL, 16);

237 + registry.usages_count = strtoull(&s[36], NULL, 16);

238 + registry.urls_count = strtoull(&s[53], NULL, 16);

239 + registry.persons_urls_count = strtoull(&s[70], NULL, 16);

240 + registry.machines_urls_count = strtoull(&s[87], NULL, 16);

241 + break;

242 +

243 + case 'P': // person

244 + m = NULL;

245 + // verify it is valid

246 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

247 + error("Registry person line %zu is wrong (len = %zu).", line, len);

248 + continue;

249 + }

250 +

251 + s[1] = s[10] = s[19] = s[28] = '\0';

252 + p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));

253 + p->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

254 + p->usages = (uint32_t)strtoul(&s[20], NULL, 16);

255 + debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);

256 + break;

257 +

258 + case 'M': // machine

259 + p = NULL;

260 + // verify it is valid

261 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

262 + error("Registry person line %zu is wrong (len = %zu).", line, len);

263 + continue;

264 + }

265 +

266 + s[1] = s[10] = s[19] = s[28] = '\0';

267 + m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));

268 + m->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

269 + m->usages = (uint32_t)strtoul(&s[20], NULL, 16);

270 + debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);

271 + break;

272 +

273 + case 'U': // person URL

274 + if(unlikely(!p)) {

275 + error("Registry: ignoring line %zu, no person loaded: %s", line, s);

276 + continue;

277 + }

278 +

279 + // verify it is valid

280 + if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') {

281 + error("Registry person URL line %zu is wrong (len = %zu).", line, len);

282 + continue;

283 + }

284 +

285 + s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0';

286 +

287 + // skip the name to find the url

288 + char *url = &s[69];

289 + while(*url && *url != '\t') url++;

290 + if(!*url) {

291 + error("Registry person URL line %zu does not have a url.", line);

292 + continue;

293 + }

294 + *url++ = '\0';

295 +

296 + // u = registry_url_allocate_nolock(url, strlen(url));

297 + u = registry_url_get(url, strlen(url));

298 +

299 + time_t first_t = strtoul(&s[2], NULL, 16);

300 +

301 + m = registry_machine_find(&s[32]);

302 + if(!m) m = registry_machine_allocate(&s[32], first_t);

303 +

304 + REGISTRY_PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t);

305 + pu->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

306 + pu->usages = (uint32_t)strtoul(&s[20], NULL, 16);

307 + pu->flags = (uint8_t)strtoul(&s[29], NULL, 16);

308 + debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->machine_name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);

309 + break;

310 +

311 + case 'V': // machine URL

312 + if(unlikely(!m)) {

313 + error("Registry: ignoring line %zu, no machine loaded: %s", line, s);

314 + continue;

315 + }

316 +

317 + // verify it is valid

318 + if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') {

319 + error("Registry person URL line %zu is wrong (len = %zu).", line, len);

320 + continue;

321 + }

322 +

323 + s[1] = s[10] = s[19] = s[28] = s[31] = '\0';

324 + // u = registry_url_allocate_nolock(&s[32], strlen(&s[32]));

325 + u = registry_url_get(&s[32], strlen(&s[32]));

326 +

327 + REGISTRY_MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16));

328 + mu->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

329 + mu->usages = (uint32_t)strtoul(&s[20], NULL, 16);

330 + mu->flags = (uint8_t)strtoul(&s[29], NULL, 16);

331 + debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags);

332 + break;

333 +

334 + default:

335 + error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s);

336 + break;

337 + }

338 + }

339 + fclose(fp);

340 +

341 + return line;

342 +}

src/registry_init.c new

+160

@@ -0,0 +1,160 @@

1 +#include "registry_internals.h"

2 +

3 +int registry_init(void) {

4 + char filename[FILENAME_MAX + 1];

5 +

6 + // registry enabled?

7 + registry.enabled = config_get_boolean("registry", "enabled", 0);

8 +

9 + // pathnames

10 + registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR "/registry");

11 + if(mkdir(registry.pathname, 0770) == -1 && errno != EEXIST)

12 + fatal("Cannot create directory '%s'.", registry.pathname);

13 +

14 + // filenames

15 + snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname);

16 + registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename);

17 + registry_get_this_machine_guid();

18 +

19 + snprintfz(filename, FILENAME_MAX, "%s/registry.db", registry.pathname);

20 + registry.db_filename = config_get("registry", "registry db file", filename);

21 +

22 + snprintfz(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname);

23 + registry.log_filename = config_get("registry", "registry log file", filename);

24 +

25 + // configuration options

26 + registry.save_registry_every_entries = (unsigned long long)config_get_number("registry", "registry save db every new entries", 1000000);

27 + registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400;

28 + registry.registry_domain = config_get("registry", "registry domain", "");

29 + registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io");

30 + registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", localhost.hostname));

31 + registry.verify_cookies_redirects = config_get_boolean("registry", "verify browser cookies support", 1);

32 +

33 + setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1);

34 + setenv("NETDATA_REGISTRY_URL", registry.registry_to_announce, 1);

35 +

36 + registry.max_url_length = (size_t)config_get_number("registry", "max URL length", 1024);

37 + if(registry.max_url_length < 10) {

38 + registry.max_url_length = 10;

39 + config_set_number("registry", "max URL length", (long long)registry.max_url_length);

40 + }

41 +

42 + registry.max_name_length = (size_t)config_get_number("registry", "max URL name length", 50);

43 + if(registry.max_name_length < 10) {

44 + registry.max_name_length = 10;

45 + config_set_number("registry", "max URL name length", (long long)registry.max_name_length);

46 + }

47 +

48 + // initialize entries counters

49 + registry.persons_count = 0;

50 + registry.machines_count = 0;

51 + registry.usages_count = 0;

52 + registry.urls_count = 0;

53 + registry.persons_urls_count = 0;

54 + registry.machines_urls_count = 0;

55 +

56 + // initialize memory counters

57 + registry.persons_memory = 0;

58 + registry.machines_memory = 0;

59 + registry.urls_memory = 0;

60 + registry.persons_urls_memory = 0;

61 + registry.machines_urls_memory = 0;

62 +

63 + // initialize locks

64 + pthread_mutex_init(&registry.lock, NULL);

65 +

66 + // create dictionaries

67 + registry.persons = dictionary_create(DICTIONARY_FLAGS);

68 + registry.machines = dictionary_create(DICTIONARY_FLAGS);

69 + avl_init(&registry.registry_urls_root_index, registry_url_compare);

70 +

71 + // load the registry database

72 + if(registry.enabled) {

73 + registry_log_open();

74 + registry_db_load();

75 + registry_log_load();

76 +

77 + if(unlikely(registry_db_should_be_saved()))

78 + registry_db_save();

79 + }

80 +

81 + return 0;

82 +}

83 +

84 +void registry_free(void) {

85 + if(!registry.enabled) return;

86 +

87 + // we need to destroy the dictionaries ourselves

88 + // since the dictionaries use memory we allocated

89 +

90 + while(registry.persons->values_index.root) {

91 + REGISTRY_PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;

92 +

93 + // fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages);

94 +

95 + while(p->person_urls->values_index.root) {

96 + REGISTRY_PERSON_URL *pu = ((NAME_VALUE *)p->person_urls->values_index.root)->value;

97 +

98 + // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags);

99 +

100 + debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid);

101 + dictionary_del(p->person_urls, pu->url->url);

102 +

103 + debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url);

104 + registry_url_unlink(pu->url);

105 +

106 + debug(D_REGISTRY, "Registry: freeing person url");

107 + freez(pu);

108 + }

109 +

110 + debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);

111 + dictionary_del(registry.persons, p->guid);

112 +

113 + debug(D_REGISTRY, "Registry: destroying URL dictionary of person '%s'", p->guid);

114 + dictionary_destroy(p->person_urls);

115 +

116 + debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid);

117 + freez(p);

118 + }

119 +

120 + while(registry.machines->values_index.root) {

121 + REGISTRY_MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value;

122 +

123 + // fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);

124 +

125 + while(m->machine_urls->values_index.root) {

126 + REGISTRY_MACHINE_URL *mu = ((NAME_VALUE *)m->machine_urls->values_index.root)->value;

127 +

128 + // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);

129 +

130 + //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url);

131 + //dictionary_destroy(mu->persons);

132 +

133 + debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid);

134 + dictionary_del(m->machine_urls, mu->url->url);

135 +

136 + debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);

137 + registry_url_unlink(mu->url);

138 +

139 + debug(D_REGISTRY, "Registry: freeing machine url");

140 + freez(mu);

141 + }

142 +

143 + debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid);

144 + dictionary_del(registry.machines, m->guid);

145 +

146 + debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid);

147 + dictionary_destroy(m->machine_urls);

148 +

149 + debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid);

150 + freez(m);

151 + }

152 +

153 + // and free the memory of remaining dictionary structures

154 +

155 + debug(D_REGISTRY, "Registry: destroying persons dictionary");

156 + dictionary_destroy(registry.persons);

157 +

158 + debug(D_REGISTRY, "Registry: destroying machines dictionary");

159 + dictionary_destroy(registry.machines);

160 +}

src/registry_internals.c new

+317

@@ -0,0 +1,317 @@

1 +#include "registry_internals.h"

2 +

3 +struct registry registry;

4 +

5 +// ----------------------------------------------------------------------------

6 +// common functions

7 +

8 +// parse a GUID and re-generated to be always lower case

9 +// this is used as a protection against the variations of GUIDs

10 +int registry_regenerate_guid(const char *guid, char *result) {

11 + uuid_t uuid;

12 + if(unlikely(uuid_parse(guid, uuid) == -1)) {

13 + info("Registry: GUID '%s' is not a valid GUID.", guid);

14 + return -1;

15 + }

16 + else {

17 + uuid_unparse_lower(uuid, result);

18 +

19 +#ifdef NETDATA_INTERNAL_CHECKS

20 + if(strcmp(guid, result))

21 + info("Registry: source GUID '%s' and re-generated GUID '%s' differ!", guid, result);

22 +#endif /* NETDATA_INTERNAL_CHECKS */

23 + }

24 +

25 + return 0;

26 +}

27 +

28 +// make sure the names of the machines / URLs do not contain any tabs

29 +// (which are used as our separator in the database files)

30 +// and are properly trimmed (before and after)

31 +static inline char *registry_fix_machine_name(char *name, size_t *len) {

32 + char *s = name?name:"";

33 +

34 + // skip leading spaces

35 + while(*s && isspace(*s)) s++;

36 +

37 + // make sure all spaces are a SPACE

38 + char *t = s;

39 + while(*t) {

40 + if(unlikely(isspace(*t)))

41 + *t = ' ';

42 +

43 + t++;

44 + }

45 +

46 + // remove trailing spaces

47 + while(--t >= s) {

48 + if(*t == ' ')

49 + *t = '\0';

50 + else

51 + break;

52 + }

53 + t++;

54 +

55 + if(likely(len))

56 + *len = (t - s);

57 +

58 + return s;

59 +}

60 +

61 +static inline char *registry_fix_url(char *url, size_t *len) {

62 + return registry_fix_machine_name(url, len);

63 +}

64 +

65 +

66 +// ----------------------------------------------------------------------------

67 +// forward definition of functions

68 +

69 +extern REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when);

70 +extern REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);

71 +

72 +

73 +// ----------------------------------------------------------------------------

74 +// HELPERS

75 +

76 +// verify the person, the machine and the URL exist in our DB

77 +REGISTRY_PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, REGISTRY_PERSON **pp, REGISTRY_MACHINE **mm) {

78 + char pbuf[GUID_LEN + 1], mbuf[GUID_LEN + 1];

79 +

80 + if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) {

81 + info("Registry Request Verification: invalid request! person: '%s', machine '%s', url '%s'", person_guid?person_guid:"UNSET", machine_guid?machine_guid:"UNSET", url?url:"UNSET");

82 + return NULL;

83 + }

84 +

85 + // normalize the url

86 + url = registry_fix_url(url, NULL);

87 +

88 + // make sure the person GUID is valid

89 + if(registry_regenerate_guid(person_guid, pbuf) == -1) {

90 + info("Registry Request Verification: invalid person GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

91 + return NULL;

92 + }

93 + person_guid = pbuf;

94 +

95 + // make sure the machine GUID is valid

96 + if(registry_regenerate_guid(machine_guid, mbuf) == -1) {

97 + info("Registry Request Verification: invalid machine GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

98 + return NULL;

99 + }

100 + machine_guid = mbuf;

101 +

102 + // make sure the machine exists

103 + REGISTRY_MACHINE *m = registry_machine_find(machine_guid);

104 + if(!m) {

105 + info("Registry Request Verification: machine not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

106 + return NULL;

107 + }

108 + if(mm) *mm = m;

109 +

110 + // make sure the person exist

111 + REGISTRY_PERSON *p = registry_person_find(person_guid);

112 + if(!p) {

113 + info("Registry Request Verification: person not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

114 + return NULL;

115 + }

116 + if(pp) *pp = p;

117 +

118 + REGISTRY_PERSON_URL *pu = dictionary_get(p->person_urls, url);

119 + if(!pu) {

120 + info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

121 + return NULL;

122 + }

123 + return pu;

124 +}

125 +

126 +

127 +// ----------------------------------------------------------------------------

128 +// REGISTRY REQUESTS

129 +

130 +REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) {

131 + debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);

132 +

133 + REGISTRY_MACHINE *m = registry_machine_get(machine_guid, when);

134 + if(!m) return NULL;

135 +

136 + // make sure the name is valid

137 + size_t namelen;

138 + name = registry_fix_machine_name(name, &namelen);

139 +

140 + size_t urllen;

141 + url = registry_fix_url(url, &urllen);

142 +

143 + REGISTRY_PERSON *p = registry_person_get(person_guid, when);

144 +

145 + REGISTRY_URL *u = registry_url_get(url, urllen);

146 + registry_person_link_to_url(p, m, u, name, namelen, when);

147 + registry_machine_link_to_url(m, u, when);

148 +

149 + registry_log('A', p, m, u, name);

150 +

151 + registry.usages_count++;

152 +

153 + return p;

154 +}

155 +

156 +REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {

157 + (void)when;

158 +

159 + REGISTRY_PERSON *p = NULL;

160 + REGISTRY_MACHINE *m = NULL;

161 + REGISTRY_PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);

162 + if(!pu || !p || !m) return NULL;

163 +

164 + // normalize the url

165 + delete_url = registry_fix_url(delete_url, NULL);

166 +

167 + // make sure the user is not deleting the url it uses

168 + if(!strcmp(delete_url, pu->url->url)) {

169 + info("Registry Delete Request: delete URL is the one currently accessed, person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);

170 + return NULL;

171 + }

172 +

173 + REGISTRY_PERSON_URL *dpu = dictionary_get(p->person_urls, delete_url);

174 + if(!dpu) {

175 + info("Registry Delete Request: URL not found for person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);

176 + return NULL;

177 + }

178 +

179 + registry_log('D', p, m, pu->url, dpu->url->url);

180 +

181 + dictionary_del(p->person_urls, dpu->url->url);

182 +

183 + registry_url_unlink(dpu->url);

184 +

185 + freez(dpu);

186 +

187 + return p;

188 +}

189 +

190 +

191 +// a structure to pass to the dictionary_get_all() callback handler

192 +struct machine_request_callback_data {

193 + REGISTRY_MACHINE *find_this_machine;

194 + REGISTRY_PERSON_URL *result;

195 +};

196 +

197 +// the callback function

198 +// this will be run for every PERSON_URL of this PERSON

199 +static int machine_request_callback(void *entry, void *data) {

200 + REGISTRY_PERSON_URL *mypu = (REGISTRY_PERSON_URL *)entry;

201 + struct machine_request_callback_data *myrdata = (struct machine_request_callback_data *)data;

202 +

203 + if(mypu->machine == myrdata->find_this_machine) {

204 + myrdata->result = mypu;

205 + return -1; // this will also stop the walk through

206 + }

207 +

208 + return 0; // continue

209 +}

210 +

211 +REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {

212 + (void)when;

213 +

214 + char mbuf[GUID_LEN + 1];

215 +

216 + REGISTRY_PERSON *p = NULL;

217 + REGISTRY_MACHINE *m = NULL;

218 + REGISTRY_PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);

219 + if(!pu || !p || !m) return NULL;

220 +

221 + // make sure the machine GUID is valid

222 + if(registry_regenerate_guid(request_machine, mbuf) == -1) {

223 + info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);

224 + return NULL;

225 + }

226 + request_machine = mbuf;

227 +

228 + // make sure the machine exists

229 + m = registry_machine_find(request_machine);

230 + if(!m) {

231 + info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, machine_guid, pu->url->url, request_machine);

232 + return NULL;

233 + }

234 +

235 + // Verify the user has in the past accessed this machine

236 + // We will walk through the PERSON_URLs to find the machine

237 + // linking to our machine

238 +

239 + // a structure to pass to the dictionary_get_all() callback handler

240 + struct machine_request_callback_data rdata = { m, NULL };

241 +

242 + // request a walk through on the dictionary

243 + // no need for locking here, the underlying dictionary has its own

244 + dictionary_get_all(p->person_urls, machine_request_callback, &rdata);

245 +

246 + if(rdata.result)

247 + return m;

248 +

249 + return NULL;

250 +}

251 +

252 +

253 +// ----------------------------------------------------------------------------

254 +// REGISTRY THIS MACHINE UNIQUE ID

255 +

256 +static inline int is_machine_guid_blacklisted(const char *guid) {

257 + // these are machine GUIDs that have been included in distribution packages.

258 + // we blacklist them here, so that the next version of netdata will generate

259 + // new ones.

260 +

261 + if(!strcmp(guid, "8a795b0c-2311-11e6-8563-000c295076a6")

262 + || !strcmp(guid, "4aed1458-1c3e-11e6-a53f-000c290fc8f5")

263 + ) {

264 + error("Blacklisted machine GUID '%s' found.", guid);

265 + return 1;

266 + }

267 +

268 + return 0;

269 +}

270 +

271 +char *registry_get_this_machine_guid(void) {

272 + if(likely(registry.machine_guid[0]))

273 + return registry.machine_guid;

274 +

275 + // read it from disk

276 + int fd = open(registry.machine_guid_filename, O_RDONLY);

277 + if(fd != -1) {

278 + char buf[GUID_LEN + 1];

279 + if(read(fd, buf, GUID_LEN) != GUID_LEN)

280 + error("Failed to read machine GUID from '%s'", registry.machine_guid_filename);

281 + else {

282 + buf[GUID_LEN] = '\0';

283 + if(registry_regenerate_guid(buf, registry.machine_guid) == -1) {

284 + error("Failed to validate machine GUID '%s' from '%s'. Ignoring it - this might mean this netdata will appear as duplicate in the registry.",

285 + buf, registry.machine_guid_filename);

286 +

287 + registry.machine_guid[0] = '\0';

288 + }

289 + else if(is_machine_guid_blacklisted(registry.machine_guid))

290 + registry.machine_guid[0] = '\0';

291 + }

292 + close(fd);

293 + }

294 +

295 + // generate a new one?

296 + if(!registry.machine_guid[0]) {

297 + uuid_t uuid;

298 +

299 + uuid_generate_time(uuid);

300 + uuid_unparse_lower(uuid, registry.machine_guid);

301 + registry.machine_guid[GUID_LEN] = '\0';

302 +

303 + // save it

304 + fd = open(registry.machine_guid_filename, O_WRONLY|O_CREAT|O_TRUNC, 444);

305 + if(fd == -1)

306 + fatal("Cannot create unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

307 +

308 + if(write(fd, registry.machine_guid, GUID_LEN) != GUID_LEN)

309 + fatal("Cannot write the unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

310 +

311 + close(fd);

312 + }

313 +

314 + setenv("NETDATA_REGISTRY_UNIQUE_ID", registry.machine_guid, 1);

315 +

316 + return registry.machine_guid;

317 +}

src/registry_internals.h new

+92

@@ -0,0 +1,92 @@

1 +#include "common.h"

2 +

3 +#ifndef NETDATA_REGISTRY_INTERNALS_H_H

4 +#define NETDATA_REGISTRY_INTERNALS_H_H

5 +

6 +#define REGISTRY_URL_FLAGS_DEFAULT 0x00

7 +#define REGISTRY_URL_FLAGS_EXPIRED 0x01

8 +

9 +#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED

10 +

11 +// ----------------------------------------------------------------------------

12 +// COMMON structures

13 +

14 +struct registry {

15 + int enabled;

16 +

17 + char machine_guid[GUID_LEN + 1];

18 +

19 + // entries counters / statistics

20 + unsigned long long persons_count;

21 + unsigned long long machines_count;

22 + unsigned long long usages_count;

23 + unsigned long long urls_count;

24 + unsigned long long persons_urls_count;

25 + unsigned long long machines_urls_count;

26 + unsigned long long log_count;

27 +

28 + // memory counters / statistics

29 + unsigned long long persons_memory;

30 + unsigned long long machines_memory;

31 + unsigned long long urls_memory;

32 + unsigned long long persons_urls_memory;

33 + unsigned long long machines_urls_memory;

34 +

35 + // configuration

36 + unsigned long long save_registry_every_entries;

37 + char *registry_domain;

38 + char *hostname;

39 + char *registry_to_announce;

40 + time_t persons_expiration; // seconds to expire idle persons

41 + int verify_cookies_redirects;

42 +

43 + size_t max_url_length;

44 + size_t max_name_length;

45 +

46 + // file/path names

47 + char *pathname;

48 + char *db_filename;

49 + char *log_filename;

50 + char *machine_guid_filename;

51 +

52 + // open files

53 + FILE *log_fp;

54 +

55 + // the database

56 + DICTIONARY *persons; // dictionary of REGISTRY_PERSON *, with key the REGISTRY_PERSON.guid

57 + DICTIONARY *machines; // dictionary of REGISTRY_MACHINE *, with key the REGISTRY_MACHINE.guid

58 +

59 + avl_tree registry_urls_root_index;

60 +

61 + pthread_mutex_t lock;

62 +};

63 +

64 +extern int registry_regenerate_guid(const char *guid, char *result);

65 +

66 +#include "registry_url.h"

67 +#include "registry_machine.h"

68 +#include "registry_person.h"

69 +#include "registry.h"

70 +

71 +extern struct registry registry;

72 +

73 +extern char *registry_get_this_machine_guid(void);

74 +

75 +// REGISTRY LOW-LEVEL REQUESTS (in registry-internals.c)

76 +extern REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when);

77 +extern REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);

78 +extern REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when);

79 +

80 +// REGISTRY LOG (in registry_log.c)

81 +extern void registry_log(const char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name);

82 +extern int registry_log_open(void);

83 +extern void registry_log_close(void);

84 +extern void registry_log_recreate(void);

85 +extern ssize_t registry_log_load(void);

86 +

87 +// REGISTRY DB (in registry_db.c)

88 +extern int registry_db_save(void);

89 +extern size_t registry_db_load(void);

90 +extern int registry_db_should_be_saved(void);

91 +

92 +#endif //NETDATA_REGISTRY_INTERNALS_H_H

src/registry_log.c new

+133

@@ -0,0 +1,133 @@

1 +#include "registry_internals.h"

2 +

3 +void registry_log(const char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name) {

4 + if(likely(registry.log_fp)) {

5 + if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",

6 + action,

7 + p->last_t,

8 + p->guid,

9 + m->guid,

10 + name,

11 + u->url) < 0))

12 + error("Registry: failed to save log. Registry data may be lost in case of abnormal restart.");

13 +

14 + // we increase the counter even on failures

15 + // so that the registry will be saved periodically

16 + registry.log_count++;

17 +

18 + // this must be outside the log_lock(), or a deadlock will happen.

19 + // registry_db_save() checks the same inside the log_lock, so only

20 + // one thread will save the db

21 + if(unlikely(registry_db_should_be_saved()))

22 + registry_db_save();

23 + }

24 +}

25 +

26 +int registry_log_open(void) {

27 + if(registry.log_fp)

28 + fclose(registry.log_fp);

29 +

30 + registry.log_fp = fopen(registry.log_filename, "a");

31 + if(registry.log_fp) {

32 + if (setvbuf(registry.log_fp, NULL, _IOLBF, 0) != 0)

33 + error("Cannot set line buffering on registry log file.");

34 + return 0;

35 + }

36 +

37 + error("Cannot open registry log file '%s'. Registry data will be lost in case of netdata or server crash.", registry.log_filename);

38 + return -1;

39 +}

40 +

41 +void registry_log_close(void) {

42 + if(registry.log_fp) {

43 + fclose(registry.log_fp);

44 + registry.log_fp = NULL;

45 + }

46 +}

47 +

48 +void registry_log_recreate(void) {

49 + if(registry.log_fp != NULL) {

50 + registry_log_close();

51 +

52 + // open it with truncate

53 + registry.log_fp = fopen(registry.log_filename, "w");

54 + if(registry.log_fp) fclose(registry.log_fp);

55 + else error("Cannot truncate registry log '%s'", registry.log_filename);

56 +

57 + registry.log_fp = NULL;

58 + registry_log_open();

59 + }

60 +}

61 +

62 +ssize_t registry_log_load(void) {

63 + ssize_t line = -1;

64 +

65 + // closing the log is required here

66 + // otherwise we will append to it the values we read

67 + registry_log_close();

68 +

69 + debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename);

70 + FILE *fp = fopen(registry.log_filename, "r");

71 + if(!fp)

72 + error("Registry: cannot open registry file: %s", registry.log_filename);

73 + else {

74 + char *s, buf[4096 + 1];

75 + line = 0;

76 + size_t len = 0;

77 +

78 + while ((s = fgets_trim_len(buf, 4096, fp, &len))) {

79 + line++;

80 +

81 + switch (s[0]) {

82 + case 'A': // accesses

83 + case 'D': // deletes

84 +

85 + // verify it is valid

86 + if (unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) {

87 + error("Registry: log line %zd is wrong (len = %zu).", line, len);

88 + continue;

89 + }

90 + s[1] = s[10] = s[47] = s[84] = '\0';

91 +

92 + // get the variables

93 + time_t when = strtoul(&s[2], NULL, 16);

94 + char *person_guid = &s[11];

95 + char *machine_guid = &s[48];

96 + char *name = &s[85];

97 +

98 + // skip the name to find the url

99 + char *url = name;

100 + while(*url && *url != '\t') url++;

101 + if(!*url) {

102 + error("Registry: log line %zd does not have a url.", line);

103 + continue;

104 + }

105 + *url++ = '\0';

106 +

107 + // make sure the person exists

108 + // without this, a new person guid will be created

109 + REGISTRY_PERSON *p = registry_person_find(person_guid);

110 + if(!p) p = registry_person_allocate(person_guid, when);

111 +

112 + if(s[0] == 'A')

113 + registry_request_access(p->guid, machine_guid, url, name, when);

114 + else

115 + registry_request_delete(p->guid, machine_guid, url, name, when);

116 +

117 + registry.log_count++;

118 + break;

119 +

120 + default:

121 + error("Registry: ignoring line %zd of filename '%s': %s.", line, registry.log_filename, s);

122 + break;

123 + }

124 + }

125 +

126 + fclose(fp);

127 + }

128 +

129 + // open the log again

130 + registry_log_open();

131 +

132 + return line;

133 +}

src/registry_machine.c new

+101

@@ -0,0 +1,101 @@

1 +#include "registry_internals.h"

2 +

3 +// ----------------------------------------------------------------------------

4 +// MACHINE

5 +

6 +REGISTRY_MACHINE *registry_machine_find(const char *machine_guid) {

7 + debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);

8 + return dictionary_get(registry.machines, machine_guid);

9 +}

10 +

11 +REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) {

12 + debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(REGISTRY_MACHINE_URL));

13 +

14 + REGISTRY_MACHINE_URL *mu = mallocz(sizeof(REGISTRY_MACHINE_URL));

15 +

16 + mu->first_t = mu->last_t = (uint32_t)when;

17 + mu->usages = 1;

18 + mu->url = u;

19 + mu->flags = REGISTRY_URL_FLAGS_DEFAULT;

20 +

21 + registry.machines_urls_memory += sizeof(REGISTRY_MACHINE_URL);

22 +

23 + debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): indexing URL in machine", m->guid, u->url);

24 + dictionary_set(m->machine_urls, u->url, mu, sizeof(REGISTRY_MACHINE_URL));

25 +

26 + registry_url_link(u);

27 +

28 + return mu;

29 +}

30 +

31 +REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) {

32 + debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(REGISTRY_MACHINE));

33 +

34 + REGISTRY_MACHINE *m = mallocz(sizeof(REGISTRY_MACHINE));

35 +

36 + strncpyz(m->guid, machine_guid, GUID_LEN);

37 +

38 + debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);

39 + m->machine_urls = dictionary_create(DICTIONARY_FLAGS);

40 +

41 + m->first_t = m->last_t = (uint32_t)when;

42 + m->usages = 0;

43 +

44 + registry.machines_memory += sizeof(REGISTRY_MACHINE);

45 +

46 + registry.machines_count++;

47 + dictionary_set(registry.machines, m->guid, m, sizeof(REGISTRY_MACHINE));

48 +

49 + return m;

50 +}

51 +

52 +// 1. validate machine GUID

53 +// 2. if it is valid, find it or create it and return it

54 +// 3. if it is not valid, return NULL

55 +REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when) {

56 + REGISTRY_MACHINE *m = NULL;

57 +

58 + if(likely(machine_guid && *machine_guid)) {

59 + // validate it is a GUID

60 + char buf[GUID_LEN + 1];

61 + if(unlikely(registry_regenerate_guid(machine_guid, buf) == -1))

62 + info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);

63 + else {

64 + machine_guid = buf;

65 + m = registry_machine_find(machine_guid);

66 + if(!m) m = registry_machine_allocate(machine_guid, when);

67 + }

68 + }

69 +

70 + return m;

71 +}

72 +

73 +

74 +// ----------------------------------------------------------------------------

75 +// LINKING OF OBJECTS

76 +

77 +REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) {

78 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): searching for URL in machine", m->guid, u->url);

79 +

80 + REGISTRY_MACHINE_URL *mu = dictionary_get(m->machine_urls, u->url);

81 + if(!mu) {

82 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): not found", m->guid, u->url);

83 + mu = registry_machine_url_allocate(m, u, when);

84 + registry.machines_urls_count++;

85 + }

86 + else {

87 + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): found", m->guid, u->url);

88 + mu->usages++;

89 + if(likely(mu->last_t < (uint32_t)when)) mu->last_t = (uint32_t)when;

90 + }

91 +

92 + m->usages++;

93 + if(likely(m->last_t < (uint32_t)when)) m->last_t = (uint32_t)when;

94 +

95 + if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) {

96 + info("registry_machine_link_to_url('%s', '%s'): accessing an expired URL.", m->guid, u->url);

97 + mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;

98 + }

99 +

100 + return mu;

101 +}

src/registry_machine.h new

+41

@@ -0,0 +1,41 @@

1 +#ifndef NETDATA_REGISTRY_MACHINE_H

2 +#define NETDATA_REGISTRY_MACHINE_H

3 +

4 +#include "registry_internals.h"

5 +

6 +// ----------------------------------------------------------------------------

7 +// MACHINE structures

8 +

9 +// For each MACHINE-URL pair we keep this

10 +struct registry_machine_url {

11 + REGISTRY_URL *url; // de-duplicated URL

12 +

13 + uint8_t flags;

14 +

15 + uint32_t first_t; // the first time we saw this

16 + uint32_t last_t; // the last time we saw this

17 + uint32_t usages; // how many times this has been accessed

18 +};

19 +typedef struct registry_machine_url REGISTRY_MACHINE_URL;

20 +

21 +// A machine

22 +struct registry_machine {

23 + char guid[GUID_LEN + 1]; // the GUID

24 +

25 + uint32_t links; // the number of REGISTRY_PERSON_URL linked to this machine

26 +

27 + DICTIONARY *machine_urls; // MACHINE_URL *

28 +

29 + uint32_t first_t; // the first time we saw this

30 + uint32_t last_t; // the last time we saw this

31 + uint32_t usages; // how many times this has been accessed

32 +};

33 +typedef struct registry_machine REGISTRY_MACHINE;

34 +

35 +extern REGISTRY_MACHINE *registry_machine_find(const char *machine_guid);

36 +extern REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when);

37 +extern REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t when);

38 +extern REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when);

39 +extern REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when);

40 +

41 +#endif //NETDATA_REGISTRY_MACHINE_H

src/registry_person.c new

+175

@@ -0,0 +1,175 @@

1 +#include "registry_internals.h"

2 +

3 +// ----------------------------------------------------------------------------

4 +// PERSON

5 +

6 +REGISTRY_PERSON *registry_person_find(const char *person_guid) {

7 + debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid);

8 + return dictionary_get(registry.persons, person_guid);

9 +}

10 +

11 +REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) {

12 + // protection from too big names

13 + if(namelen > registry.max_name_length)

14 + namelen = registry.max_name_length;

15 +

16 + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,

17 + sizeof(REGISTRY_PERSON_URL) + namelen);

18 +

19 + REGISTRY_PERSON_URL *pu = mallocz(sizeof(REGISTRY_PERSON_URL) + namelen);

20 +

21 + // a simple strcpy() should do the job

22 + // but I prefer to be safe, since the caller specified urllen

23 + strncpyz(pu->machine_name, name, namelen);

24 +

25 + pu->machine = m;

26 + pu->first_t = pu->last_t = (uint32_t)when;

27 + pu->usages = 1;

28 + pu->url = u;

29 + pu->flags = REGISTRY_URL_FLAGS_DEFAULT;

30 + m->links++;

31 +

32 + registry.persons_urls_memory += sizeof(REGISTRY_PERSON_URL) + namelen;

33 +

34 + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);

35 + dictionary_set(p->person_urls, u->url, pu, sizeof(REGISTRY_PERSON_URL));

36 + registry_url_link(u);

37 +

38 + return pu;

39 +}

40 +

41 +REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when, REGISTRY_PERSON_URL *pu) {

42 + // this function is needed to change the name of a PERSON_URL

43 +

44 + debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,

45 + sizeof(REGISTRY_PERSON_URL) + namelen);

46 +

47 + REGISTRY_PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when);

48 + tpu->first_t = pu->first_t;

49 + tpu->last_t = pu->last_t;

50 + tpu->usages = pu->usages;

51 +

52 + // ok, these are a hack - since the registry_person_url_allocate() is

53 + // adding these, we have to subtract them

54 + tpu->machine->links--;

55 + registry.persons_urls_memory -= sizeof(REGISTRY_PERSON_URL) + strlen(pu->machine_name);

56 + registry_url_unlink(u);

57 +

58 + freez(pu);

59 +

60 + return tpu;

61 +}

62 +

63 +REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when) {

64 + REGISTRY_PERSON *p = NULL;

65 +

66 + debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(REGISTRY_PERSON));

67 +

68 + p = mallocz(sizeof(REGISTRY_PERSON));

69 +

70 + if(!person_guid) {

71 + for (; ;) {

72 + uuid_t uuid;

73 + uuid_generate(uuid);

74 + uuid_unparse_lower(uuid, p->guid);

75 +

76 + debug(D_REGISTRY, "Registry: Checking if the generated person guid '%s' is unique", p->guid);

77 + if (!dictionary_get(registry.persons, p->guid)) {

78 + debug(D_REGISTRY, "Registry: generated person guid '%s' is unique", p->guid);

79 + break;

80 + }

81 + else

82 + info("Registry: generated person guid '%s' found in the registry. Retrying...", p->guid);

83 + }

84 + }

85 + else

86 + strncpyz(p->guid, person_guid, GUID_LEN);

87 +

88 + debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid);

89 + p->person_urls = dictionary_create(DICTIONARY_FLAGS);

90 +

91 + p->first_t = p->last_t = (uint32_t)when;

92 + p->usages = 0;

93 +

94 + registry.persons_memory += sizeof(REGISTRY_PERSON);

95 +

96 + registry.persons_count++;

97 + dictionary_set(registry.persons, p->guid, p, sizeof(REGISTRY_PERSON));

98 +

99 + return p;

100 +}

101 +

102 +

103 +// 1. validate person GUID

104 +// 2. if it is valid, find it

105 +// 3. if it is not valid, create a new one

106 +// 4. return it

107 +REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when) {

108 + REGISTRY_PERSON *p = NULL;

109 +

110 + if(person_guid && *person_guid) {

111 + char buf[GUID_LEN + 1];

112 + // validate it is a GUID

113 + if(unlikely(registry_regenerate_guid(person_guid, buf) == -1))

114 + info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid);

115 + else {

116 + person_guid = buf;

117 + p = registry_person_find(person_guid);

118 + }

119 + }

120 +

121 + if(!p) p = registry_person_allocate(NULL, when);

122 +

123 + return p;

124 +}

125 +

126 +// ----------------------------------------------------------------------------

127 +// LINKING OF OBJECTS

128 +

129 +REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) {

130 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);

131 +

132 + REGISTRY_PERSON_URL *pu = dictionary_get(p->person_urls, u->url);

133 + if(!pu) {

134 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);

135 + pu = registry_person_url_allocate(p, m, u, name, namelen, when);

136 + registry.persons_urls_count++;

137 + }

138 + else {

139 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);

140 + pu->usages++;

141 + if(likely(pu->last_t < (uint32_t)when)) pu->last_t = (uint32_t)when;

142 +

143 + if(pu->machine != m) {

144 + REGISTRY_MACHINE_URL *mu = dictionary_get(pu->machine->machine_urls, u->url);

145 + if(mu) {

146 + info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.",

147 + p->guid, m->guid, u->url, pu->machine->guid);

148 + mu->flags |= REGISTRY_URL_FLAGS_EXPIRED;

149 + }

150 + else {

151 + info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - but the URL is not linked to the old machine.",

152 + p->guid, m->guid, u->url, pu->machine->guid);

153 + }

154 +

155 + pu->machine->links--;

156 + pu->machine = m;

157 + }

158 +

159 + if(strcmp(pu->machine_name, name)) {

160 + // the name of the PERSON_URL has changed !

161 + pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu);

162 + }

163 + }

164 +

165 + p->usages++;

166 + if(likely(p->last_t < (uint32_t)when)) p->last_t = (uint32_t)when;

167 +

168 + if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) {

169 + info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url);

170 + pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;

171 + }

172 +

173 + return pu;

174 +}

175 +

src/registry_person.h new

+44

@@ -0,0 +1,44 @@

1 +#ifndef NETDATA_REGISTRY_PERSON_H

2 +#define NETDATA_REGISTRY_PERSON_H

3 +

4 +#include "registry_internals.h"

5 +

6 +// ----------------------------------------------------------------------------

7 +// PERSON structures

8 +

9 +// for each PERSON-URL pair we keep this

10 +struct registry_person_url {

11 + REGISTRY_URL *url; // de-duplicated URL

12 + REGISTRY_MACHINE *machine; // link the MACHINE of this URL

13 +

14 + uint8_t flags;

15 +

16 + uint32_t first_t; // the first time we saw this

17 + uint32_t last_t; // the last time we saw this

18 + uint32_t usages; // how many times this has been accessed

19 +

20 + char machine_name[1]; // the name of the machine, as known by the user

21 + // dynamically allocated to fit properly

22 +};

23 +typedef struct registry_person_url REGISTRY_PERSON_URL;

24 +

25 +// A person

26 +struct registry_person {

27 + char guid[GUID_LEN + 1]; // the person GUID

28 +

29 + DICTIONARY *person_urls; // dictionary of PERSON_URL *

30 +

31 + uint32_t first_t; // the first time we saw this

32 + uint32_t last_t; // the last time we saw this

33 + uint32_t usages; // how many times this has been accessed

34 +};

35 +typedef struct registry_person REGISTRY_PERSON;

36 +

37 +extern REGISTRY_PERSON *registry_person_find(const char *person_guid);

38 +extern REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when);

39 +extern REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when, REGISTRY_PERSON_URL *pu);

40 +extern REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when);

41 +extern REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when);

42 +extern REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when);

43 +

44 +#endif //NETDATA_REGISTRY_PERSON_H

src/registry_url.c new

+74

@@ -0,0 +1,74 @@

1 +#include "registry_internals.h"

2 +

3 +// ----------------------------------------------------------------------------

4 +// REGISTRY_URL

5 +

6 +int registry_url_compare(void *a, void *b) {

7 + if(((REGISTRY_URL *)a)->hash < ((REGISTRY_URL *)b)->hash) return -1;

8 + else if(((REGISTRY_URL *)a)->hash > ((REGISTRY_URL *)b)->hash) return 1;

9 + else return strcmp(((REGISTRY_URL *)a)->url, ((REGISTRY_URL *)b)->url);

10 +}

11 +

12 +#define registry_url_index_add(rc) (REGISTRY_URL *)avl_insert(&(registry.registry_urls_root_index), (avl *)(rc))

13 +#define registry_url_index_del(rc) (REGISTRY_URL *)avl_remove(&(registry.registry_urls_root_index), (avl *)(rc))

14 +

15 +REGISTRY_URL *registry_url_get(const char *url, size_t urllen) {

16 + // protection from too big URLs

17 + if(urllen > registry.max_url_length)

18 + urllen = registry.max_url_length;

19 +

20 + debug(D_REGISTRY, "Registry: registry_url_get('%s', %zu)", url, urllen);

21 +

22 + char buf[sizeof(REGISTRY_URL) + urllen]; // no need for +1, 1 is already in REGISTRY_URL

23 + REGISTRY_URL *n = (REGISTRY_URL *)&buf[0];

24 + n->len = (uint16_t)urllen;

25 + strncpyz(n->url, url, n->len);

26 + n->hash = simple_hash(n->url);

27 +

28 + REGISTRY_URL *u = (REGISTRY_URL *)avl_search(&(registry.registry_urls_root_index), (avl *)n);

29 + if(!u) {

30 + debug(D_REGISTRY, "Registry: registry_url_get('%s', %zu): allocating %zu bytes", url, urllen, sizeof(REGISTRY_URL) + urllen);

31 + u = callocz(1, sizeof(REGISTRY_URL) + urllen); // no need for +1, 1 is already in REGISTRY_URL

32 +

33 + // a simple strcpy() should do the job

34 + // but I prefer to be safe, since the caller specified urllen

35 + u->len = (uint16_t)urllen;

36 + strncpyz(u->url, url, u->len);

37 + u->links = 0;

38 + u->hash = simple_hash(u->url);

39 +

40 + registry.urls_memory += sizeof(REGISTRY_URL) + urllen; // no need for +1, 1 is already in REGISTRY_URL

41 +

42 + debug(D_REGISTRY, "Registry: registry_url_get('%s'): indexing it", url);

43 + n = registry_url_index_add(u);

44 + if(n != u) {

45 + error("INTERNAL ERROR: registry_url_get(): url '%s' already exists in the registry as '%s'", u->url, n->url);

46 + free(u);

47 + u = n;

48 + }

49 + else

50 + registry.urls_count++;

51 + }

52 +

53 + return u;

54 +}

55 +

56 +void registry_url_link(REGISTRY_URL *u) {

57 + u->links++;

58 + debug(D_REGISTRY, "Registry: registry_url_link('%s'): URL has now %u links", u->url, u->links);

59 +}

60 +

61 +void registry_url_unlink(REGISTRY_URL *u) {

62 + u->links--;

63 + if(!u->links) {

64 + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): No more links for this URL", u->url);

65 + REGISTRY_URL *n = registry_url_index_del(u);

66 + if(n != u)

67 + error("INTERNAL ERROR: registry_url_unlink(): request to delete url '%s', deleted url '%s'", u->url, n->url);

68 +

69 + registry.urls_memory -= sizeof(REGISTRY_URL) + n->len; // no need for +1, 1 is already in REGISTRY_URL

70 + freez(n);

71 + }

72 + else

73 + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): URL has %u links left", u->url, u->links);

74 +}

src/registry_url.h new

+28

@@ -0,0 +1,28 @@

1 +#ifndef NETDATA_REGISTRY_URL_H

2 +#define NETDATA_REGISTRY_URL_H

3 +

4 +#include "registry_internals.h"

5 +

6 +// ----------------------------------------------------------------------------

7 +// URL structures

8 +// Save memory by de-duplicating URLs

9 +// so instead of storing URLs all over the place

10 +// we store them here and we keep pointers elsewhere

11 +

12 +struct registry_url {

13 + avl avl;

14 + uint32_t hash; // the index hash

15 +

16 + uint32_t links; // the number of links to this URL - when none is left, we free it

17 +

18 + uint16_t len; // the length of the URL in bytes

19 + char url[1]; // the URL - dynamically allocated to more size

20 +};

21 +typedef struct registry_url REGISTRY_URL;

22 +

23 +extern int registry_url_compare(void *a, void *b);

24 +extern REGISTRY_URL *registry_url_get(const char *url, size_t urllen) __attribute__((returns_nonnull));

25 +extern void registry_url_link(REGISTRY_URL *u);

26 +extern void registry_url_unlink(REGISTRY_URL *u);

27 +

28 +#endif //NETDATA_REGISTRY_URL_H

src/web_client.c

+1 -57

@@ -1213,8 +1213,6 @@ cleanup:

1213 }

1214

1215

1216 -#define REGISTRY_VERIFY_COOKIES_GUID "give-me-back-this-cookie-now--please"

1217 -

1216 int web_client_api_request_v1_registry(struct web_client *w, char *url)

1217 {

1218 static uint32_t hash_action = 0, hash_access = 0, hash_hello = 0, hash_delete = 0, hash_search = 0,

@@ -1239,7 +1237,7 @@ int web_client_api_request_v1_registry(struct web_client *w, char *url)

1237 */

1238 }

1239

1242 - char person_guid[36 + 1] = "";

1240 + char person_guid[GUID_LEN + 1] = "";

1241

1242 debug(D_WEB_CLIENT, "%llu: API v1 registry with URL '%s'", w->id, url);

1243

@@ -1354,60 +1352,6 @@ int web_client_api_request_v1_registry(struct web_client *w, char *url)

1352 switch(action) {

1353 case 'A':

1354 w->tracking_required = 1;

1357 - if(registry_verify_cookies_redirects() > 0 && (!cookie || !person_guid[0])) {

1358 - buffer_flush(w->response.data);

1359 - registry_set_cookie(w, REGISTRY_VERIFY_COOKIES_GUID);

1360 - w->response.data->contenttype = CT_APPLICATION_JSON;

1361 - buffer_sprintf(w->response.data, "{ \"status\": \"redirect\", \"registry\": \"%s\" }", registry_to_announce());

1362 - return 200;

1363 -

1364 -/*

1365 - * it seems that web browsers are ignoring 307 (Moved Temporarily)

1366 - * under certain conditions, when using CORS

1367 - * so this is commented and we use application level redirects instead

1368 - *

1369 - redirects++;

1370 -

1371 - if(redirects > registry_verify_cookies_redirects()) {

1372 - buffer_flush(w->response.data);

1373 - buffer_sprintf(w->response.data, "Your browser does not support cookies");

1374 - return 400;

1375 - }

1376 -

1377 - char *encoded_url = url_encode(machine_url);

1378 - if(!encoded_url) {

1379 - error("%llu: Cannot URL encode string '%s'", w->id, machine_url);

1380 - return 500;

1381 - }

1382 -

1383 - char *encoded_name = url_encode(url_name);

1384 - if(!encoded_name) {

1385 - free(encoded_url);

1386 - error("%llu: Cannot URL encode string '%s'", w->id, url_name);

1387 - return 500;

1388 - }

1389 -

1390 - char *encoded_guid = url_encode(machine_guid);

1391 - if(!encoded_guid) {

1392 - free(encoded_url);

1393 - free(encoded_name);

1394 - error("%llu: Cannot URL encode string '%s'", w->id, machine_guid);

1395 - return 500;

1396 - }

1397 -

1398 - buffer_sprintf(w->response.header, "Location: %s/api/v1/registry?action=access&machine=%s&name=%s&url=%s&redirects=%d\r\n",

1399 - registry_to_announce(), encoded_guid, encoded_name, encoded_url, redirects);

1400 -

1401 - free(encoded_guid);

1402 - free(encoded_name);

1403 - free(encoded_url);

1404 - return 307

1405 -*/

1406 - }

1407 -

1408 - if(unlikely(cookie && person_guid[0] && !strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID)))

1409 - person_guid[0] = '\0';

1410 -

1355 return registry_request_access_json(w, person_guid, machine_guid, machine_url, url_name, now_realtime_sec());

1356

1357 case 'D':