@cryptotaxi247 / netdata-1 / commits / 5943203a6

bearer authorization API (#15321)

* bearer authorization API - untested * add machine guid to bearer token response * removed REGISTRY_URL and replaced it with STRING * eliminate url pointer from registry_machine_url * remove registry_url counters from registry * Revert "eliminate url pointer from registry_machine_url" This reverts commit 79eff56f77d12a9f840f1b30040e4123e3d9a7a2. * registry machine urls are now a double linked list * registry machine urls are now using aral * all registry objects now use aral * strings now have 64 partitions and use R/W spinlock * string to 128 partitions * fix macro without internal checks * registry now uses the bearer token when the cookie is not there * api/v1/registry sends back all nodes on each host * registry option to use mmap; optimization of registry structures * do not index the terminator byte in strings; use 256 string partitions * registry loading optimization * convert person urls to double linked list to save memory * re-organize items loading and make sure person urls are always available as machine urls too * disable registry mmap by default * keep track of all machine guids and their URLs, even if the cookie cannot be set * fix bearer parsing

Costa Tsaousis committed Jul 10, 2023 at 18:02 UTC 5943203a66b6b98e9619c8c2825aad517b209b1c

23 files changed +868 -597

CMakeLists.txt

-2

@@ -700,8 +700,6 @@ set(REGISTRY_PLUGIN_FILES

700 registry/registry_machine.h

701 registry/registry_person.c

702 registry/registry_person.h

703 - registry/registry_url.c

704 - registry/registry_url.h

703 )

704

705 set(STATSD_PLUGIN_FILES

Makefile.am

-2

@@ -661,8 +661,6 @@ REGISTRY_PLUGIN_FILES = \

661 registry/registry_machine.h \

662 registry/registry_person.c \

663 registry/registry_person.h \

664 - registry/registry_url.c \

665 - registry/registry_url.h \

664 $(NULL)

665

666 STATSD_PLUGIN_FILES = \

libnetdata/avl/avl.h

@@ -24,6 +24,11 @@ typedef struct avl_element {

24 signed char avl_balance; /* Balance factor. */

25 } avl_t;

27 +typedef struct __attribute__((packed)) avl_element_packed {

28 + struct avl_element *avl_link[2]; /* Subtrees. */

29 + signed char avl_balance; /* Balance factor. */

30 +} avl_t_packed;

31 +

32 /* An AVL tree */

33 typedef struct avl_tree_type {

34 avl_t *root;

libnetdata/dictionary/dictionary.c

-4

@@ -369,10 +369,6 @@ size_t dictionary_referenced_items(DICTIONARY *dict) {

369 return referenced_items;

370 }

371

372 -long int dictionary_stats_for_registry(DICTIONARY *dict) {

373 - if(unlikely(!dict)) return 0;

374 - return (dict->stats->memory.index + dict->stats->memory.dict);

375 -}

372 void dictionary_version_increment(DICTIONARY *dict) {

373 __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);

374 }

libnetdata/dictionary/dictionary.h

-1

@@ -310,7 +310,6 @@ void dictionary_foreach_unlock(DICTFE *dfe);

310 size_t dictionary_version(DICTIONARY *dict);

311 size_t dictionary_entries(DICTIONARY *dict);

312 size_t dictionary_referenced_items(DICTIONARY *dict);

313 -long int dictionary_stats_for_registry(DICTIONARY *dict);

313

314 // for all cases that the caller does not provide a stats structure, this is where they are accumulated.

315 extern struct dictionary_stats dictionary_stats_category_other;

libnetdata/socket/socket.h

+10 -2

@@ -11,7 +11,7 @@

12 typedef enum web_client_acl {

13 WEB_CLIENT_ACL_NONE = (0),

14 - WEB_CLIENT_ACL_NOCHECK = (0),

14 + WEB_CLIENT_ACL_NOCHECK = (0), // Don't check anything - this should work on all channels

15 WEB_CLIENT_ACL_DASHBOARD = (1 << 0),

16 WEB_CLIENT_ACL_REGISTRY = (1 << 1),

17 WEB_CLIENT_ACL_BADGE = (1 << 2),

@@ -23,9 +23,17 @@ typedef enum web_client_acl {

23 WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 8),

24 WEB_CLIENT_ACL_ACLK = (1 << 9),

25 WEB_CLIENT_ACL_WEBRTC = (1 << 10),

26 + WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 11), // allow unprotected access if bearer is not enabled in netdata

27 + WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 12), // allow access only if a valid bearer is used

28 } WEB_CLIENT_ACL;

28 -#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC)

30 +#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL)

31 +

32 +#ifdef NETDATA_DEV_MODE

33 +#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_DASHBOARD

34 +#else

35 +#define ACL_DEV_OPEN_ACCESS 0

36 +#endif

38 #define WEB_CLIENT_ACL_ALL 0xFFFF

libnetdata/string/string.c

+144 -82

@@ -8,6 +8,11 @@ typedef int32_t REFCOUNT;

8 // ----------------------------------------------------------------------------

9 // STRING implementation - dedup all STRING

11 +#define STRING_PARTITION_SHIFTS (0)

12 +#define STRING_PARTITIONS (256 >> STRING_PARTITION_SHIFTS)

13 +#define string_partition_str(str) ((uint8_t)((str)[0]) >> STRING_PARTITION_SHIFTS)

14 +#define string_partition(string) (string_partition_str((string)->str))

15 +

16 struct netdata_string {

17 uint32_t length; // the string length including the terminating '\0'

@@ -18,20 +23,22 @@ struct netdata_string {

23 const char str[]; // the string itself, is appended to this structure

24 };

21 -static struct string_hashtable {

22 - Pvoid_t JudyHSArray; // the Judy array - hashtable

23 - netdata_rwlock_t rwlock; // the R/W lock to protect the Judy array

26 +static struct string_partition {

27 + RW_SPINLOCK spinlock; // the R/W spinlock to protect the Judy array

25 - long int entries; // the number of entries in the index

26 - long int active_references; // the number of active references alive

27 - long int memory; // the memory used, without the JudyHS index

29 + Pvoid_t JudyHSArray; // the Judy array - hashtable

29 - size_t inserts; // the number of successful inserts to the index

30 - size_t deletes; // the number of successful deleted from the index

31 size_t searches; // the number of successful searches in the index

32 size_t duplications; // when a string is referenced

33 size_t releases; // when a string is unreferenced

35 + size_t inserts; // the number of successful inserts to the index

36 + size_t deletes; // the number of successful deleted from the index

37 +

38 + long int entries; // the number of entries in the index

39 + long int active_references; // the number of active references alive

40 + long int memory; // the memory used, without the JudyHS index

41 +

42 #ifdef NETDATA_INTERNAL_CHECKS

43 // internal statistics

44 size_t found_deleted_on_search;

@@ -41,50 +48,45 @@ static struct string_hashtable {

48 size_t spins;

49 #endif

44 -} string_base = {

45 - .JudyHSArray = NULL,

46 - .rwlock = NETDATA_RWLOCK_INITIALIZER,

47 -};

51 +} string_base[STRING_PARTITIONS] = { 0 };

53 #ifdef NETDATA_INTERNAL_CHECKS

50 -#define string_internal_stats_add(var, val) __atomic_add_fetch(&string_base.var, val, __ATOMIC_RELAXED)

54 +#define string_internal_stats_add(partition, var, val) __atomic_add_fetch(&string_base[partition].var, val, __ATOMIC_RELAXED)

55 #else

52 -#define string_internal_stats_add(var, val) do {;} while(0)

56 +#define string_internal_stats_add(partition, var, val) do {;} while(0)

57 #endif

55 -#define string_stats_atomic_increment(var) __atomic_add_fetch(&string_base.var, 1, __ATOMIC_RELAXED)

56 -#define string_stats_atomic_decrement(var) __atomic_sub_fetch(&string_base.var, 1, __ATOMIC_RELAXED)

59 +#define string_stats_atomic_increment(partition, var) __atomic_add_fetch(&string_base[partition].var, 1, __ATOMIC_RELAXED)

60 +#define string_stats_atomic_decrement(partition, var) __atomic_sub_fetch(&string_base[partition].var, 1, __ATOMIC_RELAXED)

62 void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) {

59 - if(inserts)

60 - *inserts = string_base.inserts;

61 -

62 - if(deletes)

63 - *deletes = string_base.deletes;

64 -

65 - if(searches)

66 - *searches = string_base.searches;

67 -

68 - if(entries)

69 - *entries = (size_t)string_base.entries;

70 -

71 - if(references)

72 - *references = (size_t)string_base.active_references;

73 -

74 - if(memory)

75 - *memory = (size_t)string_base.memory;

76 -

77 - if(duplications)

78 - *duplications = string_base.duplications;

79 -

80 - if(releases)

81 - *releases = string_base.releases;

63 + if (inserts) *inserts = 0;

64 + if (deletes) *deletes = 0;

65 + if (searches) *searches = 0;

66 + if (entries) *entries = 0;

67 + if (references) *references = 0;

68 + if (memory) *memory = 0;

69 + if (duplications) *duplications = 0;

70 + if (releases) *releases = 0;

71 +

72 + for(size_t i = 0; i < STRING_PARTITIONS ;i++) {

73 + if (inserts) *inserts += string_base[i].inserts;

74 + if (deletes) *deletes += string_base[i].deletes;

75 + if (searches) *searches += string_base[i].searches;

76 + if (entries) *entries += (size_t) string_base[i].entries;

77 + if (references) *references += (size_t) string_base[i].active_references;

78 + if (memory) *memory += (size_t) string_base[i].memory;

79 + if (duplications) *duplications += string_base[i].duplications;

80 + if (releases) *releases += string_base[i].releases;

81 + }

82 }

84 #define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST);

85 #define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST);

87 static inline bool string_entry_check_and_acquire(STRING *se) {

88 + uint8_t partition = string_partition(se);

89 +

90 REFCOUNT expected, desired, count = 0;

92 expected = __atomic_load_n(&se->refcount, __ATOMIC_SEQ_CST);

@@ -96,7 +98,7 @@ static inline bool string_entry_check_and_acquire(STRING *se) {

98 // We cannot use this.

99 // The reference counter reached value zero,

100 // so another thread is deleting this.

99 - string_internal_stats_add(spins, count - 1);

101 + string_internal_stats_add(partition, spins, count - 1);

102 return false;

103 }

104

@@ -104,11 +106,11 @@ static inline bool string_entry_check_and_acquire(STRING *se) {

106

107 } while(!__atomic_compare_exchange_n(&se->refcount, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));

108

107 - string_internal_stats_add(spins, count - 1);

109 + string_internal_stats_add(partition, spins, count - 1);

110

111 // statistics

112 // string_base.active_references is altered at the in string_strdupz() and string_freez()

111 - string_stats_atomic_increment(duplications);

113 + string_stats_atomic_increment(partition, duplications);

114

115 return true;

116 }

@@ -123,9 +125,11 @@ STRING *string_dup(STRING *string) {

125

126 string_entry_acquire(string);

127

128 + uint8_t partition = string_partition(string);

129 +

130 // statistics

127 - string_stats_atomic_increment(active_references);

128 - string_stats_atomic_increment(duplications);

131 + string_stats_atomic_increment(partition, active_references);

132 + string_stats_atomic_increment(partition, duplications);

133

134 return string;

135 }

@@ -134,26 +138,28 @@ STRING *string_dup(STRING *string) {

138 static inline STRING *string_index_search(const char *str, size_t length) {

139 STRING *string;

140

141 + uint8_t partition = string_partition_str(str);

142 +

143 // Find the string in the index

144 // With a read-lock so that multiple readers can use the index concurrently.

145

140 - netdata_rwlock_rdlock(&string_base.rwlock);

146 + rw_spinlock_read_lock(&string_base[partition].spinlock);

147

148 Pvoid_t *Rc;

143 - Rc = JudyHSGet(string_base.JudyHSArray, (void *)str, length);

149 + Rc = JudyHSGet(string_base[partition].JudyHSArray, (void *)str, length - 1);

150 if(likely(Rc)) {

151 // found in the hash table

152 string = *Rc;

153

154 if(string_entry_check_and_acquire(string)) {

155 // we can use this entry

150 - string_internal_stats_add(found_available_on_search, 1);

156 + string_internal_stats_add(partition, found_available_on_search, 1);

157 }

158 else {

159 // this entry is about to be deleted by another thread

160 // do not touch it, let it go...

161 string = NULL;

156 - string_internal_stats_add(found_deleted_on_search, 1);

162 + string_internal_stats_add(partition, found_deleted_on_search, 1);

163 }

164 }

165 else {

@@ -161,8 +167,8 @@ static inline STRING *string_index_search(const char *str, size_t length) {

167 string = NULL;

168 }

169

164 - string_stats_atomic_increment(searches);

165 - netdata_rwlock_unlock(&string_base.rwlock);

170 + string_stats_atomic_increment(partition, searches);

171 + rw_spinlock_read_unlock(&string_base[partition].spinlock);

172

173 return string;

174 }

@@ -175,12 +181,14 @@ static inline STRING *string_index_search(const char *str, size_t length) {

181 static inline STRING *string_index_insert(const char *str, size_t length) {

182 STRING *string;

183

178 - netdata_rwlock_wrlock(&string_base.rwlock);

184 + uint8_t partition = string_partition_str(str);

185 +

186 + rw_spinlock_write_lock(&string_base[partition].spinlock);

187

188 STRING **ptr;

189 {

190 JError_t J_Error;

183 - Pvoid_t *Rc = JudyHSIns(&string_base.JudyHSArray, (void *)str, length, &J_Error);

191 + Pvoid_t *Rc = JudyHSIns(&string_base[partition].JudyHSArray, (void *)str, length - 1, &J_Error);

192 if (unlikely(Rc == PJERR)) {

193 fatal(

194 "STRING: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d",

@@ -199,9 +207,9 @@ static inline STRING *string_index_insert(const char *str, size_t length) {

207 string->length = length;

208 string->refcount = 1;

209 *ptr = string;

202 - string_base.inserts++;

203 - string_base.entries++;

204 - string_base.memory += (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(length));

210 + string_base[partition].inserts++;

211 + string_base[partition].entries++;

212 + string_base[partition].memory += (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(length));

213 }

214 else {

215 // the item is already in the index

@@ -209,25 +217,27 @@ static inline STRING *string_index_insert(const char *str, size_t length) {

217

218 if(string_entry_check_and_acquire(string)) {

219 // we can use this entry

212 - string_internal_stats_add(found_available_on_insert, 1);

220 + string_internal_stats_add(partition, found_available_on_insert, 1);

221 }

222 else {

223 // this entry is about to be deleted by another thread

224 // do not touch it, let it go...

225 string = NULL;

218 - string_internal_stats_add(found_deleted_on_insert, 1);

226 + string_internal_stats_add(partition, found_deleted_on_insert, 1);

227 }

228

221 - string_stats_atomic_increment(searches);

229 + string_stats_atomic_increment(partition, searches);

230 }

231

224 - netdata_rwlock_unlock(&string_base.rwlock);

232 + rw_spinlock_write_unlock(&string_base[partition].spinlock);

233 return string;

234 }

235

236 // delete an entry from the index

237 static inline void string_index_delete(STRING *string) {

230 - netdata_rwlock_wrlock(&string_base.rwlock);

238 + uint8_t partition = string_partition(string);

239 +

240 + rw_spinlock_write_lock(&string_base[partition].spinlock);

241

242 #ifdef NETDATA_INTERNAL_CHECKS

243 if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) != 0))

@@ -236,9 +246,9 @@ static inline void string_index_delete(STRING *string) {

246

247 bool deleted = false;

248

239 - if (likely(string_base.JudyHSArray)) {

249 + if (likely(string_base[partition].JudyHSArray)) {

250 JError_t J_Error;

241 - int ret = JudyHSDel(&string_base.JudyHSArray, (void *)string->str, string->length, &J_Error);

251 + int ret = JudyHSDel(&string_base[partition].JudyHSArray, (void *)string->str, string->length - 1, &J_Error);

252 if (unlikely(ret == JERR)) {

253 netdata_log_error(

254 "STRING: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d",

@@ -253,18 +263,20 @@ static inline void string_index_delete(STRING *string) {

263 netdata_log_error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str);

264 else {

265 size_t mem_size = sizeof(STRING) + string->length;

256 - string_base.deletes++;

257 - string_base.entries--;

258 - string_base.memory -= (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(string->length));

266 + string_base[partition].deletes++;

267 + string_base[partition].entries--;

268 + string_base[partition].memory -= (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(string->length));

269 freez(string);

270 }

271

262 - netdata_rwlock_unlock(&string_base.rwlock);

272 + rw_spinlock_write_unlock(&string_base[partition].spinlock);

273 }

274

275 STRING *string_strdupz(const char *str) {

276 if(unlikely(!str || !*str)) return NULL;

277

278 + uint8_t partition = string_partition_str(str);

279 +

280 size_t length = strlen(str) + 1;

281 STRING *string = string_index_search(str, length);

282

@@ -277,7 +289,7 @@ STRING *string_strdupz(const char *str) {

289 }

290

291 // statistics

280 - string_stats_atomic_increment(active_references);

292 + string_stats_atomic_increment(partition, active_references);

293

294 return string;

295 }

@@ -285,6 +297,7 @@ STRING *string_strdupz(const char *str) {

297 void string_freez(STRING *string) {

298 if(unlikely(!string)) return;

299

300 + uint8_t partition = string_partition(string);

301 REFCOUNT refcount = string_entry_release(string);

302

303 #ifdef NETDATA_INTERNAL_CHECKS

@@ -296,8 +309,8 @@ void string_freez(STRING *string) {

309 string_index_delete(string);

310

311 // statistics

299 - string_stats_atomic_decrement(active_references);

300 - string_stats_atomic_increment(releases);

312 + string_stats_atomic_decrement(partition, active_references);

313 + string_stats_atomic_increment(partition, releases);

314 }

315

316 inline size_t string_strlen(STRING *string) {

@@ -405,6 +418,54 @@ static void string_unittest_free_char_pp(char **pp, size_t entries) {

418 freez(pp);

419 }

420

421 +static long unittest_string_entries(void) {

422 + long entries = 0;

423 + for(size_t p = 0; p < STRING_PARTITIONS ;p++)

424 + entries += string_base[p].entries;

425 +

426 + return entries;

427 +}

428 +

429 +#ifdef NETDATA_INTERNAL_CHECKS

430 +

431 +static size_t unittest_string_found_deleted_on_search(void) {

432 + size_t entries = 0;

433 + for(size_t p = 0; p < STRING_PARTITIONS ;p++)

434 + entries += string_base[p].found_deleted_on_search;

435 +

436 + return entries;

437 +}

438 +static size_t unittest_string_found_available_on_search(void) {

439 + size_t entries = 0;

440 + for(size_t p = 0; p < STRING_PARTITIONS ;p++)

441 + entries += string_base[p].found_available_on_search;

442 +

443 + return entries;

444 +}

445 +static size_t unittest_string_found_deleted_on_insert(void) {

446 + size_t entries = 0;

447 + for(size_t p = 0; p < STRING_PARTITIONS ;p++)

448 + entries += string_base[p].found_deleted_on_insert;

449 +

450 + return entries;

451 +}

452 +static size_t unittest_string_found_available_on_insert(void) {

453 + size_t entries = 0;

454 + for(size_t p = 0; p < STRING_PARTITIONS ;p++)

455 + entries += string_base[p].found_available_on_insert;

456 +

457 + return entries;

458 +}

459 +static size_t unittest_string_spins(void) {

460 + size_t entries = 0;

461 + for(size_t p = 0; p < STRING_PARTITIONS ;p++)

462 + entries += string_base[p].spins;

463 +

464 + return entries;

465 +}

466 +

467 +#endif // NETDATA_INTERNAL_CHECKS

468 +

469 int string_unittest(size_t entries) {

470 size_t errors = 0;

471

@@ -413,7 +474,7 @@ int string_unittest(size_t entries) {

474

475 // check string

476 {

416 - long int string_entries_starting = string_base.entries;

477 + long entries_starting = unittest_string_entries();

478

479 fprintf(stderr, "\nChecking strings...\n");

480

@@ -496,9 +557,10 @@ int string_unittest(size_t entries) {

557

558 freez(strings);

559

499 - if(string_base.entries != string_entries_starting + 2) {

560 + if(unittest_string_entries() != entries_starting + 2) {

561 errors++;

501 - fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, string_base.entries);

562 + fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n",

563 + entries_starting + 2, unittest_string_entries());

564 }

565 else

566 fprintf(stderr, "OK: strings dictionary has 2 items\n");

@@ -551,11 +613,11 @@ int string_unittest(size_t entries) {

613 };

614

615 #ifdef NETDATA_INTERNAL_CHECKS

554 - size_t ofound_deleted_on_search = string_base.found_deleted_on_search,

555 - ofound_available_on_search = string_base.found_available_on_search,

556 - ofound_deleted_on_insert = string_base.found_deleted_on_insert,

557 - ofound_available_on_insert = string_base.found_available_on_insert,

558 - ospins = string_base.spins;

616 + size_t ofound_deleted_on_search = unittest_string_found_deleted_on_search(),

617 + ofound_available_on_search = unittest_string_found_available_on_search(),

618 + ofound_deleted_on_insert = unittest_string_found_deleted_on_insert(),

619 + ofound_available_on_insert = unittest_string_found_available_on_insert(),

620 + ospins = unittest_string_spins();

621 #endif

622

623 size_t oinserts, odeletes, osearches, oentries, oreferences, omemory, oduplications, oreleases;

@@ -592,11 +654,11 @@ int string_unittest(size_t entries) {

654 inserts - oinserts, deletes - odeletes, searches - osearches, sentries - oentries, references - oreferences, memory - omemory, duplications - oduplications, releases - oreleases);

655

656 #ifdef NETDATA_INTERNAL_CHECKS

595 - size_t found_deleted_on_search = string_base.found_deleted_on_search,

596 - found_available_on_search = string_base.found_available_on_search,

597 - found_deleted_on_insert = string_base.found_deleted_on_insert,

598 - found_available_on_insert = string_base.found_available_on_insert,

599 - spins = string_base.spins;

657 + size_t found_deleted_on_search = unittest_string_found_deleted_on_search(),

658 + found_available_on_search = unittest_string_found_available_on_search(),

659 + found_deleted_on_insert = unittest_string_found_deleted_on_insert(),

660 + found_available_on_insert = unittest_string_found_available_on_insert(),

661 + spins = unittest_string_spins();

662

663 fprintf(stderr, "on insert: %zu ok + %zu deleted\non search: %zu ok + %zu deleted\nspins: %zu\n",

664 found_available_on_insert - ofound_available_on_insert,

registry/registry.c

+94 -41

@@ -8,6 +8,10 @@

8 #define REGISTRY_STATUS_FAILED "failed"

9 #define REGISTRY_STATUS_DISABLED "disabled"

11 +bool registry_is_valid_url(const char *url) {

12 + return url && (*url == 'h' || *url == '*');

13 +}

14 +

15 // ----------------------------------------------------------------------------

16 // REGISTRY concurrency locking

@@ -23,19 +27,19 @@ static inline void registry_unlock(void) {

27 // COOKIES

29 static void registry_set_cookie(struct web_client *w, const char *guid) {

26 - char edate[100];

30 + char e_date[100];

31 time_t et = now_realtime_sec() + registry.persons_expiration;

28 - struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);

29 - strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm);

32 + struct tm e_tm_buf, *etm = gmtime_r(&et, &e_tm_buf);

33 + strftime(e_date, sizeof(e_date), "%a, %d %b %Y %H:%M:%S %Z", etm);

31 - buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s\r\n", guid, edate);

35 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s\r\n", guid, e_date);

36 if(registry.enable_cookies_samesite_secure)

33 - buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; SameSite=None; Secure\r\n", guid, edate);

37 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; SameSite=None; Secure\r\n", guid, e_date);

39 if(registry.registry_domain && *registry.registry_domain) {

36 - buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; Domain=%s\r\n", guid, edate, registry.registry_domain);

40 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; Domain=%s\r\n", guid, e_date, registry.registry_domain);

41 if(registry.enable_cookies_samesite_secure)

38 - buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; Domain=%s; SameSite=None; Secure\r\n", guid, edate, registry.registry_domain);

42 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; Domain=%s; SameSite=None; Secure\r\n", guid, e_date, registry.registry_domain);

43 }

45 w->response.has_cookies = true;

@@ -84,37 +88,41 @@ struct registry_json_walk_person_urls_callback {

88 int count;

89 };

91 +static STRING *asterisks = NULL;

92 +

93 // callback for rendering PERSON_URLs

88 -static int registry_json_person_url_callback(void *entry, void *data) {

89 - REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry;

90 - struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;

94 +static int registry_json_person_url_callback(REGISTRY_PERSON_URL *pu, struct registry_json_walk_person_urls_callback *c) {

95 + if(unlikely(!asterisks))

96 + asterisks = string_strdupz("***");

97 +

98 struct web_client *w = c->w;

93 - if (!strcmp(pu->url->url,"***")) return 0;

100 + if (pu->url == asterisks) return 0;

101

102 buffer_json_add_array_item_array(w->response.data);

103 buffer_json_add_array_item_string(w->response.data, pu->machine->guid);

97 - buffer_json_add_array_item_string(w->response.data, pu->url->url);

104 + buffer_json_add_array_item_string(w->response.data, string2str(pu->url));

105 buffer_json_add_array_item_uint64(w->response.data, pu->last_t * (uint64_t) 1000);

106 buffer_json_add_array_item_uint64(w->response.data, pu->usages);

100 - buffer_json_add_array_item_string(w->response.data, pu->machine_name);

107 + buffer_json_add_array_item_string(w->response.data, string2str(pu->machine_name));

108 buffer_json_array_close(w->response.data);

109

110 return 1;

111 }

112

113 // callback for rendering MACHINE_URLs

107 -static int registry_json_machine_url_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) {

108 - REGISTRY_MACHINE_URL *mu = (REGISTRY_MACHINE_URL *)entry;

109 - struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;

114 +static int registry_json_machine_url_callback(REGISTRY_MACHINE_URL *mu, struct registry_json_walk_person_urls_callback *c) {

115 + if(unlikely(!asterisks))

116 + asterisks = string_strdupz("***");

117 +

118 struct web_client *w = c->w;

119 REGISTRY_MACHINE *m = c->m;

120

113 - if (!strcmp(mu->url->url,"***")) return 0;

121 + if (mu->url == asterisks) return 0;

122

123 buffer_json_add_array_item_array(w->response.data);

124 buffer_json_add_array_item_string(w->response.data, m->guid);

117 - buffer_json_add_array_item_string(w->response.data, mu->url->url);

125 + buffer_json_add_array_item_string(w->response.data, string2str(mu->url));

126 buffer_json_add_array_item_uint64(w->response.data, mu->last_t * (uint64_t) 1000);

127 buffer_json_add_array_item_uint64(w->response.data, mu->usages);

128 buffer_json_array_close(w->response.data);

@@ -130,9 +138,7 @@ struct registry_person_url_callback_verify_machine_exists_data {

138 int count;

139 };

140

133 -static inline int registry_person_url_callback_verify_machine_exists(void *entry, void *data) {

134 - struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data;

135 - REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry;

141 +static inline int registry_person_url_callback_verify_machine_exists(REGISTRY_PERSON_URL *pu, struct registry_person_url_callback_verify_machine_exists_data *d) {

142 REGISTRY_MACHINE *m = d->m;

143

144 if(pu->machine == m)

@@ -161,34 +167,54 @@ int registry_request_hello_json(RRDHOST *host, struct web_client *w) {

167 buffer_json_member_add_string(w->response.data, "cloud_base_url", registry.cloud_base_url);

168 buffer_json_member_add_boolean(w->response.data, "anonymous_statistics", netdata_anonymous_statistics_enabled);

169

170 + buffer_json_member_add_array(w->response.data, "nodes");

171 + RRDHOST *h;

172 + dfe_start_read(rrdhost_root_index, h) {

173 + buffer_json_add_array_item_object(w->response.data);

174 + buffer_json_member_add_string(w->response.data, "machine_guid", h->machine_guid);

175 + buffer_json_member_add_string(w->response.data, "hostname", rrdhost_registry_hostname(h));

176 + buffer_json_object_close(w->response.data);

177 + }

178 + dfe_done(h);

179 + buffer_json_array_close(w->response.data);

180 +

181 registry_json_footer(w);

182 return HTTP_RESP_OK;

183 }

184

185 // ----------------------------------------------------------------------------

169 -//public ACCESS request

170 -

171 -#define REGISTRY_VERIFY_COOKIES_GUID "give-me-back-this-cookie-now--please"

186 +// public ACCESS request

187

188 // the main method for registering an access

189 int registry_request_access_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) {

190 if(unlikely(!registry.enabled))

191 return registry_json_disabled(host, w, "access");

192

193 + if(!registry_is_valid_url(url)) {

194 + buffer_flush(w->response.data);

195 + buffer_strcat(w->response.data, "Invalid URL given in the request");

196 + return HTTP_RESP_BAD_REQUEST;

197 + }

198 +

199 // ------------------------------------------------------------------------

179 - // verify the browser supports cookies

200 + // verify the browser supports cookies or the bearer

201

202 if(registry.verify_cookies_redirects > 0 && !person_guid[0]) {

203 + registry_request_access(REGISTRY_VERIFY_COOKIES_GUID, machine_guid, url, name, when);

204 +

205 buffer_flush(w->response.data);

206 registry_set_cookie(w, REGISTRY_VERIFY_COOKIES_GUID);

207 w->response.data->content_type = CT_APPLICATION_JSON;

208 registry_json_header(host, w, "access", REGISTRY_STATUS_REDIRECT);

209 + buffer_json_member_add_string(w->response.data, "person_guid", REGISTRY_VERIFY_COOKIES_GUID);

210 buffer_json_member_add_string(w->response.data, "registry", registry.registry_to_announce);

211 registry_json_footer(w);

212 return HTTP_RESP_OK;

213 }

214

191 - if(unlikely(person_guid[0] && !strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID)))

215 + if(unlikely(person_guid[0] && is_dummy_person(person_guid)))

216 + // it passed the check - they gave us a different person_guid

217 + // empty the dummy one, so that we will generate a new person_guid

218 person_guid[0] = '\0';

219

220 // ------------------------------------------------------------------------

@@ -212,7 +238,8 @@ int registry_request_access_json(RRDHOST *host, struct web_client *w, char *pers

238 buffer_json_member_add_array(w->response.data, "urls");

239

240 struct registry_json_walk_person_urls_callback c = { p, NULL, w, 0 };

215 - avl_traverse(&p->person_urls, registry_json_person_url_callback, &c);

241 + for(REGISTRY_PERSON_URL *pu = p->person_urls; pu ;pu = pu->next)

242 + registry_json_person_url_callback(pu, &c);

243 buffer_json_array_close(w->response.data); // urls

244

245 registry_json_footer(w);

@@ -228,6 +255,12 @@ int registry_request_delete_json(RRDHOST *host, struct web_client *w, char *pers

255 if(!registry.enabled)

256 return registry_json_disabled(host, w, "delete");

257

258 + if(!registry_is_valid_url(url)) {

259 + buffer_flush(w->response.data);

260 + buffer_strcat(w->response.data, "Invalid URL given in the request");

261 + return HTTP_RESP_BAD_REQUEST;

262 + }

263 +

264 registry_lock();

265

266 REGISTRY_PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when);

@@ -253,6 +286,12 @@ int registry_request_search_json(RRDHOST *host, struct web_client *w, char *pers

286 if(!registry.enabled)

287 return registry_json_disabled(host, w, "search");

288

289 + if(!registry_is_valid_url(url)) {

290 + buffer_flush(w->response.data);

291 + buffer_strcat(w->response.data, "Invalid URL given in the request");

292 + return HTTP_RESP_BAD_REQUEST;

293 + }

294 +

295 registry_lock();

296

297 REGISTRY_MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when);

@@ -267,7 +306,10 @@ int registry_request_search_json(RRDHOST *host, struct web_client *w, char *pers

306

307 buffer_json_member_add_array(w->response.data, "urls");

308 struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 };

270 - dictionary_walkthrough_read(m->machine_urls, registry_json_machine_url_callback, &c);

309 +

310 + for(REGISTRY_MACHINE_URL *mu = m->machine_urls; mu ; mu = mu->next)

311 + registry_json_machine_url_callback(mu, &c);

312 +

313 buffer_json_array_close(w->response.data);

314

315 registry_json_footer(w);

@@ -279,12 +321,15 @@ int registry_request_search_json(RRDHOST *host, struct web_client *w, char *pers

321 // SWITCH REQUEST

322

323 // the main method for switching user identity

282 -int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when) {

324 +int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url __maybe_unused, char *new_person_guid, time_t when __maybe_unused) {

325 if(!registry.enabled)

326 return registry_json_disabled(host, w, "switch");

327

286 - (void)url;

287 - (void)when;

328 + if(!registry_is_valid_url(url)) {

329 + buffer_flush(w->response.data);

330 + buffer_strcat(w->response.data, "Invalid URL given in the request");

331 + return HTTP_RESP_BAD_REQUEST;

332 + }

333

334 registry_lock();

335

@@ -315,7 +360,9 @@ int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *pers

360 struct registry_person_url_callback_verify_machine_exists_data data = { m, 0 };

361

362 // verify the old person has access to this machine

318 - avl_traverse(&op->person_urls, registry_person_url_callback_verify_machine_exists, &data);

363 + for(REGISTRY_PERSON_URL *pu = op->person_urls; pu ;pu = pu->next)

364 + registry_person_url_callback_verify_machine_exists(pu, &data);

365 +

366 if(!data.count) {

367 registry_json_header(host, w, "switch", REGISTRY_STATUS_FAILED);

368 registry_json_footer(w);

@@ -325,7 +372,9 @@ int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *pers

372

373 // verify the new person has access to this machine

374 data.count = 0;

328 - avl_traverse(&np->person_urls, registry_person_url_callback_verify_machine_exists, &data);

375 + for(REGISTRY_PERSON_URL *pu = np->person_urls; pu ;pu = pu->next)

376 + registry_person_url_callback_verify_machine_exists(pu, &data);

377 +

378 if(!data.count) {

379 registry_json_header(host, w, "switch", REGISTRY_STATUS_FAILED);

380 registry_json_footer(w);

@@ -396,14 +445,12 @@ void registry_statistics(void) {

445

446 rrddim_add(stc, "persons", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

447 rrddim_add(stc, "machines", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

399 - rrddim_add(stc, "urls", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

448 rrddim_add(stc, "persons_urls", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

449 rrddim_add(stc, "machines_urls", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

450 }

451

452 rrddim_set(stc, "persons", (collected_number)registry.persons_count);

453 rrddim_set(stc, "machines", (collected_number)registry.machines_count);

406 - rrddim_set(stc, "urls", (collected_number)registry.urls_count);

454 rrddim_set(stc, "persons_urls", (collected_number)registry.persons_urls_count);

455 rrddim_set(stc, "machines_urls", (collected_number)registry.machines_urls_count);

456 rrdset_done(stc);

@@ -428,15 +475,21 @@ void registry_statistics(void) {

475

476 rrddim_add(stm, "persons", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

477 rrddim_add(stm, "machines", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

431 - rrddim_add(stm, "urls", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

478 rrddim_add(stm, "persons_urls", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

479 rrddim_add(stm, "machines_urls", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

480 }

481

436 - rrddim_set(stm, "persons", (collected_number)registry.persons_memory + dictionary_stats_for_registry(registry.persons));

437 - rrddim_set(stm, "machines", (collected_number)registry.machines_memory + dictionary_stats_for_registry(registry.machines));

438 - rrddim_set(stm, "urls", (collected_number)registry.urls_memory);

439 - rrddim_set(stm, "persons_urls", (collected_number)registry.persons_urls_memory);

440 - rrddim_set(stm, "machines_urls", (collected_number)registry.machines_urls_memory);

482 + struct aral_statistics *p_aral_stats = aral_statistics(registry.persons_aral);

483 + rrddim_set(stm, "persons", (collected_number)p_aral_stats->structures.allocated_bytes + (collected_number)p_aral_stats->malloc.allocated_bytes);

484 +

485 + struct aral_statistics *m_aral_stats = aral_statistics(registry.machines_aral);

486 + rrddim_set(stm, "machines", (collected_number)m_aral_stats->structures.allocated_bytes + (collected_number)m_aral_stats->malloc.allocated_bytes);

487 +

488 + struct aral_statistics *pu_aral_stats = aral_statistics(registry.person_urls_aral);

489 + rrddim_set(stm, "persons_urls", (collected_number)pu_aral_stats->structures.allocated_bytes + (collected_number)pu_aral_stats->malloc.allocated_bytes);

490 +

491 + struct aral_statistics *mu_aral_stats = aral_statistics(registry.machine_urls_aral);

492 + rrddim_set(stm, "machines_urls", (collected_number)mu_aral_stats->structures.allocated_bytes + (collected_number)mu_aral_stats->malloc.allocated_bytes);

493 +

494 rrdset_done(stm);

495 }

registry/registry_db.c

+137 -104

@@ -11,18 +11,15 @@ int registry_db_should_be_saved(void) {

11 // ----------------------------------------------------------------------------

12 // INTERNAL FUNCTIONS FOR SAVING REGISTRY OBJECTS

14 -static int registry_machine_save_url(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *file) {

15 - REGISTRY_MACHINE_URL *mu = entry;

16 - FILE *fp = file;

17 -

18 - debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url);

14 +static int registry_machine_save_url(REGISTRY_MACHINE_URL *mu, FILE *fp) {

15 + debug(D_REGISTRY, "REGISTRY: registry_machine_save_url('%s')", string2str(mu->url));

17 int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n",

18 mu->first_t,

19 mu->last_t,

20 mu->usages,

21 mu->flags,

25 - mu->url->url

22 + string2str(mu->url)

23 );

25 // error handling is done at registry_db_save()

@@ -35,7 +32,7 @@ static int registry_machine_save(const DICTIONARY_ITEM *item __maybe_unused, voi

32 REGISTRY_MACHINE *m = entry;

33 FILE *fp = file;

38 - debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);

35 + debug(D_REGISTRY, "REGISTRY: registry_machine_save('%s')", m->guid);

37 int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n",

38 m->first_t,

@@ -45,9 +42,13 @@ static int registry_machine_save(const DICTIONARY_ITEM *item __maybe_unused, voi

42 );

44 if(ret >= 0) {

48 - int ret2 = dictionary_walkthrough_read(m->machine_urls, registry_machine_save_url, fp);

49 - if(ret2 < 0) return ret2;

50 - ret += ret2;

45 + for(REGISTRY_MACHINE_URL *mu = m->machine_urls; mu ; mu = mu->next) {

46 + int rc = registry_machine_save_url(mu, fp);

47 + if(rc < 0)

48 + return rc;

49 +

50 + ret += rc;

51 + }

52 }

54 // error handling is done at registry_db_save()

@@ -55,11 +56,8 @@ static int registry_machine_save(const DICTIONARY_ITEM *item __maybe_unused, voi

56 return ret;

57 }

58 -static inline int registry_person_save_url(void *entry, void *file) {

59 - REGISTRY_PERSON_URL *pu = entry;

60 - FILE *fp = file;

61 -

62 - debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);

59 +static inline int registry_person_save_url(REGISTRY_PERSON_URL *pu, FILE *fp) {

60 + debug(D_REGISTRY, "REGISTRY: registry_person_save_url('%s')", string2str(pu->url));

62 int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n",

63 pu->first_t,

@@ -67,8 +65,8 @@ static inline int registry_person_save_url(void *entry, void *file) {

65 pu->usages,

66 pu->flags,

67 pu->machine->guid,

70 - pu->machine_name,

71 - pu->url->url

68 + string2str(pu->machine_name),

69 + string2str(pu->url)

70 );

72 // error handling is done at registry_db_save()

@@ -80,7 +78,7 @@ static inline int registry_person_save(const DICTIONARY_ITEM *item __maybe_unuse

78 REGISTRY_PERSON *p = entry;

79 FILE *fp = file;

83 - debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);

81 + debug(D_REGISTRY, "REGISTRY: registry_person_save('%s')", p->guid);

83 int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n",

84 p->first_t,

@@ -90,10 +88,13 @@ static inline int registry_person_save(const DICTIONARY_ITEM *item __maybe_unuse

88 );

90 if(ret >= 0) {

93 - //int ret2 = dictionary_walkthrough_read(p->person_urls, registry_person_save_url, fp);

94 - int ret2 = avl_traverse(&p->person_urls, registry_person_save_url, fp);

95 - if (ret2 < 0) return ret2;

96 - ret += ret2;

91 + for(REGISTRY_PERSON_URL *pu = p->person_urls; pu ;pu = pu->next) {

92 + int rc = registry_person_save_url(pu, fp);

93 + if(rc < 0)

94 + return rc;

95 + else

96 + ret += rc;

97 + }

98 }

100 // error handling is done at registry_db_save()

@@ -119,42 +120,42 @@ int registry_db_save(void) {

120 snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);

121 snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);

122

122 - debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);

123 + debug(D_REGISTRY, "REGISTRY: Creating file '%s'", tmp_filename);

124 FILE *fp = fopen(tmp_filename, "w");

125 if(!fp) {

125 - netdata_log_error("Registry: Cannot create file: %s", tmp_filename);

126 + netdata_log_error("REGISTRY: Cannot create file: %s", tmp_filename);

127 error_log_limit_reset();

128 return -1;

129 }

130

131 // dictionary_walkthrough_read() has its own locking, so this is safe to do

132

132 - debug(D_REGISTRY, "Saving all machines");

133 + debug(D_REGISTRY, "REGISTRY: saving all machines");

134 int bytes1 = dictionary_walkthrough_read(registry.machines, registry_machine_save, fp);

135 if(bytes1 < 0) {

135 - netdata_log_error("Registry: Cannot save registry machines - return value %d", bytes1);

136 + netdata_log_error("REGISTRY: Cannot save registry machines - return value %d", bytes1);

137 fclose(fp);

138 error_log_limit_reset();

139 return bytes1;

140 }

140 - debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);

141 + debug(D_REGISTRY, "REGISTRY: saving machines took %d bytes", bytes1);

142

143 debug(D_REGISTRY, "Saving all persons");

144 int bytes2 = dictionary_walkthrough_read(registry.persons, registry_person_save, fp);

145 if(bytes2 < 0) {

145 - netdata_log_error("Registry: Cannot save registry persons - return value %d", bytes2);

146 + netdata_log_error("REGISTRY: Cannot save registry persons - return value %d", bytes2);

147 fclose(fp);

148 error_log_limit_reset();

149 return bytes2;

150 }

150 - debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);

151 + debug(D_REGISTRY, "REGISTRY: saving persons took %d bytes", bytes2);

152

153 // save the totals

154 fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",

155 registry.persons_count,

156 registry.machines_count,

157 registry.usages_count + 1, // this is required - it is lost on db rotation

157 - registry.urls_count,

158 + 0LLU, //registry.urls_count,

159 registry.persons_urls_count,

160 registry.machines_urls_count

161 );

@@ -164,36 +165,36 @@ int registry_db_save(void) {

165 errno = 0;

166

167 // remove the .old db

167 - debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename);

168 + debug(D_REGISTRY, "REGISTRY: Removing old db '%s'", old_filename);

169 if(unlink(old_filename) == -1 && errno != ENOENT)

169 - netdata_log_error("Registry: cannot remove old registry file '%s'", old_filename);

170 + netdata_log_error("REGISTRY: cannot remove old registry file '%s'", old_filename);

171

172 // rename the db to .old

172 - debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);

173 + debug(D_REGISTRY, "REGISTRY: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);

174 if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)

174 - netdata_log_error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", registry.db_filename, old_filename);

175 + netdata_log_error("REGISTRY: cannot move file '%s' to '%s'. Saving registry DB failed!", registry.db_filename, old_filename);

176

177 else {

178 // remove the database (it is saved in .old)

178 - debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename);

179 + debug(D_REGISTRY, "REGISTRY: removing db '%s'", registry.db_filename);

180 if (unlink(registry.db_filename) == -1 && errno != ENOENT)

180 - netdata_log_error("Registry: cannot remove old registry file '%s'", registry.db_filename);

181 + netdata_log_error("REGISTRY: cannot remove old registry file '%s'", registry.db_filename);

182

183 // move the .tmp to make it active

183 - debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);

184 + debug(D_REGISTRY, "REGISTRY: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);

185 if (link(tmp_filename, registry.db_filename) == -1) {

185 - netdata_log_error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,

186 + netdata_log_error("REGISTRY: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,

187 registry.db_filename);

188

189 // move the .old back

189 - debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);

190 + debug(D_REGISTRY, "REGISTRY: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);

191 if(link(old_filename, registry.db_filename) == -1)

191 - netdata_log_error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);

192 + netdata_log_error("REGISTRY: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);

193 }

194 else {

194 - debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename);

195 + debug(D_REGISTRY, "REGISTRY: removing tmp db '%s'", tmp_filename);

196 if(unlink(tmp_filename) == -1)

196 - netdata_log_error("Registry: cannot remove tmp registry file '%s'", tmp_filename);

197 + netdata_log_error("REGISTRY: cannot remove tmp registry file '%s'", tmp_filename);

198

199 // it has been moved successfully

200 // discard the current registry log

@@ -215,75 +216,33 @@ size_t registry_db_load(void) {

216 char *s, buf[4096 + 1];

217 REGISTRY_PERSON *p = NULL;

218 REGISTRY_MACHINE *m = NULL;

218 - REGISTRY_URL *u = NULL;

219 + STRING *u = NULL;

220 size_t line = 0;

221

221 - debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename);

222 + debug(D_REGISTRY, "REGISTRY: loading active db from: '%s'", registry.db_filename);

223 FILE *fp = fopen(registry.db_filename, "r");

224 if(!fp) {

224 - netdata_log_error("Registry: cannot open registry file: '%s'", registry.db_filename);

225 + netdata_log_error("REGISTRY: cannot open registry file: '%s'", registry.db_filename);

226 return 0;

227 }

228

229 + REGISTRY_MACHINE_URL *mu;

230 size_t len = 0;

231 buf[4096] = '\0';

232 while((s = fgets_trim_len(buf, 4096, fp, &len))) {

233 line++;

234

233 - debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);

235 + debug(D_REGISTRY, "REGISTRY: read line %zu to length %zu: %s", line, len, s);

236 switch(*s) {

235 - case 'T': // totals

236 - if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {

237 - netdata_log_error("Registry totals line %zu is wrong (len = %zu).", line, len);

238 - continue;

239 - }

240 - registry.persons_count = strtoull(&s[2], NULL, 16);

241 - registry.machines_count = strtoull(&s[19], NULL, 16);

242 - registry.usages_count = strtoull(&s[36], NULL, 16);

243 - registry.urls_count = strtoull(&s[53], NULL, 16);

244 - registry.persons_urls_count = strtoull(&s[70], NULL, 16);

245 - registry.machines_urls_count = strtoull(&s[87], NULL, 16);

246 - break;

247 -

248 - case 'P': // person

249 - m = NULL;

250 - // verify it is valid

251 - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

252 - netdata_log_error("Registry person line %zu is wrong (len = %zu).", line, len);

253 - continue;

254 - }

255 -

256 - s[1] = s[10] = s[19] = s[28] = '\0';

257 - p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));

258 - p->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

259 - p->usages = (uint32_t)strtoul(&s[20], NULL, 16);

260 - debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);

261 - break;

262 -

263 - case 'M': // machine

264 - p = NULL;

265 - // verify it is valid

266 - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

267 - netdata_log_error("Registry person line %zu is wrong (len = %zu).", line, len);

268 - continue;

269 - }

270 -

271 - s[1] = s[10] = s[19] = s[28] = '\0';

272 - m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));

273 - m->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

274 - m->usages = (uint32_t)strtoul(&s[20], NULL, 16);

275 - debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);

276 - break;

277 -

237 case 'U': // person URL

238 if(unlikely(!p)) {

280 - netdata_log_error("Registry: ignoring line %zu, no person loaded: %s", line, s);

239 + netdata_log_error("REGISTRY: ignoring line %zu, no person loaded: %s", line, s);

240 continue;

241 }

242

243 // verify it is valid

244 if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') {

286 - netdata_log_error("Registry person URL line %zu is wrong (len = %zu).", line, len);

245 + netdata_log_error("REGISTRY: person URL line %zu is wrong (len = %zu).", line, len);

246 continue;

247 }

248

@@ -293,51 +252,125 @@ size_t registry_db_load(void) {

252 char *url = &s[69];

253 while(*url && *url != '\t') url++;

254 if(!*url) {

296 - netdata_log_error("Registry person URL line %zu does not have a url.", line);

255 + netdata_log_error("REGISTRY: person URL line %zu does not have a url.", line);

256 continue;

257 }

258 *url++ = '\0';

259

301 - // u = registry_url_allocate_nolock(url, strlen(url));

302 - u = registry_url_get(url, strlen(url));

260 + if(*url != 'h' && *url != '*') {

261 + netdata_log_error("REGISTRY: person URL line %zu does not have a valid url: %s", line, url);

262 + continue;

263 + }

264 +

265 + u = string_strdupz(url);

266

304 - time_t first_t = strtoul(&s[2], NULL, 16);

267 + time_t first_t = (time_t)strtoul(&s[2], NULL, 16);

268

269 m = registry_machine_find(&s[32]);

270 if(!m) m = registry_machine_allocate(&s[32], first_t);

271

309 - REGISTRY_PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t);

272 + mu = registry_machine_url_find(m, u);

273 + if(!mu) {

274 + netdata_log_error("REGISTRY: person URL line %zu was not linked to the machine it refers to", line);

275 + mu = registry_machine_url_allocate(m, u, first_t);

276 + }

277 +

278 + REGISTRY_PERSON_URL *pu = registry_person_url_index_find(p, u);

279 + if(!pu)

280 + pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t);

281 + else

282 + netdata_log_error("REGISTRY: person URL line %zu is duplicate, reusing the old one.", line);

283 +

284 pu->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

285 pu->usages = (uint32_t)strtoul(&s[20], NULL, 16);

286 pu->flags = (uint8_t)strtoul(&s[29], NULL, 16);

313 - debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->machine_name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);

287 + debug(D_REGISTRY, "REGISTRY: loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x",

288 + string2str(u), string2str(pu->machine_name), m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);

289 +

290 + string_freez(u);

291 + break;

292 +

293 + case 'P': // person

294 + m = NULL;

295 + // verify it is valid

296 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

297 + netdata_log_error("REGISTRY: person line %zu is wrong (len = %zu).", line, len);

298 + continue;

299 + }

300 +

301 + s[1] = s[10] = s[19] = s[28] = '\0';

302 + p = registry_person_allocate(&s[29], (time_t)strtoul(&s[2], NULL, 16));

303 + p->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

304 + p->usages = (uint32_t)strtoul(&s[20], NULL, 16);

305 + debug(D_REGISTRY, "REGISTRY: loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);

306 break;

307

308 case 'V': // machine URL

309 if(unlikely(!m)) {

318 - netdata_log_error("Registry: ignoring line %zu, no machine loaded: %s", line, s);

310 + netdata_log_error("REGISTRY: ignoring line %zu, no machine loaded: %s", line, s);

311 continue;

312 }

313

314 // verify it is valid

315 if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') {

324 - netdata_log_error("Registry person URL line %zu is wrong (len = %zu).", line, len);

316 + netdata_log_error("REGISTRY: person URL line %zu is wrong (len = %zu).", line, len);

317 continue;

318 }

319

320 s[1] = s[10] = s[19] = s[28] = s[31] = '\0';

329 - // u = registry_url_allocate_nolock(&s[32], strlen(&s[32]));

330 - u = registry_url_get(&s[32], strlen(&s[32]));

321

332 - REGISTRY_MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16));

322 + url = &s[32];

323 + if(*url != 'h' && *url != '*') {

324 + netdata_log_error("REGISTRY: machine URL line %zu does not have a valid url: %s", line, url);

325 + continue;

326 + }

327 +

328 + u = string_strdupz(url);

329 +

330 + mu = registry_machine_url_find(m, u);

331 + if(!mu)

332 + mu = registry_machine_url_allocate(m, u, (time_t)strtoul(&s[2], NULL, 16));

333 + else

334 + netdata_log_error("REGISTRY: machine URL line %zu is duplicate, reusing the old one.", line);

335 +

336 mu->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

337 mu->usages = (uint32_t)strtoul(&s[20], NULL, 16);

338 mu->flags = (uint8_t)strtoul(&s[29], NULL, 16);

336 - debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags);

339 + debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x",

340 + string2str(u), m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags);

341 +

342 + string_freez(u);

343 + break;

344 +

345 + case 'M': // machine

346 + p = NULL;

347 + // verify it is valid

348 + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {

349 + netdata_log_error("REGISTRY: person line %zu is wrong (len = %zu).", line, len);

350 + continue;

351 + }

352 +

353 + s[1] = s[10] = s[19] = s[28] = '\0';

354 + m = registry_machine_allocate(&s[29], (time_t)strtoul(&s[2], NULL, 16));

355 + m->last_t = (uint32_t)strtoul(&s[11], NULL, 16);

356 + m->usages = (uint32_t)strtoul(&s[20], NULL, 16);

357 + debug(D_REGISTRY, "REGISTRY: loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);

358 + break;

359 +

360 + case 'T': // totals

361 + if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {

362 + netdata_log_error("REGISTRY: totals line %zu is wrong (len = %zu).", line, len);

363 + continue;

364 + }

365 + registry.persons_count = strtoull(&s[2], NULL, 16);

366 + registry.machines_count = strtoull(&s[19], NULL, 16);

367 + registry.usages_count = strtoull(&s[36], NULL, 16);

368 + registry.persons_urls_count = strtoull(&s[70], NULL, 16);

369 + registry.machines_urls_count = strtoull(&s[87], NULL, 16);

370 break;

371

372 default:

340 - netdata_log_error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s);

373 + netdata_log_error("REGISTRY: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s);

374 break;

375 }

376 }

registry/registry_init.c

+133 -34

@@ -3,6 +3,63 @@

3 #include "daemon/common.h"

4 #include "registry_internals.h"

6 +void registry_db_stats(void) {

7 + size_t persons = 0;

8 + size_t persons_urls = 0;

9 + size_t max_urls_per_person = 0;

10 +

11 + REGISTRY_PERSON *p;

12 + dfe_start_read(registry.persons, p) {

13 + persons++;

14 + size_t urls = 0;

15 + for(REGISTRY_PERSON_URL *pu = p->person_urls ; pu ;pu = pu->next)

16 + urls++;

17 +

18 + if(urls > max_urls_per_person)

19 + max_urls_per_person = urls;

20 +

21 + persons_urls += urls;

22 + }

23 + dfe_done(p);

24 +

25 + size_t machines = 0;

26 + size_t machines_urls = 0;

27 + size_t max_urls_per_machine = 0;

28 +

29 + REGISTRY_MACHINE *m;

30 + dfe_start_read(registry.machines, m) {

31 + machines++;

32 + size_t urls = 0;

33 + for(REGISTRY_MACHINE_URL *mu = m->machine_urls ; mu ;mu = mu->next)

34 + urls++;

35 +

36 + if(urls > max_urls_per_machine)

37 + max_urls_per_machine = urls;

38 +

39 + machines_urls += urls;

40 + }

41 + dfe_done(m);

42 +

43 + netdata_log_info("REGISTRY: persons %zu, person_urls %zu, max_urls_per_person %zu, "

44 + "machines %zu, machine_urls %zu, max_urls_per_machine %zu",

45 + persons, persons_urls, max_urls_per_person,

46 + machines, machines_urls, max_urls_per_machine);

47 +}

48 +

49 +void registry_generate_curl_urls(void) {

50 + FILE *fp = fopen("/tmp/registry.curl", "w+");

51 +

52 + REGISTRY_PERSON *p;

53 + dfe_start_read(registry.persons, p) {

54 + for(REGISTRY_PERSON_URL *pu = p->person_urls ; pu ;pu = pu->next) {

55 + fprintf(fp, "do_curl '%s' '%s' '%s'\n", p->guid, pu->machine->guid, string2str(pu->url));

56 + }

57 + }

58 + dfe_done(p);

59 +

60 + fclose(fp);

61 +}

62 +

63 int registry_init(void) {

64 char filename[FILENAME_MAX + 1];

@@ -16,7 +73,7 @@ int registry_init(void) {

73 registry.enabled = 0;

74 }

19 - // pathnames

76 + // path names

77 snprintfz(filename, FILENAME_MAX, "%s/registry", netdata_configured_varlib_dir);

78 registry.pathname = config_get(CONFIG_SECTION_DIRECTORIES, "registry", filename);

79 if(mkdir(registry.pathname, 0770) == -1 && errno != EEXIST)

@@ -57,73 +114,104 @@ int registry_init(void) {

114 config_set_number(CONFIG_SECTION_REGISTRY, "max URL name length", (long long)registry.max_name_length);

115 }

116

117 + bool use_mmap = config_get_boolean(CONFIG_SECTION_REGISTRY, "use mmap", false);

118 +

119 // initialize entries counters

120 registry.persons_count = 0;

121 registry.machines_count = 0;

122 registry.usages_count = 0;

64 - registry.urls_count = 0;

123 registry.persons_urls_count = 0;

124 registry.machines_urls_count = 0;

125

68 - // initialize memory counters

69 - registry.persons_memory = 0;

70 - registry.machines_memory = 0;

71 - registry.urls_memory = 0;

72 - registry.persons_urls_memory = 0;

73 - registry.machines_urls_memory = 0;

74 -

126 // initialize locks

127 netdata_mutex_init(&registry.lock);

128

78 - // create dictionaries

79 - registry.persons = dictionary_create(REGISTRY_DICTIONARY_OPTIONS);

80 - registry.machines = dictionary_create(REGISTRY_DICTIONARY_OPTIONS);

81 - avl_init(&registry.registry_urls_root_index, registry_url_compare);

82 -

129 // load the registry database

130 if(registry.enabled) {

131 + // create dictionaries

132 + registry.persons = dictionary_create(REGISTRY_DICTIONARY_OPTIONS);

133 + registry.machines = dictionary_create(REGISTRY_DICTIONARY_OPTIONS);

134 +

135 + // initialize the allocators

136 +

137 + size_t min_page_size = 4 * 1024;

138 + size_t max_page_size = 1024 * 1024;

139 +

140 + if(use_mmap) {

141 + min_page_size = 100 * 1024 * 1024;

142 + max_page_size = 512 * 1024 * 1024;

143 + }

144 +

145 + registry.persons_aral = aral_create("registry_persons", sizeof(REGISTRY_PERSON),

146 + min_page_size / sizeof(REGISTRY_PERSON), max_page_size,

147 + &registry.aral_stats,

148 + "registry_persons",

149 + &netdata_configured_cache_dir,

150 + use_mmap, true);

151 +

152 + registry.machines_aral = aral_create("registry_machines", sizeof(REGISTRY_MACHINE),

153 + min_page_size / sizeof(REGISTRY_MACHINE), max_page_size,

154 + &registry.aral_stats,

155 + "registry_machines",

156 + &netdata_configured_cache_dir,

157 + use_mmap, true);

158 +

159 + registry.person_urls_aral = aral_create("registry_person_urls", sizeof(REGISTRY_PERSON_URL),

160 + min_page_size / sizeof(REGISTRY_PERSON_URL), max_page_size,

161 + &registry.aral_stats,

162 + "registry_person_urls",

163 + &netdata_configured_cache_dir,

164 + use_mmap, true);

165 +

166 + registry.machine_urls_aral = aral_create("registry_machine_urls", sizeof(REGISTRY_MACHINE_URL),

167 + min_page_size / sizeof(REGISTRY_MACHINE_URL), max_page_size,

168 + &registry.aral_stats,

169 + "registry_machine_urls",

170 + &netdata_configured_cache_dir,

171 + use_mmap, true);

172 +

173 + // disable cancelability to avoid enable/disable per item in the dictionary locks

174 + netdata_thread_disable_cancelability();

175 +

176 registry_log_open();

177 registry_db_load();

178 registry_log_load();

179

180 if(unlikely(registry_db_should_be_saved()))

181 registry_db_save();

91 - }

182

93 - return 0;

94 -}

95 -

96 -static int machine_urls_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) {

97 - REGISTRY_MACHINE *m = (REGISTRY_MACHINE *)data;

98 - (void)m;

183 +// registry_db_stats();

184 +// registry_generate_curl_urls();

185 +// exit(0);

186

100 - REGISTRY_MACHINE_URL *mu = (REGISTRY_MACHINE_URL *)entry;

101 -

102 - debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);

103 - registry_url_unlink(mu->url);

104 -

105 - debug(D_REGISTRY, "Registry: freeing machine url");

106 - freez(mu);

187 + netdata_thread_enable_cancelability();

188 + }

189

108 - return 1;

190 + return 0;

191 }

192

193 static int machine_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data __maybe_unused) {

194 REGISTRY_MACHINE *m = (REGISTRY_MACHINE *)entry;

113 - int ret = dictionary_walkthrough_read(m->machine_urls, machine_urls_delete_callback, m);

195

115 - dictionary_destroy(m->machine_urls);

196 + int count = 0;

197 +

198 + while(m->machine_urls) {

199 + registry_machine_url_unlink_from_machine_and_free(m, m->machine_urls);

200 + count++;

201 + }

202 +

203 freez(m);

204

118 - return ret + 1;

205 + return count + 1;

206 }

207 +

208 static int registry_person_del_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *d __maybe_unused) {

209 REGISTRY_PERSON *p = (REGISTRY_PERSON *)entry;

210

211 debug(D_REGISTRY, "Registry: registry_person_del('%s'): deleting person", p->guid);

212

125 - while(p->person_urls.root)

126 - registry_person_unlink_from_url(p, (REGISTRY_PERSON_URL *)p->person_urls.root);

213 + while(p->person_urls)

214 + registry_person_unlink_from_url(p, (REGISTRY_PERSON_URL *)p->person_urls);

215

216 //debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);

217 //dictionary_del(registry.persons, p->guid);

@@ -140,8 +228,19 @@ void registry_free(void) {

228 debug(D_REGISTRY, "Registry: destroying persons dictionary");

229 dictionary_walkthrough_read(registry.persons, registry_person_del_callback, NULL);

230 dictionary_destroy(registry.persons);

231 + registry.persons = NULL;

232

233 debug(D_REGISTRY, "Registry: destroying machines dictionary");

234 dictionary_walkthrough_read(registry.machines, machine_delete_callback, NULL);

235 dictionary_destroy(registry.machines);

236 + registry.machines = NULL;

237 +

238 + aral_destroy(registry.persons_aral);

239 + aral_destroy(registry.machines_aral);

240 + aral_destroy(registry.person_urls_aral);

241 + aral_destroy(registry.machine_urls_aral);

242 + registry.persons_aral = NULL;

243 + registry.machines_aral = NULL;

244 + registry.person_urls_aral = NULL;

245 + registry.machine_urls_aral = NULL;

246 }

registry/registry_internals.c

+31 -19

@@ -80,7 +80,7 @@ static inline char *registry_fix_url(char *url, size_t *len) {

80 // HELPERS

82 // verify the person, the machine and the URL exist in our DB

83 -REGISTRY_PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, REGISTRY_PERSON **pp, REGISTRY_MACHINE **mm) {

83 +REGISTRY_PERSON_URL *registry_verify_request(const char *person_guid, char *machine_guid, char *url, REGISTRY_PERSON **pp, REGISTRY_MACHINE **mm) {

84 char pbuf[GUID_LEN + 1], mbuf[GUID_LEN + 1];

86 if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) {

@@ -121,7 +121,10 @@ REGISTRY_PERSON_URL *registry_verify_request(char *person_guid, char *machine_gu

121 }

122 if(pp) *pp = p;

123

124 - REGISTRY_PERSON_URL *pu = registry_person_url_index_find(p, url);

124 + STRING *u = string_strdupz(url);

125 + REGISTRY_PERSON_URL *pu = registry_person_url_index_find(p, u);

126 + string_freez(u);

127 +

128 if(!pu) {

129 netdata_log_info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);

130 return NULL;

@@ -138,23 +141,28 @@ REGISTRY_PERSON_URL *registry_verify_request(char *person_guid, char *machine_gu

141 // ----------------------------------------------------------------------------

142 // REGISTRY REQUESTS

143

141 -REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) {

144 +REGISTRY_PERSON *registry_request_access(const char *person_guid, char *machine_guid, char *url, char *name, time_t when) {

145 debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);

146

144 - REGISTRY_MACHINE *m = registry_machine_get(machine_guid, when);

147 + bool is_dummy = is_dummy_person(person_guid);

148 +

149 + REGISTRY_MACHINE *m = registry_machine_find_or_create(machine_guid, when, is_dummy);

150 if(!m) return NULL;

151

152 + REGISTRY_PERSON *p = registry_person_find_or_create(person_guid, when, is_dummy);

153 +

154 // make sure the name is valid

148 - size_t namelen;

149 - name = registry_fix_machine_name(name, &namelen);

155 + size_t name_len;

156 + name = registry_fix_machine_name(name, &name_len);

157

151 - size_t urllen;

152 - url = registry_fix_url(url, &urllen);

158 + size_t url_len;

159 + url = registry_fix_url(url, &url_len);

160

154 - REGISTRY_PERSON *p = registry_person_get(person_guid, when);

161 + STRING *u = string_strdupz(url);

162 +

163 + if(!is_dummy)

164 + registry_person_link_to_url(p, m, u, name, name_len, when);

165

156 - REGISTRY_URL *u = registry_url_get(url, urllen);

157 - registry_person_link_to_url(p, m, u, name, namelen, when);

166 registry_machine_link_to_url(m, u, when);

167

168 registry_log('A', p, m, u, name);

@@ -164,7 +172,7 @@ REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid,

172 return p;

173 }

174

167 -REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {

175 +REGISTRY_PERSON *registry_request_delete(const char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {

176 (void) when;

177

178 REGISTRY_PERSON *p = NULL;

@@ -184,14 +192,17 @@ REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid,

192 }

193 */

194

187 - REGISTRY_PERSON_URL *dpu = registry_person_url_index_find(p, delete_url);

195 + STRING *d_url = string_strdupz(delete_url);

196 + REGISTRY_PERSON_URL *dpu = registry_person_url_index_find(p, d_url);

197 + string_freez(d_url);

198 +

199 if(!dpu) {

200 netdata_log_info("Registry Delete Request: URL not found for person: '%s', machine '%s', url '%s', delete url '%s'", p->guid

190 - , m->guid, pu->url->url, delete_url);

201 + , m->guid, string2str(pu->url), delete_url);

202 return NULL;

203 }

204

194 - registry_log('D', p, m, pu->url, dpu->url->url);

205 + registry_log('D', p, m, pu->url, string2str(dpu->url));

206 registry_person_unlink_from_url(p, dpu);

207

208 return p;

@@ -218,7 +229,7 @@ static int machine_request_callback(void *entry, void *data) {

229 return 0; // continue

230 }

231

221 -REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {

232 +REGISTRY_MACHINE *registry_request_machine(const char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {

233 (void)when;

234

235 char mbuf[GUID_LEN + 1];

@@ -230,7 +241,7 @@ REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid

241

242 // make sure the machine GUID is valid

243 if(regenerate_guid(request_machine, mbuf) == -1) {

233 - netdata_log_info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);

244 + netdata_log_info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, string2str(pu->url), request_machine);

245 return NULL;

246 }

247 request_machine = mbuf;

@@ -238,7 +249,7 @@ REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid

249 // make sure the machine exists

250 m = registry_machine_find(request_machine);

251 if(!m) {

241 - netdata_log_info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, machine_guid, pu->url->url, request_machine);

252 + netdata_log_info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, machine_guid, string2str(pu->url), request_machine);

253 return NULL;

254 }

255

@@ -250,7 +261,8 @@ REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid

261 struct machine_request_callback_data rdata = { m, NULL };

262

263 // request a walk through on the dictionary

253 - avl_traverse(&p->person_urls, machine_request_callback, &rdata);

264 + for(pu = p->person_urls; pu ;pu = pu->next)

265 + machine_request_callback(pu, &rdata);

266

267 if(rdata.result)

268 return m;

registry/registry_internals.h

+14 -16

@@ -10,28 +10,24 @@

11 #define REGISTRY_DICTIONARY_OPTIONS (DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_SINGLE_THREADED)

13 +#define REGISTRY_VERIFY_COOKIES_GUID "11111111-2222-3333-4444-555555555555"

14 +#define is_dummy_person(person_guid) (strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID) == 0)

15 +

16 // ----------------------------------------------------------------------------

17 // COMMON structures

19 struct registry {

20 int enabled;

21 + netdata_mutex_t lock;

23 // entries counters / statistics

24 unsigned long long persons_count;

25 unsigned long long machines_count;

26 unsigned long long usages_count;

23 - unsigned long long urls_count;

27 unsigned long long persons_urls_count;

28 unsigned long long machines_urls_count;

29 unsigned long long log_count;

28 - // memory counters / statistics

29 - unsigned long long persons_memory;

30 - unsigned long long machines_memory;

31 - unsigned long long urls_memory;

32 - unsigned long long persons_urls_memory;

33 - unsigned long long machines_urls_memory;

34 -

31 // configuration

32 unsigned long long save_registry_every_entries;

33 char *registry_domain;

@@ -50,7 +46,6 @@ struct registry {

46 char *db_filename;

47 char *log_filename;

48 char *machine_guid_filename;

53 - char *session_key_filename;

50 // open files

51 FILE *log_fp;

@@ -59,12 +54,15 @@ struct registry {

54 DICTIONARY *persons; // dictionary of REGISTRY_PERSON *, with key the REGISTRY_PERSON.guid

55 DICTIONARY *machines; // dictionary of REGISTRY_MACHINE *, with key the REGISTRY_MACHINE.guid

62 - avl_tree_type registry_urls_root_index;

57 + ARAL *persons_aral;

58 + ARAL *machines_aral;

64 - netdata_mutex_t lock;

60 + ARAL *person_urls_aral;

61 + ARAL *machine_urls_aral;

62 +

63 + struct aral_statistics aral_stats;

64 };

67 -#include "registry_url.h"

66 #include "registry_machine.h"

67 #include "registry_person.h"

68 #include "registry.h"

@@ -72,12 +70,12 @@ struct registry {

70 extern struct registry registry;

72 // REGISTRY LOW-LEVEL REQUESTS (in registry-internals.c)

75 -REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when);

76 -REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);

77 -REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when);

73 +REGISTRY_PERSON *registry_request_access(const char *person_guid, char *machine_guid, char *url, char *name, time_t when);

74 +REGISTRY_PERSON *registry_request_delete(const char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);

75 +REGISTRY_MACHINE *registry_request_machine(const char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when);

77 // REGISTRY LOG (in registry_log.c)

80 -void registry_log(char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name);

78 +void registry_log(char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *u, const char *name);

79 int registry_log_open(void);

80 void registry_log_close(void);

81 void registry_log_recreate(void);

registry/registry_log.c

+3 -3

@@ -3,7 +3,7 @@

3 #include "daemon/common.h"

4 #include "registry_internals.h"

6 -void registry_log(char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name) {

6 +void registry_log(char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *u, const char *name) {

7 if(likely(registry.log_fp)) {

8 if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",

9 action,

@@ -11,7 +11,7 @@ void registry_log(char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY

11 p->guid,

12 m->guid,

13 name,

14 - u->url) < 0))

14 + string2str(u)) < 0))

15 netdata_log_error("Registry: failed to save log. Registry data may be lost in case of abnormal restart.");

17 // we increase the counter even on failures

@@ -94,7 +94,7 @@ ssize_t registry_log_load(void) {

94 s[1] = s[10] = s[47] = s[84] = '\0';

96 // get the variables

97 - time_t when = strtoul(&s[2], NULL, 16);

97 + time_t when = (time_t)strtoul(&s[2], NULL, 16);

98 char *person_guid = &s[11];

99 char *machine_guid = &s[48];

100 char *name = &s[85];

registry/registry_machine.c

+35 -29

@@ -7,52 +7,58 @@

7 // MACHINE

9 REGISTRY_MACHINE *registry_machine_find(const char *machine_guid) {

10 - debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);

10 + debug(D_REGISTRY, "REGISTRY: registry_machine_find('%s')", machine_guid);

11 return dictionary_get(registry.machines, machine_guid);

12 }

14 -REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) {

15 - debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(REGISTRY_MACHINE_URL));

14 +REGISTRY_MACHINE_URL *registry_machine_url_find(REGISTRY_MACHINE *m, STRING *url) {

15 + REGISTRY_MACHINE_URL *mu;

17 - REGISTRY_MACHINE_URL *mu = mallocz(sizeof(REGISTRY_MACHINE_URL));

17 + for(mu = m->machine_urls; mu ;mu = mu->next)

18 + if(mu->url == url)

19 + break;

20 +

21 + return mu;

22 +}

23 +

24 +void registry_machine_url_unlink_from_machine_and_free(REGISTRY_MACHINE *m, REGISTRY_MACHINE_URL *mu) {

25 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(m->machine_urls, mu, prev, next);

26 + string_freez(mu->url);

27 + aral_freez(registry.machine_urls_aral, mu);

28 +}

29 +

30 +REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, STRING *u, time_t when) {

31 + debug(D_REGISTRY, "REGISTRY: registry_machine_url_allocate('%s', '%s'): allocating %zu bytes", m->guid, string2str(u), sizeof(REGISTRY_MACHINE_URL));

32 +

33 + REGISTRY_MACHINE_URL *mu = aral_mallocz(registry.machine_urls_aral);

35 mu->first_t = mu->last_t = (uint32_t)when;

36 mu->usages = 1;

21 - mu->url = u;

37 + mu->url = string_dup(u);

38 mu->flags = REGISTRY_URL_FLAGS_DEFAULT;

24 - registry.machines_urls_memory += sizeof(REGISTRY_MACHINE_URL);

25 -

26 - debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): indexing URL in machine", m->guid, u->url);

27 -

28 - registry.machines_urls_memory -= dictionary_stats_for_registry(m->machine_urls);

29 - dictionary_set(m->machine_urls, u->url, mu, sizeof(REGISTRY_MACHINE_URL));

30 - registry.machines_urls_memory += dictionary_stats_for_registry(m->machine_urls);

40 + debug(D_REGISTRY, "REGISTRY: registry_machine_url_allocate('%s', '%s'): indexing URL in machine", m->guid, string2str(u));

32 - registry_url_link(u);

42 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(m->machine_urls, mu, prev, next);

44 return mu;

45 }

47 REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) {

38 - debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(REGISTRY_MACHINE));

48 + debug(D_REGISTRY, "REGISTRY: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(REGISTRY_MACHINE));

40 - REGISTRY_MACHINE *m = mallocz(sizeof(REGISTRY_MACHINE));

50 + REGISTRY_MACHINE *m = aral_mallocz(registry.machines_aral);

52 strncpyz(m->guid, machine_guid, GUID_LEN);

44 - debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);

45 - m->machine_urls = dictionary_create(REGISTRY_DICTIONARY_OPTIONS);

54 + m->machine_urls = NULL;

56 m->first_t = m->last_t = (uint32_t)when;

57 m->usages = 0;

50 - registry.machines_memory += sizeof(REGISTRY_MACHINE);

59 registry.machines_count++;

53 - registry.machines_urls_memory -= dictionary_stats_for_registry(m->machine_urls);

61 dictionary_set(registry.machines, m->guid, m, sizeof(REGISTRY_MACHINE));

55 - registry.machines_urls_memory += dictionary_stats_for_registry(m->machine_urls);

63 return m;

64 }

@@ -60,14 +66,14 @@ REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t whe

66 // 1. validate machine GUID

67 // 2. if it is valid, find it or create it and return it

68 // 3. if it is not valid, return NULL

63 -REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when) {

69 +REGISTRY_MACHINE *registry_machine_find_or_create(const char *machine_guid, time_t when, bool is_dummy __maybe_unused) {

70 REGISTRY_MACHINE *m = NULL;

72 if(likely(machine_guid && *machine_guid)) {

73 // validate it is a GUID

74 char buf[GUID_LEN + 1];

75 if(unlikely(regenerate_guid(machine_guid, buf) == -1))

70 - netdata_log_info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);

76 + netdata_log_info("REGISTRY: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);

77 else {

78 machine_guid = buf;

79 m = registry_machine_find(machine_guid);

@@ -82,17 +88,17 @@ REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when) {

88 // ----------------------------------------------------------------------------

89 // LINKING OF OBJECTS

85 -REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) {

86 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): searching for URL in machine", m->guid, u->url);

91 +REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, STRING *url, time_t when) {

92 + debug(D_REGISTRY, "REGISTRY: registry_machine_link_to_url('%s', '%s'): searching for URL in machine", m->guid, string2str(url));

88 - REGISTRY_MACHINE_URL *mu = dictionary_get(m->machine_urls, u->url);

94 + REGISTRY_MACHINE_URL *mu = registry_machine_url_find(m, url);

95 if(!mu) {

90 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): not found", m->guid, u->url);

91 - mu = registry_machine_url_allocate(m, u, when);

96 + debug(D_REGISTRY, "REGISTRY: registry_machine_link_to_url('%s', '%s'): not found", m->guid, string2str(url));

97 + mu = registry_machine_url_allocate(m, url, when);

98 registry.machines_urls_count++;

99 }

100 else {

95 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): found", m->guid, u->url);

101 + debug(D_REGISTRY, "REGISTRY: registry_machine_link_to_url('%s', '%s'): found", m->guid, string2str(url));

102 mu->usages++;

103 if(likely(mu->last_t < (uint32_t)when)) mu->last_t = (uint32_t)when;

104 }

@@ -101,7 +107,7 @@ REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, REGISTRY

107 if(likely(m->last_t < (uint32_t)when)) m->last_t = (uint32_t)when;

108

109 if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) {

104 - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): accessing an expired URL.", m->guid, u->url);

110 + debug(D_REGISTRY, "REGISTRY: registry_machine_link_to_url('%s', '%s'): accessing an expired URL.", m->guid, string2str(url));

111 mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;

112 }

113

registry/registry_machine.h

+10 -5

@@ -10,13 +10,15 @@

11 // For each MACHINE-URL pair we keep this

12 struct registry_machine_url {

13 - REGISTRY_URL *url; // de-duplicated URL

13 + STRING *url; // de-duplicated URL

15 uint8_t flags;

17 uint32_t first_t; // the first time we saw this

18 uint32_t last_t; // the last time we saw this

19 uint32_t usages; // how many times this has been accessed

20 +

21 + struct registry_machine_url *prev, *next;

22 };

23 typedef struct registry_machine_url REGISTRY_MACHINE_URL;

@@ -26,7 +28,7 @@ struct registry_machine {

29 uint32_t links; // the number of REGISTRY_PERSON_URL linked to this machine

29 - DICTIONARY *machine_urls; // MACHINE_URL *

31 + REGISTRY_MACHINE_URL *machine_urls; // MACHINE_URL *

33 uint32_t first_t; // the first time we saw this

34 uint32_t last_t; // the last time we saw this

@@ -35,9 +37,12 @@ struct registry_machine {

37 typedef struct registry_machine REGISTRY_MACHINE;

39 REGISTRY_MACHINE *registry_machine_find(const char *machine_guid);

38 -REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when);

40 +REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, STRING *u, time_t when);

41 REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t when);

40 -REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when);

41 -REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when);

42 +REGISTRY_MACHINE *registry_machine_find_or_create(const char *machine_guid, time_t when, bool is_dummy);

43 +REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, STRING *url, time_t when);

44 +

45 +REGISTRY_MACHINE_URL *registry_machine_url_find(REGISTRY_MACHINE *m, STRING *url);

46 +void registry_machine_url_unlink_from_machine_and_free(REGISTRY_MACHINE *m, REGISTRY_MACHINE_URL *mu);

48 #endif //NETDATA_REGISTRY_MACHINE_H

registry/registry_person.c

+60 -82

@@ -6,103 +6,78 @@

6 // ----------------------------------------------------------------------------

7 // PERSON_URL INDEX

9 -int person_url_compare(void *a, void *b) {

10 - register uint32_t hash1 = ((REGISTRY_PERSON_URL *)a)->url->hash;

11 - register uint32_t hash2 = ((REGISTRY_PERSON_URL *)b)->url->hash;

9 +inline REGISTRY_PERSON_URL *registry_person_url_index_find(REGISTRY_PERSON *p, STRING *url) {

10 + debug(D_REGISTRY, "Registry: registry_person_url_index_find('%s', '%s')", p->guid, string2str(url));

13 - if(hash1 < hash2) return -1;

14 - else if(hash1 > hash2) return 1;

15 - else return strcmp(((REGISTRY_PERSON_URL *)a)->url->url, ((REGISTRY_PERSON_URL *)b)->url->url);

16 -}

17 -

18 -inline REGISTRY_PERSON_URL *registry_person_url_index_find(REGISTRY_PERSON *p, const char *url) {

19 - debug(D_REGISTRY, "Registry: registry_person_url_index_find('%s', '%s')", p->guid, url);

20 -

21 - char buf[sizeof(REGISTRY_URL) + strlen(url)];

22 -

23 - REGISTRY_URL *u = (REGISTRY_URL *)&buf;

24 - strcpy(u->url, url);

25 - u->hash = simple_hash(u->url);

12 + REGISTRY_PERSON_URL *pu;

13 + for(pu = p->person_urls ; pu ;pu = pu->next)

14 + if(pu->url == url)

15 + break;

27 - REGISTRY_PERSON_URL tpu = { .url = u };

28 -

29 - REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)avl_search(&p->person_urls, (void *)&tpu);

17 return pu;

18 }

20 inline REGISTRY_PERSON_URL *registry_person_url_index_add(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) {

34 - debug(D_REGISTRY, "Registry: registry_person_url_index_add('%s', '%s')", p->guid, pu->url->url);

35 - REGISTRY_PERSON_URL *tpu = (REGISTRY_PERSON_URL *)avl_insert(&(p->person_urls), (avl_t *)(pu));

36 - if(tpu != pu)

37 - netdata_log_error("Registry: registry_person_url_index_add('%s', '%s') already exists as '%s'", p->guid, pu->url->url, tpu->url->url);

38 -

39 - return tpu;

21 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(p->person_urls, pu, prev, next);

22 + return pu;

23 }

25 inline REGISTRY_PERSON_URL *registry_person_url_index_del(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) {

43 - debug(D_REGISTRY, "Registry: registry_person_url_index_del('%s', '%s')", p->guid, pu->url->url);

44 - REGISTRY_PERSON_URL *tpu = (REGISTRY_PERSON_URL *)avl_remove(&(p->person_urls), (avl_t *)(pu));

45 - if(!tpu)

46 - netdata_log_error("Registry: registry_person_url_index_del('%s', '%s') deleted nothing", p->guid, pu->url->url);

47 - else if(tpu != pu)

48 - netdata_log_error("Registry: registry_person_url_index_del('%s', '%s') deleted wrong URL '%s'", p->guid, pu->url->url, tpu->url->url);

49 -

50 - return tpu;

26 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->person_urls, pu, prev, next);

27 + return pu;

28 }

30 // ----------------------------------------------------------------------------

31 // PERSON_URL

56 -REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) {

57 - debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(REGISTRY_PERSON_URL) + namelen);

33 +REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *url, char *machine_name, size_t machine_name_len, time_t when) {

34 + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, string2str(url), sizeof(REGISTRY_PERSON_URL) + machine_name_len);

36 // protection from too big names

60 - if(namelen > registry.max_name_length)

61 - namelen = registry.max_name_length;

37 + if(machine_name_len > registry.max_name_length)

38 + machine_name_len = registry.max_name_length;

63 - REGISTRY_PERSON_URL *pu = mallocz(sizeof(REGISTRY_PERSON_URL) + namelen);

40 + REGISTRY_PERSON_URL *pu = aral_mallocz(registry.person_urls_aral);

65 - // a simple strcpy() should do the job

66 - // but I prefer to be safe, since the caller specified urllen

67 - strncpyz(pu->machine_name, name, namelen);

42 + // a simple strcpy() should do the job,

43 + // but I prefer to be safe, since the caller specified name_len

44 + pu->machine_name = string_strdupz(machine_name);

46 pu->machine = m;

47 pu->first_t = pu->last_t = (uint32_t)when;

48 pu->usages = 1;

72 - pu->url = u;

49 + pu->url = string_dup(url);

50 pu->flags = REGISTRY_URL_FLAGS_DEFAULT;

51 m->links++;

76 - registry.persons_urls_memory += sizeof(REGISTRY_PERSON_URL) + namelen;

77 -

78 - debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);

53 + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, string2str(url));

54 REGISTRY_PERSON_URL *tpu = registry_person_url_index_add(p, pu);

55 if(tpu != pu) {

81 - netdata_log_error("Registry: Attempted to add duplicate person url '%s' with name '%s' to person '%s'", u->url, name, p->guid);

82 - freez(pu);

56 + netdata_log_error("Registry: Attempted to add duplicate person url '%s' with name '%s' to person '%s'", string2str(url), machine_name, p->guid);

57 + string_freez(pu->machine_name);

58 + string_freez(pu->url);

59 + aral_freez(registry.person_urls_aral, pu);

60 pu = tpu;

61 }

85 - else

86 - registry_url_link(u);

63 return pu;

64 }

91 -void registry_person_url_free(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) {

92 - debug(D_REGISTRY, "registry_person_url_free('%s', '%s')", p->guid, pu->url->url);

66 +void registry_person_url_deindex_and_free(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) {

67 + debug(D_REGISTRY, "registry_person_url_deindex_and_free('%s', '%s')", p->guid, string2str(pu->url));

69 REGISTRY_PERSON_URL *tpu = registry_person_url_index_del(p, pu);

70 if(tpu) {

96 - registry_url_unlink(tpu->url);

71 + string_freez(tpu->machine_name);

72 + string_freez(tpu->url);

73 tpu->machine->links--;

98 - registry.persons_urls_memory -= sizeof(REGISTRY_PERSON_URL) + strlen(tpu->machine_name);

99 - freez(tpu);

74 + aral_freez(registry.person_urls_aral, tpu);

75 }

76 }

78 // this function is needed to change the name of a PERSON_URL

104 -REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when, REGISTRY_PERSON_URL *pu) {

105 - debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(REGISTRY_PERSON_URL) + namelen);

79 +REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *url, char *machine_name, size_t machine_name_len, time_t when, REGISTRY_PERSON_URL *pu) {

80 + debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, string2str(url), sizeof(REGISTRY_PERSON_URL) + machine_name_len);

82 // keep a backup

83 REGISTRY_PERSON_URL pu2 = {

@@ -111,15 +86,15 @@ REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY

86 .usages = pu->usages,

87 .flags = pu->flags,

88 .machine = pu->machine,

114 - .machine_name = ""

89 + .machine_name = NULL

90 };

92 // remove the existing one from the index

118 - registry_person_url_free(p, pu);

93 + registry_person_url_deindex_and_free(p, pu);

94 pu = &pu2;

96 // allocate a new one

122 - REGISTRY_PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when);

97 + REGISTRY_PERSON_URL *tpu = registry_person_url_allocate(p, m, url, machine_name, machine_name_len, when);

98 tpu->first_t = pu->first_t;

99 tpu->last_t = pu->last_t;

100 tpu->usages = pu->usages;

@@ -140,7 +115,7 @@ REGISTRY_PERSON *registry_person_find(const char *person_guid) {

115 REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when) {

116 debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(REGISTRY_PERSON));

117

143 - REGISTRY_PERSON *p = mallocz(sizeof(REGISTRY_PERSON));

118 + REGISTRY_PERSON *p = aral_mallocz(registry.persons_aral);

119 if(!person_guid) {

120 for(;;) {

121 uuid_t uuid;

@@ -159,14 +134,11 @@ REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when)

134 else

135 strncpyz(p->guid, person_guid, GUID_LEN);

136

162 - debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid);

163 - avl_init(&p->person_urls, person_url_compare);

137 + p->person_urls = NULL;

138

139 p->first_t = p->last_t = (uint32_t)when;

140 p->usages = 0;

141

168 - registry.persons_memory += sizeof(REGISTRY_PERSON);

169 -

142 registry.persons_count++;

143 dictionary_set(registry.persons, p->guid, p, sizeof(REGISTRY_PERSON));

144

@@ -178,23 +150,29 @@ REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when)

150 // 2. if it is valid, find it

151 // 3. if it is not valid, create a new one

152 // 4. return it

181 -REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when) {

182 - debug(D_REGISTRY, "Registry: registry_person_get('%s'): creating dictionary of urls", person_guid);

153 +REGISTRY_PERSON *registry_person_find_or_create(const char *person_guid, time_t when, bool is_dummy) {

154 + debug(D_REGISTRY, "Registry: registry_person_find_or_create('%s'): creating dictionary of urls", person_guid);

155

156 + char buf[GUID_LEN + 1];

157 REGISTRY_PERSON *p = NULL;

158

159 if(person_guid && *person_guid) {

187 - char buf[GUID_LEN + 1];

160 // validate it is a GUID

189 - if(unlikely(regenerate_guid(person_guid, buf) == -1))

161 + if(unlikely(regenerate_guid(person_guid, buf) == -1)) {

162 netdata_log_info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid);

163 + person_guid = NULL;

164 + }

165 else {

166 person_guid = buf;

167 p = registry_person_find(person_guid);

168 + if(!p && !is_dummy)

169 + person_guid = NULL;

170 }

171 }

172 + else

173 + person_guid = NULL;

174

197 - if(!p) p = registry_person_allocate(NULL, when);

175 + if(!p) p = registry_person_allocate(person_guid, when);

176

177 return p;

178 }

@@ -202,39 +180,39 @@ REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when) {

180 // ----------------------------------------------------------------------------

181 // LINKING OF OBJECTS

182

205 -REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) {

206 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);

183 +REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *url, char *machine_name, size_t machine_name_len, time_t when) {

184 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, string2str(url));

185

208 - REGISTRY_PERSON_URL *pu = registry_person_url_index_find(p, u->url);

186 + REGISTRY_PERSON_URL *pu = registry_person_url_index_find(p, url);

187 if(!pu) {

210 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);

211 - pu = registry_person_url_allocate(p, m, u, name, namelen, when);

188 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, string2str(url));

189 + pu = registry_person_url_allocate(p, m, url, machine_name, machine_name_len, when);

190 registry.persons_urls_count++;

191 }

192 else {

215 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);

193 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, string2str(url));

194 pu->usages++;

195 if(likely(pu->last_t < (uint32_t)when)) pu->last_t = (uint32_t)when;

196

197 if(pu->machine != m) {

220 - REGISTRY_MACHINE_URL *mu = dictionary_get(pu->machine->machine_urls, u->url);

198 + REGISTRY_MACHINE_URL *mu = registry_machine_url_find(pu->machine, url);

199 if(mu) {

200 debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.",

223 - p->guid, m->guid, u->url, pu->machine->guid);

201 + p->guid, m->guid, string2str(url), pu->machine->guid);

202 mu->flags |= REGISTRY_URL_FLAGS_EXPIRED;

203 }

204 else {

205 debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - but the URL is not linked to the old machine.",

228 - p->guid, m->guid, u->url, pu->machine->guid);

206 + p->guid, m->guid, string2str(url), pu->machine->guid);

207 }

208

209 pu->machine->links--;

210 pu->machine = m;

211 }

212

235 - if(strcmp(pu->machine_name, name) != 0) {

213 + if(strcmp(string2str(pu->machine_name), machine_name) != 0) {

214 // the name of the PERSON_URL has changed !

237 - pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu);

215 + pu = registry_person_url_reallocate(p, m, url, machine_name, machine_name_len, when, pu);

216 }

217 }

218

@@ -242,7 +220,7 @@ REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MA

220 if(likely(p->last_t < (uint32_t)when)) p->last_t = (uint32_t)when;

221

222 if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) {

245 - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url);

223 + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, string2str(url));

224 pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;

225 }

226

@@ -250,5 +228,5 @@ REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MA

228 }

229

230 void registry_person_unlink_from_url(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) {

253 - registry_person_url_free(p, pu);

231 + registry_person_url_deindex_and_free(p, pu);

232 }

registry/registry_person.h

+13 -17

@@ -10,19 +10,18 @@

11 // for each PERSON-URL pair we keep this

12 struct registry_person_url {

13 - avl_t avl; // binary tree node

14 -

15 - REGISTRY_URL *url; // de-duplicated URL

16 - REGISTRY_MACHINE *machine; // link the MACHINE of this URL

17 -

13 uint8_t flags;

15 + uint32_t usages; // how many times this has been accessed

16 +

17 uint32_t first_t; // the first time we saw this

18 uint32_t last_t; // the last time we saw this

22 - uint32_t usages; // how many times this has been accessed

24 - char machine_name[1]; // the name of the machine, as known by the user

25 - // dynamically allocated to fit properly

20 + REGISTRY_MACHINE *machine; // link the MACHINE of this URL

21 + STRING *machine_name; // the hostname of the machine

22 + STRING *url; // de-duplicated URL

23 +

24 + struct registry_person_url *prev, *next;

25 };

26 typedef struct registry_person_url REGISTRY_PERSON_URL;

@@ -30,32 +29,29 @@ typedef struct registry_person_url REGISTRY_PERSON_URL;

29 struct registry_person {

30 char guid[GUID_LEN + 1]; // the person GUID

33 - avl_tree_type person_urls; // dictionary of PERSON_URLs

32 + REGISTRY_PERSON_URL *person_urls; // dictionary of PERSON_URLs

34 uint32_t first_t; // the first time we saw this

35 uint32_t last_t; // the last time we saw this

36 uint32_t usages; // how many times this has been accessed

38 -

39 - //uint32_t flags;

40 - //char *email;

37 };

38 typedef struct registry_person REGISTRY_PERSON;

40 // PERSON_URL

45 -REGISTRY_PERSON_URL *registry_person_url_index_find(REGISTRY_PERSON *p, const char *url);

41 +REGISTRY_PERSON_URL *registry_person_url_index_find(REGISTRY_PERSON *p, STRING *url);

42 REGISTRY_PERSON_URL *registry_person_url_index_add(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) NEVERNULL WARNUNUSED;

43 REGISTRY_PERSON_URL *registry_person_url_index_del(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) WARNUNUSED;

49 -REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when);

50 -REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when, REGISTRY_PERSON_URL *pu);

45 +REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *url, char *machine_name, size_t machine_name_len, time_t when);

46 +REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *url, char *machine_name, size_t machine_name_len, time_t when, REGISTRY_PERSON_URL *pu);

48 // PERSON

49 REGISTRY_PERSON *registry_person_find(const char *person_guid);

50 REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when);

55 -REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when);

51 +REGISTRY_PERSON *registry_person_find_or_create(const char *person_guid, time_t when, bool is_dummy);

53 // LINKING PERSON -> PERSON_URL

58 -REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when);

54 +REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, STRING *url, char *machine_name, size_t machine_name_len, time_t when);

55 void registry_person_unlink_from_url(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu);

57 #endif //NETDATA_REGISTRY_PERSON_H

registry/registry_url.c deleted

-88

@@ -1,88 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include "daemon/common.h"

4 -#include "registry_internals.h"

5 -

6 -// ----------------------------------------------------------------------------

7 -// REGISTRY_URL

8 -

9 -int registry_url_compare(void *a, void *b) {

10 - if(((REGISTRY_URL *)a)->hash < ((REGISTRY_URL *)b)->hash) return -1;

11 - else if(((REGISTRY_URL *)a)->hash > ((REGISTRY_URL *)b)->hash) return 1;

12 - else return strcmp(((REGISTRY_URL *)a)->url, ((REGISTRY_URL *)b)->url);

13 -}

14 -

15 -inline REGISTRY_URL *registry_url_index_add(REGISTRY_URL *u) {

16 - return (REGISTRY_URL *)avl_insert(&(registry.registry_urls_root_index), (avl_t *)(u));

17 -}

18 -

19 -inline REGISTRY_URL *registry_url_index_del(REGISTRY_URL *u) {

20 - return (REGISTRY_URL *)avl_remove(&(registry.registry_urls_root_index), (avl_t *)(u));

21 -}

22 -

23 -REGISTRY_URL *registry_url_get(const char *url, size_t urllen) {

24 - // protection from too big URLs

25 - if(urllen > registry.max_url_length)

26 - urllen = registry.max_url_length;

27 -

28 - debug(D_REGISTRY, "Registry: registry_url_get('%s', %zu)", url, urllen);

29 -

30 - char buf[sizeof(REGISTRY_URL) + urllen]; // no need for +1, 1 is already in REGISTRY_URL

31 - REGISTRY_URL *n = (REGISTRY_URL *)&buf[0];

32 - n->len = (uint16_t)urllen;

33 - strncpyz(n->url, url, n->len);

34 - n->hash = simple_hash(n->url);

35 -

36 - REGISTRY_URL *u = (REGISTRY_URL *)avl_search(&(registry.registry_urls_root_index), (avl_t *)n);

37 - if(!u) {

38 - debug(D_REGISTRY, "Registry: registry_url_get('%s', %zu): allocating %zu bytes", url, urllen, sizeof(REGISTRY_URL) + urllen);

39 - u = callocz(1, sizeof(REGISTRY_URL) + urllen); // no need for +1, 1 is already in REGISTRY_URL

40 -

41 - // a simple strcpy() should do the job

42 - // but I prefer to be safe, since the caller specified urllen

43 - u->len = (uint16_t)urllen;

44 - strncpyz(u->url, url, u->len);

45 - u->links = 0;

46 - u->hash = simple_hash(u->url);

47 -

48 - registry.urls_memory += sizeof(REGISTRY_URL) + urllen; // no need for +1, 1 is already in REGISTRY_URL

49 -

50 - debug(D_REGISTRY, "Registry: registry_url_get('%s'): indexing it", url);

51 - n = registry_url_index_add(u);

52 - if(n != u) {

53 - netdata_log_error("INTERNAL ERROR: registry_url_get(): url '%s' already exists in the registry as '%s'", u->url, n->url);

54 - freez(u);

55 - u = n;

56 - }

57 - else

58 - registry.urls_count++;

59 - }

60 -

61 - return u;

62 -}

63 -

64 -void registry_url_link(REGISTRY_URL *u) {

65 - u->links++;

66 - debug(D_REGISTRY, "Registry: registry_url_link('%s'): URL has now %u links", u->url, u->links);

67 -}

68 -

69 -void registry_url_unlink(REGISTRY_URL *u) {

70 - u->links--;

71 - if(!u->links) {

72 - debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): No more links for this URL", u->url);

73 - REGISTRY_URL *n = registry_url_index_del(u);

74 - if(!n) {

75 - netdata_log_error("INTERNAL ERROR: registry_url_unlink('%s'): cannot find url in index", u->url);

76 - }

77 - else {

78 - if(n != u) {

79 - netdata_log_error("INTERNAL ERROR: registry_url_unlink('%s'): deleted different url '%s'", u->url, n->url);

80 - }

81 -

82 - registry.urls_memory -= sizeof(REGISTRY_URL) + n->len; // no need for +1, 1 is already in REGISTRY_URL

83 - freez(n);

84 - }

85 - }

86 - else

87 - debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): URL has %u links left", u->url, u->links);

88 -}

registry/registry_url.h deleted

-35

@@ -1,35 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_REGISTRY_URL_H

4 -#define NETDATA_REGISTRY_URL_H 1

5 -

6 -#include "registry_internals.h"

7 -

8 -// ----------------------------------------------------------------------------

9 -// URL structures

10 -// Save memory by de-duplicating URLs

11 -// so instead of storing URLs all over the place

12 -// we store them here and we keep pointers elsewhere

13 -

14 -struct registry_url {

15 - avl_t avl;

16 - uint32_t hash; // the index hash

17 -

18 - uint32_t links; // the number of links to this URL - when none is left, we free it

19 -

20 - uint16_t len; // the length of the URL in bytes

21 - char url[1]; // the URL - dynamically allocated to more size

22 -};

23 -typedef struct registry_url REGISTRY_URL;

24 -

25 -// REGISTRY_URL INDEX

26 -int registry_url_compare(void *a, void *b);

27 -REGISTRY_URL *registry_url_index_del(REGISTRY_URL *u) WARNUNUSED;

28 -REGISTRY_URL *registry_url_index_add(REGISTRY_URL *u) NEVERNULL WARNUNUSED;

29 -

30 -// REGISTRY_URL MANAGEMENT

31 -REGISTRY_URL *registry_url_get(const char *url, size_t urllen) NEVERNULL;

32 -void registry_url_link(REGISTRY_URL *u);

33 -void registry_url_unlink(REGISTRY_URL *u);

34 -

35 -#endif //NETDATA_REGISTRY_URL_H

web/api/web_api.c

+30 -1

@@ -2,6 +2,35 @@

3 #include "web_api.h"

5 +bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at the point the agent logs in - this should also be saved to /var/lib/netdata

6 +DICTIONARY *netdata_authorized_bearers = NULL;

7 +

8 +static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL endpoint_acl) {

9 + if(endpoint_acl == WEB_CLIENT_ACL_NOCHECK)

10 + // the endpoint is totally public

11 + return true;

12 +

13 + bool acl_allows = w->acl & endpoint_acl;

14 + if(!acl_allows)

15 + // the channel we received the request from (w->acl) is not compatible with the endpoint

16 + return false;

17 +

18 + if(!netdata_is_protected_by_bearer && !(endpoint_acl & WEB_CLIENT_ACL_BEARER_REQUIRED))

19 + // bearer protection is not enabled and is not required by the endpoint

20 + return true;

21 +

22 + if(!(endpoint_acl & (WEB_CLIENT_ACL_BEARER_REQUIRED|WEB_CLIENT_ACL_BEARER_OPTIONAL)))

23 + // endpoint does not require a bearer

24 + return true;

25 +

26 + if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)) || api_check_bearer_token(w))

27 + // the request is coming from ACLK or WEBRTC (authorized already),

28 + // or we have a valid bearer on the request

29 + return true;

30 +

31 + return false;

32 +}

33 +

34 int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_path_endpoint, struct web_api_command *api_commands) {

35 if(unlikely(!url_path_endpoint || !*url_path_endpoint)) {

36 buffer_flush(w->response.data);

@@ -13,7 +42,7 @@ int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_pat

43 for(int i = 0; api_commands[i].command ; i++) {

44 if(unlikely(hash == api_commands[i].hash && !strcmp(url_path_endpoint, api_commands[i].command))) {

16 - if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))

45 + if(unlikely(!web_client_check_acl_and_bearer(w, api_commands[i].acl)))

46 return web_client_permission_denied(w);

48 char *query_string = (char *)buffer_tostring(w->url_query_string_decoded);

web/api/web_api.h

@@ -9,6 +9,11 @@

9 #include "web/api/health/health_cmdapi.h"

10 #include "web/api/queries/weights.h"

12 +extern bool netdata_is_protected_by_bearer;

13 +extern DICTIONARY *netdata_authorized_bearers;

14 +bool api_check_bearer_token(struct web_client *w);

15 +bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);

16 +

17 struct web_api_command {

18 const char *command;

19 uint32_t hash;

web/api/web_api_v1.c

+9 -14

@@ -927,16 +927,17 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *

927 */

928 }

929

930 - char person_guid[GUID_LEN + 1] = "";

931 -

930 debug(D_WEB_CLIENT, "%llu: API v1 registry with URL '%s'", w->id, url);

931

932 // TODO

933 // The browser may send multiple cookies with our id

934

935 + char person_guid[UUID_STR_LEN] = "";

936 char *cookie = strstr(w->response.data->buffer, NETDATA_REGISTRY_COOKIE_NAME "=");

937 if(cookie)

939 - strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME)], 36);

938 + strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME)], UUID_STR_LEN - 1);

939 + else if(!extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)))

940 + person_guid[0] = '\0';

941

942 char action = '\0';

943 char *machine_guid = NULL,

@@ -1516,12 +1517,6 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc

1517 }

1518 #endif

1519

1519 -#ifdef NETDATA_DEV_MODE

1520 -#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_DASHBOARD

1521 -#else

1522 -#define ACL_DEV_OPEN_ACCESS 0

1523 -#endif

1524 -

1520 static struct web_api_command api_commands_v1[] = {

1521 { "info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info },

1522 { "data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data },

@@ -1531,7 +1526,7 @@ static struct web_api_command api_commands_v1[] = {

1526 { "contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts },

1527

1528 // registry checks the ACL by itself, so we allow everything

1534 - { "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry },

1529 + { "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry },

1530

1531 // badges can be fetched with both dashboard and badge permissions

1532 { "badge.svg", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC | WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge },

@@ -1545,16 +1540,16 @@ static struct web_api_command api_commands_v1[] = {

1540

1541 #if defined(ENABLE_ML)

1542 { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info },

1548 - { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models },

1543 + // { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models },

1544 #endif

1545

1551 - { "manage/health", 0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_mgmt_health },

1546 + {"manage/health", 0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_BEARER_REQUIRED, web_client_api_request_v1_mgmt_health },

1547 { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state },

1548 { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations },

1549 { "weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights },

1550

1556 - { "function", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function },

1557 - { "functions", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions },

1551 + {"function", 0, WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_BEARER_REQUIRED | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function },

1552 + {"functions", 0, WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_BEARER_REQUIRED | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions },

1553

1554 { "dbengine_stats", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats },

1555

web/api/web_api_v2.c

+135 -16

@@ -3,6 +3,119 @@

3 #include "web_api_v2.h"

4 #include "../rtc/webrtc.h"

6 +struct bearer_token {

7 + time_t created_s;

8 + time_t expires_s;

9 +};

10 +

11 +static void bearer_get_token(uuid_t *uuid) {

12 + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;

13 + static bool initialized = false;

14 +

15 + if(!initialized) {

16 + spinlock_lock(&spinlock);

17 + if (!netdata_authorized_bearers) {

18 + netdata_authorized_bearers = dictionary_create_advanced(

19 + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,

20 + NULL, sizeof(struct bearer_token));

21 + }

22 + spinlock_unlock(&spinlock);

23 + initialized = true;

24 + }

25 +

26 + char uuid_str[UUID_STR_LEN];

27 +

28 + uuid_generate_random(*uuid);

29 + uuid_unparse_lower(*uuid, uuid_str);

30 +

31 + struct bearer_token t = { 0 }, *z;

32 + z = dictionary_set(netdata_authorized_bearers, uuid_str, &t, sizeof(t));

33 + if(!z->created_s) {

34 + z->created_s = now_monotonic_sec();

35 + z->expires_s = z->created_s + 86400;

36 + }

37 +}

38 +

39 +#define HTTP_REQUEST_AUTHORIZATION_BEARER "\r\nAuthorization: Bearer "

40 +

41 +bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {

42 + const char *req = buffer_tostring(w->response.data);

43 + size_t req_len = buffer_strlen(w->response.data);

44 + const char *bearer = strcasestr(req, HTTP_REQUEST_AUTHORIZATION_BEARER);

45 +

46 + if(!bearer)

47 + return false;

48 +

49 + const char *token_start = bearer + sizeof(HTTP_REQUEST_AUTHORIZATION_BEARER) - 1;

50 +

51 + while(isspace(*token_start))

52 + token_start++;

53 +

54 + const char *token_end = token_start + UUID_STR_LEN - 1 + 2;

55 + if (token_end > req + req_len)

56 + return false;

57 +

58 + strncpyz(dst, token_start, dst_len - 1);

59 + uuid_t uuid;

60 + if (uuid_parse(dst, uuid) != 0)

61 + return false;

62 +

63 + return true;

64 +}

65 +

66 +bool api_check_bearer_token(struct web_client *w) {

67 + if(!netdata_authorized_bearers)

68 + return false;

69 +

70 + char token[UUID_STR_LEN];

71 + if(!extract_bearer_token_from_request(w, token, sizeof(token)))

72 + return false;

73 +

74 + struct bearer_token *z = dictionary_get(netdata_authorized_bearers, token);

75 + return z && z->expires_s > now_monotonic_sec();

76 +}

77 +

78 +int api_v2_bearer_protection(RRDHOST *host __maybe_unused, struct web_client *w __maybe_unused, char *url) {

79 + while (url) {

80 + char *value = strsep_skip_consecutive_separators(&url, "&");

81 + if (!value || !*value) continue;

82 +

83 + char *name = strsep_skip_consecutive_separators(&value, "=");

84 + if (!name || !*name) continue;

85 + if (!value || !*value) continue;

86 +

87 + if(!strcmp(name, "bearer_protection")) {

88 + if(!strcmp(value, "on") || !strcmp(value, "true") || !strcmp(value, "yes"))

89 + netdata_is_protected_by_bearer = true;

90 + else

91 + netdata_is_protected_by_bearer = false;

92 + }

93 + }

94 +

95 + BUFFER *wb = w->response.data;

96 + buffer_flush(wb);

97 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);

98 + buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);

99 + buffer_json_finalize(wb);

100 +

101 + return HTTP_RESP_OK;

102 +}

103 +

104 +int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __maybe_unused, char *url __maybe_unused) {

105 + uuid_t uuid;

106 + bearer_get_token(&uuid);

107 +

108 + BUFFER *wb = w->response.data;

109 + buffer_flush(wb);

110 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);

111 + buffer_json_member_add_string(wb, "mg", localhost->machine_guid);

112 + buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);

113 + buffer_json_member_add_uuid(wb, "token", &uuid);

114 + buffer_json_finalize(wb);

115 +

116 + return HTTP_RESP_OK;

117 +}

118 +

119 static int web_client_api_request_v2_contexts_internal(RRDHOST *host __maybe_unused, struct web_client *w, char *url, CONTEXTS_V2_MODE mode) {

120 struct api_v2_contexts_request req = { 0 };

121

@@ -441,24 +554,30 @@ static int web_client_api_request_v2_webrtc(RRDHOST *host __maybe_unused, struct

554 }

555

556 static struct web_api_command api_commands_v2[] = {

444 - {"data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data},

445 - {"info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info},

446 - {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes},

447 - {"node_instances", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances},

448 - {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts},

449 - {"weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights},

450 - {"versions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions},

451 - {"functions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_functions},

452 - {"alerts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts},

453 - {"alert_transitions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions},

454 - {"alert_config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config},

455 - {"claim", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_claim},

456 - {"q", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q},

457 -

458 - {"rtc_offer", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_webrtc},

557 + {"info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info},

558 +

559 + {"data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data},

560 + {"weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights},

561 +

562 + {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts},

563 + {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes},

564 + {"node_instances", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances},

565 + {"versions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions},

566 + {"functions", 0, WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_BEARER_REQUIRED | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_functions},

567 + {"q", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q},

568 + {"alerts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts},

569 +

570 + {"alert_transitions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions},

571 + {"alert_config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config},

572 +

573 + {"claim", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v2_claim},

574 +

575 + {"rtc_offer", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_webrtc},

576 + {"bearer_protection", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection},

577 + {"bearer_get_token", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token},

578

579 // terminator

461 - {NULL, 0, WEB_CLIENT_ACL_NONE, NULL},

580 + {NULL, 0, WEB_CLIENT_ACL_NONE, NULL},

581 };

582

583 inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint) {