@cryptotaxi247 / netdata-1 / commits / eda12f579

Implemented multihost database (#9556)

* Hard code a node for non-legacy multidb test Skip dbengine initialization for new incoming children Add code to switch to multidb ctx when accessing the dbengine * When a non-legacy streaming connection is detected, use the multidb metadata log context * Clear the superblock memory to avoid random data written in the metadata log * Activate the host detection during compaction Activate the host detection during metadata log chart updates Keep the host in the user object during replay of the HOST command * Add defaults for health / rrdpush on HOST metadata replay Check for legacy status on host creation by checking is_archived and if not conclusive, call is_legacy_child() Use defaults from the stream.conf * Count hosts only if not archived When host switches from archived to active update rrd_hosts_available Remove archived hosts from charts and info * Change parameter from "multidb disk space" to "dbengine multihost disk space" Remove unused variables Fix compilation error when dbengine is disabled Fix condition for machine_guid directory creation under cache_dir * Enable multidb disk space file creation. * Stop deleting dimensions when rotating archived metrics if the dimension is active in a different database engine. * Fix old bug in the code that confused obsolete hosts with orphan hosts. * Do not delete multi-host DB host files. * Discard dbengine state when a legacy memory mode instantiates to avoid inconsistencies. * Identify metadata that collide with non-dbengine memory mode hosts and ignore them. * Handle non-dbengine localhost with dbengine archived charts in localhost and streaming. * Ignore archived hosts in streaming. * Add documentation before merging to master. Co-authored-by: Markos Fountoulakis <markos.fountoulakis.senior@gmail.com>

Stelios Fragkakis committed Jul 28, 2020 at 15:04 UTC eda12f579f97b123ef9b890b01528223a89a70aa

24 files changed +425 -215

collectors/plugins.d/pluginsd_parser.c

@@ -274,6 +274,10 @@ PARSER_RC pluginsd_end(char **words, void *user, PLUGINSD_ACTION *plugins_actio

274 PARSER_RC pluginsd_chart(char **words, void *user, PLUGINSD_ACTION *plugins_action)

275 {

276 RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;

277 + if (unlikely(!host)) {

278 + debug(D_PLUGINSD, "Ignoring chart belonging to missing or ignored host.");

279 + return PARSER_RC_OK;

280 + }

281

282 char *type = words[1];

283 char *name = words[2];

@@ -371,6 +375,10 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION *plugins

375

376 RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;

377 RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;

378 + if (unlikely(!host)) {

379 + debug(D_PLUGINSD, "Ignoring dimension belonging to missing or ignored host.");

380 + return PARSER_RC_OK;

381 + }

382

383 if (unlikely(!id)) {

384 error(

daemon/main.c

+11 -4

@@ -41,7 +41,13 @@ void netdata_cleanup_and_exit(int ret) {

42 // free the database

43 info("EXIT: freeing database memory...");

44 +#ifdef ENABLE_DBENGINE

45 + rrdeng_prepare_exit(&multidb_ctx);

46 +#endif

47 rrdhost_free_all();

48 +#ifdef ENABLE_DBENGINE

49 + rrdeng_exit(&multidb_ctx);

50 +#endif

51 }

53 // unlink the pid

@@ -568,10 +574,10 @@ static void get_netdata_configured_variables() {

574 default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;

575 }

576

571 - default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "multidb disk space", compute_multidb_diskspace());

577 + default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", compute_multidb_diskspace());

578 if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {

573 - error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);

574 - default_multidb_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;

579 + error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);

580 + default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;

581 }

582

583 #endif

@@ -1456,7 +1462,8 @@ int main(int argc, char **argv) {

1462 // Load host labels

1463 reload_host_labels();

1464 #ifdef ENABLE_DBENGINE

1459 - metalog_commit_update_host(localhost);

1465 + if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

1466 + metalog_commit_update_host(localhost);

1467 #endif

1468

1469 // ------------------------------------------------------------------------

database/engine/README.md

+46 -17

@@ -32,35 +32,64 @@ section of your `netdata.conf`. The Agent ignores the `history` setting when usi

32 ```conf

33 [global]

34 page cache size = 32

35 - dbengine disk space = 256

35 + dbengine multihost disk space = 256

36 ```

38 -The above values are the default and minimum values for Page Cache size and DB engine disk space quota. Both numbers are

38 +The above values are the default values for Page Cache size and DB engine disk space quota. Both numbers are

39 in **MiB**.

41 The `page cache size` option determines the amount of RAM in **MiB** dedicated to caching Netdata metric values. The

42 actual page cache size will be slightly larger than this figure—see the [memory requirements](#memory-requirements)

43 section for details.

45 -The `dbengine disk space` option determines the amount of disk space in **MiB** that is dedicated to storing Netdata

46 -metric values and all related metadata describing them.

45 +The `dbengine multihost disk space` option determines the amount of disk space in **MiB** that is dedicated to storing

46 +Netdata metric values and all related metadata describing them.

47 +

48 +### Legacy configuration

49 +

50 +The deprecated `dbengine disk space` option determines the amount of disk space in **MiB** that is dedicated to storing

51 +Netdata metric values per legacy database engine instance (see [below](#Streaming-metrics-to-the-database-engine)).

52 +

53 +```conf

54 +[global]

55 + dbengine disk space = 256

56 +```

58 Use the [**database engine calculator**](https://learn.netdata.cloud/docs/agent/database/calculator) to correctly set

49 -`dbengine disk space` based on your needs. The calculator gives an accurate estimate based on how many child nodes

50 -you have, how many metrics your Agent collects, and more.

59 +`dbengine disk space`(**deprecated**) based on your needs. The calculator gives an accurate estimate based on how many

60 +child nodes you have, how many metrics your Agent collects, and more.

62 ### Streaming metrics to the database engine

54 -When streaming metrics, the Agent on the parent node creates one instance of the database engine for itself, and another

55 -instance for every child node it receives metrics from. If you have four streaming nodes, you will have five instances

56 -in total (`1 parent + 4 child nodes = 5 instances`).

64 +##### Legacy mode

58 -The Agent allocates resources for each instance separately using the `dbengine disk space` setting. If `dbengine disk

59 -space` is set to the default `256`, each instance is given 256 MiB in disk space, which means the total disk space

60 -required to store all instances is, roughly, `256 MiB * 1 parent * 4 child nodes = 1280 MiB`.

66 +When streaming metrics, the Agent on the parent node used to create one instance (legacy, version <= 1.23.2) of the

67 +database engine for itself, and another instance for every child node it receives metrics from. If you had four

68 +streaming nodes, you would have five instances in total (`1 parent + 4 child nodes = 5 instances`).

69 +

70 +The Agent allocated resources for each instance separately using the `dbengine disk space`(**deprecated**) setting. If

71 +`dbengine disk space`(**deprecated**) is set to the default `256`, each instance is given 256 MiB in disk space, which

72 +means the total disk space required to store all instances is, roughly, `256 MiB * 1 parent * 4 child nodes = 1280 MiB`.

74 See the [database engine calculator](https://learn.netdata.cloud/docs/agent/database/calculator) to help you correctly

63 -set `dbengine disk space` and undertand the toal disk space required based on your streaming setup.

75 +set `dbengine disk space`(**deprecated**) and understand the total disk space required based on your streaming setup.

76 +

77 +##### Multi host DB mode

78 +

79 +In the newer agent versions the parent and child nodes all share `page cache size` and `dbengine multihost disk space`

80 +in a single dbengine multi-host instance.

81 +

82 +##### Backward compatibility

83 +

84 +All existing metrics belonging to child nodes are automatically converted to legacy dbengine instances and the localhost

85 +metrics are transferred to the multi-host dbengine instance.

86 +

87 +All new child nodes are automatically transferred to the mult-host dbengine instance and share its page cache and disk

88 +space. If you want to migrate a child node from its legacy dbengine instance to the multi-host dbengine instance you

89 +must delete the instance's directory located in `/var/cache/netdata/MACHINE_GUID/dbengine` after stopping the netdata

90 +agent.

91 +

92 +##### Information

94 For more information about setting `memory mode` on your nodes, in addition to other streaming configurations, see

95 [streaming](/streaming/README.md).

@@ -70,8 +99,7 @@ For more information about setting `memory mode` on your nodes, in addition to o

99 Using memory mode `dbengine` we can overcome most memory restrictions and store a dataset that is much larger than the

100 available memory.

101

73 -There are explicit memory requirements **per** DB engine **instance**, meaning **per** Netdata **node** (e.g. localhost

74 -and streaming recipient nodes):

102 +There are explicit memory requirements **per** DB engine **instance**:

103

104 - The total page cache memory footprint will be an additional `#dimensions-being-collected x 4096 x 2` bytes over what

105 the user configured with `page cache size`.

@@ -83,10 +111,11 @@ and streaming recipient nodes):

111 - for very highly compressible data (compression ratio > 90%) this RAM overhead is comparable to the disk space

112 footprint.

113

86 -An important observation is that RAM usage depends on both the `page cache size` and the `dbengine disk space` options.

114 +An important observation is that RAM usage depends on both the `page cache size` and the `dbengine multihost disk space`

115 +options.

116

117 You can use our [database engine calculator](https://learn.netdata.cloud/docs/agent/database/calculator) to

89 -validate the memory requirements for your particular system(s) and configuration.

118 +validate the memory requirements for your particular system(s) and configuration (**out-of-date**).

119

120 ### File descriptor requirements

121

database/engine/metadata_log/compaction.c

+4 -3

@@ -82,7 +82,7 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)

82 RRDSET *st;

83 RRDDIM *rd;

84 BUFFER *buffer;

85 - RRDHOST *host = ctx->rrdeng_ctx->host;

85 + RRDHOST *host = NULL;

87 ret = find_object_by_guid(uuid, NULL, 0);

88 switch (ret) {

@@ -130,8 +130,9 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)

130 }

131 break;

132 case GUID_TYPE_HOST:

133 - //TODO: will be enabled when multidb is activated

134 - //RRDHOST *host = metalog_get_host_from_uuid(ctx, uuid);

133 + host = metalog_get_host_from_uuid(ctx, uuid);

134 + if (unlikely(!host))

135 + break;

136 if (ctx->current_compaction_id > host->compaction_id) {

137 host->compaction_id = ctx->current_compaction_id;

138 buffer = metalog_update_host_buffer(host);

database/engine/metadata_log/logfile.c

@@ -330,6 +330,7 @@ int create_metadata_logfile(struct metadata_logfile *metalogfile)

330 if (unlikely(ret)) {

331 fatal("posix_memalign:%s", strerror(ret));

332 }

333 + memset(superblock, 0, sizeof(*superblock));

334 (void) strncpy(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ);

335 superblock->version = RRDENG_METALOG_VER;

336

database/engine/metadata_log/metadatalog.c

+2 -2

@@ -157,9 +157,9 @@ void metalog_test_quota(struct metalog_worker_config *wc)

157 metalogfile = ctx->metadata_logfiles.last;

158 only_one_metalogfile = (metalogfile == ctx->metadata_logfiles.first) ? 1 : 0;

159 debug(D_METADATALOG, "records=%lu objects=%lu", (long unsigned)ctx->records_nr,

160 - (long unsigned)ctx->rrdeng_ctx->host->objects_nr);

160 + (long unsigned)ctx->objects_nr);

161 if (unlikely(!only_one_metalogfile &&

162 - ctx->records_nr > (ctx->rrdeng_ctx->host->objects_nr * (uint64_t)MAX_DUPLICATION_PERCENTAGE) / 100) &&

162 + ctx->records_nr > (ctx->objects_nr * (uint64_t)MAX_DUPLICATION_PERCENTAGE) / 100) &&

163 NO_QUIESCE == ctx->quiesce) {

164 metalog_do_compaction(wc);

165 }

database/engine/metadata_log/metadatalog.h

@@ -112,6 +112,8 @@ struct metalog_instance {

112 uint32_t current_compaction_id; /* Every compaction run increments this by 1 */

113 unsigned long disk_space;

114 unsigned long records_nr;

115 + unsigned long objects_nr; /* total objects (hosts, charts, dimensions) monitored in this context */

116 + uint8_t initialized; /* set to 1 to mark context initialized */

117 unsigned last_fileno; /* newest index of metadata log file */

118

119 uint8_t quiesce; /*

database/engine/metadata_log/metadatalogapi.c

+70 -29

@@ -3,6 +3,14 @@

4 #include "metadatalog.h"

6 +static inline struct metalog_instance *get_metalog_ctx(RRDHOST *host)

7 +{

8 + if (host->rrdeng_ctx)

9 + return host->rrdeng_ctx->metalog_ctx;

10 +

11 + return NULL;

12 +}

13 +

14 static inline int metalog_is_initialized(struct metalog_instance *ctx)

15 {

16 return ctx->rrdeng_ctx->metalog_ctx != NULL;

@@ -18,6 +26,16 @@ static inline void metalog_commit_deletion_record(struct metalog_instance *ctx,

26 metalog_commit_record(ctx, buffer, METALOG_COMMIT_DELETION_RECORD, NULL, 0);

27 }

29 +void metalog_upd_objcount(RRDHOST *host, int count)

30 +{

31 + struct metalog_instance *ctx = get_metalog_ctx(host);

32 +

33 + if (unlikely(!ctx))

34 + return;

35 +

36 + rrd_atomic_fetch_add(&ctx->objects_nr, count);

37 +}

38 +

39 BUFFER *metalog_update_host_buffer(RRDHOST *host)

40 {

41 BUFFER *buffer;

@@ -62,11 +80,10 @@ void metalog_commit_update_host(RRDHOST *host)

80 BUFFER *buffer;

82 /* Metadata are only available with dbengine */

65 - if (!host->rrdeng_ctx)

83 + ctx = get_metalog_ctx(host);

84 + if (!ctx)

85 return;

67 -

68 - ctx = host->rrdeng_ctx->metalog_ctx;

69 - if (!ctx) /* metadata log has not been initialized yet */

86 + if (!ctx->initialized) /* metadata log has not been initialized yet */

87 return;

89 buffer = metalog_update_host_buffer(host);

@@ -157,14 +174,15 @@ void metalog_commit_update_chart(RRDSET *st)

174 {

175 struct metalog_instance *ctx;

176 BUFFER *buffer;

160 - RRDHOST *host = st->rrdhost;

177

178 /* Metadata are only available with dbengine */

163 - if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

179 + if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

180 return;

181

166 - ctx = host->rrdeng_ctx->metalog_ctx;

167 - if (!ctx) /* metadata log has not been initialized yet */

182 + ctx = get_metalog_ctx(st->rrdhost);

183 + if (!ctx)

184 + return;

185 + if (!ctx->initialized) /* metadata log has not been initialized yet */

186 return;

187

188 buffer = metalog_update_chart_buffer(st, 0);

@@ -176,15 +194,16 @@ void metalog_commit_delete_chart(RRDSET *st)

194 {

195 struct metalog_instance *ctx;

196 BUFFER *buffer;

179 - RRDHOST *host = st->rrdhost;

197 char uuid_str[37];

198

199 /* Metadata are only available with dbengine */

183 - if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

200 + if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

201 return;

202

186 - ctx = host->rrdeng_ctx->metalog_ctx;

187 - if (!ctx) /* metadata log has not been initialized yet */

203 + ctx = get_metalog_ctx(st->rrdhost);

204 + if (!ctx)

205 + return;

206 + if (!ctx->initialized) /* metadata log has not been initialized yet */

207 return;

208 buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */

209

@@ -228,14 +247,15 @@ void metalog_commit_update_dimension(RRDDIM *rd)

247 struct metalog_instance *ctx;

248 BUFFER *buffer;

249 RRDSET *st = rd->rrdset;

231 - RRDHOST *host = st->rrdhost;

250

251 /* Metadata are only available with dbengine */

234 - if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

252 + if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

253 return;

254

237 - ctx = host->rrdeng_ctx->metalog_ctx;

238 - if (!ctx) /* metadata log has not been initialized yet */

255 + ctx = get_metalog_ctx(st->rrdhost);

256 + if (!ctx)

257 + return;

258 + if (!ctx->initialized) /* metadata log has not been initialized yet */

259 return;

260

261 buffer = metalog_update_dimension_buffer(rd);

@@ -248,15 +268,16 @@ void metalog_commit_delete_dimension(RRDDIM *rd)

268 struct metalog_instance *ctx;

269 BUFFER *buffer;

270 RRDSET *st = rd->rrdset;

251 - RRDHOST *host = st->rrdhost;

271 char uuid_str[37];

272

273 /* Metadata are only available with dbengine */

255 - if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

274 + if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)

275 return;

276

258 - ctx = host->rrdeng_ctx->metalog_ctx;

259 - if (!ctx) /* metadata log has not been initialized yet */

277 + ctx = get_metalog_ctx(st->rrdhost);

278 + if (!ctx)

279 + return;

280 + if (!ctx->initialized) /* metadata log has not been initialized yet */

281 return;

282 buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */

283

@@ -273,12 +294,15 @@ RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *host_g

294 char machine_guid[37];

295

296 uuid_unparse_lower(*host_guid, machine_guid);

297 + RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);

298 ret = find_object_by_guid(host_guid, NULL, 0);

299 if (unlikely(GUID_TYPE_HOST != ret)) {

278 - error("Host with GUID %s not found in the global map", machine_guid);

279 - return NULL;

300 + errno = 0;

301 + if (unlikely(!host))

302 + error("Host with GUID %s not found in the global map or in the list of hosts", machine_guid);

303 + else

304 + error("Host with GUID %s not found in the global map", machine_guid);

305 }

281 - RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);

306 return host;

307 }

308

@@ -292,9 +316,12 @@ RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_

316 if (unlikely(GUID_TYPE_CHART != ret))

317 return NULL;

318

295 - machine_guid = (uuid_t *)chart_object;

296 - RRDHOST *host = ctx->rrdeng_ctx->host;

319 + machine_guid = (uuid_t *)chart_object;

320 + RRDHOST *host = metalog_get_host_from_uuid(ctx, machine_guid);

321 + if (unlikely(!host))

322 + return NULL;

323 if (unlikely(uuid_compare(host->host_uuid, *machine_guid))) {

324 + errno = 0;

325 error("Metadata host machine GUID does not match the one assosiated with the chart");

326 return NULL;

327 }

@@ -311,6 +338,8 @@ RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_

338

339 RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)

340 {

341 + UNUSED(ctx);

342 +

343 GUID_TYPE ret;

344 char dim_object[49], chart_object[33], id_str[PLUGINSD_LINE_MAX], chart_fullid[RRD_ID_LENGTH_MAX + 1];

345 uuid_t *machine_guid, *chart_guid, *chart_char_guid, *dim_char_guid;

@@ -320,8 +349,12 @@ RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *me

349 return NULL;

350

351 machine_guid = (uuid_t *)dim_object;

323 - RRDHOST *host = ctx->rrdeng_ctx->host;

352 +

353 + RRDHOST *host = metalog_get_host_from_uuid(ctx, machine_guid);

354 + if (unlikely(!host))

355 + return NULL;

356 if (unlikely(uuid_compare(host->host_uuid, *machine_guid))) {

357 + errno = 0;

358 error("Metadata host machine GUID does not match the one assosiated with the dimension");

359 return NULL;

360 }

@@ -359,10 +392,11 @@ void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metr

392 uint8_t empty_chart;

393

394 rd = metalog_get_dimension_from_uuid(ctx, metric_uuid);

362 - if (!rd) { /* in the case of legacy UUID convert to multihost and try again */

395 + if (!rd) { /* in 8the case of legacy UUID convert to multihost and try again */

396 + // TODO: Check what to do since we have no host

397 uuid_t multihost_uuid;

398

365 - rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->host->machine_guid, metric_uuid, &multihost_uuid);

399 + rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->machine_guid, metric_uuid, &multihost_uuid);

400 rd = metalog_get_dimension_from_uuid(ctx, &multihost_uuid);

401 }

402 if(!rd) {

@@ -372,6 +406,10 @@ void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metr

406 st = rd->rrdset;

407 host = st->rrdhost;

408

409 + /* In case there are active metrics in a different database engine do not delete the dimension object */

410 + if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE))

411 + return;

412 +

413 /* Since the metric has no writer it will not be commited to the metadata log by rrddim_free_custom().

414 * It must be commited explicitly before calling rrddim_free_custom(). */

415 metalog_commit_delete_dimension(rd);

@@ -401,8 +439,11 @@ int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)

439

440 ctx = callocz(1, sizeof(*ctx));

441 ctx->records_nr = 0;

442 + ctx->objects_nr = 0;

443 ctx->current_compaction_id = 0;

444 ctx->quiesce = NO_QUIESCE;

445 + ctx->initialized = 0;

446 + rrdeng_parent_ctx->metalog_ctx = ctx;

447

448 memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));

449 ctx->rrdeng_ctx = rrdeng_parent_ctx;

@@ -422,7 +463,7 @@ int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)

463 if (ctx->worker_config.error) {

464 goto error_after_rrdeng_worker;

465 }

425 - rrdeng_parent_ctx->metalog_ctx = ctx; /* notify dbengine that the metadata log has finished initializing */

466 + ctx->initialized = 1; /* notify dbengine that the metadata log has finished initializing */

467 return 0;

468

469 error_after_rrdeng_worker:

database/engine/metadata_log/metadatalogapi.h

@@ -13,6 +13,7 @@ extern void metalog_commit_delete_chart(RRDSET *st);

13 extern BUFFER *metalog_update_dimension_buffer(RRDDIM *rd);

14 extern void metalog_commit_update_dimension(RRDDIM *rd);

15 extern void metalog_commit_delete_dimension(RRDDIM *rd);

16 +extern void metalog_upd_objcount(RRDHOST *host, int count);

18 extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);

19 extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);

database/engine/metadata_log/metalogpluginsd.c

+67 -19

@@ -4,15 +4,33 @@

4 #include "metadatalog.h"

5 #include "metalogpluginsd.h"

7 +extern struct config stream_config;

8 +

9 PARSER_RC metalog_pluginsd_host_action(

10 void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone,

11 char *tags)

12 {

13 + int history = 5;

14 + RRD_MEMORY_MODE mode = RRD_MEMORY_MODE_DBENGINE;

15 + int health_enabled = default_health_enabled;

16 + int rrdpush_enabled = default_rrdpush_enabled;

17 + char *rrdpush_destination = default_rrdpush_destination;

18 + char *rrdpush_api_key = default_rrdpush_api_key;

19 + char *rrdpush_send_charts_matching = default_rrdpush_send_charts_matching;

20 +

21 struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;

23 RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);

14 - if (host)

24 + if (host) {

25 + if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {

26 + error("Archived host '%s' has memory mode '%s', but the archived one is '%s'. Ignoring archived state.",

27 + host->hostname, rrd_memory_mode_name(host->rrd_memory_mode),

28 + rrd_memory_mode_name(RRD_MEMORY_MODE_DBENGINE));

29 + ((PARSER_USER_OBJECT *) user)->host = NULL; /* Ignore objects if memory mode is not dbengine */

30 + return PARSER_RC_OK;

31 + }

32 goto write_replay;

33 + }

35 if (strcmp(machine_guid, registry_get_this_machine_guid()) == 0) {

36 struct metalog_record record;

@@ -20,12 +38,32 @@ PARSER_RC metalog_pluginsd_host_action(

39 uuid_parse(machine_guid, record.uuid);

40 mlf_record_insert(metalogfile, &record);

41 + if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

42 + ((PARSER_USER_OBJECT *) user)->host = localhost;

43 + else

44 + ((PARSER_USER_OBJECT *) user)->host = NULL;

45 return PARSER_RC_OK;

46 }

26 - // Ignore HOST command for now

27 - // TODO: Remove when the next task is completed ie. accept new children in the lcoalhost / multidb

28 - return PARSER_RC_OK;

48 + // Fetch configuration options from streaming config

49 + update_every = (int)appconfig_get_number(&stream_config, machine_guid, "update every", update_every);

50 + if(update_every < 0) update_every = 1;

51 +

52 + //health_enabled = appconfig_get_boolean_ondemand(&stream_config, rpt->key, "health enabled by default", health_enabled);

53 + health_enabled = appconfig_get_boolean_ondemand(&stream_config, machine_guid, "health enabled", health_enabled);

54 +

55 + //rrdpush_enabled = appconfig_get_boolean(&stream_config, rpt->key, "default proxy enabled", rrdpush_enabled);

56 + rrdpush_enabled = appconfig_get_boolean(&stream_config, machine_guid, "proxy enabled", rrdpush_enabled);

57 +

58 + //rrdpush_destination = appconfig_get(&stream_config, rpt->key, "default proxy destination", rrdpush_destination);

59 + rrdpush_destination = appconfig_get(&stream_config, machine_guid, "proxy destination", rrdpush_destination);

60 +

61 + //rrdpush_api_key = appconfig_get(&stream_config, rpt->key, "default proxy api key", rrdpush_api_key);

62 + rrdpush_api_key = appconfig_get(&stream_config, machine_guid, "proxy api key", rrdpush_api_key);

63 +

64 + //rrdpush_send_charts_matching = appconfig_get(&stream_config, rpt->key, "default proxy send charts matching", rrdpush_send_charts_matching);

65 + rrdpush_send_charts_matching = appconfig_get(&stream_config, machine_guid, "proxy send charts matching", rrdpush_send_charts_matching);

66 +

68 host = rrdhost_create(

69 hostname

@@ -37,13 +75,13 @@ PARSER_RC metalog_pluginsd_host_action(

75 , NULL

76 , NULL

77 , update_every

40 - , 3600

41 - , RRD_MEMORY_MODE_DBENGINE

42 - , 0 // health enabled

43 - , 0 // Push enabled

44 - , NULL

45 - , NULL

46 - , NULL

78 + , history // entries

79 + , mode

80 + , health_enabled // health enabled

81 + , rrdpush_enabled // Push enabled

82 + , rrdpush_destination //destination

83 + , rrdpush_api_key // api key

84 + , rrdpush_send_charts_matching // charts matching

85 , callocz(1, sizeof(struct rrdhost_system_info))

86 , 0 // localhost

87 , 1 // archived

@@ -57,7 +95,7 @@ write_replay:

95 uuid_copy(record.uuid, host->host_uuid);

96 mlf_record_insert(metalogfile, &record);

97 }

60 -

98 + ((PARSER_USER_OBJECT *) user)->host = host;

99 return PARSER_RC_OK;

100 }

101

@@ -70,6 +108,10 @@ PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *

108 RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;

109 uuid_t *chart_uuid;

110

111 + if (unlikely(!host)) {

112 + debug(D_METADATALOG, "Ignoring chart belonging to missing or ignored host.");

113 + return PARSER_RC_OK;

114 + }

115 chart_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;

116 st = rrdset_create_custom(

117 host, type, id, name, family, context, title, units,

@@ -117,6 +159,10 @@ PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, ch

159 UNUSED(algorithm);

160 uuid_t *dim_uuid;

161

162 + if (unlikely(!st)) {

163 + debug(D_METADATALOG, "Ignoring dimension belonging to missing or ignored chart.");

164 + return PARSER_RC_OK;

165 + }

166 dim_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;

167

168 RRDDIM *rd = rrddim_add_custom(st, id, name, multiplier, divisor, algorithm_type, RRD_MEMORY_MODE_DBENGINE, 1,

@@ -155,8 +201,8 @@ PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid)

201 PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)

202 {

203 GUID_TYPE ret;

158 - struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;

159 - struct metalog_instance *ctx = state->ctx;

204 + //struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;

205 + //struct metalog_instance *ctx = state->ctx;

206 char object[49], chart_object[33], id_str[1024];

207 uuid_t *chart_guid, *chart_char_guid;

208 RRDHOST *host;

@@ -164,15 +210,16 @@ PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)

210 ret = find_object_by_guid(uuid, object, 49);

211 switch (ret) {

212 case GUID_TYPE_NOTFOUND:

167 - if (unlikely(ctx->rrdeng_ctx->host && uuid_compare(ctx->rrdeng_ctx->host->host_uuid, *uuid)))

168 - error_with_guid(uuid, "Failed to find valid context");

213 + error_with_guid(uuid, "Failed to find valid context");

214 break;

215 case GUID_TYPE_CHAR:

216 error_with_guid(uuid, "Ignoring unexpected type GUID_TYPE_CHAR");

217 break;

218 case GUID_TYPE_CHART:

219 case GUID_TYPE_DIMENSION:

175 - host = ctx->rrdeng_ctx->host;

220 + host = metalog_get_host_from_uuid(NULL, (uuid_t *) &object);

221 + if (unlikely(!host))

222 + break;

223 switch (ret) {

224 case GUID_TYPE_CHART:

225 chart_char_guid = (uuid_t *)(object + 16);

@@ -204,7 +251,7 @@ PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)

251 }

252 break;

253 case GUID_TYPE_HOST:

207 - /* Ignore for now */

254 + ((PARSER_USER_OBJECT *)user)->host = metalog_get_host_from_uuid(NULL, (uuid_t *) &object);

255 break;

256 case GUID_TYPE_NOSPACE:

257 error_with_guid(uuid, "Not enough space for object retrieval");

@@ -222,7 +269,7 @@ PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)

269 GUID_TYPE ret;

270 struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;

271 struct metalog_instance *ctx = state->ctx;

225 - RRDHOST *host = ctx->rrdeng_ctx->host;

272 + RRDHOST *host = NULL;

273 RRDSET *st;

274 RRDDIM *rd;

275

@@ -234,6 +281,7 @@ PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)

281 case GUID_TYPE_CHART:

282 st = metalog_get_chart_from_uuid(ctx, uuid);

283 if (st) {

284 + host = st->rrdhost;

285 rrdhost_wrlock(host);

286 rrdset_free(st);

287 rrdhost_unlock(host);

database/engine/rrdengine.c

+2 -2

@@ -644,7 +644,7 @@ static void delete_old_data(void *arg)

644 for (i = 0 ; i < count ; ++i) {

645 descr = extent->pages[i];

646 can_delete_metric = pg_cache_punch_hole(ctx, descr, 0, 0, &metric_id);

647 - if (unlikely(can_delete_metric && ctx->metalog_ctx)) {

647 + if (unlikely(can_delete_metric && ctx->metalog_ctx->initialized)) {

648 /*

649 * If the metric is empty, has no active writers and if the metadata log has been initialized then

650 * attempt to delete the corresponding netdata dimension.

@@ -821,7 +821,7 @@ void timer_cb(uv_timer_t* handle)

821

822 uv_stop(handle->loop);

823 uv_update_time(handle->loop);

824 - if (unlikely(!ctx->metalog_ctx))

824 + if (unlikely(!ctx->metalog_ctx->initialized))

825 return; /* Wait for the metadata log to initialize */

826 rrdeng_test_quota(wc);

827 debug(D_RRDENGINE, "%s: timeout reached.", __func__);

database/engine/rrdengine.h

+3 -2

@@ -176,7 +176,6 @@ extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of dele

176 #define QUIESCED (2) /* is set after all threads have finished running */

177

178 struct rrdengine_instance {

179 - RRDHOST *host;

179 struct metalog_instance *metalog_ctx;

180 struct rrdengine_worker_config worker_config;

181 struct completion rrdengine_completion;

@@ -185,7 +184,9 @@ struct rrdengine_instance {

184 uint8_t global_compress_alg;

185 struct transaction_commit_log commit_log;

186 struct rrdengine_datafile_list datafiles;

188 - char dbfiles_path[FILENAME_MAX+1];

187 + RRDHOST *host; /* the legacy host, or NULL for multi-host DB */

188 + char dbfiles_path[FILENAME_MAX + 1];

189 + char machine_guid[GUID_LEN + 1]; /* the unique ID of the corresponding host, or localhost for multihost DB */

190 uint64_t disk_space;

191 uint64_t max_disk_space;

192 unsigned last_fileno; /* newest index of datafile and journalfile */

database/engine/rrdengineapi.c

+32 -18

@@ -2,7 +2,7 @@

2 #include "rrdengine.h"

4 /* Default global database instance */

5 -static struct rrdengine_instance default_global_ctx;

5 +struct rrdengine_instance multidb_ctx;

7 int default_rrdeng_page_cache_mb = 32;

8 int default_rrdeng_disk_quota_mb = 256;

@@ -10,6 +10,11 @@ int default_multidb_disk_quota_mb = 256;

10 /* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */

11 uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;

13 +static inline struct rrdengine_instance *get_rrdeng_ctx_from_host(RRDHOST *host)

14 +{

15 + return host->rrdeng_ctx;

16 +}

17 +

18 /* This UUID is not unique across hosts */

19 void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid)

20 {

@@ -53,7 +58,11 @@ void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid)

58 struct pg_cache_page_index *page_index = NULL;

59 int replace_instead_of_generate = 0;

56 - ctx = rd->rrdset->rrdhost->rrdeng_ctx;

61 + ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);

62 + if (unlikely(!ctx)) {

63 + error("Failed to fetch multidb context");

64 + return;

65 + }

66 pg_cache = &ctx->pg_cache;

68 rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid);

@@ -132,7 +141,7 @@ void rrdeng_store_metric_init(RRDDIM *rd)

141 struct rrdengine_instance *ctx;

142 struct pg_cache_page_index *page_index;

143

135 - ctx = rd->rrdset->rrdhost->rrdeng_ctx;

144 + ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);

145 handle = &rd->state->handle.rrdeng;

146 handle->ctx = ctx;

147

@@ -171,6 +180,8 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)

180

181 handle = &rd->state->handle.rrdeng;

182 ctx = handle->ctx;

183 + if (unlikely(!ctx))

184 + return;

185 descr = handle->descr;

186 if (unlikely(NULL == descr)) {

187 return;

@@ -370,7 +381,7 @@ unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t e

381 struct rrdeng_region_info *region_info_array;

382 uint8_t is_first_region_initialized;

383

373 - ctx = st->rrdhost->rrdeng_ctx;

384 + ctx = get_rrdeng_ctx_from_host(st->rrdhost);

385 regions = 1;

386 *max_intervalp = max_interval = 0;

387 region_info_array = NULL;

@@ -532,7 +543,7 @@ void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_hand

543 struct rrdengine_instance *ctx;

544 unsigned pages_nr;

545

535 - ctx = rd->rrdset->rrdhost->rrdeng_ctx;

546 + ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);

547 rrdimm_handle->start_time = start_time;

548 rrdimm_handle->end_time = end_time;

549 handle = &rrdimm_handle->rrdeng;

@@ -801,6 +812,9 @@ void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_i

812 */

813 void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array)

814 {

815 + if (ctx == NULL)

816 + return;

817 +

818 struct page_cache *pg_cache = &ctx->pg_cache;

819

820 array[0] = (uint64_t)ctx->stats.metric_API_producers;

@@ -875,8 +889,7 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p

889 }

890

891 if (NULL == ctxp) {

878 - /* for testing */

879 - ctx = &default_global_ctx;

892 + ctx = &multidb_ctx;

893 memset(ctx, 0, sizeof(*ctx));

894 } else {

895 *ctxp = ctx = callocz(1, sizeof(*ctx));

@@ -892,6 +905,11 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p

905 ctx->max_disk_space = disk_space_mb * 1048576LLU;

906 strncpyz(ctx->dbfiles_path, dbfiles_path, sizeof(ctx->dbfiles_path) - 1);

907 ctx->dbfiles_path[sizeof(ctx->dbfiles_path) - 1] = '\0';

908 + if (NULL == host)

909 + strncpyz(ctx->machine_guid, registry_get_this_machine_guid(), GUID_LEN);

910 + else

911 + strncpyz(ctx->machine_guid, host->machine_guid, GUID_LEN);

912 +

913 ctx->drop_metrics_under_page_cache_pressure = rrdeng_drop_metrics_under_page_cache_pressure;

914 ctx->metric_API_max_producers = 0;

915 ctx->quiesce = NO_QUIESCE;

@@ -916,23 +934,19 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p

934 if (ctx->worker_config.error) {

935 goto error_after_rrdeng_worker;

936 }

919 - if ((strcmp(host->machine_guid, registry_get_this_machine_guid()) == 0) || (!rrdhost_flag_check(host, RRDHOST_FLAG_MULTIHOST))) {

920 - info("Metadatalog init for host %s starting...", host->hostname);

921 - error = metalog_init(ctx);

922 - if (error) {

923 - error("Failed to initialize metadata log file event loop.");

924 - goto error_after_rrdeng_worker;

925 - }

937 + error = metalog_init(ctx);

938 + if (error) {

939 + error("Failed to initialize metadata log file event loop.");

940 + goto error_after_rrdeng_worker;

941 }

927 - else

928 - info("No metadatalog init for host %s", host->hostname);

942 +

943 return 0;

944

945 error_after_rrdeng_worker:

946 finalize_rrd_files(ctx);

947 error_after_init_rrd_files:

948 free_page_cache(ctx);

935 - if (ctx != &default_global_ctx) {

949 + if (ctx != &multidb_ctx) {

950 freez(ctx);

951 *ctxp = NULL;

952 }

@@ -961,7 +975,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx)

975 metalog_exit(ctx->metalog_ctx);

976 free_page_cache(ctx);

977

964 - if (ctx != &default_global_ctx) {

978 + if (ctx != &multidb_ctx) {

979 freez(ctx);

980 }

981 rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);

database/engine/rrdengineapi.h

@@ -16,6 +16,7 @@ extern int default_rrdeng_page_cache_mb;

16 extern int default_rrdeng_disk_quota_mb;

17 extern int default_multidb_disk_quota_mb;

18 extern uint8_t rrdeng_drop_metrics_under_page_cache_pressure;

19 +extern struct rrdengine_instance multidb_ctx;

21 struct rrdeng_region_info {

22 time_t start_time;

database/engine/rrdenginelib.c

+8 -9

@@ -272,16 +272,15 @@ int compute_multidb_diskspace()

272 int rc = count_legacy_children(netdata_configured_cache_dir);

273 if (likely(rc >= 0)) {

274 computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;

275 - //info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);

275 + info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);

276

277 - // TODO: will activate the next block of code when multidb is in place

278 -// fp = fopen(multidb_disk_space_file, "w");

279 -// if (likely(fp)) {

280 -// fprintf(fp, "%d", rc * default_rrdeng_disk_quota_mb);

281 -// info("Created file '%s' to store the computed value", multidb_disk_space_file);

282 -// fclose(fp);

283 -// } else

284 -// error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);

277 + fp = fopen(multidb_disk_space_file, "w");

278 + if (likely(fp)) {

279 + fprintf(fp, "%d", computed_multidb_disk_quota_mb);

280 + info("Created file '%s' to store the computed value", multidb_disk_space_file);

281 + fclose(fp);

282 + } else

283 + error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);

284 }

285 else

286 computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;

database/rrd.h

-5

@@ -698,10 +698,6 @@ struct rrdhost {

698

699 int rrd_update_every; // the update frequency of the host

700 long rrd_history_entries; // the number of history entries for the host's charts

701 -#ifdef ENABLE_DBENGINE

702 - unsigned page_cache_mb; // Database Engine page cache size in MiB

703 - unsigned disk_space_mb; // Database Engine disk space quota in MiB

704 -#endif

701 RRD_MEMORY_MODE rrd_memory_mode; // the memory more for the charts of this host

702

703 char *cache_dir; // the directory to save RRD cache files

@@ -812,7 +808,6 @@ struct rrdhost {

808 #ifdef ENABLE_DBENGINE

809 struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host

810 uuid_t host_uuid; // Global GUID for this host

815 - unsigned long objects_nr; // Number of charts and dimensions in this host

811 uint32_t compaction_id; // The last metadata log compaction procedure that has processed

812 // this object.

813 #endif

database/rrddim.c

+2 -2

@@ -451,7 +451,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte

451 aclk_update_chart(host, st->id, ACLK_CMD_CHART);

452 #endif

453 #ifdef ENABLE_DBENGINE

454 - rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);

454 + metalog_upd_objcount(st->rrdhost, 1);

455 metalog_commit_update_dimension(rd);

456 #endif

457

@@ -525,7 +525,7 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)

525 aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);

526 #endif

527 #ifdef ENABLE_DBENGINE

528 - rrd_atomic_fetch_add(&st->rrdhost->objects_nr, -1);

528 + metalog_upd_objcount(st->rrdhost, -1);

529 #endif

530 }

531

database/rrdhost.c

+157 -75

@@ -128,18 +128,20 @@ RRDHOST *rrdhost_create(const char *hostname,

128 ) {

129 debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);

130

131 +#ifdef ENABLE_DBENGINE

132 + int is_legacy = is_archived ? 0 : (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid);

133 +#else

134 + int is_legacy = 1;

135 +#endif

136 rrd_check_wrlock();

137

138 + int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy);

139 RRDHOST *host = callocz(1, sizeof(RRDHOST));

140

141 host->rrd_update_every = (update_every > 0)?update_every:1;

142 host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);

143 host->rrd_memory_mode = memory_mode;

138 -#ifdef ENABLE_DBENGINE

139 - host->page_cache_mb = default_rrdeng_page_cache_mb;

140 - host->disk_space_mb = default_rrdeng_disk_quota_mb;

141 -#endif

142 - host->health_enabled = (memory_mode == RRD_MEMORY_MODE_NONE)? 0 : health_enabled;

144 + host->health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE) || is_archived) ? 0 : health_enabled;

145

146 host->sender = mallocz(sizeof(*host->sender));

147 sender_init(host->sender, host);

@@ -227,12 +229,15 @@ RRDHOST *rrdhost_create(const char *hostname,

229 }

230 else {

231 // this is not localhost - append our GUID to localhost path

232 + if (is_in_multihost) { // don't append to cache dir in multihost

233 + host->cache_dir = strdupz(netdata_configured_cache_dir);

234 + } else {

235 + snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_cache_dir, host->machine_guid);

236 + host->cache_dir = strdupz(filename);

237 + }

238

231 - snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_cache_dir, host->machine_guid);

232 - host->cache_dir = strdupz(filename);

233 -

234 - if(host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE ||

235 - host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

239 + if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || (

240 + host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {

241 int r = mkdir(host->cache_dir, 0775);

242 if(r != 0 && errno != EEXIST)

243 error("Host '%s': cannot create directory '%s'", host->hostname, host->cache_dir);

@@ -241,7 +246,7 @@ RRDHOST *rrdhost_create(const char *hostname,

246 snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_varlib_dir, host->machine_guid);

247 host->varlib_dir = strdupz(filename);

248

244 - if(host->health_enabled) {

249 + if(!is_archived && host->health_enabled) {

250 int r = mkdir(host->varlib_dir, 0775);

251 if(r != 0 && errno != EEXIST)

252 error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir);

@@ -249,7 +254,7 @@ RRDHOST *rrdhost_create(const char *hostname,

254

255 }

256

252 - if(host->health_enabled) {

257 + if(!is_archived && host->health_enabled) {

258 snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);

259 int r = mkdir(filename, 0775);

260 if(r != 0 && errno != EEXIST)

@@ -267,7 +272,7 @@ RRDHOST *rrdhost_create(const char *hostname,

272 // ------------------------------------------------------------------------

273 // load health configuration

274

270 - if(host->health_enabled) {

275 + if(!is_archived && host->health_enabled) {

276 rrdhost_wrlock(host);

277 health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);

278 rrdhost_unlock(host);

@@ -276,6 +281,14 @@ RRDHOST *rrdhost_create(const char *hostname,

281 health_alarm_log_open(host);

282 }

283

284 + RRDHOST *t = rrdhost_index_add(host);

285 +

286 + if(t != host) {

287 + error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", host->hostname, host->machine_guid, t->hostname, t->machine_guid);

288 + rrdhost_free(host);

289 + return NULL;

290 + }

291 +

292 if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

293 #ifdef ENABLE_DBENGINE

294 if (unlikely(-1 == uuid_parse(host->machine_guid, host->host_uuid))) {

@@ -285,26 +298,32 @@ RRDHOST *rrdhost_create(const char *hostname,

298 error("Failed to store machine GUID to global map");

299 else

300 info("Added %s to global map for host %s", host->machine_guid, host->hostname);

288 - host->objects_nr = 1;

301 host->compaction_id = 0;

302 char dbenginepath[FILENAME_MAX + 1];

303 int ret;

304

305 snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir);

306 ret = mkdir(dbenginepath, 0775);

295 - if(ret != 0 && errno != EEXIST)

307 + if (ret != 0 && errno != EEXIST)

308 error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath);

309 + else ret = 0; // succeed

310 + if (is_legacy) // initialize legacy dbengine instance as needed

311 + ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, default_rrdeng_page_cache_mb,

312 + default_rrdeng_disk_quota_mb); // may fail here for legacy dbengine initialization

313 else

298 - ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);

299 - if(ret) {

300 - error("Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",

301 - host->hostname, host->machine_guid, host->cache_dir);

314 + host->rrdeng_ctx = &multidb_ctx;

315 + if (ret) { // check legacy or multihost initialization success

316 + error(

317 + "Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",

318 + host->hostname, host->machine_guid, host->cache_dir);

319 rrdhost_free(host);

320 host = NULL;

321 //rrd_hosts_available++; //TODO: maybe we want this?

322

323 return host;

324 }

325 +

326 + metalog_upd_objcount(host, 1);

327 #else

328 fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform.");

329 #endif

@@ -325,55 +344,47 @@ RRDHOST *rrdhost_create(const char *hostname,

344 else localhost = host;

345 }

346

328 - RRDHOST *t = rrdhost_index_add(host);

329 -

330 - if(t != host) {

331 - error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", host->hostname, host->machine_guid, t->hostname, t->machine_guid);

332 - rrdhost_free(host);

333 - host = NULL;

334 - }

335 - else {

336 - info("Host '%s' (at registry as '%s') with guid '%s' initialized"

337 - ", os '%s'"

338 - ", timezone '%s'"

339 - ", tags '%s'"

340 - ", program_name '%s'"

341 - ", program_version '%s'"

342 - ", update every %d"

343 - ", memory mode %s"

344 - ", history entries %ld"

345 - ", streaming %s"

346 - " (to '%s' with api key '%s')"

347 - ", health %s"

348 - ", cache_dir '%s'"

349 - ", varlib_dir '%s'"

350 - ", health_log '%s'"

351 - ", alarms default handler '%s'"

352 - ", alarms default recipient '%s'"

353 - , host->hostname

354 - , host->registry_hostname

355 - , host->machine_guid

356 - , host->os

357 - , host->timezone

358 - , (host->tags)?host->tags:""

359 - , host->program_name

360 - , host->program_version

361 - , host->rrd_update_every

362 - , rrd_memory_mode_name(host->rrd_memory_mode)

363 - , host->rrd_history_entries

364 - , host->rrdpush_send_enabled?"enabled":"disabled"

365 - , host->rrdpush_send_destination?host->rrdpush_send_destination:""

366 - , host->rrdpush_send_api_key?host->rrdpush_send_api_key:""

367 - , host->health_enabled?"enabled":"disabled"

368 - , host->cache_dir

369 - , host->varlib_dir

370 - , host->health_log_filename

371 - , host->health_default_exec

372 - , host->health_default_recipient

373 - );

374 - }

347 + info("Host '%s' (at registry as '%s') with guid '%s' initialized"

348 + ", os '%s'"

349 + ", timezone '%s'"

350 + ", tags '%s'"

351 + ", program_name '%s'"

352 + ", program_version '%s'"

353 + ", update every %d"

354 + ", memory mode %s"

355 + ", history entries %ld"

356 + ", streaming %s"

357 + " (to '%s' with api key '%s')"

358 + ", health %s"

359 + ", cache_dir '%s'"

360 + ", varlib_dir '%s'"

361 + ", health_log '%s'"

362 + ", alarms default handler '%s'"

363 + ", alarms default recipient '%s'"

364 + , host->hostname

365 + , host->registry_hostname

366 + , host->machine_guid

367 + , host->os

368 + , host->timezone

369 + , (host->tags)?host->tags:""

370 + , host->program_name

371 + , host->program_version

372 + , host->rrd_update_every

373 + , rrd_memory_mode_name(host->rrd_memory_mode)

374 + , host->rrd_history_entries

375 + , host->rrdpush_send_enabled?"enabled":"disabled"

376 + , host->rrdpush_send_destination?host->rrdpush_send_destination:""

377 + , host->rrdpush_send_api_key?host->rrdpush_send_api_key:""

378 + , host->health_enabled?"enabled":"disabled"

379 + , host->cache_dir

380 + , host->varlib_dir

381 + , host->health_log_filename

382 + , host->health_default_exec

383 + , host->health_default_recipient

384 + );

385

376 - rrd_hosts_available++;

386 + if (!is_archived)

387 + rrd_hosts_available++;

388

389 #ifdef ENABLE_DBENGINE

390 if (likely(!is_localhost && !is_archived && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))

@@ -408,7 +419,7 @@ void rrdhost_update(RRDHOST *host

419 UNUSED(rrdpush_api_key);

420 UNUSED(rrdpush_send_charts_matching);

421

411 - host->health_enabled = health_enabled;

422 + host->health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;

423 //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?

424

425 rrdhost_system_info_free(host->system_info);

@@ -453,6 +464,33 @@ void rrdhost_update(RRDHOST *host

464

465 // update host tags

466 rrdhost_init_tags(host, tags);

467 +

468 + if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {

469 + rrdhost_flag_clear(host, RRDHOST_FLAG_ARCHIVED);

470 + if(host->health_enabled) {

471 + int r;

472 + char filename[FILENAME_MAX + 1];

473 +

474 + if (host != localhost) {

475 + r = mkdir(host->varlib_dir, 0775);

476 + if (r != 0 && errno != EEXIST)

477 + error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir);

478 + }

479 + snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);

480 + r = mkdir(filename, 0775);

481 + if(r != 0 && errno != EEXIST)

482 + error("Host '%s': cannot create directory '%s'", host->hostname, filename);

483 +

484 + rrdhost_wrlock(host);

485 + health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);

486 + rrdhost_unlock(host);

487 +

488 + health_alarm_log_load(host);

489 + health_alarm_log_open(host);

490 + }

491 + rrd_hosts_available++;

492 + info("Host %s is not in archived mode anymore", host->hostname);

493 + }

494 #ifdef ENABLE_DBENGINE

495 if (likely(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))

496 metalog_commit_update_host(host);

@@ -483,6 +521,13 @@ RRDHOST *rrdhost_find_or_create(

521

522 rrd_wrlock();

523 RRDHOST *host = rrdhost_find_by_guid(guid, 0);

524 + if (unlikely(host && RRD_MEMORY_MODE_DBENGINE != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {

525 + /* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */

526 + error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.",

527 + host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));

528 + rrdhost_free(host);

529 + host = NULL;

530 + }

531 if(!host) {

532 host = rrdhost_create(

533 hostname

@@ -555,7 +600,12 @@ restart_after_removal:

600 if(rrdhost_should_be_removed(host, protected, now)) {

601 info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid);

602

558 - if(rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST))

603 + if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST)

604 +#ifdef ENABLE_DBENGINE

605 + /* don't delete multi-host DB host files */

606 + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx)

607 +#endif

608 + )

609 rrdhost_delete_charts(host);

610 else

611 rrdhost_save_charts(host);

@@ -602,8 +652,33 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {

652 , 1

653 , 0

654 );

655 + if (unlikely(!localhost)) {

656 + rrd_unlock();

657 + return 1;

658 + }

659 +

660 +#ifdef ENABLE_DBENGINE

661 + char dbenginepath[FILENAME_MAX + 1];

662 + int ret;

663 + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", localhost->cache_dir);

664 + ret = mkdir(dbenginepath, 0775);

665 + if (ret != 0 && errno != EEXIST)

666 + error("Host '%s': cannot create directory '%s'", localhost->hostname, dbenginepath);

667 + else // Unconditionally create multihost db to support on demand host creation

668 + ret = rrdeng_init(NULL, NULL, dbenginepath, default_rrdeng_page_cache_mb, default_multidb_disk_quota_mb);

669 + if (ret) {

670 + error(

671 + "Host '%s' with machine guid '%s' failed to initialize multi-host DB engine instance at '%s'.",

672 + localhost->hostname, localhost->machine_guid, localhost->cache_dir);

673 + rrdhost_free(localhost);

674 + localhost = NULL;

675 + rrd_unlock();

676 + return 1;

677 + }

678 +#endif

679 rrd_unlock();

606 - web_client_api_v1_management_init();

680 +

681 + web_client_api_v1_management_init();

682 return localhost==NULL;

683 }

684

@@ -717,7 +792,8 @@ void rrdhost_free(RRDHOST *host) {

792 // release its children resources

793

794 #ifdef ENABLE_DBENGINE

720 - rrdeng_prepare_exit(host->rrdeng_ctx);

795 + if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx != &multidb_ctx)

796 + rrdeng_prepare_exit(host->rrdeng_ctx);

797 #endif

798 while(host->rrdset_root)

799 rrdset_free(host->rrdset_root);

@@ -749,11 +825,10 @@ void rrdhost_free(RRDHOST *host) {

825

826 health_alarm_log_free(host);

827

752 - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

828 #ifdef ENABLE_DBENGINE

829 + if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx != &multidb_ctx)

830 rrdeng_exit(host->rrdeng_ctx);

831 #endif

756 - }

832

833 // ------------------------------------------------------------------------

834 // remove it from the indexes

@@ -811,7 +886,9 @@ void rrdhost_free(RRDHOST *host) {

886

887 void rrdhost_free_all(void) {

888 rrd_wrlock();

814 - while(localhost) rrdhost_free(localhost);

889 + /* Make sure child-hosts are released before the localhost. */

890 + while(localhost->next) rrdhost_free(localhost->next);

891 + rrdhost_free(localhost);

892 rrd_unlock();

893 }

894

@@ -1396,7 +1473,12 @@ void rrdhost_cleanup_all(void) {

1473

1474 RRDHOST *host;

1475 rrdhost_foreach_read(host) {

1399 - if(host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS) && !host->receiver)

1476 + if (host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) && !host->receiver

1477 +#ifdef ENABLE_DBENGINE

1478 + /* don't delete multi-host DB host files */

1479 + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx)

1480 +#endif

1481 + )

1482 rrdhost_delete_charts(host);

1483 else

1484 rrdhost_cleanup_charts(host);

database/rrdset.c

+2 -7

@@ -392,7 +392,7 @@ void rrdset_free(RRDSET *st) {

392 break;

393 }

394 #ifdef ENABLE_DBENGINE

395 - rrd_atomic_fetch_add(&host->objects_nr, -1);

395 + metalog_upd_objcount(host, -1);

396 #endif

397

398 }

@@ -543,10 +543,6 @@ RRDSET *rrdset_create_custom(

543 if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {

544 rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);

545 changed_from_archived_to_active = 1;

546 - if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ARCHIVED)) {

547 - rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);

548 - info("Host %s is not in archived mode anymore", st->rrdhost->hostname);

549 - }

546 mark_rebuild |= META_CHART_ACTIVATED;

547 }

548 char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL,

@@ -689,7 +685,6 @@ RRDSET *rrdset_create_custom(

685 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);

686 if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {

687 rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);

692 - rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);

688 }

689 return st;

690 }

@@ -960,7 +955,7 @@ RRDSET *rrdset_create_custom(

955 }

956 #endif

957 #ifdef ENABLE_DBENGINE

963 - rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);

958 + metalog_upd_objcount(host, 1);

959 metalog_commit_update_chart(st);

960 #endif

961

streaming/README.md

+1 -1

@@ -189,7 +189,7 @@ them `/var/lib/netdata/registry/netdata.unique.id`). So, metrics for Netdata `A`

189 any number of other Netdata, will have the same `MACHINE_GUID`.

190

191 You can also use `default memory mode = dbengine` for an API key or `memory mode = dbengine` for

192 - a single host. The additional `page cache size` and `dbengine disk space` configuration options

192 + a single host. The additional `page cache size` and `dbengine multihost disk space` configuration options

193 are inherited from the global Netdata configuration.

194

195 ##### allow from

streaming/receiver.c

-19

@@ -287,25 +287,6 @@ static int rrdpush_receive(struct receiver_state *rpt)

287 }

288 netdata_mutex_unlock(&rpt->host->receiver_lock);

289 }

290 - else rrdhost_update(rpt->host

291 - , rpt->hostname

292 - , rpt->registry_hostname

293 - , rpt->machine_guid

294 - , rpt->os

295 - , rpt->timezone

296 - , rpt->tags

297 - , rpt->program_name

298 - , rpt->program_version

299 - , rpt->update_every

300 - , history

301 - , mode

302 - , (unsigned int)(health_enabled != CONFIG_BOOLEAN_NO)

303 - , (unsigned int)(rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key)

304 - , rrdpush_destination

305 - , rrdpush_api_key

306 - , rrdpush_send_charts_matching

307 - , rpt->system_info);

308 -

290

291 int ssl = 0;

292 #ifdef ENABLE_HTTPS

streaming/rrdpush.c

@@ -611,6 +611,8 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {

611 */

612 struct receiver_state *rpt = callocz(1, sizeof(*rpt));

613 RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);

614 + if (unlikely(host && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) /* Ignore archived hosts. */

615 + host = NULL;

616 if (host) {

617 netdata_mutex_lock(&host->receiver_lock);

618 if (host->receiver != NULL) {

web/api/formatters/charts2json.c

+1 -1

@@ -111,7 +111,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived

111 size_t found = 0;

112 RRDHOST *h;

113 rrdhost_foreach_read(h) {

114 - if(!rrdhost_should_be_removed(h, host, now)) {

114 + if(!rrdhost_should_be_removed(h, host, now) && !rrdhost_flag_check(h, RRDHOST_FLAG_ARCHIVED)) {

115 buffer_sprintf(wb

116 , "%s\n\t\t{"

117 "\n\t\t\t\"hostname\": \"%s\""

web/api/web_api_v1.c

@@ -778,6 +778,8 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {

778 int count = 0;

779 rrd_rdlock();

780 rrdhost_foreach_read(rc) {

781 + if (rrdhost_flag_check(rc, RRDHOST_FLAG_ARCHIVED))

782 + continue;

783 if(count > 0) buffer_strcat(wb, ",\n");

784 buffer_sprintf(wb, "\t\t\"%s\"", rc->hostname);

785 count++;