@cryptotaxi247 / netdata-1 / commits / 5871b37cf

Implemented the HOST command in metadata log replay (#9489)

Stelios Fragkakis committed Jul 17, 2020 at 00:00 UTC 5871b37cf64d92f9ef7e89d0a3813c1c4bfcf203
13 files changed +166 -11
collectors/plugins.d/pluginsd_parser.c
+27
@@ -627,6 +627,33 @@ PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_
627 return PARSER_RC_OK;
628 }
629
630 +PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION *plugins_action)
631 +{
632 + char *machine_guid = words[1];
633 + char *hostname = words[2];
634 + char *registry_hostname = words[3];
635 + char *update_every_s = words[4];
636 + char *os = words[5];
637 + char *timezone = words[6];
638 + char *tags = words[7];
639 +
640 + int update_every = 1;
641 + if (likely(update_every_s && *update_every_s))
642 + update_every = str2i(update_every_s);
643 + if (unlikely(!update_every))
644 + update_every = 1;
645 +
646 + debug(D_PLUGINSD, "HOST PARSED: guid=%s, hostname=%s, reg_host=%s, update=%d, os=%s, timezone=%s, tags=%s",
647 + machine_guid, hostname, registry_hostname, update_every, os, timezone, tags);
648 +
649 + if (plugins_action->host_action) {
650 + return plugins_action->host_action(
651 + user, machine_guid, hostname, registry_hostname, update_every, os, timezone, tags);
652 + }
653 +
654 + return PARSER_RC_OK;
655 +}
656 +
657 // New plugins.d parser
658
659 inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations)
database/engine/global_uuid_map/global_uuid_map.c
+6
@@ -155,6 +155,7 @@ GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes)
155 return GUID_TYPE_NOSPACE;
156 strncpyz(object, (char *) *PValue+1, max_bytes - 1);
157 break;
158 + case GUID_TYPE_HOST:
159 case GUID_TYPE_CHART:
160 case GUID_TYPE_DIMENSION:
161 if (unlikely(max_bytes < (size_t) value_type * 16))
@@ -221,6 +222,11 @@ int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int
222 memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16);
223 memcpy(target_object + 17, temp_uuid, 16);
224 break;
225 + case GUID_TYPE_HOST:
226 + target_object = mallocz(17);
227 + target_object[0] = object_type;
228 + memcpy(target_object + 1, (((RRDHOST *)object))->host_uuid, 16);
229 + break;
230 case GUID_TYPE_CHAR:
231 target_object = mallocz(strlen((char *) object)+2);
232 target_object[0] = object_type;
database/engine/metadata_log/compaction.c
+14 -5
@@ -79,10 +79,10 @@ static void compaction_test_quota(struct metalog_worker_config *wc)
79 static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
80 {
81 GUID_TYPE ret;
82 - RRDHOST *host = ctx->rrdeng_ctx->host;
82 RRDSET *st;
83 RRDDIM *rd;
84 BUFFER *buffer;
85 + RRDHOST *host = ctx->rrdeng_ctx->host;
86
87 ret = find_object_by_guid(uuid, NULL, 0);
88 switch (ret) {
@@ -92,6 +92,11 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
92 case GUID_TYPE_CHART:
93 st = metalog_get_chart_from_uuid(ctx, uuid);
94 if (st) {
95 + if (ctx->current_compaction_id > st->rrdhost->compaction_id) {
96 + error("Forcing compaction of HOST %s from CHART %s", st->rrdhost->hostname, st->id);
97 + compact_record_by_uuid(ctx, &st->rrdhost->host_uuid);
98 + }
99 +
100 if (ctx->current_compaction_id > st->compaction_id) {
101 st->compaction_id = ctx->current_compaction_id;
102 buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id);
@@ -106,11 +111,13 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
111 case GUID_TYPE_DIMENSION:
112 rd = metalog_get_dimension_from_uuid(ctx, uuid);
113 if (rd) {
114 + if (ctx->current_compaction_id > rd->rrdset->rrdhost->compaction_id) {
115 + error("Forcing compaction of HOST %s", rd->rrdset->rrdhost->hostname);
116 + compact_record_by_uuid(ctx, &rd->rrdset->rrdhost->host_uuid);
117 + }
118 if (ctx->current_compaction_id > rd->rrdset->compaction_id) {
110 - error("Forcing compaction of chart %s", rd->rrdset->id);
111 - rd->rrdset->compaction_id = ctx->current_compaction_id;
112 - buffer = metalog_update_chart_buffer(rd->rrdset, ctx->current_compaction_id);
113 - metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, rd->rrdset->chart_uuid, 1);
119 + error("Forcing compaction of CHART %s", rd->rrdset->id);
120 + compact_record_by_uuid(ctx, rd->rrdset->chart_uuid);
121 } else if (ctx->current_compaction_id > rd->state->compaction_id) {
122 rd->state->compaction_id = ctx->current_compaction_id;
123 buffer = metalog_update_dimension_buffer(rd);
@@ -123,6 +130,8 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
130 }
131 break;
132 case GUID_TYPE_HOST:
133 + //TODO: will be enabled when multidb is activated
134 + //RRDHOST *host = metalog_get_host_from_uuid(ctx, uuid);
135 if (ctx->current_compaction_id > host->compaction_id) {
136 host->compaction_id = ctx->current_compaction_id;
137 buffer = metalog_update_host_buffer(host);
database/engine/metadata_log/logfile.c
+3
@@ -675,6 +675,7 @@ static int scan_metalog_files(struct metalog_instance *ctx)
675 failed_to_load = matched_files;
676 goto after_failed_to_parse;
677 }
678 + parser_add_keyword(parser, PLUGINSD_KEYWORD_HOST, metalog_pluginsd_host);
679 parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid);
680 parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context);
681 parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone);
@@ -683,6 +684,8 @@ static int scan_metalog_files(struct metalog_instance *ctx)
684 parser->plugins_action->guid_action = &metalog_pluginsd_guid_action;
685 parser->plugins_action->context_action = &metalog_pluginsd_context_action;
686 parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action;
687 + parser->plugins_action->host_action = &metalog_pluginsd_host_action;
688 +
689
690 metalog_parser_object.parser = parser;
691 ctx->metalog_parser_object = &metalog_parser_object;
database/engine/metadata_log/metadatalogapi.c
+16
@@ -266,6 +266,22 @@ void metalog_commit_delete_dimension(RRDDIM *rd)
266 metalog_commit_deletion_record(ctx, buffer);
267 }
268
269 +RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *host_guid)
270 +{
271 + UNUSED(ctx);
272 + GUID_TYPE ret;
273 + char machine_guid[37];
274 +
275 + uuid_unparse_lower(*host_guid, machine_guid);
276 + ret = find_object_by_guid(host_guid, NULL, 0);
277 + if (unlikely(GUID_TYPE_HOST != ret)) {
278 + error("Host with GUID %s not found in the global map", machine_guid);
279 + return NULL;
280 + }
281 + RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
282 + return host;
283 +}
284 +
285 RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid)
286 {
287 GUID_TYPE ret;
database/engine/metadata_log/metadatalogapi.h
+1
@@ -16,6 +16,7 @@ extern void metalog_commit_delete_dimension(RRDDIM *rd);
16
17 extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);
18 extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
19 +extern RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *uuid);
20 extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
21
22 /* must call once before using anything */
database/engine/metadata_log/metalogpluginsd.c
+57
@@ -4,6 +4,63 @@
4 #include "metadatalog.h"
5 #include "metalogpluginsd.h"
6
7 +PARSER_RC metalog_pluginsd_host_action(
8 + void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone,
9 + char *tags)
10 +{
11 + struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
12 +
13 + RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
14 + if (host)
15 + goto write_replay;
16 +
17 + if (strcmp(machine_guid, registry_get_this_machine_guid()) == 0) {
18 + struct metalog_record record;
19 + struct metadata_logfile *metalogfile = state->metalogfile;
20 +
21 + uuid_parse(machine_guid, record.uuid);
22 + mlf_record_insert(metalogfile, &record);
23 + return PARSER_RC_OK;
24 + }
25 +
26 + // Ignore HOST command for now
27 + // TODO: Remove when the next task is completed ie. accept new children in the lcoalhost / multidb
28 + return PARSER_RC_OK;
29 +
30 + host = rrdhost_create(
31 + hostname
32 + , registry_hostname
33 + , machine_guid
34 + , os
35 + , timezone
36 + , tags
37 + , NULL
38 + , NULL
39 + , update_every
40 + , 3600
41 + , RRD_MEMORY_MODE_DBENGINE
42 + , 0 // health enabled
43 + , 0 // Push enabled
44 + , NULL
45 + , NULL
46 + , NULL
47 + , callocz(1, sizeof(struct rrdhost_system_info))
48 + , 0 // localhost
49 + , 1 // archived
50 + );
51 +
52 +write_replay:
53 + if (host) { /* It's a valid object */
54 + struct metalog_record record;
55 + struct metadata_logfile *metalogfile = state->metalogfile;
56 +
57 + uuid_copy(record.uuid, host->host_uuid);
58 + mlf_record_insert(metalogfile, &record);
59 + }
60 +
61 + return PARSER_RC_OK;
62 +}
63 +
64 PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
65 char *title, char *units, char *plugin, char *module, int priority,
66 int update_every, RRDSET_TYPE chart_type, char *options)
database/engine/metadata_log/metalogpluginsd.h
+2
@@ -25,5 +25,7 @@ extern PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char
25 extern PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid);
26 extern PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid);
27 extern PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid);
28 +extern PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION *plugins_action);
29 +extern PARSER_RC metalog_pluginsd_host_action(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone, char *tags);
30
31 #endif /* NETDATA_METALOGPLUGINSD_H */
database/engine/rrdengineapi.c
+9 -4
@@ -916,11 +916,16 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
916 if (ctx->worker_config.error) {
917 goto error_after_rrdeng_worker;
918 }
919 - error = metalog_init(ctx);
920 - if(error) {
921 - error("Failed to initialize metadata log file event loop.");
922 - goto error_after_rrdeng_worker;
919 + if ((strcmp(host->machine_guid, registry_get_this_machine_guid()) == 0) || (!rrdhost_flag_check(host, RRDHOST_FLAG_MULTIHOST))) {
920 + info("Metadatalog init for host %s starting...", host->hostname);
921 + error = metalog_init(ctx);
922 + if (error) {
923 + error("Failed to initialize metadata log file event loop.");
924 + goto error_after_rrdeng_worker;
925 + }
926 }
927 + else
928 + info("No metadatalog init for host %s", host->hostname);
929 return 0;
930
931 error_after_rrdeng_worker:
database/rrd.h
+8
@@ -560,6 +560,8 @@ typedef enum rrdhost_flags {
560 RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2, // delete the entire host when orphan
561 RRDHOST_FLAG_BACKEND_SEND = 1 << 3, // send it to backends
562 RRDHOST_FLAG_BACKEND_DONT_SEND = 1 << 4, // don't send it to backends
563 + RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet
564 + RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb
565 } RRDHOST_FLAGS;
566
567 #ifdef HAVE_C___ATOMIC
@@ -1195,6 +1197,12 @@ extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
1197 extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
1198
1199 extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);
1200 +extern RRDHOST *rrdhost_create(
1201 + const char *hostname, const char *registry_hostname, const char *guid, const char *os, const char *timezone,
1202 + const char *tags, const char *program_name, const char *program_version, int update_every, long entries,
1203 + RRD_MEMORY_MODE memory_mode, unsigned int health_enabled, unsigned int rrdpush_enabled, char *rrdpush_destination,
1204 + char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info,
1205 + int is_localhost, int is_archived);
1206
1207 #endif /* NETDATA_RRD_INTERNALS */
1208
database/rrdhost.c
+13 -2
@@ -123,7 +123,8 @@ RRDHOST *rrdhost_create(const char *hostname,
123 char *rrdpush_api_key,
124 char *rrdpush_send_charts_matching,
125 struct rrdhost_system_info *system_info,
126 - int is_localhost
126 + int is_localhost,
127 + int is_archived
128 ) {
129 debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
130
@@ -182,6 +183,10 @@ RRDHOST *rrdhost_create(const char *hostname,
183 avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare);
184 avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare);
185
186 + if (is_archived) {
187 + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
188 + info("Host %s is created in archived mode", hostname);
189 + }
190 if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1))
191 rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS);
192
@@ -276,6 +281,10 @@ RRDHOST *rrdhost_create(const char *hostname,
281 if (unlikely(-1 == uuid_parse(host->machine_guid, host->host_uuid))) {
282 error("Host machine GUID is not valid.");
283 }
284 + if (unlikely(find_or_generate_guid((void *) host, &host->host_uuid, GUID_TYPE_HOST, 1)))
285 + error("Failed to store machine GUID to global map");
286 + else
287 + info("Added %s to global map for host %s", host->machine_guid, host->hostname);
288 host->objects_nr = 1;
289 host->compaction_id = 0;
290 char dbenginepath[FILENAME_MAX + 1];
@@ -367,7 +376,7 @@ RRDHOST *rrdhost_create(const char *hostname,
376 rrd_hosts_available++;
377
378 #ifdef ENABLE_DBENGINE
370 - if (likely(!is_localhost && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
379 + if (likely(!is_localhost && !is_archived && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
380 metalog_commit_update_host(host);
381 #endif
382 return host;
@@ -494,6 +503,7 @@ RRDHOST *rrdhost_find_or_create(
503 , rrdpush_send_charts_matching
504 , system_info
505 , 0
506 + , 0
507 );
508 }
509 else {
@@ -590,6 +600,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
600 , default_rrdpush_send_charts_matching
601 , system_info
602 , 1
603 + , 0
604 );
605 rrd_unlock();
606 web_client_api_v1_management_init();
database/rrdset.c
+8
@@ -543,6 +543,10 @@ RRDSET *rrdset_create_custom(
543 if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
544 rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
545 changed_from_archived_to_active = 1;
546 + if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ARCHIVED)) {
547 + rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);
548 + info("Host %s is not in archived mode anymore", st->rrdhost->hostname);
549 + }
550 mark_rebuild |= META_CHART_ACTIVATED;
551 }
552 char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL,
@@ -683,6 +687,10 @@ RRDSET *rrdset_create_custom(
687 rrdhost_unlock(host);
688 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
689 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
690 + if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
691 + rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
692 + rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);
693 + }
694 return st;
695 }
696
parser/parser.h
+2
@@ -35,6 +35,8 @@ typedef struct pluginsd_action {
35 PARSER_RC (*guid_action)(void *user, uuid_t *uuid);
36 PARSER_RC (*context_action)(void *user, uuid_t *uuid);
37 PARSER_RC (*tombstone_action)(void *user, uuid_t *uuid);
38 + PARSER_RC (*host_action)(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os,
39 + char *timezone, char *tags);
40 } PLUGINSD_ACTION;
41
42 typedef enum parser_input_type {