@cryptotaxi247 / netdata-1 / commits / 87002ef11

Enable metadata persistence in all memory modes (#10742)

Stelios Fragkakis committed Mar 15, 2021 at 22:28 UTC 87002ef11f8b4689ca9ab00e1788442304569d09
6 files changed +68 -62
Makefile.am
+4 -4
@@ -376,14 +376,14 @@ RRD_PLUGIN_FILES = \
376 database/rrdsetvar.h \
377 database/rrdvar.c \
378 database/rrdvar.h \
379 + database/sqlite/sqlite_functions.c \
380 + database/sqlite/sqlite_functions.h \
381 + database/sqlite/sqlite3.c \
382 + database/sqlite/sqlite3.h \
383 $(NULL)
384
385 if ENABLE_DBENGINE
386 RRD_PLUGIN_FILES += \
383 - database/sqlite/sqlite_functions.c \
384 - database/sqlite/sqlite_functions.h \
385 - database/sqlite/sqlite3.c \
386 - database/sqlite/sqlite3.h \
387 database/engine/rrdengine.c \
388 database/engine/rrdengine.h \
389 database/engine/rrddiskprotocol.h \
database/rrd.h
+3 -3
@@ -370,9 +370,9 @@ struct rrddim_query_handle {
370 struct rrddim_volatile {
371 #ifdef ENABLE_DBENGINE
372 uuid_t *rrdeng_uuid; // database engine metric UUID
373 - uuid_t *metric_uuid; // global UUID for this metric (unique_across hosts)
373 struct pg_cache_page_index *page_index;
374 #endif
375 + uuid_t *metric_uuid; // global UUID for this metric (unique_across hosts)
376 union rrddim_collect_handle handle;
377 // ------------------------------------------------------------------------
378 // function pointers that handle data collection
@@ -856,8 +856,8 @@ struct rrdhost {
856
857 #ifdef ENABLE_DBENGINE
858 struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host
859 - uuid_t host_uuid; // Global GUID for this host
859 #endif
860 + uuid_t host_uuid; // Global GUID for this host
861
862 #ifdef ENABLE_HTTPS
863 struct netdata_ssl ssl; //Structure used to encrypt the connection
@@ -1317,7 +1317,7 @@ extern void set_host_properties(
1317
1318 #ifdef ENABLE_DBENGINE
1319 #include "database/engine/rrdengineapi.h"
1320 -#include "sqlite/sqlite_functions.h"
1320 #endif
1321 +#include "sqlite/sqlite_functions.h"
1322
1323 #endif /* NETDATA_RRD_H */
database/rrddim.c
+6 -14
@@ -232,9 +232,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
232 rc += rrddim_set_multiplier(st, rd, multiplier);
233 rc += rrddim_set_divisor(st, rd, divisor);
234 if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
235 -#ifdef ENABLE_DBENGINE
235 store_active_dimension(rd->state->metric_uuid);
237 -#endif
236 rd->state->collect_ops.init(rd);
237 rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
238 rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT);
@@ -242,14 +240,11 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
240 rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT);
241 calc_link_to_rrddim(rd);
242 }
245 - // DBENGINE available and activated?
246 -#ifdef ENABLE_DBENGINE
247 - if (likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && unlikely(rc)) {
243 + if (unlikely(rc)) {
244 debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
245 (void)sql_store_dimension(rd->state->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
246 rd->algorithm);
247 }
252 -#endif
248 rrdset_unlock(st);
249 return rd;
250 }
@@ -396,7 +391,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
391 #ifdef ENABLE_DBENGINE
392 uuid_t *dim_uuid = find_dimension_uuid(st, rd);
393 rrdeng_metric_init(rd, dim_uuid);
399 - store_active_dimension(rd->state->metric_uuid);
394 rd->state->collect_ops.init = rrdeng_store_metric_init;
395 rd->state->collect_ops.store_metric = rrdeng_store_metric_next;
396 rd->state->collect_ops.finalize = rrdeng_store_metric_finalize;
@@ -408,6 +402,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
402 rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
403 #endif
404 } else {
405 + rd->state->metric_uuid = find_dimension_uuid(st, rd);
406 + if (unlikely(!rd->state->metric_uuid))
407 + rd->state->metric_uuid = create_dimension_uuid(rd->rrdset, rd);
408 rd->state->collect_ops.init = rrddim_collect_init;
409 rd->state->collect_ops.store_metric = rrddim_collect_store_metric;
410 rd->state->collect_ops.finalize = rrddim_collect_finalize;
@@ -418,6 +415,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
415 rd->state->query_ops.latest_time = rrddim_query_latest_time;
416 rd->state->query_ops.oldest_time = rrddim_query_oldest_time;
417 }
418 + store_active_dimension(rd->state->metric_uuid);
419 rd->state->collect_ops.init(rd);
420 // append this dimension
421 if(!st->dimensions)
@@ -476,10 +474,8 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
474 if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
475 uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
476 if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
479 -#ifdef ENABLE_DBENGINE
477 /* This metric has no data and no references */
478 delete_dimension_uuid(rd->state->metric_uuid);
482 -#endif
479 }
480 }
481
@@ -503,6 +499,7 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
499 error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id);
500
501 // free(rd->annotations);
502 + freez(rd->state->metric_uuid);
503
504 RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;
505 switch(rrd_memory_mode) {
@@ -522,11 +519,6 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
519 debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
520 freez((void *)rd->id);
521 freez(rd->cache_filename);
525 -#ifdef ENABLE_DBENGINE
526 - if (rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
527 - freez(rd->state->metric_uuid);
528 - }
529 -#endif
522 freez(rd->state);
523 freez(rd);
524 break;
database/rrdhost.c
+11 -10
@@ -298,15 +298,16 @@ RRDHOST *rrdhost_create(const char *hostname,
298 return NULL;
299 }
300
301 + if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) {
302 + int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags);
303 + if (unlikely(rc))
304 + error_report("Failed to store machine GUID to the database");
305 + }
306 + else
307 + error_report("Host machine GUID %s is not valid", host->machine_guid);
308 +
309 if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
310 #ifdef ENABLE_DBENGINE
303 - if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) {
304 - int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags);
305 - if (unlikely(rc))
306 - error_report("Failed to store machine GUID to the database");
307 - }
308 - else
309 - error_report("Host machine GUID %s is not valid", host->machine_guid);
311 char dbenginepath[FILENAME_MAX + 1];
312 int ret;
313
@@ -634,11 +635,11 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
635 if (gap_when_lost_iterations_above < 1)
636 gap_when_lost_iterations_above = 1;
637
637 -#ifdef ENABLE_DBENGINE
638 if (unlikely(sql_init_database())) {
639 - return 1;
639 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
640 + return 1;
641 + info("Skipping SQLITE metadata initialization since memory mode is not db engine");
642 }
641 -#endif
643
644 health_init();
645
database/rrdset.c
+16 -28
@@ -385,6 +385,7 @@ void rrdset_free(RRDSET *st) {
385 freez(st->state->old_context);
386 free_label_list(st->state->labels.head);
387 freez(st->state);
388 + freez(st->chart_uuid);
389
390 switch(st->rrd_memory_mode) {
391 case RRD_MEMORY_MODE_SAVE:
@@ -397,10 +398,6 @@ void rrdset_free(RRDSET *st) {
398 case RRD_MEMORY_MODE_ALLOC:
399 case RRD_MEMORY_MODE_NONE:
400 case RRD_MEMORY_MODE_DBENGINE:
400 -#ifdef ENABLE_DBENGINE
401 - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
402 - freez(st->chart_uuid);
403 -#endif
401 freez(st);
402 break;
403 }
@@ -660,15 +657,12 @@ RRDSET *rrdset_create_custom(
657 sched_yield();
658 }
659 }
663 -#ifdef ENABLE_DBENGINE
664 - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
665 - (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED))) {
660 + if (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED)) {
661 debug(D_METADATALOG, "CHART [%s] metadata updated", st->id);
662 int rc = update_chart_metadata(st->chart_uuid, st, id, name);
663 if (unlikely(rc))
664 error_report("Failed to update chart metadata in the database");
665 }
671 -#endif
666 /* Fall-through during switch from archived to active so that the host lock is taken and health is linked */
667 if (!changed_from_archived_to_active)
668 return st;
@@ -925,17 +919,14 @@ RRDSET *rrdset_create_custom(
919
920 rrdsetcalc_link_matching(st);
921 rrdcalctemplate_link_matching(st);
928 -#ifdef ENABLE_DBENGINE
929 - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
930 - st->chart_uuid = find_chart_uuid(host, type, id, name);
931 - if (unlikely(!st->chart_uuid))
932 - st->chart_uuid = create_chart_uuid(st, id, name);
933 - else
934 - update_chart_metadata(st->chart_uuid, st, id, name);
922
936 - store_active_chart(st->chart_uuid);
937 - }
938 -#endif
923 + st->chart_uuid = find_chart_uuid(host, type, id, name);
924 + if (unlikely(!st->chart_uuid))
925 + st->chart_uuid = create_chart_uuid(st, id, name);
926 + else
927 + update_chart_metadata(st->chart_uuid, st, id, name);
928 +
929 + store_active_chart(st->chart_uuid);
930
931 rrdhost_cleanup_obsolete_charts(host);
932
@@ -1934,17 +1925,14 @@ void rrdset_finalize_labels(RRDSET *st)
1925 } else {
1926 replace_label_list(labels, new_labels);
1927 }
1937 -#ifdef ENABLE_DBENGINE
1938 - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1939 - netdata_rwlock_wrlock(&labels->labels_rwlock);
1940 - struct label *lbl = labels->head;
1941 - while (lbl) {
1942 - sql_store_chart_label(st->chart_uuid, (int)lbl->label_source, lbl->key, lbl->value);
1943 - lbl = lbl->next;
1944 - }
1945 - netdata_rwlock_unlock(&labels->labels_rwlock);
1928 +
1929 + netdata_rwlock_wrlock(&labels->labels_rwlock);
1930 + struct label *lbl = labels->head;
1931 + while (lbl) {
1932 + sql_store_chart_label(st->chart_uuid, (int)lbl->label_source, lbl->key, lbl->value);
1933 + lbl = lbl->next;
1934 }
1947 -#endif
1935 + netdata_rwlock_unlock(&labels->labels_rwlock);
1936
1937 st->state->new_labels = NULL;
1938 }
database/sqlite/sqlite_functions.c
+28 -3
@@ -84,7 +84,8 @@ void store_active_chart(uuid_t *chart_uuid)
84 int rc;
85
86 if (unlikely(!db_meta)) {
87 - error_report("Database has not been initialized");
87 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
88 + error_report("Database has not been initialized");
89 return;
90 }
91
@@ -111,7 +112,8 @@ void store_active_dimension(uuid_t *dimension_uuid)
112 int rc;
113
114 if (unlikely(!db_meta)) {
114 - error_report("Database has not been initialized");
115 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
116 + error_report("Database has not been initialized");
117 return;
118 }
119
@@ -143,7 +145,9 @@ int sql_init_database(void)
145 snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
146 rc = sqlite3_open(sqlite_database, &db_meta);
147 if (rc != SQLITE_OK) {
146 - error_report("Failed to initialize database at %s", sqlite_database);
148 + error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
149 + sqlite3_close(db_meta);
150 + db_meta = NULL;
151 return 1;
152 }
153
@@ -220,6 +224,9 @@ uuid_t *find_dimension_uuid(RRDSET *st, RRDDIM *rd)
224 uuid_t *uuid = NULL;
225 int rc;
226
227 + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
228 + return NULL;
229 +
230 if (unlikely(!res)) {
231 rc = sqlite3_prepare_v2(db_meta, SQL_FIND_DIMENSION_UUID, -1, &res, 0);
232 if (rc != SQLITE_OK) {
@@ -333,6 +340,9 @@ uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const c
340 uuid_t *uuid = NULL;
341 int rc;
342
343 + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
344 + return NULL;
345 +
346 if (unlikely(!res)) {
347 rc = sqlite3_prepare_v2(db_meta, SQL_FIND_CHART_UUID, -1, &res, 0);
348 if (rc != SQLITE_OK) {
@@ -390,6 +400,9 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const
400 {
401 int rc;
402
403 + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
404 + return 0;
405 +
406 rc = sql_store_chart(
407 chart_uuid, &st->rrdhost->host_uuid, st->type, id, name, st->family, st->context, st->title, st->units, st->plugin_name,
408 st->module_name, st->priority, st->update_every, st->chart_type, st->rrd_memory_mode, st->entries);
@@ -429,6 +442,8 @@ int sql_store_host(
442 int rc;
443
444 if (unlikely(!db_meta)) {
445 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
446 + return 0;
447 error_report("Database has not been initialized");
448 return 1;
449 }
@@ -499,6 +514,8 @@ int sql_store_chart(
514 int rc, param = 0;
515
516 if (unlikely(!db_meta)) {
517 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
518 + return 0;
519 error_report("Database has not been initialized");
520 return 1;
521 }
@@ -623,6 +640,8 @@ int sql_store_dimension(
640 int rc;
641
642 if (unlikely(!db_meta)) {
643 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
644 + return 0;
645 error_report("Database has not been initialized");
646 return 1;
647 }
@@ -1033,6 +1052,12 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha
1052 sqlite3_stmt *res = NULL;
1053 int rc;
1054
1055 + if (unlikely(!db_meta)) {
1056 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
1057 + error_report("Database has not been initialized");
1058 + return;
1059 + }
1060 +
1061 rc = sqlite3_prepare_v2(db_meta, SQL_INS_CHART_LABEL, -1, &res, 0);
1062 if (unlikely(rc != SQLITE_OK)) {
1063 error_report("Failed to prepare statement store chart labels");