@cryptotaxi247 / netdata-1 / commits / 9a6fd6366

Store host label information in the metadata database (#13441)

* Create new host_label table * Add generic function to store chart and host labels Add function to store host labels Cleanup old host labels * Store labels from localhost and children (streaming) Remove the host label info if the host is deleted * Delete host labels before insert

Stelios Fragkakis committed Jul 28, 2022 at 21:53 UTC 9a6fd6366fafc5585b3fe9d2be2258df4738050d

5 files changed +89 -28

collectors/plugins.d/pluginsd_parser.c

@@ -201,6 +201,7 @@ PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, DICTIONARY *new_h

201 host->host_labels = rrdlabels_create();

202

203 rrdlabels_migrate_to_these(host->host_labels, new_host_labels);

204 + sql_store_host_labels(host);

205

206 return PARSER_RC_OK;

207 }

database/rrd.h

@@ -209,6 +209,8 @@ typedef enum rrdlabel_source {

209 RRDLABEL_FLAG_NEW = (1 << 31) // marks for rrdlabels internal use - they are not exposed outside rrdlabels

210 } RRDLABEL_SRC;

211

212 +#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)

213 +

214 extern DICTIONARY *rrdlabels_create(void);

215 extern void rrdlabels_destroy(DICTIONARY *labels_dict);

216 extern void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);

database/rrdhost.c

@@ -1362,6 +1362,7 @@ void reload_host_labels(void) {

1362 rrdhost_load_auto_labels();

1363

1364 rrdlabels_remove_all_unmarked(localhost->host_labels);

1365 + sql_store_host_labels(localhost);

1366

1367 health_label_log_save(localhost);

1368

database/sqlite/sqlite_functions.c

+84 -28

@@ -42,6 +42,9 @@ const char *database_config[] = {

42 "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "

43 "date_created INT, PRIMARY KEY(host_id, system_key));",

45 + "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, "

46 + "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key));",

47 +

48 "CREATE TABLE IF NOT EXISTS chart_hash_map(chart_id blob , hash_id blob, UNIQUE (chart_id, hash_id));",

50 "CREATE TABLE IF NOT EXISTS chart_hash(hash_id blob PRIMARY KEY,type text, id text, name text, "

@@ -74,6 +77,7 @@ const char *database_cleanup[] = {

77 "DELETE FROM chart_hash WHERE hash_id NOT IN (SELECT hash_id FROM chart_hash_map);",

78 "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);",

79 "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host);",

80 + "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host);",

81 NULL

82 };

@@ -764,7 +768,7 @@ static int exec_statement_with_uuid(const char *sql, uuid_t *uuid)

768

769 rc = execute_insert(res);

770 if (likely(rc == SQLITE_DONE))

767 - result = 0;

771 + result = SQLITE_OK;

772 else

773 error_report("Failed to execute %s, rc = %d", sql, rc);

774

@@ -1599,6 +1603,45 @@ void add_migrated_file(char *path, uint64_t file_size)

1603 return;

1604 }

1605

1606 +static int sql_store_label(sqlite3_stmt *res, uuid_t *uuid, int source_type, const char *label, const char *value)

1607 +{

1608 + int rc;

1609 +

1610 + rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);

1611 + if (unlikely(rc != SQLITE_OK)) {

1612 + error_report("Failed to bind UUID parameter to store label information");

1613 + goto skip_store;

1614 + }

1615 +

1616 + rc = sqlite3_bind_int(res, 2, source_type);

1617 + if (unlikely(rc != SQLITE_OK)) {

1618 + error_report("Failed to bind type parameter to store label information");

1619 + goto skip_store;

1620 + }

1621 +

1622 + rc = sqlite3_bind_text(res, 3, label, -1, SQLITE_STATIC);

1623 + if (unlikely(rc != SQLITE_OK)) {

1624 + error_report("Failed to bind label parameter to store label information");

1625 + goto skip_store;

1626 + }

1627 +

1628 + rc = sqlite3_bind_text(res, 4, value, -1, SQLITE_STATIC);

1629 + if (unlikely(rc != SQLITE_OK)) {

1630 + error_report("Failed to bind value parameter to store label information");

1631 + goto skip_store;

1632 + }

1633 +

1634 + rc = execute_insert(res);

1635 + if (unlikely(rc != SQLITE_DONE))

1636 + error_report("Failed to store label entry, rc = %d", rc);

1637 +

1638 +skip_store:

1639 + if (unlikely(sqlite3_reset(res) != SQLITE_OK))

1640 + error_report("Failed to reset the prepared statement when storing label information");

1641 +

1642 + return rc != SQLITE_DONE;

1643 +}

1644 +

1645 #define SQL_INS_CHART_LABEL "insert or replace into chart_label " \

1646 "(chart_id, source_type, label_key, label_value, date_created) " \

1647 "values (@chart, @source, @label, @value, unixepoch());"

@@ -1622,39 +1665,35 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha

1665 }

1666 }

1667

1625 - rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);

1626 - if (unlikely(rc != SQLITE_OK)) {

1627 - error_report("Failed to bind chart_id parameter to store label information");

1628 - goto failed;

1629 - }

1668 + sql_store_label(res, chart_uuid, source_type, label, value);

1669

1631 - rc = sqlite3_bind_int(res, 2, source_type);

1632 - if (unlikely(rc != SQLITE_OK)) {

1633 - error_report("Failed to bind type parameter to store label information");

1634 - goto failed;

1635 - }

1670 + return;

1671 +}

1672

1637 - rc = sqlite3_bind_text(res, 3, label, -1, SQLITE_STATIC);

1638 - if (unlikely(rc != SQLITE_OK)) {

1639 - error_report("Failed to bind label parameter to store label information");

1640 - goto failed;

1641 - }

1673 +#define SQL_INS_HOST_LABEL "INSERT OR REPLACE INTO host_label " \

1674 + "(host_id, source_type, label_key, label_value, date_created) " \

1675 + "values (@chart, @source, @label, @value, unixepoch());"

1676

1643 - rc = sqlite3_bind_text(res, 4, value, -1, SQLITE_STATIC);

1644 - if (unlikely(rc != SQLITE_OK)) {

1645 - error_report("Failed to bind value parameter to store label information");

1646 - goto failed;

1647 - }

1677 +static void sql_store_host_label(uuid_t *host_uuid, int source_type, const char *label, const char *value)

1678 +{

1679 + static __thread sqlite3_stmt *res = NULL;

1680 + int rc;

1681

1649 - rc = execute_insert(res);

1650 - if (unlikely(rc != SQLITE_DONE))

1651 - error_report("Failed to store chart label entry, rc = %d", rc);

1682 + if (unlikely(!db_meta)) {

1683 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

1684 + error_report("Database has not been initialized");

1685 + return;

1686 + }

1687

1653 -failed:

1654 - if (unlikely(sqlite3_reset(res) != SQLITE_OK))

1655 - error_report("Failed to reset the prepared statement when storing chart label information");

1688 + if (unlikely(!res)) {

1689 + rc = prepare_statement(db_meta, SQL_INS_HOST_LABEL, &res);

1690 + if (unlikely(rc != SQLITE_OK)) {

1691 + error_report("Failed to prepare statement store chart labels");

1692 + return;

1693 + }

1694 + }

1695

1657 - return;

1696 + (void) sql_store_label(res, host_uuid, source_type, label, value);

1697 }

1698

1699 int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id,

@@ -2601,6 +2640,23 @@ void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_inf

2640 return;

2641 }

2642

2643 +static int save_host_label_callback(const char *name, const char *value, RRDLABEL_SRC label_source, void *data)

2644 +{

2645 + RRDHOST *host = (RRDHOST *)data;

2646 + sql_store_host_label(&host->host_uuid, (int)label_source & ~(RRDLABEL_FLAG_INTERNAL), name, value);

2647 + return 0;

2648 +}

2649 +

2650 +#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid;"

2651 +void sql_store_host_labels(RRDHOST *host)

2652 +{

2653 + int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid);

2654 + if (rc != SQLITE_OK)

2655 + error_report("Failed to remove old host labels for host %s", host->hostname);

2656 +

2657 + rrdlabels_walkthrough_read(host->host_labels, save_host_label_callback, host);

2658 +}

2659 +

2660 // Utils

2661 int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null)

2662 {

database/sqlite/sqlite_functions.h

@@ -111,4 +111,5 @@ int init_database_batch(sqlite3 *database, int rebuild, int init_type, const cha

111 void migrate_localhost(uuid_t *host_uuid);

112 extern void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_info *system_info);

113 extern void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);

114 +void sql_store_host_labels(RRDHOST *host);

115 #endif //NETDATA_SQLITE_FUNCTIONS_H