@cryptotaxi247 / netdata-1 / commits / 4338d8eba

Improve metadata storage of chart and host labels (#21492)

* Refactor metadata label storage with unified callback and prepared statements * Replace SQL string concatenation with prepared statements for label storage. * Simplify host and chart label update logic by removing work buffer usage. * Introduce `store_labels` function for unified label handling. * Improve code readability and maintainability by reducing redundant callback implementations. * Collect labels under spinlock and do I/O (sqlite storage) without the lock * - Added batching for label storage to minimize SQLite I/O operations. - Introduced a reusable buffer for building SQL statements. - Replaced per-label operations with multi-row inserts using prepared statements. - Enhanced scalability by adding a batch size definition. * Code cleanup * Handle bind failure by setting `errors` flag if `param` is invalid

Stelios Fragkakis committed Feb 19, 2026 at 11:49 UTC 4338d8eba1ac9b41724d94866cbed9daa97081d0

3 files changed +186 -58

src/database/rrdhost.c

+2 -5

@@ -465,11 +465,8 @@ RRDHOST *rrdhost_create(

465

466 if(!archived) {

467 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);

468 - if (is_localhost) {

469 - BUFFER *buf = buffer_create(0, NULL);

470 - store_host_info_and_metadata(host, buf);

471 - buffer_free(buf);

472 - }

468 + if (is_localhost)

469 + store_host_info_and_metadata(host);

470 rrdhost_load_rrdcontext_data(host);

471 ml_host_new(host);

472 } else

src/database/sqlite/sqlite_metadata.c

+183 -52

@@ -151,13 +151,17 @@ sqlite3 *db_meta = NULL;

151

152 #define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid"

153

154 -#define STORE_HOST_LABEL \

154 +#define SQL_STORE_HOST_LABEL \

155 "INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "

156

157 -#define STORE_CHART_LABEL \

157 +#define SQL_STORE_HOST_LABEL_CONFLICT " ON CONFLICT (host_id, label_key) " \

158 + "DO UPDATE SET source_type = excluded.source_type, label_value = excluded.label_value, date_created = UNIXEPOCH()"

159 +

160 +#define SQL_STORE_CHART_LABEL \

161 "INSERT INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "

162

160 -#define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())"

163 +#define SQL_STORE_CHART_LABEL_CONFLICT " ON CONFLICT (chart_id, label_key) " \

164 + "DO UPDATE SET source_type = excluded.source_type, label_value = excluded.label_value, date_created = UNIXEPOCH()"

165

166 #define DELETE_DIMENSION_UUID "DELETE FROM dimension WHERE dim_id = @uuid"

167

@@ -816,32 +820,166 @@ close_database:

820

821 // Metadata functions

822

819 -struct query_build {

820 - BUFFER *sql;

821 - int count;

822 - char uuid_str[UUID_STR_LEN];

823 +// Label storage types

824 +typedef enum {

825 + STORE_HOST_LABELS,

826 + STORE_CHART_LABELS,

827 +} label_store_type_t;

828 +

829 +// Structure to hold a single label entry for collection

830 +struct label_entry {

831 + char *name;

832 + char *value;

833 + RRDLABEL_SRC ls;

834 };

835

825 -static int host_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {

826 - struct query_build *lb = data;

827 - if (unlikely(!lb->count))

828 - buffer_sprintf(lb->sql, STORE_HOST_LABEL);

829 - else

830 - buffer_strcat(lb->sql, ", ");

831 - buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);

832 - lb->count++;

836 +// Context for label collection callback (no I/O, just data collection)

837 +struct label_collect_ctx {

838 + struct label_entry *entries;

839 + size_t count;

840 + size_t capacity;

841 +};

842 +

843 +#define LABEL_COLLECT_INITIAL_CAPACITY 64

844 +

845 +// Max labels per batch: SQLite default SQLITE_MAX_VARIABLE_NUMBER is 32766 in the version we are using

846 +#define LABEL_BATCH_SIZE (1024)

847 +

848 +// Callback to collect labels into an array (no SQLite I/O while spinlock is held)

849 +static int collect_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data)

850 +{

851 + struct label_collect_ctx *ctx = data;

852 +

853 + if (unlikely(!name || !value))

854 + return 1;

855 +

856 + // Grow array if needed

857 + if (ctx->count >= ctx->capacity) {

858 + ctx->capacity *= 2;

859 + ctx->entries = reallocz(ctx->entries, ctx->capacity * sizeof(*ctx->entries));

860 + }

861 +

862 + // Copy label data

863 + ctx->entries[ctx->count].name = strdupz(name);

864 + ctx->entries[ctx->count].value = strdupz(value);

865 + ctx->entries[ctx->count].ls = ls;

866 + ctx->count++;

867 +

868 return 1;

869 }

870

836 -static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {

837 - struct query_build *lb = data;

838 - if (unlikely(!lb->count))

839 - buffer_sprintf(lb->sql, STORE_CHART_LABEL);

871 +// Execute a batch of label inserts using a single multi-row INSERT statement

872 +// Returns number of errors

873 +static int store_label_batch(

874 + nd_uuid_t *uuid,

875 + struct label_entry *entries,

876 + size_t count,

877 + label_store_type_t type,

878 + BUFFER *sql)

879 +{

880 + if (count == 0)

881 + return 0;

882 +

883 + // Build SQL: INSERT INTO table (...) VALUES (?,?,?,?,UNIXEPOCH()), (?,?,?,?,UNIXEPOCH()), ... ON CONFLICT ...

884 + buffer_flush(sql);

885 +

886 + if (type == STORE_HOST_LABELS)

887 + buffer_strcat(sql, SQL_STORE_HOST_LABEL);

888 else

841 - buffer_strcat(lb->sql, ", ");

842 - buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);

843 - lb->count++;

844 - return 1;

889 + buffer_strcat(sql, SQL_STORE_CHART_LABEL);

890 +

891 + for (size_t i = 0; i < count; i++) {

892 + if (i > 0)

893 + buffer_strcat(sql, ", ");

894 + buffer_strcat(sql, "(?, ?, ?, ?, UNIXEPOCH())");

895 + }

896 +

897 + if (type == STORE_HOST_LABELS)

898 + buffer_strcat(sql, SQL_STORE_HOST_LABEL_CONFLICT);

899 + else

900 + buffer_strcat(sql, SQL_STORE_CHART_LABEL_CONFLICT);

901 +

902 + sqlite3_stmt *stmt = NULL;

903 +

904 + if (!PREPARE_STATEMENT(db_meta, buffer_tostring(sql), &stmt))

905 + return 1;

906 +

907 + int errors = 0;

908 +

909 + // Bind all parameters: 4 per label (uuid, source, key, value)

910 + int param = 0;

911 + for (size_t i = 0; i < count; i++) {

912 + SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_blob(stmt, ++param, uuid, sizeof(*uuid), SQLITE_STATIC));

913 + SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(stmt, ++param, (int)(entries[i].ls & ~(RRDLABEL_FLAG_INTERNAL))));

914 + SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(stmt, ++param, entries[i].name, -1, SQLITE_STATIC));

915 + SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_text(stmt, ++param, entries[i].value, -1, SQLITE_STATIC));

916 + }

917 + param = 0;

918 +

919 + int rc = sqlite3_step_monitored(stmt);

920 + if (unlikely(rc != SQLITE_DONE)) {

921 + errors = 1;

922 + error_report("Failed to store label batch, rc = %d", rc);

923 + }

924 +

925 +bind_fail:

926 + if (param)

927 + errors = 1;

928 + REPORT_BIND_FAIL(stmt, param);

929 + SQLITE_FINALIZE(stmt);

930 +

931 + return errors;

932 +}

933 +

934 +// Store labels for a host or chart using batched prepared statements

935 +// Two-phase approach: collect labels first (while spinlock held), then execute SQLite (after spinlock released)

936 +static int store_labels(nd_uuid_t *uuid, RRDLABELS *labels, label_store_type_t type, BUFFER *work_buffer)

937 +{

938 + if (unlikely(!uuid || !labels))

939 + return 0;

940 +

941 + // Phase 1: Collect labels while spinlock is held (fast, no I/O)

942 + struct label_collect_ctx collect_ctx = {

943 + .entries = mallocz(LABEL_COLLECT_INITIAL_CAPACITY * sizeof(*collect_ctx.entries)),

944 + .count = 0,

945 + .capacity = LABEL_COLLECT_INITIAL_CAPACITY,

946 + };

947 +

948 + rrdlabels_walkthrough_read(labels, collect_label_callback, &collect_ctx);

949 + // Spinlock is now released

950 +

951 + // Phase 2: Execute SQLite operations in batches (no spinlock held)

952 + int errors = 0;

953 +

954 + if (collect_ctx.count > 0) {

955 + bool free_buffer = false;

956 + if (!work_buffer) {

957 + work_buffer = buffer_create(256 + collect_ctx.count * 30, NULL);

958 + free_buffer = true;

959 + }

960 +

961 + size_t remaining = collect_ctx.count;

962 + size_t offset = 0;

963 +

964 + while (remaining > 0) {

965 + size_t batch_count = (remaining > LABEL_BATCH_SIZE) ? LABEL_BATCH_SIZE : remaining;

966 + errors += store_label_batch(uuid, &collect_ctx.entries[offset], batch_count, type, work_buffer);

967 + offset += batch_count;

968 + remaining -= batch_count;

969 + }

970 +

971 + if (free_buffer)

972 + buffer_free(work_buffer);

973 + }

974 +

975 + // Cleanup collected labels

976 + for (size_t i = 0; i < collect_ctx.count; i++) {

977 + freez(collect_ctx.entries[i].name);

978 + freez(collect_ctx.entries[i].value);

979 + }

980 + freez(collect_ctx.entries);

981 +

982 + return errors ? 1 : 0;

983 }

984

985 static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer)

@@ -852,11 +990,7 @@ static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer)

990 if (new_version == old_version)

991 return 0;

992

855 - struct query_build tmp = {.sql = work_buffer, .count = 0};

856 - uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);

857 - rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);

858 - buffer_strcat(work_buffer, " ON CONFLICT (chart_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");

859 - int rc = db_execute(db_meta, buffer_tostring(work_buffer), NULL);

993 + int rc = store_labels(&st->chart_uuid, st->rrdlabels, STORE_CHART_LABELS, work_buffer);

994 if (likely(!rc))

995 st->rrdlabels_last_saved_version = new_version;

996

@@ -2028,7 +2162,7 @@ size_t populate_metrics_from_database(void *mrg, void (*populate_cb)(void *mrg,

2162 }

2163 #endif

2164

2031 -static void metadata_scan_host(struct meta_config_s *config, RRDHOST *host, BUFFER *work_buffer, bool is_worker)

2165 +static void metadata_scan_host(struct meta_config_s *config, RRDHOST *host, bool is_worker, BUFFER *work_buffer)

2166 {

2167 static bool skip_models = false;

2168 RRDSET *st;

@@ -2052,8 +2186,6 @@ static void metadata_scan_host(struct meta_config_s *config, RRDHOST *host, BUFF

2186

2187 rrdset_flag_clear(st, RRDSET_FLAG_METADATA_UPDATE);

2188

2055 - buffer_flush(work_buffer);

2056 -

2189 if (is_worker)

2190 worker_is_busy(UV_EVENT_STORE_CHART);

2191

@@ -2327,6 +2459,7 @@ static void meta_store_host_labels(RRDHOST *host, BUFFER *work_buffer)

2459 {

2460 rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_LABELS);

2461

2462 + // Delete existing labels first to handle label removal

2463 int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_id.uuid);

2464 if (unlikely(rc)) {

2465 error_report("METADATA: 'host:%s': failed to delete old host labels", rrdhost_hostname(host));

@@ -2334,18 +2467,10 @@ static void meta_store_host_labels(RRDHOST *host, BUFFER *work_buffer)

2467 return;

2468 }

2469

2337 - buffer_flush(work_buffer);

2338 -

2339 - struct query_build tmp = {.sql = work_buffer, .count = 0};

2340 - uuid_unparse_lower(host->host_id.uuid, tmp.uuid_str);

2341 - rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);

2342 - buffer_strcat(

2343 - work_buffer,

2344 - " ON CONFLICT (host_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");

2345 - rc = db_execute(db_meta, buffer_tostring(work_buffer), NULL);

2346 -

2470 + // Store all current labels using prepared statements

2471 + rc = store_labels(&host->host_id.uuid, host->rrdlabels, STORE_HOST_LABELS, work_buffer);

2472 if (unlikely(rc)) {

2348 - error_report("METADATA: 'host:%s': failed to update metadata host labels", rrdhost_hostname(host));

2473 + error_report("METADATA: 'host:%s': failed to store host labels", rrdhost_hostname(host));

2474 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);

2475 }

2476 }

@@ -2364,7 +2489,7 @@ static void store_host_claim_id(RRDHOST *host)

2489 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID | RRDHOST_FLAG_METADATA_UPDATE);

2490 }

2491

2367 -void store_host_info_and_metadata(RRDHOST *host, BUFFER *work_buffer)

2492 +static void store_host_info_and_metadata_with_buffer(RRDHOST *host, BUFFER *work_buffer)

2493 {

2494 // Store labels (if needed)

2495 if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_LABELS)))

@@ -2379,7 +2504,13 @@ void store_host_info_and_metadata(RRDHOST *host, BUFFER *work_buffer)

2504 store_host_and_system_info(host);

2505 }

2506

2382 -static void store_hosts_metadata(struct meta_config_s *config, BUFFER *work_buffer, bool is_worker)

2507 +// Public API - creates buffer internally if needed

2508 +void store_host_info_and_metadata(RRDHOST *host)

2509 +{

2510 + store_host_info_and_metadata_with_buffer(host, NULL);

2511 +}

2512 +

2513 +static void store_hosts_metadata(struct meta_config_s *config, bool is_worker)

2514 {

2515 RRDHOST *host;

2516 size_t host_count = 0;

@@ -2393,6 +2524,9 @@ static void store_hosts_metadata(struct meta_config_s *config, BUFFER *work_buff

2524 host_count = 1; // avoid division by zero

2525 }

2526

2527 + // Reusable buffer for building SQL statements

2528 + BUFFER *work_buffer = buffer_create(1024, NULL);

2529 +

2530 size_t count = 0;

2531 dfe_start_reentrant(rrdhost_root_index, host)

2532 {

@@ -2409,18 +2543,20 @@ static void store_hosts_metadata(struct meta_config_s *config, BUFFER *work_buff

2543 worker_is_busy(UV_EVENT_STORE_HOST);

2544

2545 // store labels, claim_id, host and system info (if needed)

2412 - store_host_info_and_metadata(host, work_buffer);

2546 + store_host_info_and_metadata_with_buffer(host, work_buffer);

2547

2548 if (is_worker)

2549 worker_is_idle();

2550

2417 - metadata_scan_host(config, host, work_buffer, is_worker);

2551 + metadata_scan_host(config, host, is_worker, work_buffer);

2552

2553 if (!is_worker)

2554 nd_log_daemon(NDLP_INFO, "METADATA: Progress of metadata storage: %6.2f%% completed", (100.0 * count / host_count));

2555 }

2556 dfe_done(host);

2557

2558 + buffer_free(work_buffer);

2559 +

2560 if (!is_worker) {

2561 COMPUTE_DURATION(report_duration, "us", started_ut, now_monotonic_usec());

2562 nd_log_daemon(

@@ -2449,7 +2585,6 @@ static void start_metadata_hosts(uv_work_t *req)

2585 worker_data_t *worker = req->data;

2586 struct meta_config_s *config = worker->config;

2587

2452 - BUFFER *work_buffer = worker->work_buffer;

2588 usec_t all_started_ut = now_monotonic_usec();

2589

2590 store_sql_statements((struct judy_list_t *)worker->pending_sql_statement, true, false);

@@ -2461,7 +2596,7 @@ static void start_metadata_hosts(uv_work_t *req)

2596

2597 worker_is_busy(UV_EVENT_METADATA_STORE);

2598

2464 - store_hosts_metadata(config, work_buffer, true);

2599 + store_hosts_metadata(config, true);

2600

2601 COMPUTE_DURATION(report_duration, "us", all_started_ut, now_monotonic_usec());

2602 nd_log_daemon(NDLP_DEBUG, "Checking all hosts completed in %s", report_duration);

@@ -2512,7 +2647,6 @@ static void metadata_event_loop(void *arg)

2647 nd_log(NDLS_DAEMON, NDLP_DEBUG, "Starting metadata sync thread");

2648 config->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_FIRST_CHECK;

2649

2515 - BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);

2650 worker_data_t *worker;

2651 Pvoid_t *Pvalue;

2652 struct judy_list_t *pending_alert_list = NULL;

@@ -2598,8 +2732,6 @@ static void metadata_event_loop(void *arg)

2732 worker->pending_ctx_cleanup_list = pending_ctx_cleanup_list;

2733 worker->pending_uuid_deletion = pending_uuid_deletion;

2734 worker->pending_sql_statement = pending_sql_statement;

2601 -

2602 - worker->work_buffer = work_buffer;

2735 pending_alert_list = NULL;

2736 pending_ctx_cleanup_list = NULL;

2737 pending_uuid_deletion = NULL;

@@ -2734,7 +2866,6 @@ static void metadata_event_loop(void *arg)

2866 freez(pending_uuid_deletion);

2867 }

2868

2737 - buffer_free(work_buffer);

2869 release_cmd_pool(&config->cmd_pool);

2870 worker_unregister();

2871 service_exits();

src/database/sqlite/sqlite_metadata.h

+1 -1

@@ -59,7 +59,7 @@ void commit_alert_transitions(RRDHOST *host);

59 //void metadata_sync_shutdown_background(void);

60 //void metadata_sync_shutdown_background_wait(void);

61 void metadata_queue_ctx_host_cleanup(nd_uuid_t *host_uuid, const char *context);

62 -void store_host_info_and_metadata(RRDHOST *host, BUFFER *work_buffer);

62 +void store_host_info_and_metadata(RRDHOST *host);

63 void metadata_execute_store_statement(sqlite3_stmt *stmt);

64 size_t populate_metrics_from_database(void *mrg, void (*populate_cb)(void *mrg, Word_t section, nd_uuid_t *uuid));