load rrdcontext dimensions in batches (#19304)
* load rrdcontext dimensions in batches * Batch load all dimensions of a host --------- Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Costa Tsaousis committed
Dec 31, 2024 at 17:00 UTC
50b2adeb39781bf6779560a254319604be0657ee
5 files changed
+50
-15
src/database/contexts/internal.h
+1
@@ -443,6 +443,7 @@ void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool
443
#define rrdcontext_lock(rc) spinlock_lock(&((rc)->spinlock))
444
#define rrdcontext_unlock(rc) spinlock_unlock(&((rc)->spinlock))
445
446
+void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
447
void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function);
448
void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function);
449
src/database/contexts/metric.c
+2
-2
@@ -2,7 +2,7 @@
2
3
#include "internal.h"
4
5
-static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
5
+void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
6
7
inline const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma) {
8
RRDMETRIC *rm = rrdmetric_acquired_value(rma);
@@ -225,7 +225,7 @@ void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) {
225
}
226
227
// trigger post-processing of the rrdmetric, escalating changes to the rrdinstance it belongs
228
-static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) {
228
+void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) {
229
if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD)))
230
rrdmetric_set_archived(rm);
231
src/database/contexts/worker.c
+37
-8
@@ -28,8 +28,26 @@ void load_instance_labels_on_demand(nd_uuid_t *uuid, void *data) {
28
ctx_get_label_list(uuid, rrdinstance_load_clabel, data);
29
}
30
31
-static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
32
- RRDINSTANCE *ri = data;
31
+static void rrdinstance_load_dimension_callback(SQL_DIMENSION_DATA *sd, void *data) {
32
+ RRDHOST *host = data;
33
+ RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, sd->context);
34
+ if(!rca) {
35
+ nd_log(NDLS_DAEMON, NDLP_ERR,
36
+ "RRDCONTEXT: context '%s' is not found in host '%s'",
37
+ sd->context, rrdhost_hostname(host));
38
+ return;
39
+ }
40
+ RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
41
+
42
+ RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_get_and_acquire_item(rc->rrdinstances, sd->chart_id);
43
+ if(!ria) {
44
+ rrdcontext_release(rca);
45
+ nd_log(NDLS_DAEMON, NDLP_ERR,
46
+ "RRDCONTEXT: instance '%s' of context '%s' is not found in host '%s'",
47
+ sd->chart_id, sd->context, rrdhost_hostname(host));
48
+ return;
49
+ }
50
+ RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
51
52
RRDMETRIC trm = {
53
.id = string_strdupz(sd->id),
@@ -41,9 +59,12 @@ static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
59
uuid_copy(trm.uuid, sd->dim_id);
60
61
dictionary_set(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
62
+
63
+ rrdinstance_release(ria);
64
+ rrdcontext_release(rca);
65
}
66
46
-static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
67
+static void rrdinstance_load_instance_callback(SQL_CHART_DATA *sc, void *data) {
68
RRDHOST *host = data;
69
70
RRDCONTEXT tc = {
@@ -74,10 +95,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
95
uuid_copy(tri.uuid, sc->chart_id);
96
97
RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, sc->id, &tri, sizeof(tri));
77
- RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
98
79
- ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri);
80
- rrdinstance_trigger_updates(ri, __FUNCTION__ );
99
rrdinstance_release(ria);
100
rrdcontext_release(rca);
101
}
@@ -106,12 +124,23 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) {
124
return;
125
126
ctx_get_context_list(&host->host_id.uuid, rrdcontext_load_context_callback, host);
109
- ctx_get_chart_list(&host->host_id.uuid, rrdinstance_load_chart_callback, host);
127
+ ctx_get_chart_list(&host->host_id.uuid, rrdinstance_load_instance_callback, host);
128
+ ctx_get_dimension_list(&host->host_id.uuid, rrdinstance_load_dimension_callback, host);
129
130
RRDCONTEXT *rc;
131
dfe_start_read(host->rrdctx.contexts, rc) {
113
- rrdcontext_trigger_updates(rc, __FUNCTION__ );
132
+ RRDINSTANCE *ri;
133
+ dfe_start_read(rc->rrdinstances, ri) {
134
+ RRDMETRIC *rm;
135
+ dfe_start_read(ri->rrdmetrics, rm) {
136
+ rrdmetric_trigger_updates(rm, __FUNCTION__ );
137
}
138
+ dfe_done(rm);
139
+ rrdinstance_trigger_updates(ri, __FUNCTION__ );
140
+ }
141
+ dfe_done(ri);
142
+ rrdcontext_trigger_updates(rc, __FUNCTION__ );
143
+ }
144
dfe_done(rc);
145
146
rrdcontext_garbage_collect_single_host(host, false);
src/database/sqlite/sqlite_context.c
+6
-4
@@ -115,9 +115,9 @@ done:
115
}
116
117
// Dimension list
118
-#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END " \
119
- "FROM dimension d WHERE d.chart_id = @id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC"
120
-void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)
118
+#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END, c.type||'.'||c.id, c.context " \
119
+ "FROM dimension d, chart c WHERE c.host_id = @host_id AND d.chart_id = c.chart_id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC"
120
+void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)
121
{
122
static __thread sqlite3_stmt *res = NULL;
123
@@ -125,7 +125,7 @@ void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION
125
return;
126
127
int param = 0;
128
- SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC));
128
+ SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC));
129
130
SQL_DIMENSION_DATA dimension_data;
131
@@ -135,6 +135,8 @@ void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION
135
dimension_data.id = (char *) sqlite3_column_text(res, 1);
136
dimension_data.name = (char *) sqlite3_column_text(res, 2);
137
dimension_data.hidden = sqlite3_column_int(res, 3);
138
+ dimension_data.chart_id = (char *) sqlite3_column_text(res, 4);
139
+ dimension_data.context = (char *) sqlite3_column_text(res, 5);
140
dict_cb(&dimension_data, data);
141
}
142
src/database/sqlite/sqlite_context.h
+4
-1
@@ -25,6 +25,9 @@ typedef struct ctx_dimension {
25
char *id;
26
char *name;
27
bool hidden;
28
+
29
+ char *context;
30
+ char *chart_id;
31
} SQL_DIMENSION_DATA;
32
33
typedef struct ctx_label {
@@ -56,7 +59,7 @@ void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEX
59
60
void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data);
61
void ctx_get_label_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data);
59
-void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data);
62
+void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data);
63
64
int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);
65