@cryptotaxi247 / netdata-1 / commits / 65bc43d9c

Add data query support for archived charts (#10771)

Stelios Fragkakis committed Mar 22, 2021 at 09:47 UTC 65bc43d9cbc17d6ac356672cdc24ffe74fc463e5
14 files changed +345 -71
database/engine/rrdengineapi.c
+30
@@ -691,6 +691,36 @@ time_t rrdeng_metric_oldest_time(RRDDIM *rd)
691 return page_index->oldest_time / USEC_PER_SEC;
692 }
693
694 +int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t)
695 +{
696 + struct page_cache *pg_cache;
697 + struct rrdengine_instance *ctx;
698 + Pvoid_t *PValue;
699 + struct pg_cache_page_index *page_index = NULL;
700 +
701 + ctx = get_rrdeng_ctx_from_host(localhost);
702 + if (unlikely(!ctx)) {
703 + error("Failed to fetch multidb context");
704 + return 1;
705 + }
706 + pg_cache = &ctx->pg_cache;
707 +
708 + uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
709 + PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, dim_uuid, sizeof(uuid_t));
710 + if (likely(NULL != PValue)) {
711 + page_index = *PValue;
712 + }
713 + uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
714 +
715 + if (likely(page_index)) {
716 + *first_entry_t = page_index->oldest_time / USEC_PER_SEC;
717 + *last_entry_t = page_index->latest_time / USEC_PER_SEC;
718 + return 0;
719 + }
720 +
721 + return 1;
722 +}
723 +
724 /* Also gets a reference for the page */
725 void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrdeng_page_descr **ret_descr)
726 {
database/engine/rrdengineapi.h
+1
@@ -59,5 +59,6 @@ extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *db
59
60 extern int rrdeng_exit(struct rrdengine_instance *ctx);
61 extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
62 +extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t);
63
64 #endif /* NETDATA_RRDENGINEAPI_H */
database/rrd.h
+11 -1
@@ -41,10 +41,17 @@ struct pg_cache_page_index;
41 #include "aclk/aclk.h"
42 #endif
43
44 +enum {
45 + CONTEXT_FLAGS_ARCHIVE = 0x01,
46 + CONTEXT_FLAGS_CHART = 0x02,
47 + CONTEXT_FLAGS_CONTEXT = 0x04
48 +};
49 +
50 struct context_param {
51 RRDDIM *rd;
52 time_t first_entry_t;
53 time_t last_entry_t;
54 + uint8_t flags;
55 };
56
57 #define META_CHART_UPDATED 1
@@ -533,7 +540,10 @@ struct rrdset {
540 size_t counter; // the number of times we added values to this database
541 size_t counter_done; // the number of times rrdset_done() has been called
542
536 - time_t last_accessed_time; // the last time this RRDSET has been accessed
543 + union {
544 + time_t last_accessed_time; // the last time this RRDSET has been accessed
545 + time_t last_entry_t; // the last_entry_t computed for transient RRDSET
546 + };
547 time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream
548
549 char *plugin_name; // the name of the plugin that generated this
database/sqlite/sqlite_functions.c
+212 -13
@@ -755,7 +755,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb)
755 rc = sqlite3_bind_blob(res_chart, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
756 if (unlikely(rc != SQLITE_OK)) {
757 error_report("Failed to bind host parameter to fetch archived charts");
758 - return;
758 + goto failed;
759 }
760
761 rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION, -1, &res_dim, 0);
@@ -905,25 +905,41 @@ failed:
905 return;
906 }
907
908 -#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname;"
908 +#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;"
909 +#define SELECT_HOST_BY_UUID "select host_id, registry_hostname, update_every, os, timezone, tags from host where host_id = @host_id ;"
910
911 RRDHOST *sql_create_host_by_uuid(char *hostname)
912 {
913 int rc;
914 RRDHOST *host = NULL;
915 + uuid_t host_uuid;
916
917 sqlite3_stmt *res = NULL;
918
917 - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0);
918 - if (unlikely(rc != SQLITE_OK)) {
919 - error_report("Failed to prepare statement to fetch host");
920 - return NULL;
919 + rc = uuid_parse(hostname, host_uuid);
920 + if (!rc) {
921 + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_BY_UUID, -1, &res, 0);
922 + if (unlikely(rc != SQLITE_OK)) {
923 + error_report("Failed to prepare statement to fetch host by uuid");
924 + return NULL;
925 + }
926 + rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);
927 + if (unlikely(rc != SQLITE_OK)) {
928 + error_report("Failed to bind host_id parameter to fetch host information");
929 + goto failed;
930 + }
931 }
922 -
923 - rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC);
924 - if (unlikely(rc != SQLITE_OK)) {
925 - error_report("Failed to bind hostname parameter to fetch host information");
926 - return NULL;
932 + else {
933 + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0);
934 + if (unlikely(rc != SQLITE_OK)) {
935 + error_report("Failed to prepare statement to fetch host by hostname");
936 + return NULL;
937 + }
938 + rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC);
939 + if (unlikely(rc != SQLITE_OK)) {
940 + error_report("Failed to bind hostname parameter to fetch host information");
941 + goto failed;
942 + }
943 }
944
945 rc = sqlite3_step(res);
@@ -938,13 +954,17 @@ RRDHOST *sql_create_host_by_uuid(char *hostname)
954 host = callocz(1, sizeof(RRDHOST));
955
956 set_host_properties(host, sqlite3_column_int(res, 2), RRD_MEMORY_MODE_DBENGINE, hostname,
941 - (char *) sqlite3_column_text(res, 1), (const char *) uuid_str,
957 + (char *) sqlite3_column_text(res, 1), (const char *) uuid_str,
958 (char *) sqlite3_column_text(res, 3), (char *) sqlite3_column_text(res, 5),
959 (char *) sqlite3_column_text(res, 4), NULL, NULL);
960
961 uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
962
947 - host->system_info = NULL;
963 + host->system_info = callocz(1, sizeof(*host->system_info));;
964 + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
965 +#ifdef ENABLE_DBENGINE
966 + host->rrdeng_ctx = &multidb_ctx;
967 +#endif
968
969 failed:
970 rc = sqlite3_finalize(res);
@@ -1098,3 +1118,182 @@ failed:
1118
1119 return;
1120 }
1121 +
1122 +int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id,
1123 + uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid)
1124 +{
1125 +#ifdef ENABLE_DBENGINE
1126 + int rc;
1127 + uuid_t legacy_uuid;
1128 + uuid_t multihost_legacy_uuid;
1129 + time_t dim_first_entry_t, dim_last_entry_t;
1130 +
1131 + rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t);
1132 + if (unlikely(rc)) {
1133 + rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid);
1134 + rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t);
1135 + if (likely(rc)) {
1136 + rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid);
1137 + rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t);
1138 + if (likely(!rc))
1139 + uuid_copy(*rrdeng_uuid, multihost_legacy_uuid);
1140 + }
1141 + else
1142 + uuid_copy(*rrdeng_uuid, legacy_uuid);
1143 + }
1144 + else
1145 + uuid_copy(*rrdeng_uuid, *uuid);
1146 +
1147 + if (likely(!rc)) {
1148 + *first_entry_t = MIN(*first_entry_t, dim_first_entry_t);
1149 + *last_entry_t = MAX(*last_entry_t, dim_last_entry_t);
1150 + }
1151 + return rc;
1152 +#else
1153 + UNUSED(machine_guid);
1154 + UNUSED(chart_id);
1155 + UNUSED(dim_id);
1156 + UNUSED(uuid);
1157 + UNUSED(first_entry_t);
1158 + UNUSED(last_entry_t);
1159 + UNUSED(rrdeng_uuid);
1160 + return 1;
1161 +#endif
1162 +}
1163 +
1164 +#ifdef ENABLE_DBENGINE
1165 +static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
1166 +{
1167 + RRDDIM *rd = callocz(1, sizeof(*rd));
1168 + rd->rrdset = st;
1169 + rd->last_stored_value = NAN;
1170 + rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
1171 + rd->state = mallocz(sizeof(*rd->state));
1172 + rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
1173 + rd->state->query_ops.init = rrdeng_load_metric_init;
1174 + rd->state->query_ops.next_metric = rrdeng_load_metric_next;
1175 + rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
1176 + rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
1177 + rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
1178 + rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
1179 + rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t));
1180 + uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid);
1181 + rd->state->metric_uuid = rd->state->rrdeng_uuid;
1182 + rd->id = strdupz(id);
1183 + rd->name = strdupz(name);
1184 + return rd;
1185 +}
1186 +#endif
1187 +
1188 +#define SELECT_CHART_CONTEXT "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id from chart c, " \
1189 + "dimension d, host h " \
1190 + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.context = @context " \
1191 + "order by c.chart_id asc, c.type||c.id desc;"
1192 +
1193 +#define SELECT_CHART_SINGLE "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, c.context from chart c, " \
1194 + "dimension d, host h " \
1195 + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \
1196 + "order by c.chart_id asc, c.type||'.'||c.id desc;"
1197 +
1198 +void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart)
1199 +{
1200 +#ifdef ENABLE_DBENGINE
1201 + int rc;
1202 +
1203 + if (unlikely(!param_list) || host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1204 + return;
1205 +
1206 + if (unlikely(!(*param_list))) {
1207 + *param_list = mallocz(sizeof(struct context_param));
1208 + (*param_list)->first_entry_t = LONG_MAX;
1209 + (*param_list)->last_entry_t = 0;
1210 + (*param_list)->rd = NULL;
1211 + (*param_list)->flags = CONTEXT_FLAGS_ARCHIVE;
1212 + if (chart)
1213 + (*param_list)->flags |= CONTEXT_FLAGS_CHART;
1214 + else
1215 + (*param_list)->flags |= CONTEXT_FLAGS_CONTEXT;
1216 + }
1217 +
1218 + sqlite3_stmt *res = NULL;
1219 +
1220 + if (context)
1221 + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_CONTEXT, -1, &res, 0);
1222 + else
1223 + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_SINGLE, -1, &res, 0);
1224 + if (unlikely(rc != SQLITE_OK)) {
1225 + error_report("Failed to prepare statement to fetch host archived charts");
1226 + return;
1227 + }
1228 +
1229 + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
1230 + if (unlikely(rc != SQLITE_OK)) {
1231 + error_report("Failed to bind host parameter to fetch archived charts");
1232 + goto failed;
1233 + }
1234 +
1235 + if (context)
1236 + rc = sqlite3_bind_text(res, 2, context, -1, SQLITE_STATIC);
1237 + else
1238 + rc = sqlite3_bind_text(res, 2, chart, -1, SQLITE_STATIC);
1239 + if (unlikely(rc != SQLITE_OK)) {
1240 + error_report("Failed to bind host parameter to fetch archived charts");
1241 + goto failed;
1242 + }
1243 +
1244 + RRDSET *st = NULL;
1245 + char machine_guid[GUID_LEN + 1];
1246 + uuid_unparse_lower(host->host_uuid, machine_guid);
1247 + uuid_t rrdeng_uuid;
1248 + uuid_t chart_id;
1249 +
1250 + while (sqlite3_step(res) == SQLITE_ROW) {
1251 + char id[512];
1252 + sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1));
1253 +
1254 + if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) {
1255 + st = callocz(1, sizeof(*st));
1256 + char n[RRD_ID_LENGTH_MAX + 1];
1257 +
1258 + snprintfz(
1259 + n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4),
1260 + (char *)sqlite3_column_text(res, 3));
1261 + st->name = strdupz(n);
1262 + st->update_every = sqlite3_column_int(res, 6);
1263 + st->counter = 0;
1264 + if (chart) {
1265 + st->context = strdupz((char *)sqlite3_column_text(res, 8));
1266 + strncpyz(st->id, chart, RRD_ID_LENGTH_MAX);
1267 + }
1268 + uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7));
1269 + st->last_entry_t = 0;
1270 + st->rrdhost = host;
1271 + }
1272 +
1273 + if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1),
1274 + (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t,
1275 + &rrdeng_uuid)))
1276 + continue;
1277 +
1278 + st->counter++;
1279 + st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t);
1280 +
1281 + RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
1282 + rd->next = (*param_list)->rd;
1283 + (*param_list)->rd = rd;
1284 + }
1285 + if (likely(st && context && !st->context))
1286 + st->context = strdupz(context);
1287 +
1288 +failed:
1289 + rc = sqlite3_finalize(res);
1290 + if (unlikely(rc != SQLITE_OK))
1291 + error_report("Failed to finalize the prepared statement when reading archived charts");
1292 +#else
1293 + UNUSED(param_list);
1294 + UNUSED(host);
1295 + UNUSED(context);
1296 + UNUSED(chart);
1297 +#endif
1298 + return;
1299 +}
database/sqlite/sqlite_functions.h
+1
@@ -59,4 +59,5 @@ extern void db_unlock(void);
59 extern void db_lock(void);
60 extern void delete_dimension_uuid(uuid_t *dimension_uuid);
61 extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value);
62 +extern void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart);
63 #endif //NETDATA_SQLITE_FUNCTIONS_H
web/api/formatters/json/json.c
+8 -2
@@ -5,8 +5,14 @@
5 #define JSON_DATES_JS 1
6 #define JSON_DATES_TIMESTAMP 2
7
8 -void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd) {
9 - rrdset_check_rdlock(r->st);
8 +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list)
9 +{
10 + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
11 +
12 + int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
13 +
14 + if (should_lock)
15 + rrdset_check_rdlock(r->st);
16
17 //info("RRD2JSON(): %s: BEGIN", r->st->id);
18 int row_annotations = 0, dates, dates_with_new = 0;
web/api/formatters/json/json.h
+1 -1
@@ -5,6 +5,6 @@
5
6 #include "../rrd2json.h"
7
8 -extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd);
8 +extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list);
9
10 #endif //NETDATA_API_FORMATTER_JSON_H
web/api/formatters/json_wrapper.c
+19 -9
@@ -2,8 +2,16 @@
2
3 #include "json_wrapper.h"
4
5 -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_label_key) {
6 - rrdset_check_rdlock(r->st);
5 +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
6 + struct context_param *context_param_list, char *chart_label_key)
7 +{
8 +
9 + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
10 + int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
11 + uint8_t context_mode = (!context_param_list || (context_param_list->flags & CONTEXT_FLAGS_CONTEXT));
12 +
13 + if (should_lock)
14 + rrdset_check_rdlock(r->st);
15
16 long rows = rrdr_rows(r);
17 long c, i;
@@ -22,7 +30,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
30 sq[0] = '"';
31 }
32
25 - rrdset_rdlock(r->st);
33 + if (should_lock)
34 + rrdset_rdlock(r->st);
35 buffer_sprintf(wb, "{\n"
36 " %sapi%s: 1,\n"
37 " %sid%s: %s%s%s,\n"
@@ -35,16 +44,17 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
44 " %safter%s: %u,\n"
45 " %sdimension_names%s: ["
46 , kq, kq
38 - , kq, kq, sq, temp_rd?r->st->context:r->st->id, sq
39 - , kq, kq, sq, temp_rd?r->st->context:r->st->name, sq
47 + , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq
48 + , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq
49 , kq, kq, r->update_every
50 , kq, kq, r->st->update_every
42 - , kq, kq, (uint32_t)rrdset_first_entry_t_nolock(r->st)
43 - , kq, kq, (uint32_t)rrdset_last_entry_t_nolock(r->st)
51 + , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t(r->st))
52 + , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t(r->st))
53 , kq, kq, (uint32_t)r->before
54 , kq, kq, (uint32_t)r->after
55 , kq, kq);
47 - rrdset_unlock(r->st);
56 + if (should_lock)
57 + rrdset_unlock(r->st);
58
59 for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
60 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
@@ -89,7 +99,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
99 buffer_strcat(wb, "],\n");
100
101 // Composite charts
92 - if (temp_rd) {
102 + if (context_mode && temp_rd) {
103 buffer_sprintf(
104 wb,
105 " %schart_ids%s: [",
web/api/formatters/json_wrapper.h
+1 -1
@@ -5,7 +5,7 @@
5
6 #include "rrd2json.h"
7
8 -extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_key);
8 +extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key);
9 extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
10
11 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
web/api/formatters/rrd2json.c
+19 -18
@@ -65,6 +65,7 @@ void build_context_param_list(struct context_param **param_list, RRDSET *st)
65 *param_list = mallocz(sizeof(struct context_param));
66 (*param_list)->first_entry_t = LONG_MAX;
67 (*param_list)->last_entry_t = 0;
68 + (*param_list)->flags = 0;
69 (*param_list)->rd = NULL;
70 }
71
@@ -214,9 +215,9 @@ int rrdset2anything_api_v1(
215 , struct context_param *context_param_list
216 , char *chart_label_key
217 ) {
217 - time_t last_accessed_time = now_realtime_sec();
218 - st->last_accessed_time = last_accessed_time;
218
219 + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
220 + st->last_accessed_time = now_realtime_sec();
221
222 RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list);
223 if(!r) {
@@ -238,7 +239,7 @@ int rrdset2anything_api_v1(
239 case DATASOURCE_SSV:
240 if(options & RRDR_OPTION_JSON_WRAP) {
241 wb->contenttype = CT_APPLICATION_JSON;
241 - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
242 + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
243 rrdr2ssv(r, wb, options, "", " ", "");
244 rrdr_json_wrapper_end(r, wb, format, options, 1);
245 }
@@ -251,7 +252,7 @@ int rrdset2anything_api_v1(
252 case DATASOURCE_SSV_COMMA:
253 if(options & RRDR_OPTION_JSON_WRAP) {
254 wb->contenttype = CT_APPLICATION_JSON;
254 - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
255 + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
256 rrdr2ssv(r, wb, options, "", ",", "");
257 rrdr_json_wrapper_end(r, wb, format, options, 1);
258 }
@@ -264,7 +265,7 @@ int rrdset2anything_api_v1(
265 case DATASOURCE_JS_ARRAY:
266 if(options & RRDR_OPTION_JSON_WRAP) {
267 wb->contenttype = CT_APPLICATION_JSON;
267 - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
268 + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
269 rrdr2ssv(r, wb, options, "[", ",", "]");
270 rrdr_json_wrapper_end(r, wb, format, options, 0);
271 }
@@ -277,7 +278,7 @@ int rrdset2anything_api_v1(
278 case DATASOURCE_CSV:
279 if(options & RRDR_OPTION_JSON_WRAP) {
280 wb->contenttype = CT_APPLICATION_JSON;
280 - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
281 + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
282 rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
283 rrdr_json_wrapper_end(r, wb, format, options, 1);
284 }
@@ -290,7 +291,7 @@ int rrdset2anything_api_v1(
291 case DATASOURCE_CSV_MARKDOWN:
292 if(options & RRDR_OPTION_JSON_WRAP) {
293 wb->contenttype = CT_APPLICATION_JSON;
293 - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
294 + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
295 rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
296 rrdr_json_wrapper_end(r, wb, format, options, 1);
297 }
@@ -303,7 +304,7 @@ int rrdset2anything_api_v1(
304 case DATASOURCE_CSV_JSON_ARRAY:
305 wb->contenttype = CT_APPLICATION_JSON;
306 if(options & RRDR_OPTION_JSON_WRAP) {
306 - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
307 + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
308 buffer_strcat(wb, "[\n");
309 rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
310 buffer_strcat(wb, "\n]");
@@ -320,7 +321,7 @@ int rrdset2anything_api_v1(
321 case DATASOURCE_TSV:
322 if(options & RRDR_OPTION_JSON_WRAP) {
323 wb->contenttype = CT_APPLICATION_JSON;
323 - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
324 + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
325 rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
326 rrdr_json_wrapper_end(r, wb, format, options, 1);
327 }
@@ -333,7 +334,7 @@ int rrdset2anything_api_v1(
334 case DATASOURCE_HTML:
335 if(options & RRDR_OPTION_JSON_WRAP) {
336 wb->contenttype = CT_APPLICATION_JSON;
336 - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
337 + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
338 buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
339 rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
340 buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
@@ -351,9 +352,9 @@ int rrdset2anything_api_v1(
352 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
353
354 if(options & RRDR_OPTION_JSON_WRAP)
354 - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
355 + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
356
356 - rrdr2json(r, wb, options, 1, temp_rd);
357 + rrdr2json(r, wb, options, 1, context_param_list);
358
359 if(options & RRDR_OPTION_JSON_WRAP)
360 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -363,9 +364,9 @@ int rrdset2anything_api_v1(
364 wb->contenttype = CT_APPLICATION_JSON;
365
366 if(options & RRDR_OPTION_JSON_WRAP)
366 - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
367 + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
368
368 - rrdr2json(r, wb, options, 1, temp_rd);
369 + rrdr2json(r, wb, options, 1, context_param_list);
370
371 if(options & RRDR_OPTION_JSON_WRAP)
372 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -374,9 +375,9 @@ int rrdset2anything_api_v1(
375 case DATASOURCE_JSONP:
376 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
377 if(options & RRDR_OPTION_JSON_WRAP)
377 - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
378 + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
379
379 - rrdr2json(r, wb, options, 0, temp_rd);
380 + rrdr2json(r, wb, options, 0, context_param_list);
381
382 if(options & RRDR_OPTION_JSON_WRAP)
383 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -387,9 +388,9 @@ int rrdset2anything_api_v1(
388 wb->contenttype = CT_APPLICATION_JSON;
389
390 if(options & RRDR_OPTION_JSON_WRAP)
390 - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
391 + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
392
392 - rrdr2json(r, wb, options, 0, temp_rd);
393 + rrdr2json(r, wb, options, 0, context_param_list);
394
395 if(options & RRDR_OPTION_JSON_WRAP)
396 rrdr_json_wrapper_end(r, wb, format, options, 0);
web/api/queries/query.c
+19 -8
@@ -281,8 +281,14 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI
281
282 // ----------------------------------------------------------------------------
283
284 -static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims, RRDDIM *temp_rd) {
285 - rrdset_check_rdlock(r->st);
284 +static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,
285 + struct context_param *context_param_list)
286 +{
287 + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
288 + int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
289 +
290 + if (should_lock)
291 + rrdset_check_rdlock(r->st);
292
293 if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
294
@@ -1060,10 +1066,11 @@ static RRDR *rrd2rrdr_fixedstep(
1066 // -------------------------------------------------------------------------
1067 // disable the not-wanted dimensions
1068
1063 - rrdset_check_rdlock(st);
1069 + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
1070 + rrdset_check_rdlock(st);
1071
1072 if(dimensions)
1066 - rrdr_disable_not_selected_dimensions(r, options, dimensions, temp_rd);
1073 + rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
1074
1075
1076 // -------------------------------------------------------------------------
@@ -1435,11 +1442,11 @@ static RRDR *rrd2rrdr_variablestep(
1442
1443 // -------------------------------------------------------------------------
1444 // disable the not-wanted dimensions
1438 -
1439 - rrdset_check_rdlock(st);
1445 + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
1446 + rrdset_check_rdlock(st);
1447
1448 if(dimensions)
1442 - rrdr_disable_not_selected_dimensions(r, options, dimensions, temp_rd);
1449 + rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
1450
1451
1452 // -------------------------------------------------------------------------
@@ -1591,8 +1598,12 @@ RRDR *rrd2rrdr(
1598 if (first_entry_t > after_requested)
1599 first_entry_t = after_requested;
1600
1594 - if (context_param_list)
1601 + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
1602 rebuild_context_param_list(context_param_list, after_requested);
1603 + st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
1604 + if (unlikely(!st))
1605 + return NULL;
1606 + }
1607
1608 #ifdef ENABLE_DBENGINE
1609 if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
web/api/queries/rrdr.c
+2 -1
@@ -108,7 +108,8 @@ RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param
108 RRDR *r = callocz(1, sizeof(RRDR));
109 r->st = st;
110
111 - rrdr_lock_rrdset(r);
111 + if (!context_param_list || (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)))
112 + rrdr_lock_rrdset(r);
113
114 RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
115 RRDDIM *rd;
web/api/web_api_v1.c
+13
@@ -511,6 +511,8 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
511 rrdhost_unlock(host);
512 if (likely(context_param_list && context_param_list->rd)) // Just set the first one
513 st = context_param_list->rd->rrdset;
514 + else
515 + sql_build_context_param_list(&context_param_list, host, context, NULL);
516 }
517 else {
518 st = rrdset_find(host, chart);
@@ -518,6 +520,17 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
520 st = rrdset_find_byname(host, chart);
521 if (likely(st))
522 st->last_accessed_time = now_realtime_sec();
523 + else
524 + sql_build_context_param_list(&context_param_list, host, NULL, chart);
525 + }
526 +
527 + if (!st) {
528 + if (likely(context_param_list && context_param_list->rd && context_param_list->rd->rrdset))
529 + st = context_param_list->rd->rrdset;
530 + else {
531 + free_context_param_list(&context_param_list);
532 + context_param_list = NULL;
533 + }
534 }
535
536 if (!st && !context_param_list) {
web/server/web_client.c
+8 -17
@@ -1374,34 +1374,25 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1374 uint32_t hash = simple_hash(tok);
1375
1376 host = rrdhost_find_by_hostname(tok, hash);
1377 - if(!host) host = rrdhost_find_by_guid(tok, hash);
1378 -
1379 -#ifdef ENABLE_DBENGINE
1380 - int release_host = 0;
1377 + if (!host)
1378 + host = rrdhost_find_by_guid(tok, hash);
1379 if (!host) {
1380 host = sql_create_host_by_uuid(tok);
1381 if (likely(host)) {
1384 - rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
1385 - release_host = 1;
1386 - }
1387 - }
1388 - if(host) {
1389 - int rc = web_client_process_url(host, w, url);
1390 - if (release_host) {
1382 + int rc = web_client_process_url(host, w, url);
1383 freez(host->hostname);
1392 - freez((char *) host->os);
1393 - freez((char *) host->tags);
1394 - freez((char *) host->timezone);
1384 + freez((char *)host->os);
1385 + freez((char *)host->tags);
1386 + freez((char *)host->timezone);
1387 freez(host->program_name);
1388 freez(host->program_version);
1389 freez(host->registry_hostname);
1390 + freez(host->system_info);
1391 freez(host);
1392 + return rc;
1393 }
1400 - return rc;
1394 }
1402 -#else
1395 if (host) return web_client_process_url(host, w, url);
1404 -#endif
1396 }
1397
1398 buffer_flush(w->response.data);