dbengine free from RRDSET and RRDDIM (#13772)
* dbengine free from RRDSET and RRDDIM * fix for excess parameters to query ops * add comment about ML * update_every from int to uint32_t * rrddim_mem storage engine working * fixes for update_every_s * working dbengine * a lot of changes in dbengine regarding timestamps * better logging of not sequential points * rrdset_done() now gives aligned timestamps for higher tiers * dont change the end_time of descriptors, because they cant be loaded back * fixes for cmake * fixes for db mode ram * Global counters for dbengine loading errors. Ensure dbengine store metrics always has aligned metrics or breaks the page when storing new data. * update lgtm config * fixes for 32-bit systems * update unittests * Don't try to find and create a host on the fly if not already in memory * Remove unused functions * print backtrace in case of fatal * always set ctx to page_index * detect ctx and metric uuid discrepancies * use legacy uuid if multihost is not available * fix for last commit * prevent repeating log * Do not try to access archived charts when executing a data query * Remove unused function * log inconsistent collections once every 10 mins Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Costa Tsaousis committed
Oct 13, 2022 at 08:05 UTC
afe1b704857a7307547341a1027c019bbe68e910
37 files changed
+1013
-827
.lgtm.yml
+2
@@ -13,10 +13,12 @@ path_classifiers:
13
- collectors/python.d.plugin/python_modules/pyyaml2/
14
- collectors/python.d.plugin/python_modules/pyyaml3/
15
- ml/kmeans/dlib/
16
+ - ml/dlib/dlib/
17
- ml/json/
18
- web/gui/lib/
19
- web/gui/src/
20
- web/gui/css/
21
+ - web/gui/dashboard/lib/
22
- libnetdata/libjudy/
23
test:
24
- tests/
CMakeLists.txt
+43
-15
@@ -9,31 +9,25 @@ project(netdata C CXX)
9
find_package(Threads REQUIRED)
10
find_package(PkgConfig REQUIRED)
11
12
+include(CheckFunctionExists)
13
+include(CheckLibraryExists)
14
+
15
# default is "Debug"
16
#set(CMAKE_BUILD_TYPE "Release")
17
18
# set this to see the compilation commands
16
-# set(CMAKE_VERBOSE_MAKEFILE 1)
17
-
19
+#set(CMAKE_VERBOSE_MAKEFILE 1)
20
21
# -----------------------------------------------------------------------------
22
# Set compilation options according to build type
23
24
+set(CMAKE_C_STANDARD 11)
25
+
26
IF("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
27
+ set(CXX_DEFAULT_CFLAGS "-O0 -g -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_DEV_MODE=1 -fstack-protector-all -fno-omit-frame-pointer")
28
message(STATUS "building for: debugging")
24
-
25
- ## unfortunately these produce errors
26
- #include(CheckCXXCompilerFlag)
27
- #CHECK_CXX_COMPILER_FLAG("-Wformat-signedness" CXX_FORMAT_SIGNEDNESS)
28
- #CHECK_CXX_COMPILER_FLAG("-Werror=format-security" CXX_FORMAT_SECURITY)
29
- #CHECK_CXX_COMPILER_FLAG("-fstack-protector-all" CXX_STACK_PROTECTOR)
30
- set(CXX_FORMAT_SIGNEDNESS "-Wformat-signedness")
31
- set(CXX_FORMAT_SECURITY "-Werror=format-security")
32
- set(CXX_STACK_PROTECTOR "-fstack-protector-all")
33
- set(CXX_FLAGS_DEBUG "-O0")
34
- set(CMAKE_C_STANDARD 99)
35
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1 ${CXX_FORMAT_SIGNEDNESS} ${CXX_FORMAT_SECURITY} ${CXX_STACK_PROTECTOR} ${CXX_FLAGS_DEBUG}")
29
ELSE()
30
+ set(CXX_DEFAULT_CFLAGS "-O2")
31
message(STATUS "building for: release")
32
cmake_policy(SET CMP0069 "NEW")
33
include(CheckIPOSupported)
@@ -46,6 +40,7 @@ ELSE()
40
ENDIF()
41
ENDIF()
42
43
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -Wformat-signedness -Werror=format-security -Wunused-result ${CXX_DEFAULT_CFLAGS}")
44
45
# -----------------------------------------------------------------------------
46
# O/S Detection
@@ -70,6 +65,12 @@ message(STATUS "system name: ${CMAKE_SYSTEM_NAME}")
65
set(GENERATED_CONFIG_H_DIR ${CMAKE_BINARY_DIR})
66
set(GENERATED_CONFIG_H ${GENERATED_CONFIG_H_DIR}/config.h)
67
68
+# -----------------------------------------------------------------------------
69
+# Math
70
+
71
+set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} m)
72
+set(NETDATA_REQUIRED_DEFINES "-DSTORAGE_WITH_MATH=1 ${NETDATA_REQUIRED_DEFINES}")
73
+
74
# -----------------------------------------------------------------------------
75
# Detect libuuid
76
@@ -85,6 +86,7 @@ pkg_check_modules(ZLIB REQUIRED zlib)
86
set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${ZLIB_CFLAGS_OTHER})
87
set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${ZLIB_LIBRARIES})
88
set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
89
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DNETDATA_WITH_ZLIB=1")
90
91
# -----------------------------------------------------------------------------
92
# libuv multi-platform support library with a focus on asynchronous I/O
@@ -101,6 +103,7 @@ pkg_check_modules(LIBLZ4 REQUIRED liblz4)
103
set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBLZ4_CFLAGS_OTHER})
104
set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBLZ4_LIBRARIES})
105
set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS})
106
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_COMPRESSION=1")
107
108
# -----------------------------------------------------------------------------
109
# Judy General purpose dynamic array
@@ -114,6 +117,7 @@ pkg_check_modules(OPENSSL REQUIRED openssl)
117
set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${OPENSSL_CFLAGS_OTHER})
118
set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${OPENSSL_LIBRARIES})
119
set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIRS})
120
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_HTTPS=1")
121
122
# -----------------------------------------------------------------------------
123
# JSON-C used to health
@@ -1023,7 +1027,11 @@ set(ML_FILES
1027
ml/ml-dummy.c
1028
)
1029
1030
+# -----------------------------------------------------------------------------
1031
+# ML
1032
+
1033
IF(ENABLE_ML)
1034
+ message(STATUS "ML: enabled")
1035
list(APPEND ML_FILES
1036
ml/Config.h
1037
ml/Config.cc
@@ -1041,6 +1049,8 @@ IF(ENABLE_ML)
1049
ml/ml.cc
1050
ml/ml-private.h
1051
)
1052
+ELSE()
1053
+ message(STATUS "ML: disabled")
1054
ENDIF()
1055
1056
set(NETDATA_FILES
@@ -1731,7 +1741,6 @@ if (HAVE_SYS_SYSMACROS_H)
1741
endif()
1742
1743
if (CRYPTO_FOUND)
1734
- include(CheckLibraryExists)
1744
set(HAVE_CRYPTO True)
1745
FIND_LIBRARY(CRYPTO_LIBRARY_LOCATION NAMES crypto)
1746
check_library_exists(crypto X509_VERIFY_PARAM_set1_host ${CRYPTO_LIBRARY_LOCATION} HAVE_X509_VERIFY_PARAM_set1_host)
@@ -1763,4 +1772,23 @@ ELSE()
1772
SET(MAYBE_UNUSED_MACRO "")
1773
ENDIF()
1774
1775
+# -----------------------------------------------------------------------------
1776
+
1777
+check_library_exists(c clock_gettime "" HAVE_CLOCK_GETTIME)
1778
+
1779
+IF(NOT HAVE_CLOCK_GETTIME)
1780
+ CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_CLOCK_GETTIME)
1781
+ENDIF()
1782
+
1783
+IF(HAVE_CLOCK_GETTIME)
1784
+ message("-- clock_gettime(): found")
1785
+ set(NETDATA_REQUIRED_DEFINES "-DHAVE_CLOCK_GETTIME=1 ${NETDATA_REQUIRED_DEFINES}")
1786
+ELSE()
1787
+ message("-- clock_gettime(): not found")
1788
+ENDIF()
1789
+
1790
+# -----------------------------------------------------------------------------
1791
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${NETDATA_REQUIRED_DEFINES}")
1792
+message("CFLAGS=\"${CMAKE_C_FLAGS}\"")
1793
+
1794
configure_file(config.cmake.h.in config.h)
aclk/aclk_query.c
+6
-11
@@ -99,7 +99,6 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
99
}
100
}
101
102
- RRDHOST *temp_host = NULL;
102
if (!strncmp(query->data.http_api_v2.query, NODE_ID_QUERY, strlen(NODE_ID_QUERY))) {
103
char *node_uuid = query->data.http_api_v2.query + strlen(NODE_ID_QUERY);
104
char nodeid[UUID_STR_LEN];
@@ -114,14 +113,11 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
113
114
query_host = node_id_2_rrdhost(nodeid);
115
if (!query_host) {
117
- temp_host = sql_create_host_by_uuid(nodeid);
118
- if (!temp_host) {
119
- error_report("Host with node_id \"%s\" not found! Returning 404 to Cloud!", nodeid);
120
- retval = 1;
121
- w->response.code = 404;
122
- aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_NODE_NOT_FOUND, CLOUD_EMSG_NODE_NOT_FOUND, NULL, 0);
123
- goto cleanup;
124
- }
116
+ error_report("Host with node_id \"%s\" not found! Returning 404 to Cloud!", nodeid);
117
+ retval = 1;
118
+ w->response.code = 404;
119
+ aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_NODE_NOT_FOUND, CLOUD_EMSG_NODE_NOT_FOUND, NULL, 0);
120
+ goto cleanup;
121
}
122
}
123
@@ -142,8 +138,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
138
}
139
140
// execute the query
145
- t = aclk_web_api_v1_request(query_host ? query_host : temp_host, w, mysep ? mysep + 1 : "noop");
146
- free_temporary_host(temp_host);
141
+ t = aclk_web_api_v1_request(query_host, w, mysep ? mysep + 1 : "noop");
142
size = (w->mode == WEB_CLIENT_MODE_FILECOPY) ? w->response.rlen : w->response.data->len;
143
sent = size;
144
collectors/statsd.plugin/statsd.c
+4
-1
@@ -1421,7 +1421,10 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1421
app->default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
1422
}
1423
else if (!strcmp(name, "memory mode")) {
1424
- app->rrd_memory_mode = rrd_memory_mode_id(value);
1424
+ // this is not supported anymore
1425
+ // with the implementation of storage engines, all charts have the same storage engine always
1426
+ // app->rrd_memory_mode = rrd_memory_mode_id(value);
1427
+ ;
1428
}
1429
else if (!strcmp(name, "history")) {
1430
app->rrd_history_entries = atol(value);
configure.ac
+4
@@ -317,6 +317,10 @@ AM_CONDITIONAL([MACOS], [test "${build_target}" = "macos"])
317
AM_CONDITIONAL([LINUX], [test "${build_target}" = "linux"])
318
AC_MSG_RESULT([Host OS: ${build_target}])
319
320
+# -----------------------------------------------------------------------------
321
+# backtrace
322
+
323
+AC_SEARCH_LIBS([backtrace], [execinfo], [AC_DEFINE([HAVE_BACKTRACE], [1], [backtrace availability])])
324
325
# -----------------------------------------------------------------------------
326
# pthreads
daemon/unit_test.c
+4
-2
@@ -1816,6 +1816,8 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
1816
// feed it with the test data
1817
for (i = 0 ; i < CHARTS ; ++i) {
1818
for (j = 0 ; j < DIMS ; ++j) {
1819
+ rd[i][j]->tiers[0]->collect_ops.change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every);
1820
+
1821
rd[i][j]->last_collected_time.tv_sec =
1822
st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
1823
rd[i][j]->last_collected_time.tv_usec =
@@ -1859,7 +1861,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1861
time_now = time_start + (c + 1) * update_every;
1862
for (i = 0 ; i < CHARTS ; ++i) {
1863
for (j = 0; j < DIMS; ++j) {
1862
- rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, TIER_QUERY_FETCH_SUM);
1864
+ rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every);
1865
for (k = 0; k < QUERY_BATCH; ++k) {
1866
last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
1867
j * REGION_POINTS[current_region] + c + k;
@@ -2316,7 +2318,7 @@ static void query_dbengine_chart(void *arg)
2318
time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
2319
}
2320
2319
- rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before, TIER_QUERY_FETCH_SUM);
2321
+ rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before);
2322
++thread_info->queries_nr;
2323
for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
2324
generatedv = generate_dbengine_chart_value(i, j, time_now);
database/engine/journalfile.c
+51
-14
@@ -311,20 +311,46 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
311
}
312
continue;
313
}
314
- uint64_t start_time = jf_metric_data->descr[i].start_time;
315
- uint64_t end_time = jf_metric_data->descr[i].end_time;
314
+ uint64_t start_time_ut = jf_metric_data->descr[i].start_time_ut;
315
+ uint64_t end_time_ut = jf_metric_data->descr[i].end_time_ut;
316
+ size_t entries = jf_metric_data->descr[i].page_length / page_type_size[page_type];
317
+ time_t update_every_s = (entries > 1) ? ((end_time_ut - start_time_ut) / USEC_PER_SEC / (entries - 1)) : 0;
318
+
319
+ if (unlikely(start_time_ut > end_time_ut)) {
320
+ ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter++;
321
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut < end_time_ut)
322
+ ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut = end_time_ut;
323
+ continue;
324
+ }
325
317
- if (unlikely(start_time > end_time)) {
318
- error("Invalid page encountered, start time %"PRIu64" > end time %"PRIu64"", start_time , end_time);
326
+ if (unlikely(start_time_ut == end_time_ut && entries != 1)) {
327
+ ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter++;
328
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut < end_time_ut)
329
+ ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut = end_time_ut;
330
continue;
331
}
332
322
- if (unlikely(start_time == end_time)) {
323
- size_t entries = jf_metric_data->descr[i].page_length / page_type_size[page_type];
324
- if (unlikely(entries > 1)) {
325
- error("Invalid page encountered, start time %"PRIu64" = end time but %zu entries were found", start_time, entries);
326
- continue;
327
- }
333
+ if (unlikely(!entries)) {
334
+ ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter++;
335
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut < end_time_ut)
336
+ ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut = end_time_ut;
337
+ continue;
338
+ }
339
+
340
+ if(entries > 1 && update_every_s == 0) {
341
+ ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter++;
342
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut < end_time_ut)
343
+ ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut = end_time_ut;
344
+ continue;
345
+ }
346
+
347
+ if(start_time_ut + update_every_s * USEC_PER_SEC * (entries - 1) != end_time_ut) {
348
+ ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter++;
349
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut < end_time_ut)
350
+ ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut = end_time_ut;
351
+
352
+ // let this be
353
+ // end_time_ut = start_time_ut + update_every_s * USEC_PER_SEC * (entries - 1);
354
}
355
356
temp_id = (uuid_t *)jf_metric_data->descr[i].uuid;
@@ -340,7 +366,7 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
366
uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
367
PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, temp_id, sizeof(uuid_t), PJE0);
368
fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
343
- *PValue = page_index = create_page_index(temp_id);
369
+ *PValue = page_index = create_page_index(temp_id, ctx);
370
page_index->prev = pg_cache->metrics_index.last_page_index;
371
pg_cache->metrics_index.last_page_index = page_index;
372
uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
@@ -348,21 +374,32 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
374
375
descr = pg_cache_create_descr();
376
descr->page_length = jf_metric_data->descr[i].page_length;
351
- descr->start_time = start_time;
352
- descr->end_time = end_time;
377
+ descr->start_time_ut = start_time_ut;
378
+ descr->end_time_ut = end_time_ut;
379
+ descr->update_every_s = (update_every_s > 0) ? (uint32_t)update_every_s : (page_index->latest_update_every_s);
380
descr->id = &page_index->id;
381
descr->extent = extent;
382
descr->type = page_type;
383
extent->pages[valid_pages++] = descr;
384
pg_cache_insert(ctx, page_index, descr);
385
+
386
+ if(page_index->latest_time_ut == descr->end_time_ut)
387
+ page_index->latest_update_every_s = descr->update_every_s;
388
+
389
+ if(descr->update_every_s == 0)
390
+ fatal(
391
+ "DBENGINE: page descriptor update every is zero, end_time_ut = %llu, start_time_ut = %llu, entries = %zu",
392
+ (unsigned long long)end_time_ut, (unsigned long long)start_time_ut, entries);
393
}
394
395
extent->number_of_pages = valid_pages;
396
397
if (likely(valid_pages))
398
df_extent_insert(extent);
364
- else
399
+ else {
400
freez(extent);
401
+ ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter++;
402
+ }
403
}
404
405
/*
database/engine/pagecache.c
+59
-54
@@ -127,12 +127,13 @@ struct rrdeng_page_descr *pg_cache_create_descr(void)
127
128
descr = rrdeng_page_descr_mallocz();
129
descr->page_length = 0;
130
- descr->start_time = INVALID_TIME;
131
- descr->end_time = INVALID_TIME;
130
+ descr->start_time_ut = INVALID_TIME;
131
+ descr->end_time_ut = INVALID_TIME;
132
descr->id = NULL;
133
descr->extent = NULL;
134
descr->pg_cache_descr_state = 0;
135
descr->pg_cache_descr = NULL;
136
+ descr->update_every_s = 0;
137
138
return descr;
139
}
@@ -476,7 +477,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
477
uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
478
479
uv_rwlock_wrlock(&page_index->lock);
479
- ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
480
+ ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time_ut / USEC_PER_SEC), PJE0);
481
if (unlikely(0 == ret)) {
482
uv_rwlock_wrunlock(&page_index->lock);
483
if (unlikely(debug_flags & D_RRDENGINE)) {
@@ -506,7 +507,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
507
while (!pg_cache_try_get_unsafe(descr, 1)) {
508
debug(D_RRDENGINE, "%s: Waiting for locked page:", __func__);
509
if (unlikely(debug_flags & D_RRDENGINE))
509
- print_page_cache_descr(descr);
510
+ print_page_cache_descr(descr, "", true);
511
pg_cache_wait_event_unsafe(descr);
512
}
513
}
@@ -517,7 +518,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
518
while (unlikely(pg_cache_descr->flags & RRD_PAGE_DIRTY)) {
519
debug(D_RRDENGINE, "%s: Found dirty page, waiting for it to be flushed:", __func__);
520
if (unlikely(debug_flags & D_RRDENGINE))
520
- print_page_cache_descr(descr);
521
+ print_page_cache_descr(descr, "", true);
522
pg_cache_wait_event_unsafe(descr);
523
}
524
}
@@ -548,8 +549,8 @@ static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t
549
{
550
usec_t pg_start, pg_end;
551
551
- pg_start = descr->start_time;
552
- pg_end = descr->end_time;
552
+ pg_start = descr->start_time_ut;
553
+ pg_end = descr->end_time_ut;
554
555
return (pg_start < start_time && pg_end >= start_time) ||
556
(pg_start >= start_time && pg_start <= end_time);
@@ -557,7 +558,7 @@ static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t
558
559
static inline int is_point_in_time_in_page(struct rrdeng_page_descr *descr, usec_t point_in_time)
560
{
560
- return (point_in_time >= descr->start_time && point_in_time <= descr->end_time);
561
+ return (point_in_time >= descr->start_time_ut && point_in_time <= descr->end_time_ut);
562
}
563
564
/* The caller must hold the page index lock */
@@ -592,14 +593,14 @@ static inline struct rrdeng_page_descr *
593
/* Update metric oldest and latest timestamps efficiently when adding new values */
594
void pg_cache_add_new_metric_time(struct pg_cache_page_index *page_index, struct rrdeng_page_descr *descr)
595
{
595
- usec_t oldest_time = page_index->oldest_time;
596
- usec_t latest_time = page_index->latest_time;
596
+ usec_t oldest_time = page_index->oldest_time_ut;
597
+ usec_t latest_time = page_index->latest_time_ut;
598
598
- if (unlikely(oldest_time == INVALID_TIME || descr->start_time < oldest_time)) {
599
- page_index->oldest_time = descr->start_time;
599
+ if (unlikely(oldest_time == INVALID_TIME || descr->start_time_ut < oldest_time)) {
600
+ page_index->oldest_time_ut = descr->start_time_ut;
601
}
601
- if (likely(descr->end_time > latest_time || latest_time == INVALID_TIME)) {
602
- page_index->latest_time = descr->end_time;
602
+ if (likely(descr->end_time_ut > latest_time || latest_time == INVALID_TIME)) {
603
+ page_index->latest_time_ut = descr->end_time_ut;
604
}
605
}
606
@@ -618,23 +619,23 @@ void pg_cache_update_metric_times(struct pg_cache_page_index *page_index)
619
firstPValue = JudyLFirst(page_index->JudyL_array, &firstIndex, PJE0);
620
if (likely(NULL != firstPValue)) {
621
descr = *firstPValue;
621
- oldest_time = descr->start_time;
622
+ oldest_time = descr->start_time_ut;
623
}
624
lastIndex = (Word_t)-1;
625
lastPValue = JudyLLast(page_index->JudyL_array, &lastIndex, PJE0);
626
if (likely(NULL != lastPValue)) {
627
descr = *lastPValue;
627
- latest_time = descr->end_time;
628
+ latest_time = descr->end_time_ut;
629
}
630
uv_rwlock_rdunlock(&page_index->lock);
631
632
if (unlikely(NULL == firstPValue)) {
633
fatal_assert(NULL == lastPValue);
633
- page_index->oldest_time = page_index->latest_time = INVALID_TIME;
634
+ page_index->oldest_time_ut = page_index->latest_time_ut = INVALID_TIME;
635
return;
636
}
636
- page_index->oldest_time = oldest_time;
637
- page_index->latest_time = latest_time;
637
+ page_index->oldest_time_ut = oldest_time;
638
+ page_index->latest_time_ut = latest_time;
639
}
640
641
/* If index is NULL lookup by UUID (descr->id) */
@@ -669,7 +670,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
670
}
671
672
uv_rwlock_wrlock(&page_index->lock);
672
- PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
673
+ PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time_ut / USEC_PER_SEC), PJE0);
674
*PValue = descr;
675
++page_index->page_count;
676
pg_cache_add_new_metric_time(page_index, descr);
@@ -681,7 +682,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
682
uv_rwlock_wrunlock(&pg_cache->pg_cache_rwlock);
683
}
684
684
-usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time)
685
+usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut)
686
{
687
struct page_cache *pg_cache = &ctx->pg_cache;
688
struct rrdeng_page_descr *descr = NULL;
@@ -699,25 +700,25 @@ usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
700
}
701
702
uv_rwlock_rdlock(&page_index->lock);
702
- descr = find_first_page_in_time_range(page_index, start_time, end_time);
703
+ descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
704
if (NULL == descr) {
705
uv_rwlock_rdunlock(&page_index->lock);
706
return INVALID_TIME;
707
}
708
uv_rwlock_rdunlock(&page_index->lock);
708
- return descr->start_time;
709
+ return descr->start_time_ut;
710
}
711
712
/**
713
* Return page information for the first page before point_in_time that satisfies the filter.
714
* @param ctx DB context
715
* @param page_index page index of a metric
715
- * @param point_in_time the pages that are searched must be older than this timestamp
716
+ * @param point_in_time_ut the pages that are searched must be older than this timestamp
717
* @param filter decides if the page satisfies the caller's criteria
718
* @param page_info the result of the search is set in this pointer
719
*/
720
void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
720
- usec_t point_in_time, pg_cache_page_info_filter_t *filter,
721
+ usec_t point_in_time_ut, pg_cache_page_info_filter_t *filter,
722
struct rrdeng_page_info *page_info)
723
{
724
struct page_cache *pg_cache = &ctx->pg_cache;
@@ -728,7 +729,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
729
(void)pg_cache;
730
fatal_assert(NULL != page_index);
731
731
- Index = (Word_t)(point_in_time / USEC_PER_SEC);
732
+ Index = (Word_t)(point_in_time_ut / USEC_PER_SEC);
733
uv_rwlock_rdlock(&page_index->lock);
734
do {
735
PValue = JudyLPrev(page_index->JudyL_array, &Index, PJE0);
@@ -736,12 +737,12 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
737
} while (descr != NULL && !filter(descr));
738
if (unlikely(NULL == descr)) {
739
page_info->page_length = 0;
739
- page_info->start_time = INVALID_TIME;
740
- page_info->end_time = INVALID_TIME;
740
+ page_info->start_time_ut = INVALID_TIME;
741
+ page_info->end_time_ut = INVALID_TIME;
742
} else {
743
page_info->page_length = descr->page_length;
743
- page_info->start_time = descr->start_time;
744
- page_info->end_time = descr->end_time;
744
+ page_info->start_time_ut = descr->start_time_ut;
745
+ page_info->end_time_ut = descr->end_time_ut;
746
}
747
uv_rwlock_rdunlock(&page_index->lock);
748
}
@@ -750,7 +751,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
751
* Searches for an unallocated page without triggering disk I/O. Attempts to reserve the page and get a reference.
752
* @param ctx DB context
753
* @param id lookup by UUID
753
- * @param start_time exact starting time in usec
754
+ * @param start_time_ut exact starting time in usec
755
* @param ret_page_indexp Sets the page index pointer (*ret_page_indexp) for the given UUID.
756
* @return the page descriptor or NULL on failure. It can fail if:
757
* 1. The page is already allocated to the page cache.
@@ -758,7 +759,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
759
* 3. It did not succeed to reserve a spot in the page cache.
760
*/
761
struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id,
761
- usec_t start_time)
762
+ usec_t start_time_ut)
763
{
764
struct page_cache *pg_cache = &ctx->pg_cache;
765
struct rrdeng_page_descr *descr = NULL;
@@ -781,7 +782,7 @@ struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_
782
}
783
784
uv_rwlock_rdlock(&page_index->lock);
784
- Index = (Word_t)(start_time / USEC_PER_SEC);
785
+ Index = (Word_t)(start_time_ut / USEC_PER_SEC);
786
PValue = JudyLGet(page_index->JudyL_array, Index, PJE0);
787
if (likely(NULL != PValue)) {
788
descr = *PValue;
@@ -818,15 +819,15 @@ struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_
819
* Does not get a reference.
820
* @param ctx DB context
821
* @param id UUID
821
- * @param start_time inclusive starting time in usec
822
- * @param end_time inclusive ending time in usec
822
+ * @param start_time_ut inclusive starting time in usec
823
+ * @param end_time_ut inclusive ending time in usec
824
* @param page_info_arrayp It allocates (*page_arrayp) and populates it with information of pages that overlap
825
* with the time range [start_time,end_time]. The caller must free (*page_info_arrayp) with freez().
826
* If page_info_arrayp is set to NULL nothing was allocated.
827
* @param ret_page_indexp Sets the page index pointer (*ret_page_indexp) for the given UUID.
828
* @return the number of pages that overlap with the time range [start_time,end_time].
829
*/
829
-unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time,
830
+unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut,
831
struct rrdeng_page_info **page_info_arrayp, struct pg_cache_page_index **ret_page_indexp)
832
{
833
struct page_cache *pg_cache = &ctx->pg_cache;
@@ -854,14 +855,14 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
855
}
856
857
uv_rwlock_rdlock(&page_index->lock);
857
- descr = find_first_page_in_time_range(page_index, start_time, end_time);
858
+ descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
859
if (NULL == descr) {
860
uv_rwlock_rdunlock(&page_index->lock);
861
debug(D_RRDENGINE, "%s: No page was found to attempt preload.", __func__);
862
*ret_page_indexp = NULL;
863
return 0;
864
} else {
864
- Index = (Word_t)(descr->start_time / USEC_PER_SEC);
865
+ Index = (Word_t)(descr->start_time_ut / USEC_PER_SEC);
866
}
867
if (page_info_arrayp) {
868
page_info_array_max_size = PAGE_CACHE_MAX_PRELOAD_PAGES * sizeof(struct rrdeng_page_info);
@@ -869,7 +870,7 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
870
}
871
872
for (count = 0, preload_count = 0 ;
872
- descr != NULL && is_page_in_time_range(descr, start_time, end_time) ;
873
+ descr != NULL && is_page_in_time_range(descr, start_time_ut, end_time_ut) ;
874
PValue = JudyLNext(page_index->JudyL_array, &Index, PJE0),
875
descr = unlikely(NULL == PValue) ? NULL : *PValue) {
876
/* Iterate all pages in range */
@@ -881,8 +882,8 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
882
page_info_array_max_size += PAGE_CACHE_MAX_PRELOAD_PAGES * sizeof(struct rrdeng_page_info);
883
*page_info_arrayp = reallocz(*page_info_arrayp, page_info_array_max_size);
884
}
884
- (*page_info_arrayp)[count].start_time = descr->start_time;
885
- (*page_info_arrayp)[count].end_time = descr->end_time;
885
+ (*page_info_arrayp)[count].start_time_ut = descr->start_time_ut;
886
+ (*page_info_arrayp)[count].end_time_ut = descr->end_time_ut;
887
(*page_info_arrayp)[count].page_length = descr->page_length;
888
}
889
++count;
@@ -974,7 +975,7 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
975
*/
976
struct rrdeng_page_descr *
977
pg_cache_lookup(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
977
- usec_t point_in_time)
978
+ usec_t point_in_time_ut)
979
{
980
struct page_cache *pg_cache = &ctx->pg_cache;
981
struct rrdeng_page_descr *descr = NULL;
@@ -1003,15 +1004,15 @@ struct rrdeng_page_descr *
1004
page_not_in_cache = 0;
1005
uv_rwlock_rdlock(&page_index->lock);
1006
while (1) {
1006
- Index = (Word_t)(point_in_time / USEC_PER_SEC);
1007
+ Index = (Word_t)(point_in_time_ut / USEC_PER_SEC);
1008
PValue = JudyLLast(page_index->JudyL_array, &Index, PJE0);
1009
if (likely(NULL != PValue)) {
1010
descr = *PValue;
1011
}
1012
if (NULL == PValue ||
1013
0 == descr->page_length ||
1013
- (INVALID_TIME != point_in_time &&
1014
- !is_point_in_time_in_page(descr, point_in_time))) {
1014
+ (INVALID_TIME != point_in_time_ut &&
1015
+ !is_point_in_time_in_page(descr, point_in_time_ut))) {
1016
/* non-empty page not found */
1017
uv_rwlock_rdunlock(&page_index->lock);
1018
@@ -1038,7 +1039,7 @@ struct rrdeng_page_descr *
1039
1040
debug(D_RRDENGINE, "%s: Waiting for page to be asynchronously read from disk:", __func__);
1041
if(unlikely(debug_flags & D_RRDENGINE))
1041
- print_page_cache_descr(descr);
1042
+ print_page_cache_descr(descr, "", true);
1043
while (!(pg_cache_descr->flags & RRD_PAGE_POPULATED)) {
1044
pg_cache_wait_event_unsafe(descr);
1045
}
@@ -1053,7 +1054,7 @@ struct rrdeng_page_descr *
1054
uv_rwlock_rdunlock(&page_index->lock);
1055
debug(D_RRDENGINE, "%s: Waiting for page to be unlocked:", __func__);
1056
if(unlikely(debug_flags & D_RRDENGINE))
1056
- print_page_cache_descr(descr);
1057
+ print_page_cache_descr(descr, "", true);
1058
if (!(flags & RRD_PAGE_POPULATED))
1059
page_not_in_cache = 1;
1060
pg_cache_wait_event_unsafe(descr);
@@ -1081,7 +1082,7 @@ struct rrdeng_page_descr *
1082
*/
1083
struct rrdeng_page_descr *
1084
pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
1084
- usec_t start_time, usec_t end_time)
1085
+ usec_t start_time_ut, usec_t end_time_ut)
1086
{
1087
struct page_cache *pg_cache = &ctx->pg_cache;
1088
struct rrdeng_page_descr *descr = NULL;
@@ -1110,7 +1111,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
1111
uv_rwlock_rdlock(&page_index->lock);
1112
int retry_count = 0;
1113
while (1) {
1113
- descr = find_first_page_in_time_range(page_index, start_time, end_time);
1114
+ descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
1115
if (NULL == descr || 0 == descr->page_length || retry_count == default_rrdeng_page_fetch_retries) {
1116
/* non-empty page not found */
1117
if (retry_count == default_rrdeng_page_fetch_retries)
@@ -1140,7 +1141,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
1141
1142
debug(D_RRDENGINE, "%s: Waiting for page to be asynchronously read from disk:", __func__);
1143
if(unlikely(debug_flags & D_RRDENGINE))
1143
- print_page_cache_descr(descr);
1144
+ print_page_cache_descr(descr, "", true);
1145
while (!(pg_cache_descr->flags & RRD_PAGE_POPULATED)) {
1146
pg_cache_wait_event_unsafe(descr);
1147
}
@@ -1155,7 +1156,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
1156
uv_rwlock_rdunlock(&page_index->lock);
1157
debug(D_RRDENGINE, "%s: Waiting for page to be unlocked:", __func__);
1158
if(unlikely(debug_flags & D_RRDENGINE))
1158
- print_page_cache_descr(descr);
1159
+ print_page_cache_descr(descr, "", true);
1160
if (!(flags & RRD_PAGE_POPULATED))
1161
page_not_in_cache = 1;
1162
@@ -1180,7 +1181,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
1181
return descr;
1182
}
1183
1183
-struct pg_cache_page_index *create_page_index(uuid_t *id)
1184
+struct pg_cache_page_index *create_page_index(uuid_t *id, struct rrdengine_instance *ctx)
1185
{
1186
struct pg_cache_page_index *page_index;
1187
@@ -1188,11 +1189,15 @@ struct pg_cache_page_index *create_page_index(uuid_t *id)
1189
page_index->JudyL_array = (Pvoid_t) NULL;
1190
uuid_copy(page_index->id, *id);
1191
fatal_assert(0 == uv_rwlock_init(&page_index->lock));
1191
- page_index->oldest_time = INVALID_TIME;
1192
- page_index->latest_time = INVALID_TIME;
1192
+ page_index->oldest_time_ut = INVALID_TIME;
1193
+ page_index->latest_time_ut = INVALID_TIME;
1194
page_index->prev = NULL;
1195
page_index->page_count = 0;
1196
+ page_index->refcount = 0;
1197
page_index->writers = 0;
1198
+ page_index->ctx = ctx;
1199
+ page_index->alignment = NULL;
1200
+ page_index->latest_update_every_s = default_rrd_update_every;
1201
1202
return page_index;
1203
}
database/engine/pagecache.h
+40
-29
@@ -60,18 +60,19 @@ struct rrdeng_page_descr {
60
volatile unsigned long pg_cache_descr_state;
61
62
/* page information */
63
- usec_t start_time;
64
- usec_t end_time;
65
- uint32_t page_length;
63
+ usec_t start_time_ut;
64
+ usec_t end_time_ut;
65
+ uint32_t update_every_s:24;
66
uint8_t type;
67
+ uint32_t page_length;
68
};
69
70
#define PAGE_INFO_SCRATCH_SZ (8)
71
struct rrdeng_page_info {
72
uint8_t scratch[PAGE_INFO_SCRATCH_SZ]; /* scratch area to be used by page-cache users */
73
73
- usec_t start_time;
74
- usec_t end_time;
74
+ usec_t start_time_ut;
75
+ usec_t end_time_ut;
76
uint32_t page_length;
77
};
78
@@ -80,6 +81,11 @@ typedef int pg_cache_page_info_filter_t(struct rrdeng_page_descr *);
81
82
#define PAGE_CACHE_MAX_PRELOAD_PAGES (256)
83
84
+struct pg_alignment {
85
+ uint32_t page_length;
86
+ uint32_t refcount;
87
+};
88
+
89
/* maps time ranges to pages */
90
struct pg_cache_page_index {
91
uuid_t id;
@@ -89,6 +95,7 @@ struct pg_cache_page_index {
95
*/
96
Pvoid_t JudyL_array;
97
Word_t page_count;
98
+ unsigned short refcount;
99
unsigned short writers;
100
uv_rwlock_t lock;
101
@@ -96,13 +103,17 @@ struct pg_cache_page_index {
103
* Only one effective writer, data deletion workqueue.
104
* It's also written during the DB loading phase.
105
*/
99
- usec_t oldest_time;
106
+ usec_t oldest_time_ut;
107
108
/*
109
* Only one effective writer, data collection thread.
110
* It's also written by the data deletion workqueue when data collection is disabled for this metric.
111
*/
105
- usec_t latest_time;
112
+ usec_t latest_time_ut;
113
+
114
+ struct rrdengine_instance *ctx;
115
+ struct pg_alignment *alignment;
116
+ uint32_t latest_update_every_s;
117
118
struct pg_cache_page_index *prev;
119
};
@@ -171,22 +182,22 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
182
uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
183
uint8_t remove_dirty, uint8_t is_exclusive_holder, uuid_t *metric_id);
184
usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
174
- usec_t start_time, usec_t end_time);
185
+ usec_t start_time_ut, usec_t end_time_ut);
186
void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
176
- usec_t point_in_time, pg_cache_page_info_filter_t *filter,
187
+ usec_t point_in_time_ut, pg_cache_page_info_filter_t *filter,
188
struct rrdeng_page_info *page_info);
189
struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id,
179
- usec_t start_time);
180
-extern unsigned
181
- pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time,
190
+ usec_t start_time_ut);
191
+unsigned
192
+ pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut,
193
struct rrdeng_page_info **page_info_arrayp, struct pg_cache_page_index **ret_page_indexp);
183
-extern struct rrdeng_page_descr *
194
+struct rrdeng_page_descr *
195
pg_cache_lookup(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
185
- usec_t point_in_time);
186
-extern struct rrdeng_page_descr *
196
+ usec_t point_in_time_ut);
197
+struct rrdeng_page_descr *
198
pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
188
- usec_t start_time, usec_t end_time);
189
-struct pg_cache_page_index *create_page_index(uuid_t *id);
199
+ usec_t start_time_ut, usec_t end_time_ut);
200
+struct pg_cache_page_index *create_page_index(uuid_t *id, struct rrdengine_instance *ctx);
201
void init_page_cache(struct rrdengine_instance *ctx);
202
void free_page_cache(struct rrdengine_instance *ctx);
203
void pg_cache_add_new_metric_time(struct pg_cache_page_index *page_index, struct rrdeng_page_descr *descr);
@@ -204,41 +215,41 @@ struct rrdeng_page_descr *rrdeng_page_descr_mallocz(void);
215
void rrdeng_page_descr_freez(struct rrdeng_page_descr *descr);
216
217
static inline void
207
- pg_cache_atomic_get_pg_info(struct rrdeng_page_descr *descr, usec_t *end_timep, uint32_t *page_lengthp)
218
+ pg_cache_atomic_get_pg_info(struct rrdeng_page_descr *descr, usec_t *end_time_ut_p, uint32_t *page_lengthp)
219
{
209
- usec_t end_time, old_end_time;
220
+ usec_t end_time_ut, old_end_time_ut;
221
uint32_t page_length;
222
223
if (NULL == descr->extent) {
224
/* this page is currently being modified, get consistent info locklessly */
225
do {
215
- end_time = descr->end_time;
226
+ end_time_ut = descr->end_time_ut;
227
__sync_synchronize();
217
- old_end_time = end_time;
228
+ old_end_time_ut = end_time_ut;
229
page_length = descr->page_length;
230
__sync_synchronize();
220
- end_time = descr->end_time;
231
+ end_time_ut = descr->end_time_ut;
232
__sync_synchronize();
222
- } while ((end_time != old_end_time || (end_time & 1) != 0));
233
+ } while ((end_time_ut != old_end_time_ut || (end_time_ut & 1) != 0));
234
224
- *end_timep = end_time;
235
+ *end_time_ut_p = end_time_ut;
236
*page_lengthp = page_length;
237
} else {
227
- *end_timep = descr->end_time;
238
+ *end_time_ut_p = descr->end_time_ut;
239
*page_lengthp = descr->page_length;
240
}
241
}
242
243
/* The caller must hold a reference to the page and must have already set the new data */
233
-static inline void pg_cache_atomic_set_pg_info(struct rrdeng_page_descr *descr, usec_t end_time, uint32_t page_length)
244
+static inline void pg_cache_atomic_set_pg_info(struct rrdeng_page_descr *descr, usec_t end_time_ut, uint32_t page_length)
245
{
235
- fatal_assert(!(end_time & 1));
246
+ fatal_assert(!(end_time_ut & 1));
247
__sync_synchronize();
237
- descr->end_time |= 1; /* mark start of uncertainty period by adding 1 microsecond */
248
+ descr->end_time_ut |= 1; /* mark start of uncertainty period by adding 1 microsecond */
249
__sync_synchronize();
250
descr->page_length = page_length;
251
__sync_synchronize();
241
- descr->end_time = end_time; /* mark end of uncertainty period */
252
+ descr->end_time_ut = end_time_ut; /* mark end of uncertainty period */
253
}
254
255
#endif /* NETDATA_PAGECACHE_H */
database/engine/rrddiskprotocol.h
+2
-2
@@ -46,8 +46,8 @@ struct rrdeng_extent_page_descr {
46
47
uint8_t uuid[UUID_SZ];
48
uint32_t page_length;
49
- uint64_t start_time;
50
- uint64_t end_time;
49
+ uint64_t start_time_ut;
50
+ uint64_t end_time_ut;
51
} __attribute__ ((packed));
52
53
/*
database/engine/rrdengine.c
+71
-8
@@ -206,8 +206,8 @@ void read_cached_extent_cb(struct rrdengine_worker_config* wc, unsigned idx, str
206
/* care, we don't hold the descriptor mutex */
207
if (!uuid_compare(*extent->pages[j]->id, *descr->id) &&
208
extent->pages[j]->page_length == descr->page_length &&
209
- extent->pages[j]->start_time == descr->start_time &&
210
- extent->pages[j]->end_time == descr->end_time) {
209
+ extent->pages[j]->start_time_ut == descr->start_time_ut &&
210
+ extent->pages[j]->end_time_ut == descr->end_time_ut) {
211
break;
212
}
213
page_offset += extent->pages[j]->page_length;
@@ -378,8 +378,8 @@ after_crc_check:
378
/* care, we don't hold the descriptor mutex */
379
if (!uuid_compare(*(uuid_t *) header->descr[i].uuid, *descrj->id) &&
380
header->descr[i].page_length == descrj->page_length &&
381
- header->descr[i].start_time == descrj->start_time &&
382
- header->descr[i].end_time == descrj->end_time) {
381
+ header->descr[i].start_time_ut == descrj->start_time_ut &&
382
+ header->descr[i].end_time_ut == descrj->end_time_ut) {
383
descr = descrj;
384
break;
385
}
@@ -387,7 +387,7 @@ after_crc_check:
387
is_prefetched_page = 0;
388
if (!descr) { /* This extent page has not been requested. Try populating it for locality (best effort). */
389
descr = pg_cache_lookup_unpopulated_and_lock(ctx, (uuid_t *)header->descr[i].uuid,
390
- header->descr[i].start_time);
390
+ header->descr[i].start_time_ut);
391
if (!descr)
392
continue; /* Failed to reserve a suitable page */
393
is_prefetched_page = 1;
@@ -820,8 +820,8 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct
820
header->descr[i].type = descr->type;
821
uuid_copy(*(uuid_t *)header->descr[i].uuid, *descr->id);
822
header->descr[i].page_length = descr->page_length;
823
- header->descr[i].start_time = descr->start_time;
824
- header->descr[i].end_time = descr->end_time;
823
+ header->descr[i].start_time_ut = descr->start_time_ut;
824
+ header->descr[i].end_time_ut = descr->end_time_ut;
825
pos += sizeof(header->descr[i]);
826
}
827
for (i = 0 ; i < count ; ++i) {
@@ -1043,7 +1043,70 @@ static void rrdeng_cleanup_finished_threads(struct rrdengine_worker_config* wc)
1043
/* return 0 on success */
1044
int init_rrd_files(struct rrdengine_instance *ctx)
1045
{
1046
- return init_data_files(ctx);
1046
+ int ret = init_data_files(ctx);
1047
+
1048
+ BUFFER *wb = buffer_create(1000);
1049
+ size_t all_errors = 0;
1050
+ usec_t now = now_realtime_usec();
1051
+
1052
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter) {
1053
+ buffer_sprintf(wb, "%s%zu pages had start time > end time (latest: %llu secs ago)"
1054
+ , (all_errors)?", ":""
1055
+ , ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter
1056
+ , (now - ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut) / USEC_PER_SEC
1057
+ );
1058
+ all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter;
1059
+ }
1060
+
1061
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter) {
1062
+ buffer_sprintf(wb, "%s%zu pages had start time = end time with more than 1 entries (latest: %llu secs ago)"
1063
+ , (all_errors)?", ":""
1064
+ , ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter
1065
+ , (now - ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut) / USEC_PER_SEC
1066
+ );
1067
+ all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter;
1068
+ }
1069
+
1070
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter) {
1071
+ buffer_sprintf(wb, "%s%zu pages had zero points (latest: %llu secs ago)"
1072
+ , (all_errors)?", ":""
1073
+ , ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter
1074
+ , (now - ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut) / USEC_PER_SEC
1075
+ );
1076
+ all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter;
1077
+ }
1078
+
1079
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter) {
1080
+ buffer_sprintf(wb, "%s%zu pages had update every == 0 with entries > 1 (latest: %llu secs ago)"
1081
+ , (all_errors)?", ":""
1082
+ , ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter
1083
+ , (now - ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut) / USEC_PER_SEC
1084
+ );
1085
+ all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter;
1086
+ }
1087
+
1088
+ if(ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter) {
1089
+ buffer_sprintf(wb, "%s%zu pages had a different number of points compared to their timestamps (latest: %llu secs ago; these page have been loaded)"
1090
+ , (all_errors)?", ":""
1091
+ , ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter
1092
+ , (now - ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut) / USEC_PER_SEC
1093
+ );
1094
+ all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter;
1095
+ }
1096
+
1097
+ if(ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter) {
1098
+ buffer_sprintf(wb, "%s%zu extents have been dropped because they didn't have any valid pages"
1099
+ , (all_errors)?", ":""
1100
+ , ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter
1101
+ );
1102
+ all_errors += ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter;
1103
+ }
1104
+
1105
+ if(all_errors)
1106
+ info("DBENGINE: tier %d: %s", ctx->tier, buffer_tostring(wb));
1107
+
1108
+ buffer_free(wb);
1109
+ return ret;
1110
}
1111
1112
void finalize_rrd_files(struct rrdengine_instance *ctx)
database/engine/rrdengine.h
+19
-9
@@ -37,29 +37,25 @@ struct rrdengine_instance;
37
#define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
38
39
struct rrdeng_collect_handle {
40
- struct rrdeng_metric_handle *metric_handle;
40
+ struct pg_cache_page_index *page_index;
41
struct rrdeng_page_descr *descr;
42
unsigned long page_correlation_id;
43
- struct rrdengine_instance *ctx;
43
// set to 1 when this dimension is not page aligned with the other dimensions in the chart
44
uint8_t unaligned_page;
45
};
46
47
struct rrdeng_query_handle {
49
- struct rrdeng_metric_handle *metric_handle;
48
struct rrdeng_page_descr *descr;
49
struct rrdengine_instance *ctx;
50
struct pg_cache_page_index *page_index;
53
- time_t next_page_time;
54
- time_t now;
51
+ time_t wanted_start_time_s;
52
+ time_t now_s;
53
unsigned position;
54
unsigned entries;
57
- TIER_QUERY_FETCH tier_query_fetch_type;
55
storage_number *page;
59
- usec_t page_end_time;
56
+ usec_t page_end_time_ut;
57
uint32_t page_length;
61
- usec_t dt;
62
- time_t dt_sec;
58
+ time_t dt_s;
59
};
60
61
typedef enum {
@@ -230,6 +226,15 @@ extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of dele
226
#define SET_QUIESCE (1) /* set it before shutting down the instance, quiesce long running operations */
227
#define QUIESCED (2) /* is set after all threads have finished running */
228
229
+typedef enum {
230
+ LOAD_ERRORS_PAGE_FLIPPED_TIME = 0,
231
+ LOAD_ERRORS_PAGE_EQUAL_TIME = 1,
232
+ LOAD_ERRORS_PAGE_ZERO_ENTRIES = 2,
233
+ LOAD_ERRORS_PAGE_UPDATE_ZERO = 3,
234
+ LOAD_ERRORS_PAGE_FLEXY_TIME = 4,
235
+ LOAD_ERRORS_DROPPED_EXTENT = 5,
236
+} INVALID_PAGE_ID;
237
+
238
struct rrdengine_instance {
239
struct metalog_instance *metalog_ctx;
240
struct rrdengine_worker_config worker_config;
@@ -254,6 +259,11 @@ struct rrdengine_instance {
259
uint8_t page_type; /* Default page type for this context */
260
261
struct rrdengine_statistics stats;
262
+
263
+ struct {
264
+ size_t counter;
265
+ usec_t latest_end_time_ut;
266
+ } load_errors[6];
267
};
268
269
void *dbengine_page_alloc(void);
database/engine/rrdengineapi.c
+388
-196
@@ -1,5 +1,6 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
#include "rrdengine.h"
3
+#include "../storage_engine.h"
4
5
/* Default global database instance */
6
struct rrdengine_instance multidb_ctx_storage_tier0;
@@ -35,14 +36,41 @@ int default_multidb_disk_quota_mb = 256;
36
/* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
37
uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
38
39
+
40
+// ----------------------------------------------------------------------------
41
+// helpers
42
+
43
static inline struct rrdengine_instance *get_rrdeng_ctx_from_host(RRDHOST *host, int tier) {
44
if(tier < 0 || tier >= RRD_STORAGE_TIERS) tier = 0;
45
if(!host->storage_instance[tier]) tier = 0;
46
return (struct rrdengine_instance *)host->storage_instance[tier];
47
}
48
49
+// ----------------------------------------------------------------------------
50
+// metrics groups
51
+
52
+STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid __maybe_unused) {
53
+ return callocz(1, sizeof(struct pg_alignment));
54
+}
55
+
56
+void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
57
+ if(!smg) return;
58
+
59
+ struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
60
+ struct pg_alignment *pa = (struct pg_alignment *)smg;
61
+ struct page_cache *pg_cache = &ctx->pg_cache;
62
+
63
+ uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
64
+ if(pa->refcount == 0)
65
+ freez(pa);
66
+ uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
67
+}
68
+
69
+// ----------------------------------------------------------------------------
70
+// metric handle for legacy dbs
71
+
72
/* This UUID is not unique across hosts */
45
-void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid)
73
+void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
74
{
75
EVP_MD_CTX *evpctx;
76
unsigned char hash_value[EVP_MAX_MD_SIZE];
@@ -75,97 +103,134 @@ void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uu
103
memcpy(ret_uuid, hash_value, sizeof(uuid_t));
104
}
105
78
-struct rrdeng_metric_handle {
79
- RRDDIM *rd;
80
- struct rrdengine_instance *ctx;
81
- uuid_t *rrdeng_uuid; // database engine metric UUID
82
- struct pg_cache_page_index *page_index;
83
-};
106
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_legacy(STORAGE_INSTANCE *db_instance, const char *rd_id, const char *st_id, STORAGE_METRICS_GROUP *smg) {
107
+ uuid_t legacy_uuid;
108
+ rrdeng_generate_legacy_uuid(rd_id, st_id, &legacy_uuid);
109
+ return rrdeng_metric_get(db_instance, &legacy_uuid, smg);
110
+}
111
+
112
+// ----------------------------------------------------------------------------
113
+// metric handle
114
+
115
+void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle) {
116
+ struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
117
+ struct rrdengine_instance *ctx = page_index->ctx;
118
+ struct page_cache *pg_cache = &ctx->pg_cache;
119
+
120
+ uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
121
+ page_index->refcount--;
122
+ if(page_index->alignment && page_index->refcount == 0) {
123
+ page_index->alignment->refcount--;
124
+ page_index->alignment = NULL;
125
+ }
126
+ uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
127
85
-void rrdeng_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle) {
128
freez(db_metric_handle);
129
}
130
89
-STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance) {
131
+STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg) {
132
struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
91
- struct page_cache *pg_cache;
92
- uuid_t legacy_uuid;
93
- uuid_t multihost_legacy_uuid;
94
- Pvoid_t *PValue;
133
+ struct pg_alignment *pa = (struct pg_alignment *)smg;
134
+ struct page_cache *pg_cache = &ctx->pg_cache;
135
struct pg_cache_page_index *page_index = NULL;
96
- int is_multihost_child = 0;
97
- RRDHOST *host = rd->rrdset->rrdhost;
98
-
99
- pg_cache = &ctx->pg_cache;
100
-
101
- rrdeng_generate_legacy_uuid(rrddim_id(rd), (char *)rrdset_id(rd->rrdset), &legacy_uuid);
102
- if (host != localhost && is_storage_engine_shared((STORAGE_INSTANCE *)ctx))
103
- is_multihost_child = 1;
136
137
uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
106
- PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &legacy_uuid, sizeof(uuid_t));
138
+ Pvoid_t *PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, uuid, sizeof(uuid_t));
139
if (likely(NULL != PValue)) {
140
page_index = *PValue;
141
+ page_index->refcount++;
142
+
143
+ if(pa) {
144
+ if(page_index->alignment && page_index->alignment != pa)
145
+ fatal("DBENGINE: page_index has a different alignment.");
146
+
147
+ if(!page_index->alignment) {
148
+ page_index->alignment = pa;
149
+ pa->refcount++;
150
+ }
151
+ }
152
}
153
uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
111
- if (is_multihost_child || NULL == PValue) {
112
- /* First time we see the legacy UUID or metric belongs to child host in multi-host DB.
113
- * Drop legacy support, normal path */
114
-
115
- uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
116
- PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &rd->metric_uuid, sizeof(uuid_t));
117
- if (likely(NULL != PValue)) {
118
- page_index = *PValue;
119
- }
120
- uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
121
- if (NULL == PValue) {
122
- uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
123
- PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0);
124
- fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
125
- *PValue = page_index = create_page_index(&rd->metric_uuid);
126
- page_index->prev = pg_cache->metrics_index.last_page_index;
127
- pg_cache->metrics_index.last_page_index = page_index;
128
- uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
129
- }
130
- } else {
131
- /* There are legacy UUIDs in the database, implement backward compatibility */
154
133
- rrdeng_convert_legacy_uuid_to_multihost(rd->rrdset->rrdhost->machine_guid, &legacy_uuid,
134
- &multihost_legacy_uuid);
155
+ return (STORAGE_METRIC_HANDLE *)page_index;
156
+}
157
+
158
+STORAGE_METRIC_HANDLE *rrdeng_metric_create(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg) {
159
+ internal_fatal(!db_instance, "DBENGINE: db_instance is NULL");
160
+
161
+ struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
162
+ struct pg_alignment *pa = (struct pg_alignment *)smg;
163
+ struct pg_cache_page_index *page_index;
164
+ struct page_cache *pg_cache = &ctx->pg_cache;
165
+
166
+ uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
167
+ Pvoid_t *PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, uuid, sizeof(uuid_t), PJE0);
168
+ fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
169
+ *PValue = page_index = create_page_index(uuid, ctx);
170
+ page_index->prev = pg_cache->metrics_index.last_page_index;
171
+ pg_cache->metrics_index.last_page_index = page_index;
172
+ page_index->alignment = pa;
173
+ if(pa)
174
+ pa->refcount++;
175
+ uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
176
+
177
+ return (STORAGE_METRIC_HANDLE *)page_index;
178
+}
179
136
- int need_to_store = uuid_compare(rd->metric_uuid, multihost_legacy_uuid);
180
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
181
+ STORAGE_METRIC_HANDLE *db_metric_handle;
182
138
- uuid_copy(rd->metric_uuid, multihost_legacy_uuid);
183
+ db_metric_handle = rrdeng_metric_get(db_instance, &rd->metric_uuid, smg);
184
+ if(!db_metric_handle) {
185
+ db_metric_handle = rrdeng_metric_get_legacy(db_instance, rrddim_id(rd), rrdset_id(rd->rrdset), smg);
186
+ if(db_metric_handle) {
187
+ struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
188
+ uuid_copy(rd->metric_uuid, page_index->id);
189
+ }
190
+ }
191
+ if(!db_metric_handle)
192
+ db_metric_handle = rrdeng_metric_create(db_instance, &rd->metric_uuid, smg);
193
140
- if (unlikely(need_to_store && !ctx->tier))
141
- (void)sql_store_dimension(&rd->metric_uuid, &rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm);
194
+#ifdef NETDATA_INTERNAL_CHECKS
195
+ struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
196
+ if(uuid_compare(rd->metric_uuid, page_index->id) != 0) {
197
+ char uuid1[UUID_STR_LEN + 1];
198
+ char uuid2[UUID_STR_LEN + 1];
199
+
200
+ uuid_unparse(rd->metric_uuid, uuid1);
201
+ uuid_unparse(page_index->id, uuid2);
202
+ fatal("DBENGINE: uuids do not match, asked for metric '%s', but got page_index of metric '%s'", uuid1, uuid2);
203
}
204
144
- struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle));
145
- mh->rd = rd;
146
- mh->ctx = ctx;
147
- mh->rrdeng_uuid = &page_index->id;
148
- mh->page_index = page_index;
149
- return (STORAGE_METRIC_HANDLE *)mh;
205
+ struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
206
+ if(page_index->ctx != ctx)
207
+ fatal("DBENGINE: mixed up rrdengine instances, asked for metric from %p, got from %p", ctx, page_index->ctx);
208
+#endif
209
+
210
+ return db_metric_handle;
211
}
212
213
+
214
+// ----------------------------------------------------------------------------
215
+// collect ops
216
+
217
/*
218
* Gets a handle for storing metrics to the database.
219
* The handle must be released with rrdeng_store_metric_final().
220
*/
156
-STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle) {
157
- struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
158
-
221
+STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every) {
222
+ struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
223
struct rrdeng_collect_handle *handle;
160
- struct pg_cache_page_index *page_index;
224
+
225
+ if(!page_index->alignment)
226
+ fatal("DBENGINE: metric group is required for collect operations");
227
228
handle = callocz(1, sizeof(struct rrdeng_collect_handle));
163
- handle->metric_handle = metric_handle;
164
- handle->ctx = metric_handle->ctx;
229
+ handle->page_index = page_index;
230
handle->descr = NULL;
231
handle->unaligned_page = 0;
232
+ page_index->latest_update_every_s = update_every;
233
168
- page_index = metric_handle->page_index;
234
uv_rwlock_wrlock(&page_index->lock);
235
++page_index->writers;
236
uv_rwlock_wrunlock(&page_index->lock);
@@ -213,7 +278,7 @@ static int page_has_only_empty_metrics(struct rrdeng_page_descr *descr)
278
void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle) {
279
struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
280
// struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
216
- struct rrdengine_instance *ctx = handle->ctx;
281
+ struct rrdengine_instance *ctx = handle->page_index->ctx;
282
struct rrdeng_page_descr *descr = handle->descr;
283
284
if (unlikely(!ctx)) return;
@@ -226,9 +291,7 @@ void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_h
291
292
page_is_empty = page_has_only_empty_metrics(descr);
293
if (page_is_empty) {
229
- debug(D_RRDENGINE, "Page has empty metrics only, deleting:");
230
- if (unlikely(debug_flags & D_RRDENGINE))
231
- print_page_cache_descr(descr);
294
+ print_page_cache_descr(descr, "Page has empty metrics only, deleting", true);
295
pg_cache_put(ctx, descr);
296
pg_cache_punch_hole(ctx, descr, 1, 0, NULL);
297
} else
@@ -241,8 +304,8 @@ void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_h
304
handle->descr = NULL;
305
}
306
244
-void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
245
- usec_t point_in_time,
307
+static void rrdeng_store_metric_next_internal(STORAGE_COLLECT_HANDLE *collection_handle,
308
+ usec_t point_in_time_ut,
309
NETDATA_DOUBLE n,
310
NETDATA_DOUBLE min_value,
311
NETDATA_DOUBLE max_value,
@@ -251,11 +314,10 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
314
SN_FLAGS flags)
315
{
316
struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
254
- struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
255
- struct rrdengine_instance *ctx = handle->ctx;
317
+ struct pg_cache_page_index *page_index = handle->page_index;
318
+ struct rrdengine_instance *ctx = handle->page_index->ctx;
319
struct page_cache *pg_cache = &ctx->pg_cache;
320
struct rrdeng_page_descr *descr = handle->descr;
258
- RRDDIM *rd = metric_handle->rd;
321
322
void *page;
323
uint8_t must_flush_unaligned_page = 0, perfect_page_alignment = 0;
@@ -263,21 +325,33 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
325
if (descr) {
326
/* Make alignment decisions */
327
266
- if (descr->page_length == rd->rrdset->rrddim_page_alignment) {
328
+#ifdef NETDATA_INTERNAL_CHECKS
329
+ if(descr->end_time_ut + page_index->latest_update_every_s * USEC_PER_SEC != point_in_time_ut) {
330
+ char buffer[200 + 1];
331
+ snprintfz(buffer, 200,
332
+ "metrics collected are %s, end_time_ut = %llu, point_in_time_ut = %llu, update_every = %u, delta = %llu",
333
+ (point_in_time_ut / USEC_PER_SEC - descr->end_time_ut / USEC_PER_SEC > page_index->latest_update_every_s)?"far apart":"not aligned",
334
+ descr->end_time_ut / USEC_PER_SEC,
335
+ point_in_time_ut / USEC_PER_SEC,
336
+ page_index->latest_update_every_s,
337
+ point_in_time_ut / USEC_PER_SEC - descr->end_time_ut / USEC_PER_SEC);
338
+ print_page_cache_descr(descr, buffer, false);
339
+ }
340
+#endif
341
+
342
+ if (descr->page_length == page_index->alignment->page_length) {
343
/* this is the leading dimension that defines chart alignment */
344
perfect_page_alignment = 1;
345
}
346
/* is the metric far enough out of alignment with the others? */
271
- if (unlikely(descr->page_length + PAGE_POINT_SIZE_BYTES(descr) < rd->rrdset->rrddim_page_alignment)) {
347
+ if (unlikely(descr->page_length + PAGE_POINT_SIZE_BYTES(descr) < page_index->alignment->page_length)) {
348
handle->unaligned_page = 1;
273
- debug(D_RRDENGINE, "Metric page is not aligned with chart:");
274
- if (unlikely(debug_flags & D_RRDENGINE))
275
- print_page_cache_descr(descr);
349
+ print_page_cache_descr(descr, "Metric page is not aligned with chart", true);
350
}
351
if (unlikely(handle->unaligned_page &&
352
/* did the other metrics change page? */
279
- rd->rrdset->rrddim_page_alignment <= PAGE_POINT_SIZE_BYTES(descr))) {
280
- debug(D_RRDENGINE, "Flushing unaligned metric page.");
353
+ page_index->alignment->page_length <= PAGE_POINT_SIZE_BYTES(descr))) {
354
+ print_page_cache_descr(descr, "must_flush_unaligned_page = 1", true);
355
must_flush_unaligned_page = 1;
356
handle->unaligned_page = 0;
357
}
@@ -285,16 +359,21 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
359
if (unlikely(NULL == descr ||
360
descr->page_length + PAGE_POINT_SIZE_BYTES(descr) > RRDENG_BLOCK_SIZE ||
361
must_flush_unaligned_page)) {
288
- rrdeng_store_metric_flush_current_page(collection_handle);
362
290
- page = rrdeng_create_page(ctx, &metric_handle->page_index->id, &descr);
363
+ if(descr) {
364
+ print_page_cache_descr(descr, "flushing metric", true);
365
+ rrdeng_store_metric_flush_current_page(collection_handle);
366
+ }
367
+
368
+ page = rrdeng_create_page(ctx, &page_index->id, &descr);
369
fatal_assert(page);
370
371
+ descr->update_every_s = page_index->latest_update_every_s;
372
handle->descr = descr;
373
374
handle->page_correlation_id = rrd_atomic_fetch_add(&pg_cache->committed_page_index.latest_corr_id, 1);
375
297
- if (0 == rd->rrdset->rrddim_page_alignment) {
376
+ if (0 == page_index->alignment->page_length) {
377
/* this is the leading dimension that defines chart alignment */
378
perfect_page_alignment = 1;
379
}
@@ -329,13 +408,13 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
408
break;
409
}
410
332
- pg_cache_atomic_set_pg_info(descr, point_in_time, descr->page_length + PAGE_POINT_SIZE_BYTES(descr));
411
+ pg_cache_atomic_set_pg_info(descr, point_in_time_ut, descr->page_length + PAGE_POINT_SIZE_BYTES(descr));
412
413
if (perfect_page_alignment)
335
- rd->rrdset->rrddim_page_alignment = descr->page_length;
336
- if (unlikely(INVALID_TIME == descr->start_time)) {
414
+ page_index->alignment->page_length = descr->page_length;
415
+ if (unlikely(INVALID_TIME == descr->start_time_ut)) {
416
unsigned long new_metric_API_producers, old_metric_API_max_producers, ret_metric_API_max_producers;
338
- descr->start_time = point_in_time;
417
+ descr->start_time_ut = point_in_time_ut;
418
419
new_metric_API_producers = rrd_atomic_add_fetch(&ctx->stats.metric_API_producers, 1);
420
while (unlikely(new_metric_API_producers > (old_metric_API_max_producers = ctx->metric_API_max_producers))) {
@@ -349,20 +428,111 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
428
}
429
}
430
352
- pg_cache_insert(ctx, metric_handle->page_index, descr);
431
+ pg_cache_insert(ctx, page_index, descr);
432
} else {
354
- pg_cache_add_new_metric_time(metric_handle->page_index, descr);
433
+ pg_cache_add_new_metric_time(page_index, descr);
434
+ }
435
+
436
+// {
437
+// unsigned char u[16] = { 0x0C, 0x0A, 0x40, 0xD6, 0x2A, 0x43, 0x4A, 0x7C, 0x95, 0xF7, 0xD1, 0x1E, 0x0C, 0x9E, 0x8A, 0xE7 };
438
+// if(uuid_compare(u, page_index->id) == 0) {
439
+// char buffer[100];
440
+// snprintfz(buffer, 100, "store system.cpu, collect:%u, page_index first:%u, last:%u",
441
+// (uint32_t)(point_in_time / USEC_PER_SEC),
442
+// (uint32_t)(page_index->oldest_time / USEC_PER_SEC),
443
+// (uint32_t)(page_index->latest_time / USEC_PER_SEC));
444
+//
445
+// print_page_cache_descr(descr, buffer, false);
446
+// }
447
+// }
448
+}
449
+
450
+void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
451
+ usec_t point_in_time_ut,
452
+ NETDATA_DOUBLE n,
453
+ NETDATA_DOUBLE min_value,
454
+ NETDATA_DOUBLE max_value,
455
+ uint16_t count,
456
+ uint16_t anomaly_count,
457
+ SN_FLAGS flags)
458
+{
459
+ struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
460
+ struct pg_cache_page_index *page_index = handle->page_index;
461
+ struct rrdeng_page_descr *descr = handle->descr;
462
+
463
+ if(likely(descr)) {
464
+ usec_t last_point_in_time_ut = descr->end_time_ut;
465
+ usec_t update_every_ut = page_index->latest_update_every_s * USEC_PER_SEC;
466
+ size_t points_gap = (point_in_time_ut <= last_point_in_time_ut) ?
467
+ (size_t)0 :
468
+ (size_t)((point_in_time_ut - last_point_in_time_ut) / update_every_ut);
469
+
470
+ if(unlikely(points_gap != 1)) {
471
+ if (unlikely(points_gap <= 0)) {
472
+ time_t now = now_realtime_sec();
473
+ static __thread size_t counter = 0;
474
+ static __thread time_t last_time_logged = 0;
475
+ counter++;
476
+
477
+ if(now - last_time_logged > 600) {
478
+ error("DBENGINE: collected point is in the past (repeated %zu times in the last %zu secs). Ignoring these data collection points.",
479
+ counter, (size_t)(last_time_logged?(now - last_time_logged):0));
480
+
481
+ last_time_logged = now;
482
+ counter = 0;
483
+ }
484
+ return;
485
+ }
486
+
487
+ size_t point_size = PAGE_POINT_SIZE_BYTES(descr);
488
+ size_t page_size_in_points = RRDENG_BLOCK_SIZE / point_size;
489
+ size_t used_points = descr->page_length / point_size;
490
+ size_t remaining_points_in_page = page_size_in_points - used_points;
491
+
492
+ bool new_point_is_aligned = true;
493
+ if(unlikely((point_in_time_ut - last_point_in_time_ut) / points_gap != update_every_ut))
494
+ new_point_is_aligned = false;
495
+
496
+ if(unlikely(points_gap > remaining_points_in_page || !new_point_is_aligned)) {
497
+// char buffer[200];
498
+// snprintfz(buffer, 200, "data collection skipped %zu points, last stored point %llu, new point %llu, update every %d. Cutting page.",
499
+// points_gap, last_point_in_time_ut / USEC_PER_SEC, point_in_time_ut / USEC_PER_SEC, page_index->latest_update_every_s);
500
+// print_page_cache_descr(descr, buffer, false);
501
+
502
+ rrdeng_store_metric_flush_current_page(collection_handle);
503
+ }
504
+ else {
505
+// char buffer[200];
506
+// snprintfz(buffer, 200, "data collection skipped %zu points, last stored point %llu, new point %llu, update every %d. Filling the gap.",
507
+// points_gap, last_point_in_time_ut / USEC_PER_SEC, point_in_time_ut / USEC_PER_SEC, page_index->latest_update_every_s);
508
+// print_page_cache_descr(descr, buffer, false);
509
+
510
+ // loop to fill the gap
511
+ usec_t step_ut = page_index->latest_update_every_s * USEC_PER_SEC;
512
+ usec_t last_point_filled_ut = last_point_in_time_ut + step_ut;
513
+
514
+ while (last_point_filled_ut < point_in_time_ut) {
515
+ rrdeng_store_metric_next_internal(
516
+ collection_handle, last_point_filled_ut, NAN, NAN, NAN,
517
+ 1, 0, SN_EMPTY_SLOT);
518
+
519
+ last_point_filled_ut += step_ut;
520
+ }
521
+ }
522
+ }
523
}
524
+
525
+ rrdeng_store_metric_next_internal(collection_handle, point_in_time_ut, n, min_value, max_value, count, anomaly_count, flags);
526
}
527
528
+
529
/*
530
* Releases the database reference from the handle for storing metrics.
531
* Returns 1 if it's safe to delete the dimension.
532
*/
533
int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
534
struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
364
- struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
365
- struct pg_cache_page_index *page_index = metric_handle->page_index;
535
+ struct pg_cache_page_index *page_index = handle->page_index;
536
537
uint8_t can_delete_metric = 0;
538
@@ -377,6 +547,18 @@ int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
547
return can_delete_metric;
548
}
549
550
+void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every) {
551
+ struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
552
+ struct pg_cache_page_index *page_index = handle->page_index;
553
+ rrdeng_store_metric_flush_current_page(collection_handle);
554
+ uv_rwlock_rdlock(&page_index->lock);
555
+ page_index->latest_update_every_s = update_every;
556
+ uv_rwlock_rdunlock(&page_index->lock);
557
+}
558
+
559
+// ----------------------------------------------------------------------------
560
+// query ops
561
+
562
//static inline uint32_t *pginfo_to_dt(struct rrdeng_page_info *page_info)
563
//{
564
// return (uint32_t *)&page_info->scratch[0];
@@ -391,49 +573,45 @@ int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
573
* Gets a handle for loading metrics from the database.
574
* The handle must be released with rrdeng_load_metric_final().
575
*/
394
-void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type)
576
+void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle, time_t start_time_s, time_t end_time_s)
577
{
396
- struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
397
- struct rrdengine_instance *ctx = metric_handle->ctx;
398
- RRDDIM *rd = metric_handle->rd;
578
+ struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
579
+ struct rrdengine_instance *ctx = page_index->ctx;
580
581
// fprintf(stderr, "%s: %s/%s start time %ld, end time %ld\n", __FUNCTION__ , rd->rrdset->name, rd->name, start_time, end_time);
582
583
struct rrdeng_query_handle *handle;
584
unsigned pages_nr;
585
405
- rrdimm_handle->start_time = start_time;
406
- rrdimm_handle->end_time = end_time;
586
+ if(!page_index->latest_update_every_s)
587
+ page_index->latest_update_every_s = default_rrd_update_every;
588
+
589
+ rrdimm_handle->start_time_s = start_time_s;
590
+ rrdimm_handle->end_time_s = end_time_s;
591
592
handle = callocz(1, sizeof(struct rrdeng_query_handle));
409
- handle->next_page_time = start_time;
410
- handle->now = start_time;
411
- handle->tier_query_fetch_type = tier_query_fetch_type;
412
- // TODO we should store the dt of each page in each page
413
- // this will produce wrong values for dt in case the user changes
414
- // the update every of the charts or the tier grouping iterations
415
- handle->dt_sec = get_tier_grouping(ctx->tier) * (time_t)rd->update_every;
416
- handle->dt = handle->dt_sec * USEC_PER_SEC;
593
+ handle->wanted_start_time_s = start_time_s;
594
+ handle->now_s = start_time_s;
595
handle->position = 0;
596
handle->ctx = ctx;
419
- handle->metric_handle = metric_handle;
597
handle->descr = NULL;
598
+ handle->dt_s = page_index->latest_update_every_s;
599
rrdimm_handle->handle = (STORAGE_QUERY_HANDLE *)handle;
422
- pages_nr = pg_cache_preload(ctx, metric_handle->rrdeng_uuid, start_time * USEC_PER_SEC, end_time * USEC_PER_SEC,
600
+ pages_nr = pg_cache_preload(ctx, &page_index->id, start_time_s * USEC_PER_SEC, end_time_s * USEC_PER_SEC,
601
NULL, &handle->page_index);
602
if (unlikely(NULL == handle->page_index || 0 == pages_nr))
603
// there are no metrics to load
426
- handle->next_page_time = INVALID_TIME;
604
+ handle->wanted_start_time_s = INVALID_TIME;
605
}
606
429
-static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
607
+static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle, bool debug_this __maybe_unused) {
608
struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
609
610
struct rrdengine_instance *ctx = handle->ctx;
611
struct rrdeng_page_descr *descr = handle->descr;
612
613
uint32_t page_length;
436
- usec_t page_end_time;
614
+ usec_t page_end_time_ut;
615
unsigned position;
616
617
if (likely(descr)) {
@@ -445,14 +623,15 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
623
624
pg_cache_put(ctx, descr);
625
handle->descr = NULL;
448
- handle->next_page_time = (handle->page_end_time / USEC_PER_SEC) + 1;
626
+ handle->wanted_start_time_s = (time_t)((handle->page_end_time_ut / USEC_PER_SEC) + handle->dt_s);
627
450
- if (unlikely(handle->next_page_time > rrdimm_handle->end_time))
628
+ if (unlikely(handle->wanted_start_time_s > rrdimm_handle->end_time_s))
629
return 1;
630
}
631
454
- usec_t next_page_time = handle->next_page_time * USEC_PER_SEC;
455
- descr = pg_cache_lookup_next(ctx, handle->page_index, &handle->page_index->id, next_page_time, rrdimm_handle->end_time * USEC_PER_SEC);
632
+ usec_t wanted_start_time_ut = handle->wanted_start_time_s * USEC_PER_SEC;
633
+ descr = pg_cache_lookup_next(ctx, handle->page_index, &handle->page_index->id,
634
+ wanted_start_time_ut, rrdimm_handle->end_time_s * USEC_PER_SEC);
635
if (NULL == descr)
636
return 1;
637
@@ -461,77 +640,116 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
640
#endif
641
642
handle->descr = descr;
464
- pg_cache_atomic_get_pg_info(descr, &page_end_time, &page_length);
465
- if (unlikely(INVALID_TIME == descr->start_time || INVALID_TIME == page_end_time))
643
+ pg_cache_atomic_get_pg_info(descr, &page_end_time_ut, &page_length);
644
+ if (unlikely(INVALID_TIME == descr->start_time_ut || INVALID_TIME == page_end_time_ut || 0 == descr->update_every_s)) {
645
+ error("DBENGINE: discarding invalid page descriptor (start_time = %llu, end_time = %llu, update_every_s = %d)",
646
+ descr->start_time_ut, page_end_time_ut, descr->update_every_s);
647
return 1;
648
+ }
649
468
- if (unlikely(descr->start_time != page_end_time && next_page_time > descr->start_time)) {
650
+ if (unlikely(descr->start_time_ut != page_end_time_ut && wanted_start_time_ut > descr->start_time_ut)) {
651
// we're in the middle of the page somewhere
652
unsigned entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
471
- position = ((uint64_t)(next_page_time - descr->start_time)) * (entries - 1) /
472
- (page_end_time - descr->start_time);
653
+ position = ((uint64_t)(wanted_start_time_ut - descr->start_time_ut)) * (entries - 1) /
654
+ (page_end_time_ut - descr->start_time_ut);
655
}
656
else
657
position = 0;
658
477
- handle->page_end_time = page_end_time;
659
+ handle->page_end_time_ut = page_end_time_ut;
660
handle->page_length = page_length;
661
+ handle->entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
662
handle->page = descr->pg_cache_descr->page;
480
- usec_t entries = handle->entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
481
- if (likely(entries > 1))
482
- handle->dt = (page_end_time - descr->start_time) / (entries - 1);
483
- else {
484
- // TODO we should store the dt of each page in each page
485
- // now we keep the dt of whatever was before
486
- ;
487
- }
488
-
489
- handle->dt_sec = (time_t)(handle->dt / USEC_PER_SEC);
663
+ handle->dt_s = descr->update_every_s;
664
handle->position = position;
665
666
+// if(debug_this)
667
+// info("DBENGINE: rrdeng_load_page_next(), "
668
+// "position:%d, "
669
+// "start_time_ut:%llu, "
670
+// "page_end_time_ut:%llu, "
671
+// "next_page_time_ut:%llu, "
672
+// "in_out:%s"
673
+// , position
674
+// , descr->start_time_ut
675
+// , page_end_time_ut
676
+// ,
677
+// wanted_start_time_ut, in_out?"true":"false"
678
+// );
679
+
680
return 0;
681
}
682
683
// Returns the metric and sets its timestamp into current_time
684
// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
685
// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
498
-STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle) {
499
- struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
686
+STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrddim_handle) {
687
+ struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrddim_handle->handle;
688
// struct rrdeng_metric_handle *metric_handle = handle->metric_handle;
689
502
- STORAGE_POINT sp;
690
struct rrdeng_page_descr *descr = handle->descr;
691
+ time_t now = handle->now_s + handle->dt_s;
692
+
693
+// bool debug_this = false;
694
+// {
695
+// unsigned char u[16] = { 0x0C, 0x0A, 0x40, 0xD6, 0x2A, 0x43, 0x4A, 0x7C, 0x95, 0xF7, 0xD1, 0x1E, 0x0C, 0x9E, 0x8A, 0xE7 };
696
+// if(uuid_compare(u, handle->page_index->id) == 0) {
697
+// char buffer[100];
698
+// snprintfz(buffer, 100, "load system.cpu, now:%u, dt:%u, position:%u page_index first:%u, last:%u",
699
+// (uint32_t)(now),
700
+// (uint32_t)(handle->dt_s),
701
+// (uint32_t)(handle->position),
702
+// (uint32_t)(handle->page_index->oldest_time / USEC_PER_SEC),
703
+// (uint32_t)(handle->page_index->latest_time / USEC_PER_SEC));
704
+//
705
+// print_page_cache_descr(descr, buffer, false);
706
+// debug_this = true;
707
+// }
708
+// }
709
+
710
+ STORAGE_POINT sp;
711
unsigned position = handle->position + 1;
505
- time_t now = handle->now + handle->dt_sec;
712
storage_number_tier1_t tier1_value;
713
508
- if (unlikely(INVALID_TIME == handle->next_page_time)) {
509
- handle->next_page_time = INVALID_TIME;
510
- handle->now = now;
511
- storage_point_empty(sp, now - handle->dt_sec, now);
714
+ if (unlikely(INVALID_TIME == handle->wanted_start_time_s)) {
715
+ handle->wanted_start_time_s = INVALID_TIME;
716
+ handle->now_s = now;
717
+ storage_point_empty(sp, now - handle->dt_s, now);
718
return sp;
719
}
720
721
if (unlikely(!descr || position >= handle->entries)) {
722
// We need to get a new page
517
- if(rrdeng_load_page_next(rrdimm_handle)) {
723
+ if(rrdeng_load_page_next(rrddim_handle, false)) {
724
// next calls will not load any more metrics
519
- handle->next_page_time = INVALID_TIME;
520
- handle->now = now;
521
- storage_point_empty(sp, now - handle->dt_sec, now);
725
+ handle->wanted_start_time_s = INVALID_TIME;
726
+ handle->now_s = now;
727
+ storage_point_empty(sp, now - handle->dt_s, now);
728
return sp;
729
}
730
731
descr = handle->descr;
732
position = handle->position;
527
- now = (time_t)((descr->start_time + position * handle->dt) / USEC_PER_SEC);
733
+ now = (time_t)((descr->start_time_ut / USEC_PER_SEC) + position * descr->update_every_s);
734
+
735
+// if(debug_this) {
736
+// char buffer[100];
737
+// snprintfz(buffer, 100, "NEW PAGE system.cpu, now:%u, dt:%u, position:%u page_index first:%u, last:%u",
738
+// (uint32_t)(now),
739
+// (uint32_t)(handle->dt_s),
740
+// (uint32_t)(handle->position),
741
+// (uint32_t)(handle->page_index->oldest_time / USEC_PER_SEC),
742
+// (uint32_t)(handle->page_index->latest_time / USEC_PER_SEC));
743
+//
744
+// print_page_cache_descr(descr, buffer, false);
745
+// }
746
}
747
530
- sp.start_time = now - handle->dt_sec;
748
+ sp.start_time = now - handle->dt_s;
749
sp.end_time = now;
750
751
handle->position = position;
534
- handle->now = now;
752
+ handle->now_s = now;
753
754
switch(descr->type) {
755
case PAGE_METRICS: {
@@ -566,18 +784,26 @@ STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle)
784
break;
785
}
786
569
- if (unlikely(now >= rrdimm_handle->end_time)) {
787
+ if (unlikely(now >= rrddim_handle->end_time_s)) {
788
// next calls will not load any more metrics
571
- handle->next_page_time = INVALID_TIME;
789
+ handle->wanted_start_time_s = INVALID_TIME;
790
}
791
792
+// if(debug_this)
793
+// info("DBENGINE: returning point: "
794
+// "time from %ld to %ld // query from %ld to %ld // wanted_start_time_s %ld"
795
+// , sp.start_time, sp.end_time
796
+// , rrddim_handle->start_time_s, rrddim_handle->end_time_s
797
+// , handle->wanted_start_time_s
798
+// );
799
+
800
return sp;
801
}
802
803
int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle)
804
{
805
struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
580
- return (INVALID_TIME == handle->next_page_time);
806
+ return (INVALID_TIME == handle->wanted_start_time_s);
807
}
808
809
/*
@@ -602,46 +828,12 @@ void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle)
828
}
829
830
time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
605
- struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
606
-
607
- struct pg_cache_page_index *page_index = metric_handle->page_index;
608
- return page_index->latest_time / USEC_PER_SEC;
831
+ struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
832
+ return (time_t)(page_index->latest_time_ut / USEC_PER_SEC);
833
}
834
time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
611
- struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
612
-
613
- struct pg_cache_page_index *page_index = metric_handle->page_index;
614
- return page_index->oldest_time / USEC_PER_SEC;
615
-}
616
-
617
-int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier)
618
-{
619
- struct page_cache *pg_cache;
620
- struct rrdengine_instance *ctx;
621
- Pvoid_t *PValue;
622
- struct pg_cache_page_index *page_index = NULL;
623
-
624
- ctx = get_rrdeng_ctx_from_host(localhost, tier);
625
- if (unlikely(!ctx)) {
626
- error("Failed to fetch multidb context");
627
- return 1;
628
- }
629
- pg_cache = &ctx->pg_cache;
630
-
631
- uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
632
- PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, dim_uuid, sizeof(uuid_t));
633
- if (likely(NULL != PValue)) {
634
- page_index = *PValue;
635
- }
636
- uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
637
-
638
- if (likely(page_index)) {
639
- *first_entry_t = page_index->oldest_time / USEC_PER_SEC;
640
- *last_entry_t = page_index->latest_time / USEC_PER_SEC;
641
- return 0;
642
- }
643
-
644
- return 1;
835
+ struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
836
+ return (time_t)(page_index->oldest_time_ut / USEC_PER_SEC);
837
}
838
839
int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t)
@@ -666,8 +858,8 @@ int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time
858
uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
859
860
if (likely(page_index)) {
669
- *first_entry_t = page_index->oldest_time / USEC_PER_SEC;
670
- *last_entry_t = page_index->latest_time / USEC_PER_SEC;
861
+ *first_entry_t = page_index->oldest_time_ut / USEC_PER_SEC;
862
+ *last_entry_t = page_index->latest_time_ut / USEC_PER_SEC;
863
return 0;
864
}
865
@@ -694,7 +886,7 @@ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrde
886
887
debug(D_RRDENGINE, "Created new page:");
888
if (unlikely(debug_flags & D_RRDENGINE))
697
- print_page_cache_descr(descr);
889
+ print_page_cache_descr(descr, "", true);
890
rrdeng_page_descr_mutex_unlock(ctx, descr);
891
*ret_descr = descr;
892
return page;
@@ -766,13 +958,13 @@ void *rrdeng_get_latest_page(struct rrdengine_instance *ctx, uuid_t *id, void **
958
}
959
960
/* Gets a reference for the page */
769
-void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time, void **handle)
961
+void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time_ut, void **handle)
962
{
963
struct rrdeng_page_descr *descr;
964
struct page_cache_descr *pg_cache_descr;
965
966
debug(D_RRDENGINE, "Reading existing page:");
775
- descr = pg_cache_lookup(ctx, NULL, id, point_in_time);
967
+ descr = pg_cache_lookup(ctx, NULL, id, point_in_time_ut);
968
if (NULL == descr) {
969
*handle = NULL;
970
@@ -1009,13 +1201,13 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
1201
size_t points = descr->page_length / PAGE_POINT_SIZE_BYTES(descr);
1202
1203
if(likely(points > 1))
1012
- update_every_usec = (descr->end_time - descr->start_time) / (points - 1);
1204
+ update_every_usec = (descr->end_time_ut - descr->start_time_ut) / (points - 1);
1205
else {
1206
update_every_usec = default_rrd_update_every * get_tier_grouping(ctx->tier) * USEC_PER_SEC;
1207
stats.single_point_pages++;
1208
}
1209
1018
- time_t duration_secs = (time_t)((descr->end_time - descr->start_time + update_every_usec)/USEC_PER_SEC);
1210
+ time_t duration_secs = (time_t)((descr->end_time_ut - descr->start_time_ut + update_every_usec)/USEC_PER_SEC);
1211
1212
stats.extents_pages++;
1213
stats.pages_uncompressed_bytes += descr->page_length;
@@ -1027,11 +1219,11 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
1219
stats.page_types[descr->type].pages_duration_secs += duration_secs;
1220
stats.page_types[descr->type].points += points;
1221
1030
- if(!stats.first_t || (descr->start_time - update_every_usec) < stats.first_t)
1031
- stats.first_t = (descr->start_time - update_every_usec) / USEC_PER_SEC;
1222
+ if(!stats.first_t || (descr->start_time_ut - update_every_usec) < stats.first_t)
1223
+ stats.first_t = (descr->start_time_ut - update_every_usec) / USEC_PER_SEC;
1224
1033
- if(!stats.last_t || descr->end_time > stats.last_t)
1034
- stats.last_t = descr->end_time / USEC_PER_SEC;
1225
+ if(!stats.last_t || descr->end_time_ut > stats.last_t)
1226
+ stats.last_t = descr->end_time_ut / USEC_PER_SEC;
1227
}
1228
}
1229
}
@@ -1071,7 +1263,7 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
1263
}
1264
}
1265
1074
- stats.sizeof_metric = struct_natural_alignment(sizeof(struct pg_cache_page_index));
1266
+ stats.sizeof_metric = struct_natural_alignment(sizeof(struct pg_cache_page_index) + sizeof(struct pg_alignment));
1267
stats.sizeof_page = struct_natural_alignment(sizeof(struct rrdeng_page_descr));
1268
stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile));
1269
stats.sizeof_page_in_cache = struct_natural_alignment(sizeof(struct page_cache_descr));
database/engine/rrdengineapi.h
+18
-11
@@ -25,7 +25,7 @@ extern size_t page_type_size[];
25
#define PAGE_POINT_SIZE_BYTES(x) page_type_size[(x)->type]
26
27
struct rrdeng_region_info {
28
- time_t start_time;
28
+ time_t start_time_s;
29
int update_every;
30
unsigned points;
31
};
@@ -34,20 +34,24 @@ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrde
34
void rrdeng_commit_page(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
35
Word_t page_correlation_id);
36
void *rrdeng_get_latest_page(struct rrdengine_instance *ctx, uuid_t *id, void **handle);
37
-void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time, void **handle);
37
+void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time_ut, void **handle);
38
void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle);
39
40
-void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid);
40
+void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid);
41
void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid,
42
uuid_t *ret_uuid);
43
44
45
-STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
46
-void rrdeng_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle);
45
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
46
+STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
47
+STORAGE_METRIC_HANDLE *rrdeng_metric_create(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
48
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_legacy(STORAGE_INSTANCE *db_instance, const char *rd_id, const char *st_id, STORAGE_METRICS_GROUP *smg);
49
+void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
50
48
-STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle);
51
+STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
52
void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
50
-void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE n,
53
+void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
54
+void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
55
NETDATA_DOUBLE min_value,
56
NETDATA_DOUBLE max_value,
57
uint16_t count,
@@ -55,12 +59,13 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
59
SN_FLAGS flags);
60
int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
61
58
-unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
62
+unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time_s, time_t end_time_s,
63
struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
64
65
void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle,
62
- time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
63
-STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle);
66
+ time_t start_time_s, time_t end_time_s);
67
+STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrddim_handle);
68
+
69
70
int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle);
71
void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
@@ -75,9 +80,11 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
80
81
int rrdeng_exit(struct rrdengine_instance *ctx);
82
void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
78
-int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier);
83
int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t);
84
85
+extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
86
+extern void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
87
+
88
typedef struct rrdengine_size_statistics {
89
size_t default_granularity_secs;
90
database/engine/rrdenginelib.c
+37
-20
@@ -4,28 +4,45 @@
4
#define BUFSIZE (512)
5
6
/* Caller must hold descriptor lock */
7
-void print_page_cache_descr(struct rrdeng_page_descr *descr)
7
+void print_page_cache_descr(struct rrdeng_page_descr *descr, const char *msg, bool log_debug)
8
{
9
- struct page_cache_descr *pg_cache_descr = descr->pg_cache_descr;
10
- char uuid_str[UUID_STR_LEN];
11
- char str[BUFSIZE + 1];
12
- int pos = 0;
9
+ if(log_debug && !(debug_flags & D_RRDENGINE))
10
+ return;
11
14
- uuid_unparse_lower(*descr->id, uuid_str);
15
- pos += snprintfz(str, BUFSIZE - pos, "page(%p) id=%s\n"
16
- "--->len:%"PRIu32" time:%"PRIu64"->%"PRIu64" xt_offset:",
17
- pg_cache_descr->page, uuid_str,
18
- descr->page_length,
19
- (uint64_t)descr->start_time,
20
- (uint64_t)descr->end_time);
21
- if (!descr->extent) {
22
- pos += snprintfz(str + pos, BUFSIZE - pos, "N/A");
23
- } else {
24
- pos += snprintfz(str + pos, BUFSIZE - pos, "%"PRIu64, descr->extent->offset);
12
+ BUFFER *wb = buffer_create(512);
13
+
14
+ if(!descr) {
15
+ buffer_sprintf(wb, "DBENGINE: %s : descr is NULL", msg);
16
}
17
+ else {
18
+ struct page_cache_descr *pg_cache_descr = descr->pg_cache_descr;
19
+ char uuid_str[UUID_STR_LEN];
20
+
21
+ uuid_unparse_lower(*descr->id, uuid_str);
22
+ buffer_sprintf(wb, "DBENGINE: %s : page(%p) metric:%s, len:%"PRIu32", time:%"PRIu64"->%"PRIu64", update_every:%u, type:%u, xt_offset:",
23
+ msg,
24
+ pg_cache_descr->page, uuid_str,
25
+ descr->page_length,
26
+ (uint64_t)descr->start_time_ut,
27
+ (uint64_t)descr->end_time_ut,
28
+ (uint32_t)descr->update_every_s,
29
+ (uint32_t)descr->type
30
+ );
31
+ if (!descr->extent) {
32
+ buffer_strcat(wb, "N/A");
33
+ } else {
34
+ buffer_sprintf(wb, "%"PRIu64, descr->extent->offset);
35
+ }
36
+
37
+ buffer_sprintf(wb, ", flags:0x%2.2lX refcnt:%u", pg_cache_descr->flags, pg_cache_descr->refcnt);
38
+ }
39
+
40
+ if(log_debug)
41
+ debug(D_RRDENGINE, "%s", buffer_tostring(wb));
42
+ else
43
+ internal_error(true, "%s", buffer_tostring(wb));
44
27
- snprintfz(str + pos, BUFSIZE - pos, " flags:0x%2.2lX refcnt:%u\n\n", pg_cache_descr->flags, pg_cache_descr->refcnt);
28
- debug(D_RRDENGINE, "%s", str);
45
+ buffer_free(wb);
46
}
47
48
void print_page_descr(struct rrdeng_page_descr *descr)
@@ -39,8 +56,8 @@ void print_page_descr(struct rrdeng_page_descr *descr)
56
"--->len:%"PRIu32" time:%"PRIu64"->%"PRIu64" xt_offset:",
57
uuid_str,
58
descr->page_length,
42
- (uint64_t)descr->start_time,
43
- (uint64_t)descr->end_time);
59
+ (uint64_t)descr->start_time_ut,
60
+ (uint64_t)descr->end_time_ut);
61
if (!descr->extent) {
62
pos += snprintfz(str + pos, BUFSIZE - pos, "N/A");
63
} else {
database/engine/rrdenginelib.h
+1
-1
@@ -83,7 +83,7 @@ static inline void crc32set(void *crcp, uLong crc)
83
*(uint32_t *)crcp = crc;
84
}
85
86
-void print_page_cache_descr(struct rrdeng_page_descr *page_cache_descr);
86
+void print_page_cache_descr(struct rrdeng_page_descr *descr, const char *msg, bool log_debug);
87
void print_page_descr(struct rrdeng_page_descr *descr);
88
int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size);
89
int open_file_for_io(char *path, int flags, uv_file *file, int direct);
database/ram/rrddim_mem.c
+61
-11
@@ -2,18 +2,67 @@
2
3
#include "rrddim_mem.h"
4
5
+static Pvoid_t rrddim_JudyHS_array = NULL;
6
+static netdata_rwlock_t rrddim_JudyHS_rwlock = NETDATA_RWLOCK_INITIALIZER;
7
+
8
+
9
+// ----------------------------------------------------------------------------
10
+// metrics groups
11
+
12
+STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid __maybe_unused) {
13
+ return NULL;
14
+}
15
+
16
+void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance __maybe_unused, STORAGE_METRICS_GROUP *smg __maybe_unused) {
17
+ // if(!smg) return; // smg may be NULL
18
+ ;
19
+}
20
+
21
// ----------------------------------------------------------------------------
22
// RRDDIM legacy data collection functions
23
8
-STORAGE_METRIC_HANDLE *rrddim_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance __maybe_unused) {
24
+STORAGE_METRIC_HANDLE *
25
+rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance __maybe_unused, STORAGE_METRICS_GROUP *smg __maybe_unused) {
26
+ STORAGE_METRIC_HANDLE *t = rrddim_metric_get(db_instance, &rd->metric_uuid, smg);
27
+ if(!t) {
28
+ netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock);
29
+ Pvoid_t *PValue = JudyHSIns(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0);
30
+ fatal_assert(NULL == *PValue);
31
+ *PValue = rd;
32
+ t = (STORAGE_METRIC_HANDLE *)rd;
33
+ netdata_rwlock_unlock(&rrddim_JudyHS_rwlock);
34
+ }
35
+
36
+ if((RRDDIM *)t != rd)
37
+ fatal("RRDDIM_MEM: incorrect pointer returned from index.");
38
+
39
return (STORAGE_METRIC_HANDLE *)rd;
40
}
41
12
-void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused) {
13
- ;
42
+STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid, STORAGE_METRICS_GROUP *smg __maybe_unused) {
43
+ RRDDIM *rd = NULL;
44
+ netdata_rwlock_rdlock(&rrddim_JudyHS_rwlock);
45
+ Pvoid_t *PValue = JudyHSGet(rrddim_JudyHS_array, uuid, sizeof(uuid_t));
46
+ if (likely(NULL != PValue))
47
+ rd = *PValue;
48
+ netdata_rwlock_unlock(&rrddim_JudyHS_rwlock);
49
+
50
+ return (STORAGE_METRIC_HANDLE *)rd;
51
+}
52
+
53
+void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused) {
54
+ RRDDIM *rd = (RRDDIM *)db_metric_handle;
55
+
56
+ netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock);
57
+ JudyHSDel(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0);
58
+ netdata_rwlock_unlock(&rrddim_JudyHS_rwlock);
59
+}
60
+
61
+void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every __maybe_unused) {
62
+ rrddim_store_metric_flush(collection_handle);
63
}
64
16
-STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle) {
65
+STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every __maybe_unused) {
66
RRDDIM *rd = (RRDDIM *)db_metric_handle;
67
rd->db[rd->rrdset->current_entry] = pack_storage_number(NAN, SN_FLAG_NONE);
68
struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle));
@@ -41,8 +90,11 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec
90
91
void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
92
struct mem_collect_handle *ch = (struct mem_collect_handle *)collection_handle;
93
+
94
RRDDIM *rd = ch->rd;
45
- memset(rd->db, 0, rd->rrdset->entries * sizeof(storage_number));
95
+ for(int i = 0; i < rd->rrdset->entries ;i++)
96
+ rd->db[i] = SN_EMPTY_SLOT;
97
+
98
}
99
100
int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
@@ -134,14 +186,12 @@ static inline time_t rrddim_slot2time(RRDDIM *rd, size_t slot) {
186
// ----------------------------------------------------------------------------
187
// RRDDIM legacy database query functions
188
137
-void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type) {
138
- UNUSED(tier_query_fetch_type);
139
-
189
+void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time) {
190
RRDDIM *rd = (RRDDIM *)db_metric_handle;
191
192
handle->rd = rd;
143
- handle->start_time = start_time;
144
- handle->end_time = end_time;
193
+ handle->start_time_s = start_time;
194
+ handle->end_time_s = end_time;
195
struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle));
196
h->slot = rrddim_time2slot(rd, start_time);
197
h->last_slot = rrddim_time2slot(rd, end_time);
@@ -200,7 +250,7 @@ STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle) {
250
251
int rrddim_query_is_finished(struct rrddim_query_handle *handle) {
252
struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;
203
- return (h->next_timestamp > handle->end_time);
253
+ return (h->next_timestamp > handle->end_time_s);
254
}
255
256
void rrddim_query_finalize(struct rrddim_query_handle *handle) {
database/ram/rrddim_mem.h
+9
-4
@@ -20,10 +20,15 @@ struct mem_query_handle {
20
size_t last_slot;
21
};
22
23
-STORAGE_METRIC_HANDLE *rrddim_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
24
-void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle);
23
+STORAGE_METRIC_HANDLE *rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
24
+STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
25
+void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
26
26
-STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle);
27
+STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
28
+void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
29
+
30
+STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
31
+void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
32
void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number,
33
NETDATA_DOUBLE min_value,
34
NETDATA_DOUBLE max_value,
@@ -33,7 +38,7 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec
38
void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
39
int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
40
36
-void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
41
+void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
42
STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle);
43
int rrddim_query_is_finished(struct rrddim_query_handle *handle);
44
void rrddim_query_finalize(struct rrddim_query_handle *handle);
database/rrd.h
+12
-4
@@ -11,6 +11,7 @@ extern "C" {
11
// to enable type checking at compile time
12
typedef struct storage_instance STORAGE_INSTANCE;
13
typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
14
+typedef struct storage_alignment STORAGE_METRICS_GROUP;
15
16
// forward typedefs
17
typedef struct rrdhost RRDHOST;
@@ -356,8 +357,8 @@ typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
357
// iterator state for RRD dimension data queries
358
struct rrddim_query_handle {
359
RRDDIM *rd;
359
- time_t start_time;
360
- time_t end_time;
360
+ time_t start_time_s;
361
+ time_t end_time_s;
362
STORAGE_QUERY_HANDLE* handle;
363
};
364
@@ -401,7 +402,7 @@ typedef struct storage_point {
402
// function pointers that handle data collection
403
struct rrddim_collect_ops {
404
// an initialization function to run before starting collection
404
- STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle);
405
+ STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
406
407
// run this to store each metric into the database
408
void (*store_metric)(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number, NETDATA_DOUBLE min_value,
@@ -413,12 +414,17 @@ struct rrddim_collect_ops {
414
// a finalization function to run after collection is over
415
// returns 1 if it's safe to delete the dimension
416
int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle);
417
+
418
+ void (*change_collection_frequency)(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
419
+
420
+ STORAGE_METRICS_GROUP *(*metrics_group_get)(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
421
+ void (*metrics_group_release)(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *sa);
422
};
423
424
// function pointers that handle database queries
425
struct rrddim_query_ops {
426
// run this before starting a series of next_metric() database queries
421
- void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
427
+ void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
428
429
// run this to load each metric number from the database
430
STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
@@ -557,6 +563,8 @@ struct rrdset {
563
// netdata will interpolate values for gaps lower than this
564
// TODO - use the global - all charts have the same value
565
566
+ STORAGE_METRICS_GROUP *storage_metrics_groups[RRD_STORAGE_TIERS];
567
+
568
// ------------------------------------------------------------------------
569
// linking to siblings and parents
570
database/rrddim.c
+4
-4
@@ -101,7 +101,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
101
rd->tiers[tier]->mode = eng->id;
102
rd->tiers[tier]->collect_ops = eng->api.collect_ops;
103
rd->tiers[tier]->query_ops = eng->api.query_ops;
104
- rd->tiers[tier]->db_metric_handle = eng->api.init(rd, host->storage_instance[tier]);
104
+ rd->tiers[tier]->db_metric_handle = eng->api.metric_get_or_create(rd, host->storage_instance[tier], rd->rrdset->storage_metrics_groups[tier]);
105
storage_point_unset(rd->tiers[tier]->virtual_point);
106
initialized++;
107
@@ -120,7 +120,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
120
size_t initialized = 0;
121
for (int tier = 0; tier < storage_tiers; tier++) {
122
if (rd->tiers[tier]) {
123
- rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
123
+ rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle, st->update_every * storage_tiers_grouping_iterations[tier]);
124
initialized++;
125
}
126
}
@@ -217,7 +217,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
217
218
STORAGE_ENGINE* eng = storage_engine_get(rd->tiers[tier]->mode);
219
if(eng)
220
- eng->api.free(rd->tiers[tier]->db_metric_handle);
220
+ eng->api.metric_release(rd->tiers[tier]->db_metric_handle);
221
222
freez(rd->tiers[tier]);
223
rd->tiers[tier] = NULL;
@@ -253,7 +253,7 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
253
for(int tier = 0; tier < storage_tiers ;tier++) {
254
if (rd->tiers[tier])
255
rd->tiers[tier]->db_collection_handle =
256
- rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
256
+ rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle, st->update_every * storage_tiers_grouping_iterations[tier]);
257
}
258
259
rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
database/rrdset.c
+87
-52
@@ -3,6 +3,7 @@
3
#define NETDATA_RRD_INTERNALS
4
#include "rrd.h"
5
#include <sched.h>
6
+#include "storage_engine.h"
7
8
// ----------------------------------------------------------------------------
9
@@ -109,7 +110,7 @@ struct rrdset_constructor {
110
enum {
111
RRDSET_REACT_NONE = 0,
112
RRDSET_REACT_NEW = (1 << 0),
112
- RRDSET_REACT_CHART_ARCHIVED_TO_LIVE = (1 << 1),
113
+ RRDSET_REACT_UPDATED = (1 << 1),
114
RRDSET_REACT_PLUGIN_UPDATED = (1 << 2),
115
RRDSET_REACT_MODULE_UPDATED = (1 << 3),
116
RRDSET_REACT_CHART_ACTIVATED = (1 << 4),
@@ -176,6 +177,17 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
177
uuid_generate(st->chart_uuid);
178
update_chart_metadata(&st->chart_uuid, st, string2str(st->parts.id), string2str(st->parts.name));
179
180
+ // initialize the db tiers
181
+ {
182
+ RRD_MEMORY_MODE wanted_mode = ctr->memory_mode;
183
+ for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
184
+ STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
185
+ if(!eng) continue;
186
+
187
+ st->storage_metrics_groups[tier] = eng->api.collect_ops.metrics_group_get(host->storage_instance[tier], &st->chart_uuid);
188
+ }
189
+ }
190
+
191
rrddim_index_init(st);
192
193
// chart variables - we need this for data collection to work (collector given chart variables) - not only health
@@ -203,6 +215,17 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
215
216
rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID);
217
218
+ // cleanup storage engines
219
+ {
220
+ RRD_MEMORY_MODE wanted_mode = st->rrd_memory_mode;
221
+ for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
222
+ STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
223
+ if(!eng) continue;
224
+
225
+ eng->api.collect_ops.metrics_group_release(host->storage_instance[tier], st->storage_metrics_groups[tier]);
226
+ }
227
+ }
228
+
229
// remove it from the name index
230
rrdset_index_del_name(host, st);
231
@@ -281,15 +304,27 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
304
}
305
306
if (rrdset_reset_name(st, (ctr->name && *ctr->name) ? ctr->name : ctr->id) == 2)
284
- ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
307
+ ctr->react_action |= RRDSET_REACT_UPDATED;
308
309
if (unlikely(st->priority != ctr->priority)) {
310
st->priority = ctr->priority;
288
- ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
311
+ ctr->react_action |= RRDSET_REACT_UPDATED;
312
}
290
- if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && st->update_every != ctr->update_every)) {
313
+
314
+ if (unlikely(st->update_every != ctr->update_every)) {
315
st->update_every = ctr->update_every;
292
- ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
316
+
317
+ // switch update every to the storage engine
318
+ RRDDIM *rd;
319
+ rrddim_foreach_read(rd, st) {
320
+ for (int tier = 0; tier < storage_tiers; tier++) {
321
+ if (rd->tiers[tier] && rd->tiers[tier]->db_collection_handle)
322
+ rd->tiers[tier]->collect_ops.change_collection_frequency(rd->tiers[tier]->db_collection_handle, st->update_every);
323
+ }
324
+ }
325
+ rrddim_foreach_done(rd);
326
+
327
+ ctr->react_action |= RRDSET_REACT_UPDATED;
328
}
329
330
if(ctr->plugin && *ctr->plugin) {
@@ -312,7 +347,7 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
347
STRING *old_title = st->title;
348
st->title = rrd_string_strdupz(ctr->title);
349
if(old_title != st->title)
315
- ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
350
+ ctr->react_action |= RRDSET_REACT_UPDATED;
351
string_freez(old_title);
352
}
353
@@ -320,7 +355,7 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
355
STRING *old_units = st->units;
356
st->units = rrd_string_strdupz(ctr->units);
357
if(old_units != st->units)
323
- ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
358
+ ctr->react_action |= RRDSET_REACT_UPDATED;
359
string_freez(old_units);
360
}
361
@@ -328,13 +363,13 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
363
STRING *old_context = st->context;
364
st->context = rrd_string_strdupz(ctr->context);
365
if(old_context != st->context)
331
- ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
366
+ ctr->react_action |= RRDSET_REACT_UPDATED;
367
string_freez(old_context);
368
}
369
370
if(st->chart_type != ctr->chart_type) {
371
st->chart_type = ctr->chart_type;
337
- ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
372
+ ctr->react_action |= RRDSET_REACT_UPDATED;
373
}
374
375
rrdset_update_permanent_labels(st);
@@ -357,7 +392,7 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
392
rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
393
}
394
360
- if(ctr->react_action & (RRDSET_REACT_CHART_ARCHIVED_TO_LIVE | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
395
+ if(ctr->react_action & (RRDSET_REACT_UPDATED | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
396
debug(D_METADATALOG, "CHART [%s] metadata updated", rrdset_id(st));
397
if(unlikely(update_chart_metadata(&st->chart_uuid, st, ctr->id, ctr->name)))
398
error_report("Failed to update chart metadata in the database");
@@ -998,10 +1033,38 @@ static inline time_t tier_next_point_time(RRDDIM *rd, struct rrddim_tier *t, tim
1033
return now + loop - ((now + loop) % loop);
1034
}
1035
1001
-void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut) {
1036
+void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {
1037
if (unlikely(!t->next_point_time))
1038
t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
1039
1040
+ if(unlikely(sp.start_time > t->next_point_time)) {
1041
+ if (likely(!storage_point_is_unset(t->virtual_point))) {
1042
+
1043
+ t->collect_ops.store_metric(
1044
+ t->db_collection_handle,
1045
+ t->next_point_time * USEC_PER_SEC,
1046
+ t->virtual_point.sum,
1047
+ t->virtual_point.min,
1048
+ t->virtual_point.max,
1049
+ t->virtual_point.count,
1050
+ t->virtual_point.anomaly_count,
1051
+ t->virtual_point.flags);
1052
+ }
1053
+ else {
1054
+ t->collect_ops.store_metric(
1055
+ t->db_collection_handle,
1056
+ t->next_point_time * USEC_PER_SEC,
1057
+ NAN,
1058
+ NAN,
1059
+ NAN,
1060
+ 0,
1061
+ 0, SN_FLAG_NONE);
1062
+ }
1063
+
1064
+ t->virtual_point.count = 0; // make the point unset
1065
+ t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
1066
+ }
1067
+
1068
// merge the dates into our virtual point
1069
if (unlikely(sp.start_time < t->virtual_point.start_time))
1070
t->virtual_point.start_time = sp.start_time;
@@ -1027,34 +1090,6 @@ void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, u
1090
t->virtual_point = sp;
1091
}
1092
}
1030
-
1031
- if(unlikely(sp.end_time >= t->next_point_time)) {
1032
- if (likely(!storage_point_is_unset(t->virtual_point))) {
1033
-
1034
- t->collect_ops.store_metric(
1035
- t->db_collection_handle,
1036
- now_ut,
1037
- t->virtual_point.sum,
1038
- t->virtual_point.min,
1039
- t->virtual_point.max,
1040
- t->virtual_point.count,
1041
- t->virtual_point.anomaly_count,
1042
- t->virtual_point.flags);
1043
- }
1044
- else {
1045
- t->collect_ops.store_metric(
1046
- t->db_collection_handle,
1047
- now_ut,
1048
- NAN,
1049
- NAN,
1050
- NAN,
1051
- 0,
1052
- 0, SN_FLAG_NONE);
1053
- }
1054
-
1055
- t->virtual_point.count = 0;
1056
- t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
1057
- }
1093
}
1094
1095
static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
@@ -1062,30 +1097,30 @@ static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1097
// store the metric on tier 0
1098
rd->tiers[0]->collect_ops.store_metric(rd->tiers[0]->db_collection_handle, point_end_time_ut, n, 0, 0, 1, 0, flags);
1099
1100
+ time_t now = (time_t)(point_end_time_ut / USEC_PER_SEC);
1101
+
1102
+ STORAGE_POINT sp = {
1103
+ .start_time = now - rd->update_every,
1104
+ .end_time = now,
1105
+ .min = n,
1106
+ .max = n,
1107
+ .sum = n,
1108
+ .count = 1,
1109
+ .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,
1110
+ .flags = flags
1111
+ };
1112
+
1113
for(int tier = 1; tier < storage_tiers ;tier++) {
1114
if(unlikely(!rd->tiers[tier])) continue;
1115
1116
struct rrddim_tier *t = rd->tiers[tier];
1117
1070
- time_t now = (time_t)(point_end_time_ut / USEC_PER_SEC);
1071
-
1118
if(!t->last_collected_ut) {
1119
// we have not collected this tier before
1120
// let's fill any gap that may exist
1121
rrdr_fill_tier_gap_from_smaller_tiers(rd, tier, now);
1122
}
1123
1078
- STORAGE_POINT sp = {
1079
- .start_time = now - rd->update_every,
1080
- .end_time = now,
1081
- .min = n,
1082
- .max = n,
1083
- .sum = n,
1084
- .count = 1,
1085
- .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,
1086
- .flags = flags
1087
- };
1088
-
1124
t->last_collected_ut = point_end_time_ut;
1125
store_metric_at_tier(rd, t, sp, point_end_time_ut);
1126
}
database/sqlite/sqlite_functions.c
-307
@@ -1330,98 +1330,6 @@ failed:
1330
return;
1331
}
1332
1333
-void free_temporary_host(RRDHOST *host)
1334
-{
1335
- if (host) {
1336
- string_freez(host->hostname);
1337
- string_freez(host->os);
1338
- string_freez(host->tags);
1339
- string_freez(host->timezone);
1340
- string_freez(host->program_name);
1341
- string_freez(host->program_version);
1342
- string_freez(host->registry_hostname);
1343
- freez(host->system_info);
1344
- freez(host);
1345
- }
1346
-}
1347
-
1348
-#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;"
1349
-#define SELECT_HOST_BY_UUID "select h.host_id, h.registry_hostname, h.update_every, h.os, h.timezone, h.tags from host h, node_instance ni " \
1350
- "where (ni.host_id = @host_id or ni.node_id = @host_id) AND ni.host_id = h.host_id;"
1351
-
1352
-RRDHOST *sql_create_host_by_uuid(char *hostname)
1353
-{
1354
- int rc;
1355
- RRDHOST *host = NULL;
1356
- uuid_t host_uuid;
1357
-
1358
- sqlite3_stmt *res = NULL;
1359
-
1360
- rc = uuid_parse(hostname, host_uuid);
1361
- if (!rc) {
1362
- rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_BY_UUID, -1, &res, 0);
1363
- if (unlikely(rc != SQLITE_OK)) {
1364
- error_report("Failed to prepare statement to fetch host by uuid");
1365
- return NULL;
1366
- }
1367
- rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);
1368
- if (unlikely(rc != SQLITE_OK)) {
1369
- error_report("Failed to bind host_id parameter to fetch host information");
1370
- goto failed;
1371
- }
1372
- }
1373
- else {
1374
- rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0);
1375
- if (unlikely(rc != SQLITE_OK)) {
1376
- error_report("Failed to prepare statement to fetch host by hostname");
1377
- return NULL;
1378
- }
1379
- rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC);
1380
- if (unlikely(rc != SQLITE_OK)) {
1381
- error_report("Failed to bind hostname parameter to fetch host information");
1382
- goto failed;
1383
- }
1384
- }
1385
-
1386
- rc = sqlite3_step_monitored(res);
1387
- if (unlikely(rc != SQLITE_ROW)) {
1388
- error_report("Failed to find hostname %s", hostname);
1389
- goto failed;
1390
- }
1391
-
1392
- char uuid_str[GUID_LEN + 1];
1393
- uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 0)), uuid_str);
1394
-
1395
- host = callocz(1, sizeof(RRDHOST));
1396
-
1397
- set_host_properties(host, sqlite3_column_int(res, 2),
1398
- RRD_MEMORY_MODE_DBENGINE,
1399
- (char *) sqlite3_column_text(res, 1),
1400
- (char *) sqlite3_column_text(res, 3),
1401
- (char *) sqlite3_column_text(res, 5),
1402
- (char *) sqlite3_column_text(res, 4),
1403
- NULL, 0, NULL, NULL);
1404
-
1405
- uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
1406
-
1407
- host->system_info = callocz(1, sizeof(*host->system_info));;
1408
- rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
1409
-
1410
-#ifdef ENABLE_DBENGINE
1411
- if(dbengine_enabled) {
1412
- for (int tier = 0; tier < storage_tiers; tier++)
1413
- host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
1414
- }
1415
-#endif
1416
-
1417
-failed:
1418
- rc = sqlite3_finalize(res);
1419
- if (unlikely(rc != SQLITE_OK))
1420
- error_report("Failed to finalize the prepared statement when reading host information");
1421
-
1422
- return host;
1423
-}
1424
-
1333
void db_execute(const char *cmd)
1334
{
1335
int rc;
@@ -1586,221 +1494,6 @@ static void sql_store_host_label(uuid_t *host_uuid, int source_type, const char
1494
(void) sql_store_label(res, host_uuid, source_type, label, value);
1495
}
1496
1589
-int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id,
1590
- uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid, int tier)
1591
-{
1592
-#ifdef ENABLE_DBENGINE
1593
- int rc;
1594
- uuid_t legacy_uuid;
1595
- uuid_t multihost_legacy_uuid;
1596
- time_t dim_first_entry_t, dim_last_entry_t;
1597
-
1598
- rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
1599
- if (unlikely(rc)) {
1600
- rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid);
1601
- rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
1602
- if (likely(rc)) {
1603
- rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid);
1604
- rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
1605
- if (likely(!rc))
1606
- uuid_copy(*rrdeng_uuid, multihost_legacy_uuid);
1607
- }
1608
- else
1609
- uuid_copy(*rrdeng_uuid, legacy_uuid);
1610
- }
1611
- else
1612
- uuid_copy(*rrdeng_uuid, *uuid);
1613
-
1614
- if (likely(!rc)) {
1615
- *first_entry_t = MIN(*first_entry_t, dim_first_entry_t);
1616
- *last_entry_t = MAX(*last_entry_t, dim_last_entry_t);
1617
- }
1618
- return rc;
1619
-#else
1620
- UNUSED(machine_guid);
1621
- UNUSED(chart_id);
1622
- UNUSED(dim_id);
1623
- UNUSED(uuid);
1624
- UNUSED(first_entry_t);
1625
- UNUSED(last_entry_t);
1626
- UNUSED(rrdeng_uuid);
1627
- return 1;
1628
-#endif
1629
-}
1630
-#include "../storage_engine.h"
1631
-#ifdef ENABLE_DBENGINE
1632
-static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
1633
-{
1634
- STORAGE_ENGINE *eng = storage_engine_get(RRD_MEMORY_MODE_DBENGINE);
1635
-
1636
- if (unlikely(!eng))
1637
- return NULL;
1638
-
1639
- RRDDIM *rd = onewayalloc_callocz(owa, 1, sizeof(*rd));
1640
- rd->rrdset = st;
1641
- rd->update_every = st->update_every;
1642
- rd->last_stored_value = NAN;
1643
- rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
1644
-
1645
- uuid_copy(rd->metric_uuid, *metric_uuid);
1646
- rd->id = string_strdupz(id);
1647
- rd->name = string_strdupz(name);
1648
-
1649
- for(int tier = 0; tier < storage_tiers ;tier++) {
1650
- rd->tiers[tier] = onewayalloc_callocz(owa, 1, sizeof(*rd->tiers[tier]));
1651
- rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
1652
- rd->tiers[tier]->tier_grouping = get_tier_grouping(tier);
1653
- rd->tiers[tier]->mode = RRD_MEMORY_MODE_DBENGINE;
1654
- rd->tiers[tier]->query_ops.init = rrdeng_load_metric_init;
1655
- rd->tiers[tier]->query_ops.next_metric = rrdeng_load_metric_next;
1656
- rd->tiers[tier]->query_ops.is_finished = rrdeng_load_metric_is_finished;
1657
- rd->tiers[tier]->query_ops.finalize = rrdeng_load_metric_finalize;
1658
- rd->tiers[tier]->query_ops.latest_time = rrdeng_metric_latest_time;
1659
- rd->tiers[tier]->query_ops.oldest_time = rrdeng_metric_oldest_time;
1660
- rd->tiers[tier]->db_metric_handle = eng->api.init(rd, st->rrdhost->storage_instance[tier]);
1661
- }
1662
-
1663
- return rd;
1664
-}
1665
-#endif
1666
-
1667
-#define SELECT_CHART_CONTEXT "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, " \
1668
- "c.context, CASE WHEN d.options = 'hidden' THEN 1 else 0 END from chart c, " \
1669
- "dimension d, host h " \
1670
- "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.context = @context " \
1671
- "order by c.chart_id asc, c.type||c.id desc;"
1672
-
1673
-#define SELECT_CHART_SINGLE "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, " \
1674
- "c.context, CASE WHEN d.options = 'hidden' THEN 1 else 0 END from chart c, " \
1675
- "dimension d, host h " \
1676
- "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \
1677
- "order by c.chart_id asc, c.type||'.'||c.id desc;"
1678
-
1679
-void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart)
1680
-{
1681
-#ifdef ENABLE_DBENGINE
1682
- int rc;
1683
-
1684
- if (unlikely(!param_list) || host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1685
- return;
1686
-
1687
- if (unlikely(!(*param_list))) {
1688
- *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
1689
- (*param_list)->first_entry_t = LONG_MAX;
1690
- (*param_list)->last_entry_t = 0;
1691
- (*param_list)->rd = NULL;
1692
- (*param_list)->flags = CONTEXT_FLAGS_ARCHIVE;
1693
- if (chart)
1694
- (*param_list)->flags |= CONTEXT_FLAGS_CHART;
1695
- else
1696
- (*param_list)->flags |= CONTEXT_FLAGS_CONTEXT;
1697
- }
1698
-
1699
- sqlite3_stmt *res = NULL;
1700
-
1701
- if (context)
1702
- rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_CONTEXT, -1, &res, 0);
1703
- else
1704
- rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_SINGLE, -1, &res, 0);
1705
- if (unlikely(rc != SQLITE_OK)) {
1706
- error_report("Failed to prepare statement to fetch host archived charts");
1707
- return;
1708
- }
1709
-
1710
- rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
1711
- if (unlikely(rc != SQLITE_OK)) {
1712
- error_report("Failed to bind host parameter to fetch archived charts");
1713
- goto failed;
1714
- }
1715
-
1716
- if (context)
1717
- rc = sqlite3_bind_text(res, 2, context, -1, SQLITE_STATIC);
1718
- else
1719
- rc = sqlite3_bind_text(res, 2, chart, -1, SQLITE_STATIC);
1720
- if (unlikely(rc != SQLITE_OK)) {
1721
- error_report("Failed to bind host parameter to fetch archived charts");
1722
- goto failed;
1723
- }
1724
-
1725
- RRDSET *st = NULL;
1726
- char machine_guid[GUID_LEN + 1];
1727
- uuid_unparse_lower(host->host_uuid, machine_guid);
1728
- uuid_t rrdeng_uuid;
1729
- uuid_t chart_id;
1730
-
1731
- while (sqlite3_step_monitored(res) == SQLITE_ROW) {
1732
- char id[512];
1733
- sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1));
1734
-
1735
- if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) {
1736
- if (unlikely(st && !st->counter)) {
1737
- string_freez(st->context);
1738
- string_freez(st->name);
1739
- string_freez(st->id);
1740
- onewayalloc_freez(owa, st);
1741
- }
1742
- st = onewayalloc_callocz(owa, 1, sizeof(*st));
1743
- char n[RRD_ID_LENGTH_MAX + 1];
1744
-
1745
- snprintfz(
1746
- n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4),
1747
- (char *)sqlite3_column_text(res, 3));
1748
- st->name = string_strdupz(n);
1749
- st->update_every = sqlite3_column_int(res, 6);
1750
- st->counter = 0;
1751
- if (chart) {
1752
- st->context = string_strdupz((char *)sqlite3_column_text(res, 8));
1753
- st->id = string_strdupz(chart);
1754
- } else
1755
- st->id = string_strdupz(n);
1756
-
1757
- uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7));
1758
- st->last_entry_t = 0;
1759
- st->rrdhost = host;
1760
- }
1761
-
1762
- if (unlikely(find_dimension_first_last_t(machine_guid, (char *)rrdset_name(st), (char *)sqlite3_column_text(res, 1),
1763
- (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t,
1764
- &rrdeng_uuid, 0)))
1765
- continue;
1766
-
1767
- st->counter++;
1768
- st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t);
1769
-
1770
- RRDDIM *rd = create_rrdim_entry(owa, st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
1771
- if (unlikely(!rd))
1772
- continue;
1773
- if (sqlite3_column_int(res, 9) == 1)
1774
- rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN);
1775
- rd->next = (*param_list)->rd;
1776
- (*param_list)->rd = rd;
1777
- }
1778
- if (st) {
1779
- if (!st->counter) {
1780
- string_freez(st->context);
1781
- string_freez(st->name);
1782
- string_freez(st->id);
1783
- onewayalloc_freez(owa,st);
1784
- }
1785
- else
1786
- if (!st->context && context)
1787
- st->context = string_strdupz(context);
1788
- }
1789
-
1790
-failed:
1791
- rc = sqlite3_finalize(res);
1792
- if (unlikely(rc != SQLITE_OK))
1793
- error_report("Failed to finalize the prepared statement when reading archived charts");
1794
-#else
1795
- UNUSED(param_list);
1796
- UNUSED(host);
1797
- UNUSED(context);
1798
- UNUSED(chart);
1799
-#endif
1800
- return;
1801
-}
1802
-
1803
-
1497
#define SQL_STORE_CLAIM_ID "insert into node_instance " \
1498
"(host_id, claim_id, date_created) values (@host_id, @claim_id, unixepoch()) " \
1499
"on conflict(host_id) do update set claim_id = excluded.claim_id;"
database/sqlite/sqlite_functions.h
-3
@@ -85,7 +85,6 @@ int find_uuid_type(uuid_t *uuid);
85
86
void sql_rrdset2json(RRDHOST *host, BUFFER *wb);
87
88
-RRDHOST *sql_create_host_by_uuid(char *guid);
88
int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement);
89
int execute_insert(sqlite3_stmt *res);
90
void db_execute(const char *cmd);
@@ -94,7 +93,6 @@ void add_migrated_file(char *path, uint64_t file_size);
93
void db_unlock(void);
94
void db_lock(void);
95
void delete_dimension_uuid(uuid_t *dimension_uuid);
97
-void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart);
96
void store_claim_id(uuid_t *host_id, uuid_t *claim_id);
97
int update_node_id(uuid_t *host_id, uuid_t *node_id);
98
int get_node_id(uuid_t *host_id, uuid_t *node_id);
@@ -104,7 +102,6 @@ struct node_instance_list *get_node_list(void);
102
void sql_load_node_id(RRDHOST *host);
103
int sql_set_dimension_option(uuid_t *dim_uuid, char *option);
104
char *get_hostname_by_node_id(char *node_id);
107
-void free_temporary_host(RRDHOST *host);
105
int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[]);
106
void migrate_localhost(uuid_t *host_uuid);
107
void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_info *system_info);
database/storage_engine.c
+38
-14
@@ -10,7 +10,10 @@
10
.init = rrddim_collect_init,\
11
.store_metric = rrddim_collect_store_metric,\
12
.flush = rrddim_store_metric_flush,\
13
- .finalize = rrddim_collect_finalize\
13
+ .finalize = rrddim_collect_finalize, \
14
+ .change_collection_frequency = rrddim_store_metric_change_collection_frequency, \
15
+ .metrics_group_get = rrddim_metrics_group_get, \
16
+ .metrics_group_release = rrddim_metrics_group_release, \
17
}
18
19
#define im_query_ops { \
@@ -27,8 +30,11 @@ static STORAGE_ENGINE engines[] = {
30
.id = RRD_MEMORY_MODE_NONE,
31
.name = RRD_MEMORY_MODE_NONE_NAME,
32
.api = {
30
- .init = rrddim_metric_init,
31
- .free = rrddim_metric_free,
33
+ .metric_get = rrddim_metric_get,
34
+ .metric_get_or_create = rrddim_metric_get_or_create,
35
+ .metric_release = rrddim_metric_release,
36
+ .group_get = rrddim_metrics_group_get,
37
+ .group_release = rrddim_metrics_group_release,
38
.collect_ops = im_collect_ops,
39
.query_ops = im_query_ops
40
}
@@ -37,8 +43,11 @@ static STORAGE_ENGINE engines[] = {
43
.id = RRD_MEMORY_MODE_RAM,
44
.name = RRD_MEMORY_MODE_RAM_NAME,
45
.api = {
40
- .init = rrddim_metric_init,
41
- .free = rrddim_metric_free,
46
+ .metric_get = rrddim_metric_get,
47
+ .metric_get_or_create = rrddim_metric_get_or_create,
48
+ .metric_release = rrddim_metric_release,
49
+ .group_get = rrddim_metrics_group_get,
50
+ .group_release = rrddim_metrics_group_release,
51
.collect_ops = im_collect_ops,
52
.query_ops = im_query_ops
53
}
@@ -47,8 +56,11 @@ static STORAGE_ENGINE engines[] = {
56
.id = RRD_MEMORY_MODE_MAP,
57
.name = RRD_MEMORY_MODE_MAP_NAME,
58
.api = {
50
- .init = rrddim_metric_init,
51
- .free = rrddim_metric_free,
59
+ .metric_get = rrddim_metric_get,
60
+ .metric_get_or_create = rrddim_metric_get_or_create,
61
+ .metric_release = rrddim_metric_release,
62
+ .group_get = rrddim_metrics_group_get,
63
+ .group_release = rrddim_metrics_group_release,
64
.collect_ops = im_collect_ops,
65
.query_ops = im_query_ops
66
}
@@ -57,8 +69,11 @@ static STORAGE_ENGINE engines[] = {
69
.id = RRD_MEMORY_MODE_SAVE,
70
.name = RRD_MEMORY_MODE_SAVE_NAME,
71
.api = {
60
- .init = rrddim_metric_init,
61
- .free = rrddim_metric_free,
72
+ .metric_get = rrddim_metric_get,
73
+ .metric_get_or_create = rrddim_metric_get_or_create,
74
+ .metric_release = rrddim_metric_release,
75
+ .group_get = rrddim_metrics_group_get,
76
+ .group_release = rrddim_metrics_group_release,
77
.collect_ops = im_collect_ops,
78
.query_ops = im_query_ops
79
}
@@ -67,8 +82,11 @@ static STORAGE_ENGINE engines[] = {
82
.id = RRD_MEMORY_MODE_ALLOC,
83
.name = RRD_MEMORY_MODE_ALLOC_NAME,
84
.api = {
70
- .init = rrddim_metric_init,
71
- .free = rrddim_metric_free,
85
+ .metric_get = rrddim_metric_get,
86
+ .metric_get_or_create = rrddim_metric_get_or_create,
87
+ .metric_release = rrddim_metric_release,
88
+ .group_get = rrddim_metrics_group_get,
89
+ .group_release = rrddim_metrics_group_release,
90
.collect_ops = im_collect_ops,
91
.query_ops = im_query_ops
92
}
@@ -78,13 +96,19 @@ static STORAGE_ENGINE engines[] = {
96
.id = RRD_MEMORY_MODE_DBENGINE,
97
.name = RRD_MEMORY_MODE_DBENGINE_NAME,
98
.api = {
81
- .init = rrdeng_metric_init,
82
- .free = rrdeng_metric_free,
99
+ .metric_get = rrdeng_metric_get,
100
+ .metric_get_or_create = rrdeng_metric_get_or_create,
101
+ .metric_release = rrdeng_metric_release,
102
+ .group_get = rrdeng_metrics_group_get,
103
+ .group_release = rrdeng_metrics_group_release,
104
.collect_ops = {
105
.init = rrdeng_store_metric_init,
106
.store_metric = rrdeng_store_metric_next,
107
.flush = rrdeng_store_metric_flush_current_page,
87
- .finalize = rrdeng_store_metric_finalize
108
+ .finalize = rrdeng_store_metric_finalize,
109
+ .change_collection_frequency = rrdeng_store_metric_change_collection_frequency,
110
+ .metrics_group_get = rrdeng_metrics_group_get,
111
+ .metrics_group_release = rrdeng_metrics_group_release,
112
},
113
.query_ops = {
114
.init = rrdeng_load_metric_init,
database/storage_engine.h
+10
-2
@@ -10,8 +10,16 @@ typedef struct storage_engine STORAGE_ENGINE;
10
// ------------------------------------------------------------------------
11
// function pointers for all APIs provided by a storge engine
12
typedef struct storage_engine_api {
13
- STORAGE_METRIC_HANDLE *(*init)(RRDDIM *rd, STORAGE_INSTANCE *instance);
14
- void (*free)(STORAGE_METRIC_HANDLE *);
13
+ // metric management
14
+ STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
15
+ STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *instance, STORAGE_METRICS_GROUP *smg);
16
+ void (*metric_release)(STORAGE_METRIC_HANDLE *);
17
+
18
+ // metrics groups management
19
+ STORAGE_METRICS_GROUP *(*group_get)(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
20
+ void (*group_release)(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
21
+
22
+ // operations
23
struct rrddim_collect_ops collect_ops;
24
struct rrddim_query_ops query_ops;
25
} STORAGE_ENGINE_API;
exporting/process_data.c
+1
-1
@@ -125,7 +125,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
125
size_t counter = 0;
126
NETDATA_DOUBLE sum = 0;
127
128
- for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before, TIER_QUERY_FETCH_SUM); !rd->tiers[0]->query_ops.is_finished(&handle);) {
128
+ for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before); !rd->tiers[0]->query_ops.is_finished(&handle);) {
129
STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
130
131
if (unlikely(storage_point_is_empty(sp))) {
libnetdata/clocks/clocks.c
+2
-2
@@ -11,14 +11,14 @@ usec_t clock_monotonic_resolution = 1000;
11
usec_t clock_realtime_resolution = 1000;
12
13
#ifndef HAVE_CLOCK_GETTIME
14
-inline int clock_gettime(clockid_t clk_id, struct timespec *ts) {
14
+inline int clock_gettime(clockid_t clk_id __maybe_unused, struct timespec *ts) {
15
struct timeval tv;
16
if(unlikely(gettimeofday(&tv, NULL) == -1)) {
17
error("gettimeofday() failed.");
18
return -1;
19
}
20
ts->tv_sec = tv.tv_sec;
21
- ts->tv_nsec = (tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC;
21
+ ts->tv_nsec = (long)((tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC);
22
return 0;
23
}
24
#endif
libnetdata/log/log.c
+20
@@ -3,6 +3,10 @@
3
#include <daemon/main.h>
4
#include "../libnetdata.h"
5
6
+#ifdef HAVE_BACKTRACE
7
+#include <execinfo.h>
8
+#endif
9
+
10
int web_server_is_multithreaded = 1;
11
12
const char *program_name = "";
@@ -857,6 +861,18 @@ static void crash_netdata(void) {
861
}
862
#endif
863
864
+#ifdef HAVE_BACKTRACE
865
+#define BT_BUF_SIZE 100
866
+static void print_call_stack(void) {
867
+ int nptrs;
868
+ void *buffer[BT_BUF_SIZE];
869
+
870
+ nptrs = backtrace(buffer, BT_BUF_SIZE);
871
+ if(nptrs)
872
+ backtrace_symbols_fd(buffer, nptrs, fileno(stderr));
873
+}
874
+#endif
875
+
876
void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
877
// save a copy of errno - just in case this function generates a new error
878
int __errno = errno;
@@ -904,6 +920,10 @@ void fatal_int( const char *file, const char *function, const unsigned long line
920
snprintfz(action_result, 60, "%s:%s", program_name, strncmp(thread_tag, "STREAM_RECEIVER", strlen("STREAM_RECEIVER")) ? thread_tag : "[x]");
921
send_statistics("FATAL", action_result, action_data);
922
923
+#ifdef HAVE_BACKTRACE
924
+ print_call_stack();
925
+#endif
926
+
927
#ifdef NETDATA_INTERNAL_CHECKS
928
crash_netdata();
929
#endif
libnetdata/storage_number/storage_number.c
+2
-1
@@ -176,7 +176,8 @@ int print_netdata_double(char *str, NETDATA_DOUBLE value) {
176
#ifdef STORAGE_WITH_MATH
177
fractional = modfndd(value, &integral) * 10000000.0;
178
#else
179
- fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL);
179
+ integral = (NETDATA_DOUBLE)((unsigned long long)(value * 10000000ULL) / 10000000ULL);
180
+ fractional = (NETDATA_DOUBLE)((unsigned long long)(value * 10000000ULL) % 10000000ULL);
181
#endif
182
183
unsigned long long integral_int = (unsigned long long)integral;
ml/Query.h
+1
-1
@@ -20,7 +20,7 @@ public:
20
}
21
22
void init(time_t AfterT, time_t BeforeT) {
23
- Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT, TIER_QUERY_FETCH_SUM);
23
+ Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT);
24
Initialized = true;
25
}
26
streaming/receiver.c
+2
-5
@@ -126,18 +126,15 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins
126
return PARSER_RC_ERROR;
127
}
128
129
-#define CLAIMED_ID_MIN_WORDS 3
129
PARSER_RC streaming_claimed_id(char **words, void *user, PLUGINSD_ACTION *plugins_action)
130
{
131
UNUSED(plugins_action);
132
134
- int i;
133
uuid_t uuid;
134
RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host;
135
138
- for (i = 0; words[i]; i++) ;
139
- if (i != CLAIMED_ID_MIN_WORDS) {
140
- error("Command CLAIMED_ID came malformed %d parameters are expected but %d received", CLAIMED_ID_MIN_WORDS - 1, i - 1);
136
+ if (!words[1] || !words[2]) {
137
+ error("Command CLAIMED_ID came malformed, uuid = '%s', claim_id = '%s'", words[1]?words[1]:"[unset]", words[2]?words[2]:"[unset]");
138
return PARSER_RC_ERROR;
139
}
140
web/api/formatters/rrd2json.c
+1
-1
@@ -27,7 +27,7 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc
27
if(archive_mode) {
28
STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
29
if (eng)
30
- eng->api.free(temp_rd->tiers[tier]->db_metric_handle);
30
+ eng->api.metric_release(temp_rd->tiers[tier]->db_metric_handle);
31
}
32
33
onewayalloc_freez(owa, temp_rd->tiers[tier]);
web/api/queries/query.c
+14
-3
@@ -966,7 +966,7 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
966
967
ops->tier = ops->plan.data[plan_id].tier;
968
ops->tier_ptr = ops->rd->tiers[ops->tier];
969
- ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch);
969
+ ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before);
970
ops->next_metric = ops->tier_ptr->query_ops.next_metric;
971
ops->is_finished = ops->tier_ptr->query_ops.is_finished;
972
ops->finalize = ops->tier_ptr->query_ops.finalize;
@@ -1156,6 +1156,10 @@ static inline void rrd2rrdr_do_dimension(
1156
, time_t after_wanted
1157
, time_t before_wanted
1158
){
1159
+// bool debug_this = false;
1160
+// if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
1161
+// debug_this = true;
1162
+
1163
time_t max_date = 0,
1164
min_date = 0;
1165
@@ -1210,6 +1214,9 @@ static inline void rrd2rrdr_do_dimension(
1214
new_point = QUERY_POINT_EMPTY;
1215
new_point.start_time = last1_point.end_time;
1216
new_point.end_time = now_end_time;
1217
+//
1218
+// if(debug_this) info("QUERY: is finished() returned true");
1219
+//
1220
break;
1221
}
1222
@@ -1225,6 +1232,10 @@ static inline void rrd2rrdr_do_dimension(
1232
new_point.anomaly = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0;
1233
query_point_set_id(new_point, ops.db_total_points_read);
1234
1235
+// if(debug_this)
1236
+// info("QUERY: got point %zu, from time %ld to %ld // now from %ld to %ld // query from %ld to %ld",
1237
+// new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
1238
+//
1239
// set the right value to the point we got
1240
if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
1241
@@ -1268,7 +1279,7 @@ static inline void rrd2rrdr_do_dimension(
1279
1280
// check if the db is advancing the query
1281
if(unlikely(new_point.end_time <= last1_point.end_time)) {
1271
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld",
1282
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before the last point %zu end time %ld, now is %ld to %ld",
1283
rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time,
1284
last1_point.id, last1_point.end_time, now_start_time, now_end_time);
1285
@@ -1466,7 +1477,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
1477
long before_wanted = smaller_tier_last_time;
1478
1479
struct rrddim_tier *tmp = rd->tiers[tr];
1469
- tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE);
1480
+ tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted);
1481
1482
size_t points = 0;
1483
web/api/tests/valid_urls.c
-13
@@ -19,19 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
19
return NULL;
20
}
21
22
-
23
-RRDHOST *sql_create_host_by_uuid(char *hostname)
24
-{
25
- (void) hostname;
26
- return NULL;
27
-}
28
-
29
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
30
-{
31
- (void) hostname;
32
- return NULL;
33
-}
34
-
22
void repr(char *result, int result_size, char const *buf, int size)
23
{
24
int n;
web/api/tests/web_api.c
-12
@@ -19,18 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
19
return NULL;
20
}
21
22
-RRDHOST *sql_create_host_by_uuid(char *hostname)
23
-{
24
- (void) hostname;
25
- return NULL;
26
-}
27
-
28
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
29
-{
30
- (void) hostname;
31
- return NULL;
32
-}
33
-
22
void repr(char *result, int result_size, char const *buf, int size)
23
{
24
int n;
web/api/web_api_v1.c
-6
@@ -721,10 +721,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
721
722
if (likely(context_param_list && context_param_list->rd)) // Just set the first one
723
st = context_param_list->rd->rrdset;
724
- else {
725
- if (!chart_label_key && !chart_labels_filter)
726
- sql_build_context_param_list(owa, &context_param_list, host, context, NULL);
727
- }
724
}
725
else {
726
st = rrdset_find(host, chart);
@@ -732,8 +728,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
728
st = rrdset_find_byname(host, chart);
729
if (likely(st))
730
st->last_accessed_time = now_realtime_sec();
735
- else
736
- sql_build_context_param_list(owa, &context_param_list, host, NULL, chart);
731
}
732
733
if (!st) {
web/server/web_client.c
-8
@@ -1308,14 +1308,6 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1308
host = rrdhost_find_by_hostname(tok);
1309
if (!host)
1310
host = rrdhost_find_by_guid(tok);
1311
- if (!host) {
1312
- host = sql_create_host_by_uuid(tok);
1313
- if (likely(host)) {
1314
- int rc = web_client_process_url(host, w, url);
1315
- free_temporary_host(host);
1316
- return rc;
1317
- }
1318
- }
1311
if (host) return web_client_process_url(host, w, url);
1312
}
1313