allow the cache to grow when huge queries are running that exceed the cache size (#14247)
* allow the cache to grow when huge queries are running that exceed the size of the cache * queue preloading of queries * finalize prepared queries on timeout
Costa Tsaousis committed
Jan 12, 2023 at 00:41 UTC
a844215ac9adf197397b23c7bcd884079940249b
2 files changed
+38
-9
database/engine/cache.c
+16
-7
@@ -273,14 +273,20 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {
273
size_t wanted_cache_size;
274
size_t per1000;
275
276
+ size_t dirty = __atomic_load_n(&cache->dirty.stats->size, __ATOMIC_RELAXED);
277
+ size_t hot = __atomic_load_n(&cache->hot.stats->size, __ATOMIC_RELAXED);
278
+
279
if(cache->config.options & PGC_OPTIONS_AUTOSCALE) {
280
size_t dirty_max = __atomic_load_n(&cache->dirty.stats->max_size, __ATOMIC_RELAXED);
281
size_t hot_max = __atomic_load_n(&cache->hot.stats->max_size, __ATOMIC_RELAXED);
279
- size_t dirty = __atomic_load_n(&cache->dirty.stats->size, __ATOMIC_RELAXED);
280
- size_t hot = __atomic_load_n(&cache->hot.stats->size, __ATOMIC_RELAXED);
282
283
+ // our promise to users
284
size_t max_size1 = MAX(hot_max, hot) * 2;
285
+
286
+ // protection against slow flushing
287
size_t max_size2 = hot_max + ((dirty_max < hot_max / 2) ? hot_max / 2 : dirty_max * 2);
288
+
289
+ // the final wanted cache size
290
wanted_cache_size = MIN(max_size1, max_size2);
291
292
if(cache->config.dynamic_target_size_cb) {
@@ -292,12 +298,15 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {
298
if (wanted_cache_size < hot + dirty + cache->config.clean_size)
299
wanted_cache_size = hot + dirty + cache->config.clean_size;
300
}
295
- else {
296
- size_t dirty = __atomic_load_n(&cache->dirty.stats->size, __ATOMIC_RELAXED);
297
- size_t hot = __atomic_load_n(&cache->hot.stats->size, __ATOMIC_RELAXED);
298
-
301
+ else
302
wanted_cache_size = hot + dirty + cache->config.clean_size;
300
- }
303
+
304
+ // protection again huge queries
305
+ // if huge queries are running, or huge amounts need to be saved
306
+ // allow the cache to grow more (hot pages in main cache are also referenced)
307
+ size_t referenced_size = __atomic_load_n(&cache->stats.referenced_size, __ATOMIC_RELAXED);
308
+ if(unlikely(wanted_cache_size < referenced_size * 2 / 3))
309
+ wanted_cache_size = referenced_size * 2 / 3;
310
311
current_cache_size = __atomic_load_n(&cache->stats.size, __ATOMIC_RELAXED);
312
web/api/queries/query.c
+22
-2
@@ -2094,10 +2094,24 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2094
now_realtime_timeval(&query_start_time);
2095
2096
QUERY_ENGINE_OPS **ops = onewayalloc_callocz(r->internal.owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
2097
- for(size_t c = 0, max = qt->query.used; c < max ; c++)
2098
- ops[c] = rrd2rrdr_query_prep(r, c);
2097
+
2098
+ size_t capacity = libuv_worker_threads * 2;
2099
+ size_t max_queries_to_prepare = (qt->query.used > (capacity - 1)) ? (capacity - 1) : qt->query.used;
2100
+ size_t queries_prepared = 0;
2101
+ while(queries_prepared < max_queries_to_prepare) {
2102
+ // preload another query
2103
+ ops[queries_prepared] = rrd2rrdr_query_prep(r, queries_prepared);
2104
+ queries_prepared++;
2105
+ }
2106
2107
for(size_t c = 0, max = qt->query.used; c < max ; c++) {
2108
+
2109
+ if(queries_prepared < max) {
2110
+ // preload another query
2111
+ ops[queries_prepared] = rrd2rrdr_query_prep(r, queries_prepared);
2112
+ queries_prepared++;
2113
+ }
2114
+
2115
// set the query target dimension options to rrdr
2116
r->od[c] = qt->query.array[c].dimension.options;
2117
@@ -2149,6 +2163,12 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2163
log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)",
2164
(NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout);
2165
r->result_options |= RRDR_RESULT_OPTION_CANCEL;
2166
+
2167
+ for(size_t i = c + 1; i < queries_prepared ; i++) {
2168
+ if(ops[i])
2169
+ query_planer_finalize_remaining_plans(ops[i]);
2170
+ }
2171
+
2172
break;
2173
}
2174
}