@cryptotaxi247 / netdata-1 / commits / 55d1f00bb

DBENGINE v2 - improvements part 12 (#14379)

* parallel initialization of tiers * do not spawn multiple dbengine event loops * user configurable dbengine parallel initialization * size netdata based on the real cpu cores available on the system netdata runs, not on the system monitored * user configurable system cpus * move cpuset parsing to os.c/.h * fix replication of misaligned chart dimensions * give a different path to each tier thread * statically allocate the path into the initialization structure * use aral for reusing dbengine pages * dictionaries uses ARAL for fixed sized values * fix compilation without internal checks * journal v2 index uses aral * test to see judy allocations * judy allocations using aral * Add config option to select if dbengine will use direct I/O (default is yes) * V1 journafiles will use uv_fs_read instead of mmap (respect the direct I/O setting) * Remove sqlite3IsMemdb as it is unused * Fix compilation error when --disable-dbengine is used * use aral for dbengine work_cmds * changed aral API to support new features * pgc and mrg aral overheads * rrdeng opcodes using aral * better structuring and naming * dbegnine query handles using aral * page descriptors using aral * remove obsolete linking * extent io descriptors using aral * aral keeps one last page alive * add missing return value * added judy aral overhead * pdc now uses aral * page_details now use aral * epdl and deol using aral - make sure ARALs are initialized before spawning the event loop * remove unused linking * pgc now uses one aral per partition * aral measure maximum allocation queue * aral to allocate pages in parallel * aral parallel pages allocation when needed * aral cleanup * track page allocation and page population separately --------- Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

Costa Tsaousis committed Feb 2, 2023 at 00:14 UTC 55d1f00bb7c2403b451947b2a225b5d1f6be9183
55 files changed +1223 -1272
aclk/aclk.c
+1 -1
@@ -310,7 +310,7 @@ static void puback_callback(uint16_t packet_id)
310
311 static int read_query_thread_count()
312 {
313 - int threads = MIN(get_system_cpus()/2, 6);
313 + int threads = MIN(get_netdata_cpus()/2, 6);
314 threads = MAX(threads, 2);
315 threads = config_get_number(CONFIG_SECTION_CLOUD, "query thread count", threads);
316 if(threads < 1) {
collectors/cgroups.plugin/sys_fs_cgroup.c
+4 -41
@@ -3516,52 +3516,15 @@ static inline char *cgroup_chart_type(char *buffer, const char *id, size_t len)
3516 return buffer;
3517 }
3518
3519 -static inline unsigned long long cpuset_str2ull(char **s) {
3520 - unsigned long long n = 0;
3521 - char c;
3522 - for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) {
3523 - n *= 10;
3524 - n += c - '0';
3525 - }
3526 - return n;
3527 -}
3528 -
3519 static inline void update_cpu_limits(char **filename, unsigned long long *value, struct cgroup *cg) {
3520 if(*filename) {
3521 int ret = -1;
3522
3523 if(value == &cg->cpuset_cpus) {
3534 - static char *buf = NULL;
3535 - static size_t buf_size = 0;
3536 -
3537 - if(!buf) {
3538 - buf_size = 100U + 6 * get_system_cpus(); // taken from kernel/cgroup/cpuset.c
3539 - buf = mallocz(buf_size + 1);
3540 - }
3541 -
3542 - ret = read_file(*filename, buf, buf_size);
3543 -
3544 - if(!ret) {
3545 - char *s = buf;
3546 - unsigned long long ncpus = 0;
3547 -
3548 - // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use
3549 - while(*s) {
3550 - unsigned long long n = cpuset_str2ull(&s);
3551 - ncpus++;
3552 - if(*s == ',') {
3553 - s++;
3554 - continue;
3555 - }
3556 - if(*s == '-') {
3557 - s++;
3558 - unsigned long long m = cpuset_str2ull(&s);
3559 - ncpus += m - n; // calculate the number of cpus in the region
3560 - }
3561 - s++;
3562 - }
3563 -
3564 - if(likely(ncpus)) *value = ncpus;
3524 + unsigned long ncpus = read_cpuset_cpus(*filename, get_system_cpus());
3525 + if(ncpus) {
3526 + *value = ncpus;
3527 + ret = 0;
3528 }
3529 }
3530 else if(value == &cg->cpu_cfs_period) {
collectors/diskspace.plugin/plugin_diskspace.c
+1 -1
@@ -319,7 +319,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
319 , SIMPLE_PATTERN_EXACT
320 );
321
322 - dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors);
322 + dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors, 0);
323 }
324
325 struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point);
collectors/plugins.d/pluginsd_parser.c
+1 -1
@@ -525,7 +525,7 @@ static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __may
525 }
526
527 void inflight_functions_init(PARSER *parser) {
528 - parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions);
528 + parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions, 0);
529 dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser);
530 dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser);
531 dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser);
collectors/proc.plugin/proc_self_mountinfo.c
+1 -1
@@ -229,7 +229,7 @@ struct mountinfo *mountinfo_read(int do_statvfs) {
229 // create a dictionary to track uniqueness
230 DICTIONARY *dict = dictionary_create_advanced(
231 DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE,
232 - &dictionary_stats_category_collectors);
232 + &dictionary_stats_category_collectors, 0);
233
234 unsigned long l, lines = procfile_lines(ff);
235 for(l = 0; l < lines ;l++) {
collectors/proc.plugin/proc_spl_kstat_zfs.c
+1 -1
@@ -322,7 +322,7 @@ int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt)
322 snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs");
323 dirname = config_get("plugin:proc:" ZFS_PROC_POOLS, "directory to monitor", filename);
324
325 - zfs_pools = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
325 + zfs_pools = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
326
327 do_zfs_pool_state = 1;
328 }
collectors/proc.plugin/sys_block_zram.c
+1 -1
@@ -267,7 +267,7 @@ int do_sys_block_zram(int update_every, usec_t dt) {
267 }
268 procfile_close(ff);
269
270 - devices = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
270 + devices = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
271 device_count = init_devices(devices, (unsigned int)zram_id, update_every);
272 }
273
collectors/statsd.plugin/statsd.c
+10 -10
@@ -595,7 +595,7 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
595 }
596
597 if (unlikely(!m->set.dict)) {
598 - m->set.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
598 + m->set.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
599 dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m);
600 m->set.unique = 0;
601 }
@@ -635,7 +635,7 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value
635 statsd_reset_metric(m);
636
637 if (unlikely(!m->dictionary.dict)) {
638 - m->dictionary.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
638 + m->dictionary.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
639 dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m);
640 m->dictionary.unique = 0;
641 }
@@ -1337,7 +1337,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1337 else if(app) {
1338 if(!strcmp(s, "dictionary")) {
1339 if(!app->dict)
1340 - app->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
1340 + app->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
1341
1342 dict = app->dict;
1343 }
@@ -2422,13 +2422,13 @@ void *statsd_main(void *ptr) {
2422
2423 netdata_thread_cleanup_push(statsd_main_cleanup, ptr);
2424
2425 - statsd.gauges.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
2426 - statsd.meters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
2427 - statsd.counters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
2428 - statsd.histograms.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
2429 - statsd.dictionaries.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
2430 - statsd.sets.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
2431 - statsd.timers.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
2425 + statsd.gauges.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
2426 + statsd.meters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
2427 + statsd.counters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
2428 + statsd.histograms.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
2429 + statsd.dictionaries.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
2430 + statsd.sets.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
2431 + statsd.timers.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
2432
2433 dictionary_register_insert_callback(statsd.gauges.dict, dictionary_metric_insert_callback, &statsd.gauges);
2434 dictionary_register_insert_callback(statsd.meters.dict, dictionary_metric_insert_callback, &statsd.meters);
collectors/tc.plugin/plugin_tc.c
+2 -2
@@ -98,7 +98,7 @@ static bool tc_class_conflict_callback(const DICTIONARY_ITEM *item __maybe_unuse
98
99 static void tc_class_index_init(struct tc_device *d) {
100 if(!d->classes) {
101 - d->classes = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
101 + d->classes = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
102
103 dictionary_register_delete_callback(d->classes, tc_class_free_callback, d);
104 dictionary_register_conflict_callback(d->classes, tc_class_conflict_callback, d);
@@ -146,7 +146,7 @@ static void tc_device_index_init() {
146 if(!tc_device_root_index) {
147 tc_device_root_index = dictionary_create_advanced(
148 DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED | DICT_OPTION_ADD_IN_FRONT,
149 - &dictionary_stats_category_collectors);
149 + &dictionary_stats_category_collectors, 0);
150
151 dictionary_register_insert_callback(tc_device_root_index, tc_device_add_callback, NULL);
152 dictionary_register_delete_callback(tc_device_root_index, tc_device_free_callback, NULL);
daemon/common.c
+35
@@ -19,3 +19,38 @@ int32_t netdata_configured_utc_offset = 0;
19 int netdata_ready;
20 int netdata_cloud_setting;
21
22 +long get_netdata_cpus(void) {
23 + static long processors = 0;
24 +
25 + if(processors)
26 + return processors;
27 +
28 + long cores_proc_stat = get_system_cpus_with_cache(false, true);
29 + long cores_cpuset_v1 = (long)read_cpuset_cpus("/sys/fs/cgroup/cpuset/cpuset.cpus", cores_proc_stat);
30 + long cores_cpuset_v2 = (long)read_cpuset_cpus("/sys/fs/cgroup/cpuset.cpus", cores_proc_stat);
31 +
32 + if(cores_cpuset_v2)
33 + processors = cores_cpuset_v2;
34 + else if(cores_cpuset_v1)
35 + processors = cores_cpuset_v1;
36 + else
37 + processors = cores_proc_stat;
38 +
39 + long cores_user_configured = config_get_number(CONFIG_SECTION_GLOBAL, "cpu cores", processors);
40 +
41 + errno = 0;
42 + internal_error(true,
43 + "System CPUs: %ld, ("
44 + "system: %ld, cgroups cpuset v1: %ld, cgroups cpuset v2: %ld, netdata.conf: %ld"
45 + ")"
46 + , processors
47 + , cores_proc_stat
48 + , cores_cpuset_v1
49 + , cores_cpuset_v2
50 + , cores_user_configured
51 + );
52 +
53 + processors = cores_user_configured;
54 +
55 + return processors;
56 +}
daemon/common.h
+2
@@ -106,4 +106,6 @@ extern int netdata_anonymous_statistics_enabled;
106 extern int netdata_ready;
107 extern int netdata_cloud_setting;
108
109 +long get_netdata_cpus(void);
110 +
111 #endif /* NETDATA_COMMON_H */
daemon/event_loop.c
+1
@@ -24,6 +24,7 @@ void register_libuv_worker_jobs() {
24 worker_register_job_name(UV_EVENT_DBENGINE_EXTENT_DECOMPRESSION, "extent decompression");
25 worker_register_job_name(UV_EVENT_DBENGINE_EXTENT_PAGE_LOOKUP, "page lookup");
26 worker_register_job_name(UV_EVENT_DBENGINE_EXTENT_PAGE_POPULATION, "page populate");
27 + worker_register_job_name(UV_EVENT_DBENGINE_EXTENT_PAGE_ALLOCATION, "page allocate");
28
29 // flushing related
30 worker_register_job_name(UV_EVENT_DBENGINE_FLUSH_MAIN_CACHE, "flush main");
daemon/event_loop.h
+1
@@ -16,6 +16,7 @@ enum event_loop_job {
16 UV_EVENT_DBENGINE_EXTENT_DECOMPRESSION,
17 UV_EVENT_DBENGINE_EXTENT_PAGE_LOOKUP,
18 UV_EVENT_DBENGINE_EXTENT_PAGE_POPULATION,
19 + UV_EVENT_DBENGINE_EXTENT_PAGE_ALLOCATION,
20
21 // flushing related
22 UV_EVENT_DBENGINE_FLUSH_MAIN_CACHE,
daemon/global_statistics.c
+18 -10
@@ -243,9 +243,6 @@ static void global_statistics_charts(void) {
243 global_statistics_copy(&gs, GLOBAL_STATS_RESET_WEB_USEC_MAX);
244 getrusage(RUSAGE_SELF, &me);
245
246 - size_t aral_structures, aral_malloc_allocated, aral_malloc_used, aral_mmap_allocated, aral_mmap_used;
247 - aral_get_size_statistics(&aral_structures, &aral_malloc_allocated, &aral_malloc_used, &aral_mmap_allocated, &aral_mmap_used);
248 -
246 // ----------------------------------------------------------------
247
248 {
@@ -296,6 +293,7 @@ static void global_statistics_charts(void) {
293 static RRDDIM *rd_buffers = NULL;
294 static RRDDIM *rd_workers = NULL;
295 static RRDDIM *rd_aral = NULL;
296 + static RRDDIM *rd_judy = NULL;
297 static RRDDIM *rd_other = NULL;
298
299 if (unlikely(!st_memory)) {
@@ -327,6 +325,7 @@ static void global_statistics_charts(void) {
325 rd_buffers = rrddim_add(st_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
326 rd_workers = rrddim_add(st_memory, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
327 rd_aral = rrddim_add(st_memory, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
328 + rd_judy = rrddim_add(st_memory, "judy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
329 rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
330 }
331
@@ -342,7 +341,9 @@ static void global_statistics_charts(void) {
341 netdata_buffers_statistics.buffers_streaming +
342 netdata_buffers_statistics.cbuffers_streaming +
343 netdata_buffers_statistics.buffers_web +
345 - replication_allocated_buffers();
344 + replication_allocated_buffers() +
345 + aral_by_size_overhead() +
346 + judy_aral_overhead();
347
348 size_t strings = 0;
349 string_statistics(NULL, NULL, NULL, NULL, NULL, &strings, NULL, NULL);
@@ -360,7 +361,8 @@ static void global_statistics_charts(void) {
361 rrddim_set_by_pointer(st_memory, rd_replication, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + (collected_number)replication_allocated_memory());
362 rrddim_set_by_pointer(st_memory, rd_buffers, (collected_number)buffers);
363 rrddim_set_by_pointer(st_memory, rd_workers, (collected_number) workers_allocated_memory());
363 - rrddim_set_by_pointer(st_memory, rd_aral, (collected_number) aral_structures);
364 + rrddim_set_by_pointer(st_memory, rd_aral, (collected_number) aral_by_size_structures());
365 + rrddim_set_by_pointer(st_memory, rd_judy, (collected_number) judy_aral_structures());
366 rrddim_set_by_pointer(st_memory, rd_other, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other));
367
368 rrdset_done(st_memory);
@@ -381,6 +383,7 @@ static void global_statistics_charts(void) {
383 static RRDDIM *rd_buffers_replication = NULL;
384 static RRDDIM *rd_buffers_web = NULL;
385 static RRDDIM *rd_buffers_aral = NULL;
386 + static RRDDIM *rd_buffers_judy = NULL;
387
388 if (unlikely(!st_memory_buffers)) {
389 st_memory_buffers = rrdset_create_localhost(
@@ -410,6 +413,7 @@ static void global_statistics_charts(void) {
413 rd_buffers_replication = rrddim_add(st_memory_buffers, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
414 rd_buffers_web = rrddim_add(st_memory_buffers, "web", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
415 rd_buffers_aral = rrddim_add(st_memory_buffers, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
416 + rd_buffers_judy = rrddim_add(st_memory_buffers, "judy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
417 }
418
419 rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory());
@@ -424,7 +428,8 @@ static void global_statistics_charts(void) {
428 rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming);
429 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers());
430 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web);
427 - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aral, (collected_number)(aral_malloc_allocated + aral_mmap_allocated) - (collected_number)(aral_malloc_used + aral_mmap_used));
431 + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aral, (collected_number)aral_by_size_overhead());
432 + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_judy, (collected_number)judy_aral_overhead());
433
434 rrdset_done(st_memory_buffers);
435 }
@@ -1743,7 +1748,7 @@ static void dbengine2_statistics_charts(void) {
1748
1749 struct rrdeng_buffer_sizes buffers = rrdeng_get_buffer_sizes();
1750 size_t buffers_total_size = buffers.handles + buffers.xt_buf + buffers.xt_io + buffers.pdc + buffers.descriptors +
1746 - buffers.opcodes + buffers.wal + buffers.workers + buffers.epdl + buffers.deol + buffers.pd + buffers.pages;
1751 + buffers.opcodes + buffers.wal + buffers.workers + buffers.epdl + buffers.deol + buffers.pd + buffers.pgc + buffers.mrg;
1752
1753 #ifdef PDC_USE_JULYL
1754 buffers_total_size += buffers.julyl;
@@ -1796,6 +1801,8 @@ static void dbengine2_statistics_charts(void) {
1801
1802 {
1803 static RRDSET *st_pgc_buffers = NULL;
1804 + static RRDDIM *rd_pgc_buffers_pgc = NULL;
1805 + static RRDDIM *rd_pgc_buffers_mrg = NULL;
1806 static RRDDIM *rd_pgc_buffers_opcodes = NULL;
1807 static RRDDIM *rd_pgc_buffers_handles = NULL;
1808 static RRDDIM *rd_pgc_buffers_descriptors = NULL;
@@ -1807,7 +1814,6 @@ static void dbengine2_statistics_charts(void) {
1814 static RRDDIM *rd_pgc_buffers_epdl = NULL;
1815 static RRDDIM *rd_pgc_buffers_deol = NULL;
1816 static RRDDIM *rd_pgc_buffers_pd = NULL;
1810 - static RRDDIM *rd_pgc_buffers_pages = NULL;
1817 #ifdef PDC_USE_JULYL
1818 static RRDDIM *rd_pgc_buffers_julyl = NULL;
1819 #endif
@@ -1827,6 +1833,8 @@ static void dbengine2_statistics_charts(void) {
1833 localhost->rrd_update_every,
1834 RRDSET_TYPE_STACKED);
1835
1836 + rd_pgc_buffers_pgc = rrddim_add(st_pgc_buffers, "pgc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1837 + rd_pgc_buffers_mrg = rrddim_add(st_pgc_buffers, "mrg", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1838 rd_pgc_buffers_opcodes = rrddim_add(st_pgc_buffers, "opcodes", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1839 rd_pgc_buffers_handles = rrddim_add(st_pgc_buffers, "query handles", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1840 rd_pgc_buffers_descriptors = rrddim_add(st_pgc_buffers, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1834,7 +1842,6 @@ static void dbengine2_statistics_charts(void) {
1842 rd_pgc_buffers_workers = rrddim_add(st_pgc_buffers, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1843 rd_pgc_buffers_pdc = rrddim_add(st_pgc_buffers, "pdc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1844 rd_pgc_buffers_pd = rrddim_add(st_pgc_buffers, "pd", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1837 - rd_pgc_buffers_pages = rrddim_add(st_pgc_buffers, "pages", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1845 rd_pgc_buffers_xt_io = rrddim_add(st_pgc_buffers, "extent io", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1846 rd_pgc_buffers_xt_buf = rrddim_add(st_pgc_buffers, "extent buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1847 rd_pgc_buffers_epdl = rrddim_add(st_pgc_buffers, "epdl", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -1845,6 +1852,8 @@ static void dbengine2_statistics_charts(void) {
1852 }
1853 priority++;
1854
1855 + rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pgc, (collected_number)buffers.pgc);
1856 + rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_mrg, (collected_number)buffers.mrg);
1857 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_opcodes, (collected_number)buffers.opcodes);
1858 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_handles, (collected_number)buffers.handles);
1859 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_descriptors, (collected_number)buffers.descriptors);
@@ -1852,7 +1861,6 @@ static void dbengine2_statistics_charts(void) {
1861 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_workers, (collected_number)buffers.workers);
1862 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pdc, (collected_number)buffers.pdc);
1863 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pd, (collected_number)buffers.pd);
1855 - rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_pages, (collected_number)buffers.pages);
1864 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_xt_io, (collected_number)buffers.xt_io);
1865 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_xt_buf, (collected_number)buffers.xt_buf);
1866 rrddim_set_by_pointer(st_pgc_buffers, rd_pgc_buffers_epdl, (collected_number)buffers.epdl);
daemon/main.c
+3 -1
@@ -1801,7 +1801,7 @@ int main(int argc, char **argv) {
1801 #endif
1802
1803 // set libuv worker threads
1804 - libuv_worker_threads = get_system_cpus() * 2;
1804 + libuv_worker_threads = (int)get_netdata_cpus() * 2;
1805
1806 if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS)
1807 libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
@@ -1866,6 +1866,8 @@ int main(int argc, char **argv) {
1866 // initialize the log files
1867 open_all_log_files();
1868
1869 + aral_judy_init();
1870 +
1871 get_system_timezone();
1872
1873 // --------------------------------------------------------------------
database/engine/cache.c
+64 -25
@@ -106,7 +106,7 @@ struct pgc {
106 } config;
107
108 #ifdef PGC_WITH_ARAL
109 - ARAL *aral;
109 + ARAL **aral;
110 #endif
111
112 PGC_CACHE_LINE_PADDING(0);
@@ -311,7 +311,7 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {
311 if(unlikely(wanted_cache_size < referenced_size * 2 / 3))
312 wanted_cache_size = referenced_size * 2 / 3;
313
314 - current_cache_size = __atomic_load_n(&cache->stats.size, __ATOMIC_RELAXED);
314 + current_cache_size = __atomic_load_n(&cache->stats.size, __ATOMIC_RELAXED); // + pgc_aral_overhead();
315
316 per1000 = (size_t)((unsigned long long)current_cache_size * 1000ULL / (unsigned long long)wanted_cache_size);
317
@@ -430,7 +430,7 @@ static void pgc_section_pages_static_aral_init(void) {
430 "pgc_section",
431 sizeof(struct section_pages),
432 0,
433 - 4096,
433 + 65536, NULL,
434 NULL, NULL, false, false);
435
436 netdata_spinlock_unlock(&spinlock);
@@ -851,7 +851,7 @@ static inline bool acquired_page_get_for_deletion_or_release_it(PGC *cache __may
851 // ----------------------------------------------------------------------------
852 // Indexing
853
854 -static inline void free_this_page(PGC *cache, PGC_PAGE *page) {
854 +static inline void free_this_page(PGC *cache, PGC_PAGE *page, size_t partition __maybe_unused) {
855 // call the callback to free the user supplied memory
856 cache->config.pgc_free_clean_cb(cache, (PGC_ENTRY){
857 .section = page->section,
@@ -874,7 +874,7 @@ static inline void free_this_page(PGC *cache, PGC_PAGE *page) {
874
875 // free our memory
876 #ifdef PGC_WITH_ARAL
877 - aral_freez(cache->aral, page);
877 + aral_freez(cache->aral[partition], page);
878 #else
879 freez(page);
880 #endif
@@ -942,7 +942,7 @@ static inline void remove_and_free_page_not_in_any_queue_and_acquired_for_deleti
942 pgc_index_write_lock(cache, partition);
943 remove_this_page_from_index_unsafe(cache, page, partition);
944 pgc_index_write_unlock(cache, partition);
945 - free_this_page(cache, page);
945 + free_this_page(cache, page, partition);
946 }
947
948 static inline bool make_acquired_page_clean_and_evict_or_page_release(PGC *cache, PGC_PAGE *page) {
@@ -1137,7 +1137,7 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic
1137 next = page->link.next;
1138
1139 size_t page_size = page->assumed_size;
1140 - free_this_page(cache, page);
1140 + free_this_page(cache, page, partition);
1141
1142 __atomic_sub_fetch(&cache->stats.evicting_entries, 1, __ATOMIC_RELAXED);
1143 __atomic_sub_fetch(&cache->stats.evicting_size, page_size, __ATOMIC_RELAXED);
@@ -1156,7 +1156,7 @@ static bool evict_pages_with_filter(PGC *cache, size_t max_skip, size_t max_evic
1156 pgc_index_write_lock(cache, partition);
1157 remove_this_page_from_index_unsafe(cache, page, partition);
1158 pgc_index_write_unlock(cache, partition);
1159 - free_this_page(cache, page);
1159 + free_this_page(cache, page, partition);
1160
1161 __atomic_sub_fetch(&cache->stats.evicting_entries, 1, __ATOMIC_RELAXED);
1162 __atomic_sub_fetch(&cache->stats.evicting_size, page_size, __ATOMIC_RELAXED);
@@ -1191,8 +1191,10 @@ premature_exit:
1191 static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {
1192 __atomic_add_fetch(&cache->stats.workers_add, 1, __ATOMIC_RELAXED);
1193
1194 + size_t partition = pgc_indexing_partition(cache, entry->metric_id);
1195 +
1196 #ifdef PGC_WITH_ARAL
1195 - PGC_PAGE *allocation = aral_mallocz(cache->aral);
1197 + PGC_PAGE *allocation = aral_mallocz(cache->aral[partition]);
1198 #endif
1199 PGC_PAGE *page;
1200 size_t spins = 0;
@@ -1201,7 +1203,6 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {
1203 if(++spins > 1)
1204 __atomic_add_fetch(&cache->stats.insert_spins, 1, __ATOMIC_RELAXED);
1205
1204 - size_t partition = pgc_indexing_partition(cache, entry->metric_id);
1206 pgc_index_write_lock(cache, partition);
1207
1208 size_t mem_before_judyl = 0, mem_after_judyl = 0;
@@ -1299,7 +1300,7 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {
1300
1301 #ifdef PGC_WITH_ARAL
1302 if(allocation)
1302 - aral_freez(cache->aral, allocation);
1303 + aral_freez(cache->aral[partition], allocation);
1304 #endif
1305
1306 __atomic_sub_fetch(&cache->stats.workers_add, 1, __ATOMIC_RELAXED);
@@ -1757,7 +1758,7 @@ PGC *pgc_create(const char *name,
1758 cache->config.max_pages_per_inline_eviction = (max_pages_per_inline_eviction < 2) ? 2 : max_pages_per_inline_eviction;
1759 cache->config.max_skip_pages_per_inline_eviction = (max_skip_pages_per_inline_eviction < 2) ? 2 : max_skip_pages_per_inline_eviction;
1760 cache->config.max_flushes_inline = (max_flushes_inline < 1) ? 1 : max_flushes_inline;
1760 - cache->config.partitions = partitions < 1 ? (size_t)get_system_cpus() : partitions;
1761 + cache->config.partitions = partitions < 1 ? (size_t)get_netdata_cpus() : partitions;
1762 cache->config.additional_bytes_per_page = additional_bytes_per_page;
1763
1764 cache->config.max_workers_evict_inline = max_inline_evictors;
@@ -1787,20 +1788,40 @@ PGC *pgc_create(const char *name,
1788 cache->clean.linked_list_in_sections_judy = false;
1789 cache->clean.stats = &cache->stats.queues.clean;
1790
1791 + pgc_section_pages_static_aral_init();
1792 +
1793 #ifdef PGC_WITH_ARAL
1791 - cache->aral = aral_create(name,
1792 - sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page,
1793 - 0,
1794 - 4096,
1795 - NULL, NULL, false, false);
1794 + cache->aral = callocz(cache->config.partitions, sizeof(ARAL *));
1795 + for(size_t part = 0; part < cache->config.partitions ; part++) {
1796 + char buf[100 +1];
1797 + snprintfz(buf, 100, "%s[%zu]", name, part);
1798 + cache->aral[part] = aral_create(
1799 + buf,
1800 + sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page,
1801 + 0,
1802 + 16384,
1803 + aral_statistics(pgc_section_pages_aral),
1804 + NULL, NULL, false, false);
1805 + }
1806 #endif
1807
1798 - pgc_section_pages_static_aral_init();
1808 pointer_index_init(cache);
1809
1810 return cache;
1811 }
1812
1813 +struct aral_statistics *pgc_aral_statistics(void) {
1814 + return aral_statistics(pgc_section_pages_aral);
1815 +}
1816 +
1817 +size_t pgc_aral_structures(void) {
1818 + return aral_structures(pgc_section_pages_aral);
1819 +}
1820 +
1821 +size_t pgc_aral_overhead(void) {
1822 + return aral_overhead(pgc_section_pages_aral);
1823 +}
1824 +
1825 void pgc_flush_all_hot_and_dirty_pages(PGC *cache, Word_t section) {
1826 all_hot_pages_to_dirty(cache, section);
1827
@@ -1822,9 +1843,17 @@ void pgc_destroy(PGC *cache) {
1843 error("DBENGINE CACHE: there are %zu referenced cache pages - leaving the cache allocated", PGC_REFERENCED_PAGES(cache));
1844 else {
1845 pointer_destroy_index(cache);
1846 +
1847 + for(size_t part = 0; part < cache->config.partitions ; part++)
1848 + netdata_rwlock_destroy(&cache->index[part].rwlock);
1849 +
1850 #ifdef PGC_WITH_ARAL
1826 - aral_destroy(cache->aral);
1851 + for(size_t part = 0; part < cache->config.partitions ; part++)
1852 + aral_destroy(cache->aral[part]);
1853 +
1854 + freez(cache->aral);
1855 #endif
1856 +
1857 freez(cache);
1858 }
1859 }
@@ -2059,6 +2088,10 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2088 return;
2089 }
2090
2091 + ARAL *ar_mi = aral_by_size_acquire(sizeof(struct jv2_metrics_info));
2092 + ARAL *ar_pi = aral_by_size_acquire(sizeof(struct jv2_page_info));
2093 + ARAL *ar_ei = aral_by_size_acquire(sizeof(struct jv2_extents_info));
2094 +
2095 for(PGC_PAGE *page = sp->base; page ; page = page->link.next) {
2096 struct extent_io_data *xio = (struct extent_io_data *)page->custom_data;
2097 if(xio->fileno != datafile_fileno) continue;
@@ -2091,7 +2124,7 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2124
2125 struct jv2_extents_info *ei;
2126 if(!*PValue) {
2094 - ei = callocz(1, sizeof(struct jv2_extents_info));
2127 + ei = aral_mallocz(ar_ei); // callocz(1, sizeof(struct jv2_extents_info));
2128 ei->pos = xio->pos;
2129 ei->bytes = xio->bytes;
2130 ei->number_of_pages = 1;
@@ -2115,11 +2148,13 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2148
2149 struct jv2_metrics_info *mi;
2150 if(!*PValue) {
2118 - mi = callocz(1, sizeof(struct jv2_metrics_info));
2151 + mi = aral_mallocz(ar_mi); // callocz(1, sizeof(struct jv2_metrics_info));
2152 mi->uuid = mrg_metric_uuid(main_mrg, (METRIC *)page->metric_id);
2153 mi->first_time_s = page->start_time_s;
2154 mi->last_time_s = page->end_time_s;
2155 mi->number_of_pages = 1;
2156 + mi->page_list_header = 0;
2157 + mi->JudyL_pages_by_start_time = NULL;
2158 *PValue = mi;
2159
2160 count_of_unique_metrics++;
@@ -2138,7 +2173,7 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2173 fatal("Corrupted JudyL metric pages");
2174
2175 if(!*PValue) {
2141 - struct jv2_page_info *pi = callocz(1, (sizeof(struct jv2_page_info)));
2176 + struct jv2_page_info *pi = aral_mallocz(ar_pi); // callocz(1, (sizeof(struct jv2_page_info)));
2177 pi->start_time_s = page->start_time_s;
2178 pi->end_time_s = page->end_time_s;
2179 pi->update_every_s = page->update_every_s;
@@ -2182,11 +2217,11 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2217 page_transition_unlock(cache, pi->page);
2218 pgc_page_hot_to_dirty_and_release(cache, pi->page);
2219 // make_acquired_page_clean_and_evict_or_page_release(cache, pi->page);
2185 - freez(pi);
2220 + aral_freez(ar_pi, pi);
2221 }
2222
2223 JudyLFreeArray(&mi->JudyL_pages_by_start_time, PJE0);
2189 - freez(mi);
2224 + aral_freez(ar_mi, mi);
2225 }
2226 JudyLFreeArray(&JudyL_metrics, PJE0);
2227 }
@@ -2197,11 +2232,15 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2232 Word_t extent_pos = 0;
2233 while ((PValue = JudyLFirstThenNext(JudyL_extents_pos, &extent_pos, &extent_pos_first))) {
2234 struct jv2_extents_info *ei = *PValue;
2200 - freez(ei);
2235 + aral_freez(ar_ei, ei);
2236 }
2237 JudyLFreeArray(&JudyL_extents_pos, PJE0);
2238 }
2239
2240 + aral_by_size_release(ar_ei);
2241 + aral_by_size_release(ar_pi);
2242 + aral_by_size_release(ar_mi);
2243 +
2244 __atomic_sub_fetch(&cache->stats.workers_jv2_flush, 1, __ATOMIC_RELAXED);
2245 }
2246
database/engine/cache.h
+4
@@ -242,4 +242,8 @@ bool pgc_flush_pages(PGC *cache, size_t max_flushes);
242 struct pgc_statistics pgc_get_statistics(PGC *cache);
243 size_t pgc_hot_and_dirty_entries(PGC *cache);
244
245 +struct aral_statistics *pgc_aral_statistics(void);
246 +size_t pgc_aral_structures(void);
247 +size_t pgc_aral_overhead(void);
248 +
249 #endif // DBENGINE_CACHE_H
database/engine/datafile.c
+3 -3
@@ -257,7 +257,7 @@ int create_data_file(struct rrdengine_datafile *datafile)
257 char path[RRDENG_PATH_MAX];
258
259 generate_datafilepath(datafile, path, sizeof(path));
260 - fd = open_file_direct_io(path, O_CREAT | O_RDWR | O_TRUNC, &file);
260 + fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, use_direct_io);
261 if (fd < 0) {
262 ctx_fs_error(ctx);
263 return fd;
@@ -340,7 +340,7 @@ static int load_data_file(struct rrdengine_datafile *datafile)
340 char path[RRDENG_PATH_MAX];
341
342 generate_datafilepath(datafile, path, sizeof(path));
343 - fd = open_file_direct_io(path, O_RDWR, &file);
343 + fd = open_file_for_io(path, O_RDWR, &file, use_direct_io);
344 if (fd < 0) {
345 ctx_fs_error(ctx);
346 return fd;
@@ -392,7 +392,7 @@ static int scan_data_files(struct rrdengine_instance *ctx)
392 {
393 int ret;
394 unsigned tier, no, matched_files, i,failed_to_load;
395 - static uv_fs_t req;
395 + uv_fs_t req;
396 uv_dirent_t dent;
397 struct rrdengine_datafile **datafiles, *datafile;
398 struct rrdengine_journalfile *journalfile;
database/engine/journalfile.c
+21 -37
@@ -495,7 +495,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi
495 char path[RRDENG_PATH_MAX];
496
497 journalfile_v1_generate_path(datafile, path, sizeof(path));
498 - fd = open_file_direct_io(path, O_CREAT | O_RDWR | O_TRUNC, &file);
498 + fd = open_file_for_io(path, O_CREAT | O_RDWR | O_TRUNC, &file, use_direct_io);
499 if (fd < 0) {
500 ctx_fs_error(ctx);
501 return fd;
@@ -704,7 +704,7 @@ static unsigned journalfile_replay_transaction(struct rrdengine_instance *ctx, s
704 static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile)
705 {
706 uv_file file;
707 - uint64_t file_size;//, data_file_size;
707 + uint64_t file_size;
708 int ret;
709 uint64_t pos, pos_i, max_id, id;
710 unsigned size_bytes;
@@ -714,33 +714,26 @@ static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx,
714
715 file = journalfile->file;
716 file_size = journalfile->unsafe.pos;
717 - //data_file_size = journalfile->datafile->pos; TODO: utilize this?
717
718 max_id = 1;
720 - bool journal_is_mmapped = (journalfile->data != NULL);
721 - if (unlikely(!journal_is_mmapped)) {
722 - ret = posix_memalign((void *)&buf, RRDFILE_ALIGNMENT, READAHEAD_BYTES);
723 - if (unlikely(ret))
724 - fatal("DBENGINE: posix_memalign:%s", strerror(ret));
725 - }
726 - else
727 - buf = journalfile->data + sizeof(struct rrdeng_jf_sb);
728 - for (pos = sizeof(struct rrdeng_jf_sb) ; pos < file_size ; pos += READAHEAD_BYTES) {
719 + ret = posix_memalign((void *)&buf, RRDFILE_ALIGNMENT, READAHEAD_BYTES);
720 + if (unlikely(ret))
721 + fatal("DBENGINE: posix_memalign:%s", strerror(ret));
722 +
723 + for (pos = sizeof(struct rrdeng_jf_sb); pos < file_size; pos += READAHEAD_BYTES) {
724 size_bytes = MIN(READAHEAD_BYTES, file_size - pos);
730 - if (unlikely(!journal_is_mmapped)) {
731 - iov = uv_buf_init(buf, size_bytes);
732 - ret = uv_fs_read(NULL, &req, file, &iov, 1, pos, NULL);
733 - if (ret < 0) {
734 - error("DBENGINE: uv_fs_read: pos=%" PRIu64 ", %s", pos, uv_strerror(ret));
735 - uv_fs_req_cleanup(&req);
736 - goto skip_file;
737 - }
738 - fatal_assert(req.result >= 0);
725 + iov = uv_buf_init(buf, size_bytes);
726 + ret = uv_fs_read(NULL, &req, file, &iov, 1, pos, NULL);
727 + if (ret < 0) {
728 + error("DBENGINE: uv_fs_read: pos=%" PRIu64 ", %s", pos, uv_strerror(ret));
729 uv_fs_req_cleanup(&req);
740 - ctx_io_read_op_bytes(ctx, size_bytes);
730 + goto skip_file;
731 }
732 + fatal_assert(req.result >= 0);
733 + uv_fs_req_cleanup(&req);
734 + ctx_io_read_op_bytes(ctx, size_bytes);
735
743 - for (pos_i = 0 ; pos_i < size_bytes ; ) {
736 + for (pos_i = 0; pos_i < size_bytes;) {
737 unsigned max_size;
738
739 max_size = pos + size_bytes - pos_i;
@@ -752,12 +745,9 @@ static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx,
745 pos_i += ret;
746 max_id = MAX(max_id, id);
747 }
755 - if (likely(journal_is_mmapped))
756 - buf += size_bytes;
748 }
749 skip_file:
759 - if (unlikely(!journal_is_mmapped))
760 - posix_memfree(buf);
750 + posix_memfree(buf);
751 return max_id;
752 }
753
@@ -1400,18 +1390,15 @@ int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfil
1390 uint64_t file_size, max_id;
1391 char path[RRDENG_PATH_MAX];
1392
1403 - // Do not try to load the latest file (always rebuild and live migrate)
1393 + // Do not try to load the latest file
1394 if (datafile->fileno != ctx_last_fileno_get(ctx)) {
1405 - if (!journalfile_v2_load(ctx, journalfile, datafile)) {
1406 -// unmap_journal_file(journalfile);
1395 + if (likely(!journalfile_v2_load(ctx, journalfile, datafile)))
1396 return 0;
1408 - }
1397 }
1398
1399 journalfile_v1_generate_path(datafile, path, sizeof(path));
1400
1413 - // If it is not the last file, open read only
1414 - fd = open_file_direct_io(path, O_RDWR, &file);
1401 + fd = open_file_for_io(path, O_RDWR, &file, use_direct_io);
1402 if (fd < 0) {
1403 ctx_fs_error(ctx);
1404 return fd;
@@ -1432,16 +1419,13 @@ int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfil
1419 journalfile->file = file;
1420 journalfile->unsafe.pos = file_size;
1421
1435 - journalfile->data = netdata_mmap(path, file_size, MAP_SHARED, 0, !(datafile->fileno == ctx_last_fileno_get(ctx)), NULL);
1436 - info("DBENGINE: loading journal file '%s' using %s.", path, journalfile->data?"MMAP":"uv_fs_read");
1422 + info("DBENGINE: loading journal file '%s'", path);
1423
1424 max_id = journalfile_iterate_transactions(ctx, journalfile);
1425
1426 __atomic_store_n(&ctx->atomic.transaction_id, MAX(__atomic_load_n(&ctx->atomic.transaction_id, __ATOMIC_RELAXED), max_id + 1), __ATOMIC_RELAXED);
1427
1428 info("DBENGINE: journal file '%s' loaded (size:%"PRIu64").", path, file_size);
1443 - if (likely(journalfile->data))
1444 - netdata_munmap(journalfile->data, file_size);
1429
1430 bool is_last_file = (ctx_last_fileno_get(ctx) == journalfile->datafile->fileno);
1431 if (is_last_file && journalfile->datafile->pos <= rrdeng_target_data_file_size(ctx) / 3) {
database/engine/journalfile.h
-1
@@ -47,7 +47,6 @@ struct rrdengine_journalfile {
47 } unsafe;
48
49 uv_file file;
50 - void *data;
50 struct rrdengine_datafile *datafile;
51 };
52
database/engine/metric.c
+19 -7
@@ -24,6 +24,8 @@ struct metric {
24 // YOU HAVE TO INITIALIZE IT YOURSELF !
25 };
26
27 +static struct aral_statistics mrg_aral_statistics;
28 +
29 struct mrg {
30 ARAL *aral[MRG_PARTITIONS];
31
@@ -312,16 +314,18 @@ MRG *mrg_create(void) {
314 MRG *mrg = callocz(1, sizeof(MRG));
315
316 for(size_t i = 0; i < MRG_PARTITIONS ; i++) {
317 + netdata_rwlock_init(&mrg->index[i].rwlock);
318 +
319 char buf[ARAL_MAX_NAME + 1];
320 snprintfz(buf, ARAL_MAX_NAME, "mrg[%zu]", i);
317 - netdata_rwlock_init(&mrg->index[i].rwlock);
321
319 - mrg->aral[i] = aral_create("mrg",
320 - sizeof(METRIC),
321 - 0,
322 - 512,
323 - NULL, NULL, false,
324 - false);
322 + mrg->aral[i] = aral_create(buf,
323 + sizeof(METRIC),
324 + 0,
325 + 16384,
326 + &mrg_aral_statistics,
327 + NULL, NULL, false,
328 + false);
329 }
330
331 mrg->stats.size = sizeof(MRG);
@@ -329,6 +333,14 @@ MRG *mrg_create(void) {
333 return mrg;
334 }
335
336 +size_t mrg_aral_structures(void) {
337 + return aral_structures_from_stats(&mrg_aral_statistics);
338 +}
339 +
340 +size_t mrg_aral_overhead(void) {
341 + return aral_overhead_from_stats(&mrg_aral_statistics);
342 +}
343 +
344 void mrg_destroy(MRG *mrg __maybe_unused) {
345 // no destruction possible
346 // we can't traverse the metrics list
database/engine/metric.h
+2
@@ -73,5 +73,7 @@ bool mrg_metric_set_writer(MRG *mrg, METRIC *metric);
73 bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric);
74
75 struct mrg_statistics mrg_get_statistics(MRG *mrg);
76 +size_t mrg_aral_structures(void);
77 +size_t mrg_aral_overhead(void);
78
79 #endif // DBENGINE_METRIC_H
database/engine/pagecache.c
+58 -68
@@ -1055,77 +1055,67 @@ size_t dynamic_extent_cache_size(void) {
1055 return target_size;
1056 }
1057
1058 -void init_page_cache(void)
1058 +void pgc_and_mrg_initialize(void)
1059 {
1060 - static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
1061 - static bool initialized = false;
1060 + main_mrg = mrg_create();
1061
1063 - netdata_spinlock_lock(&spinlock);
1064 - if (!initialized) {
1065 - initialized = true;
1062 + size_t target_cache_size = (size_t)default_rrdeng_page_cache_mb * 1024ULL * 1024ULL;
1063 + size_t main_cache_size = (target_cache_size / 100) * 95;
1064 + size_t open_cache_size = 0;
1065 + size_t extent_cache_size = (target_cache_size / 100) * 5;
1066
1067 - main_mrg = mrg_create();
1068 -
1069 - size_t target_cache_size = (size_t)default_rrdeng_page_cache_mb * 1024ULL * 1024ULL;
1070 - size_t main_cache_size = (target_cache_size / 100) * 95;
1071 - size_t open_cache_size = 0;
1072 - size_t extent_cache_size = (target_cache_size / 100) * 5;
1073 -
1074 - if(extent_cache_size < 3 * 1024 * 1024) {
1075 - extent_cache_size = 3 * 1024 * 1024;
1076 - main_cache_size = target_cache_size - extent_cache_size;
1077 - }
1078 -
1079 - main_cache = pgc_create(
1080 - "main_cache",
1081 - main_cache_size,
1082 - main_cache_free_clean_page_callback,
1083 - (size_t) rrdeng_pages_per_extent,
1084 - main_cache_flush_dirty_page_init_callback,
1085 - main_cache_flush_dirty_page_callback,
1086 - 10,
1087 - 10240, // if there are that many threads, evict so many at once!
1088 - 1000, //
1089 - 5, // don't delay too much other threads
1090 - PGC_OPTIONS_AUTOSCALE, // AUTOSCALE = 2x max hot pages
1091 - 0, // 0 = as many as the system cpus
1092 - 0
1093 - );
1094 -
1095 - open_cache = pgc_create(
1096 - "open_cache",
1097 - open_cache_size, // the default is 1MB
1098 - open_cache_free_clean_page_callback,
1099 - 1,
1100 - NULL,
1101 - open_cache_flush_dirty_page_callback,
1102 - 10,
1103 - 10240, // if there are that many threads, evict that many at once!
1104 - 1000, //
1105 - 3, // don't delay too much other threads
1106 - PGC_OPTIONS_AUTOSCALE | PGC_OPTIONS_EVICT_PAGES_INLINE | PGC_OPTIONS_FLUSH_PAGES_INLINE,
1107 - 0, // 0 = as many as the system cpus
1108 - sizeof(struct extent_io_data)
1109 - );
1110 - pgc_set_dynamic_target_cache_size_callback(open_cache, dynamic_open_cache_size);
1111 -
1112 - extent_cache = pgc_create(
1113 - "extent_cache",
1114 - extent_cache_size,
1115 - extent_cache_free_clean_page_callback,
1116 - 1,
1117 - NULL,
1118 - extent_cache_flush_dirty_page_callback,
1119 - 5,
1120 - 10, // it will lose up to that extents at once!
1121 - 100, //
1122 - 2, // don't delay too much other threads
1123 - PGC_OPTIONS_AUTOSCALE | PGC_OPTIONS_EVICT_PAGES_INLINE | PGC_OPTIONS_FLUSH_PAGES_INLINE,
1124 - 0, // 0 = as many as the system cpus
1125 - 0
1126 - );
1127 - pgc_set_dynamic_target_cache_size_callback(extent_cache, dynamic_extent_cache_size);
1067 + if(extent_cache_size < 3 * 1024 * 1024) {
1068 + extent_cache_size = 3 * 1024 * 1024;
1069 + main_cache_size = target_cache_size - extent_cache_size;
1070 }
1071
1130 - netdata_spinlock_unlock(&spinlock);
1072 + main_cache = pgc_create(
1073 + "main_cache",
1074 + main_cache_size,
1075 + main_cache_free_clean_page_callback,
1076 + (size_t) rrdeng_pages_per_extent,
1077 + main_cache_flush_dirty_page_init_callback,
1078 + main_cache_flush_dirty_page_callback,
1079 + 10,
1080 + 10240, // if there are that many threads, evict so many at once!
1081 + 1000, //
1082 + 5, // don't delay too much other threads
1083 + PGC_OPTIONS_AUTOSCALE, // AUTOSCALE = 2x max hot pages
1084 + 0, // 0 = as many as the system cpus
1085 + 0
1086 + );
1087 +
1088 + open_cache = pgc_create(
1089 + "open_cache",
1090 + open_cache_size, // the default is 1MB
1091 + open_cache_free_clean_page_callback,
1092 + 1,
1093 + NULL,
1094 + open_cache_flush_dirty_page_callback,
1095 + 10,
1096 + 10240, // if there are that many threads, evict that many at once!
1097 + 1000, //
1098 + 3, // don't delay too much other threads
1099 + PGC_OPTIONS_AUTOSCALE | PGC_OPTIONS_EVICT_PAGES_INLINE | PGC_OPTIONS_FLUSH_PAGES_INLINE,
1100 + 0, // 0 = as many as the system cpus
1101 + sizeof(struct extent_io_data)
1102 + );
1103 + pgc_set_dynamic_target_cache_size_callback(open_cache, dynamic_open_cache_size);
1104 +
1105 + extent_cache = pgc_create(
1106 + "extent_cache",
1107 + extent_cache_size,
1108 + extent_cache_free_clean_page_callback,
1109 + 1,
1110 + NULL,
1111 + extent_cache_flush_dirty_page_callback,
1112 + 5,
1113 + 10, // it will lose up to that extents at once!
1114 + 100, //
1115 + 2, // don't delay too much other threads
1116 + PGC_OPTIONS_AUTOSCALE | PGC_OPTIONS_EVICT_PAGES_INLINE | PGC_OPTIONS_FLUSH_PAGES_INLINE,
1117 + 0, // 0 = as many as the system cpus
1118 + 0
1119 + );
1120 + pgc_set_dynamic_target_cache_size_callback(extent_cache, dynamic_extent_cache_size);
1121 }
database/engine/pagecache.h
+1 -6
@@ -33,11 +33,6 @@ struct page_descr_with_data {
33 struct page_descr_with_data *prev;
34 struct page_descr_with_data *next;
35 } link;
36 -
37 - struct {
38 - struct page_descr_with_data *prev;
39 - struct page_descr_with_data *next;
40 - } cache;
36 };
37
38 #define PAGE_INFO_SCRATCH_SZ (8)
@@ -62,7 +57,7 @@ void rrdeng_prep_wait(struct page_details_control *pdc);
57 void rrdeng_prep_query(struct page_details_control *pdc);
58 void pg_cache_preload(struct rrdeng_query_handle *handle);
59 struct pgc_page *pg_cache_lookup_next(struct rrdengine_instance *ctx, struct page_details_control *pdc, time_t now_s, time_t last_update_every_s, size_t *entries);
65 -void init_page_cache(void);
60 +void pgc_and_mrg_initialize(void);
61
62 void pgc_open_add_hot_page(Word_t section, Word_t metric_id, time_t start_time_s, time_t end_time_s, time_t update_every_s, struct rrdengine_datafile *datafile, uint64_t extent_offset, unsigned extent_size, uint32_t page_length);
63
database/engine/pdc.c
+64 -257
@@ -18,22 +18,12 @@ struct extent_page_details_list {
18 struct extent_page_details_list *prev;
19 struct extent_page_details_list *next;
20 } query;
21 -
22 - struct {
23 - struct extent_page_details_list *prev;
24 - struct extent_page_details_list *next;
25 - } cache;
21 };
22
23 typedef struct datafile_extent_offset_list {
24 uv_file file;
25 unsigned fileno;
26 Pvoid_t extent_pd_list_by_extent_offset_JudyL;
32 -
33 - struct {
34 - struct datafile_extent_offset_list *prev;
35 - struct datafile_extent_offset_list *next;
36 - } cache;
27 } DEOL;
28
29 // ----------------------------------------------------------------------------
@@ -41,315 +31,129 @@ typedef struct datafile_extent_offset_list {
31
32 static struct {
33 struct {
44 - SPINLOCK spinlock;
45 - PDC *available_items;
46 - size_t available;
47 - } protected;
34 + ARAL *ar;
35 + } pdc;
36
37 struct {
50 - size_t allocated;
51 - } atomics;
52 -} pdc_globals = {
53 - .protected = {
54 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
55 - .available_items = NULL,
56 - .available = 0,
57 - },
58 - .atomics = {
59 - .allocated = 0,
60 - },
61 -};
38 + ARAL *ar;
39 + } pd;
40
63 -void pdc_cleanup1(void) {
64 - PDC *item = NULL;
65 -
66 - if(!netdata_spinlock_trylock(&pdc_globals.protected.spinlock))
67 - return;
68 -
69 - if(pdc_globals.protected.available_items && pdc_globals.protected.available > (size_t)libuv_worker_threads) {
70 - item = pdc_globals.protected.available_items;
71 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(pdc_globals.protected.available_items, item, cache.prev, cache.next);
72 - pdc_globals.protected.available--;
73 - }
74 -
75 - netdata_spinlock_unlock(&pdc_globals.protected.spinlock);
41 + struct {
42 + ARAL *ar;
43 + } epdl;
44
77 - if(item) {
78 - freez(item);
79 - __atomic_sub_fetch(&pdc_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
80 - }
45 + struct {
46 + ARAL *ar;
47 + } deol;
48 +} pdc_globals = {};
49 +
50 +void pdc_init(void) {
51 + pdc_globals.pdc.ar = aral_create(
52 + "dbengine-pdc",
53 + sizeof(PDC),
54 + 0,
55 + 65536,
56 + NULL,
57 + NULL, NULL, false, false
58 + );
59 }
60
61 PDC *pdc_get(void) {
84 - PDC *pdc = NULL;
85 -
86 - netdata_spinlock_lock(&pdc_globals.protected.spinlock);
87 -
88 - if(likely(pdc_globals.protected.available_items)) {
89 - pdc = pdc_globals.protected.available_items;
90 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(pdc_globals.protected.available_items, pdc, cache.prev, cache.next);
91 - pdc_globals.protected.available--;
92 - }
93 -
94 - netdata_spinlock_unlock(&pdc_globals.protected.spinlock);
95 -
96 - if(unlikely(!pdc)) {
97 - pdc = mallocz(sizeof(PDC));
98 - __atomic_add_fetch(&pdc_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
99 - }
100 -
62 + PDC *pdc = aral_mallocz(pdc_globals.pdc.ar);
63 memset(pdc, 0, sizeof(PDC));
64 return pdc;
65 }
66
67 static void pdc_release(PDC *pdc) {
106 - if(unlikely(!pdc)) return;
107 -
108 - netdata_spinlock_lock(&pdc_globals.protected.spinlock);
109 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(pdc_globals.protected.available_items, pdc, cache.prev, cache.next);
110 - pdc_globals.protected.available++;
111 - netdata_spinlock_unlock(&pdc_globals.protected.spinlock);
68 + aral_freez(pdc_globals.pdc.ar, pdc);
69 }
70
71 size_t pdc_cache_size(void) {
115 - return __atomic_load_n(&pdc_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(PDC);
72 + return aral_overhead(pdc_globals.pdc.ar) + aral_structures(pdc_globals.pdc.ar);
73 }
74
75 // ----------------------------------------------------------------------------
76 // PD cache
77
121 -static struct {
122 - struct {
123 - SPINLOCK spinlock;
124 - struct page_details *available_items;
125 - size_t available;
126 - } protected;
127 -
128 - struct {
129 - size_t allocated;
130 - } atomics;
131 -} page_details_globals = {
132 - .protected = {
133 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
134 - .available_items = NULL,
135 - .available = 0,
136 - },
137 - .atomics = {
138 - .allocated = 0,
139 - },
140 -};
141 -
142 -void page_details_cleanup1(void) {
143 - struct page_details *item = NULL;
144 -
145 - if(!netdata_spinlock_trylock(&page_details_globals.protected.spinlock))
146 - return;
147 -
148 - if(page_details_globals.protected.available_items && page_details_globals.protected.available > (size_t)libuv_worker_threads * 2) {
149 - item = page_details_globals.protected.available_items;
150 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_details_globals.protected.available_items, item, cache.prev, cache.next);
151 - page_details_globals.protected.available--;
152 - }
153 -
154 - netdata_spinlock_unlock(&page_details_globals.protected.spinlock);
155 -
156 - if(item) {
157 - freez(item);
158 - __atomic_sub_fetch(&page_details_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
159 - }
78 +void page_details_init(void) {
79 + pdc_globals.pd.ar = aral_create(
80 + "dbengine-pd",
81 + sizeof(struct page_details),
82 + 0,
83 + 65536,
84 + NULL,
85 + NULL, NULL, false, false
86 + );
87 }
88
89 struct page_details *page_details_get(void) {
163 - struct page_details *pd = NULL;
164 -
165 - netdata_spinlock_lock(&page_details_globals.protected.spinlock);
166 -
167 - if(likely(page_details_globals.protected.available_items)) {
168 - pd = page_details_globals.protected.available_items;
169 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_details_globals.protected.available_items, pd, cache.prev, cache.next);
170 - page_details_globals.protected.available--;
171 - }
172 -
173 - netdata_spinlock_unlock(&page_details_globals.protected.spinlock);
174 -
175 - if(unlikely(!pd)) {
176 - pd = mallocz(sizeof(struct page_details));
177 - __atomic_add_fetch(&page_details_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
178 - }
179 -
90 + struct page_details *pd = aral_mallocz(pdc_globals.pd.ar);
91 memset(pd, 0, sizeof(struct page_details));
92 return pd;
93 }
94
95 static void page_details_release(struct page_details *pd) {
185 - if(unlikely(!pd)) return;
186 -
187 - netdata_spinlock_lock(&page_details_globals.protected.spinlock);
188 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(page_details_globals.protected.available_items, pd, cache.prev, cache.next);
189 - page_details_globals.protected.available++;
190 - netdata_spinlock_unlock(&page_details_globals.protected.spinlock);
96 + aral_freez(pdc_globals.pd.ar, pd);
97 }
98
99 size_t pd_cache_size(void) {
194 - return __atomic_load_n(&page_details_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(struct page_details);
100 + return aral_overhead(pdc_globals.pd.ar) + aral_structures(pdc_globals.pd.ar);
101 }
102
103 // ----------------------------------------------------------------------------
104 // epdl cache
105
200 -static struct {
201 - struct {
202 - SPINLOCK spinlock;
203 - EPDL *available_items;
204 - size_t available;
205 - } protected;
206 -
207 - struct {
208 - size_t allocated;
209 - } atomics;
210 -} epdl_globals = {
211 - .protected = {
212 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
213 - .available_items = NULL,
214 - .available = 0,
215 - },
216 - .atomics = {
217 - .allocated = 0,
218 - },
219 -};
220 -
221 -void epdl_cleanup1(void) {
222 - EPDL *item = NULL;
223 -
224 - if(!netdata_spinlock_trylock(&epdl_globals.protected.spinlock))
225 - return;
226 -
227 - if(epdl_globals.protected.available_items && epdl_globals.protected.available > 100) {
228 - item = epdl_globals.protected.available_items;
229 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(epdl_globals.protected.available_items, item, cache.prev, cache.next);
230 - epdl_globals.protected.available--;
231 - }
232 -
233 - netdata_spinlock_unlock(&epdl_globals.protected.spinlock);
234 -
235 - if(item) {
236 - freez(item);
237 - __atomic_sub_fetch(&epdl_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
238 - }
106 +void epdl_init(void) {
107 + pdc_globals.epdl.ar = aral_create(
108 + "dbengine-epdl",
109 + sizeof(EPDL),
110 + 0,
111 + 65536,
112 + NULL,
113 + NULL, NULL, false, false
114 + );
115 }
116
117 static EPDL *epdl_get(void) {
242 - EPDL *epdl = NULL;
243 -
244 - netdata_spinlock_lock(&epdl_globals.protected.spinlock);
245 -
246 - if(likely(epdl_globals.protected.available_items)) {
247 - epdl = epdl_globals.protected.available_items;
248 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(epdl_globals.protected.available_items, epdl, cache.prev, cache.next);
249 - epdl_globals.protected.available--;
250 - }
251 -
252 - netdata_spinlock_unlock(&epdl_globals.protected.spinlock);
253 -
254 - if(unlikely(!epdl)) {
255 - epdl = mallocz(sizeof(EPDL));
256 - __atomic_add_fetch(&epdl_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
257 - }
258 -
118 + EPDL *epdl = aral_mallocz(pdc_globals.epdl.ar);
119 memset(epdl, 0, sizeof(EPDL));
120 return epdl;
121 }
122
123 static void epdl_release(EPDL *epdl) {
264 - if(unlikely(!epdl)) return;
265 -
266 - netdata_spinlock_lock(&epdl_globals.protected.spinlock);
267 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(epdl_globals.protected.available_items, epdl, cache.prev, cache.next);
268 - epdl_globals.protected.available++;
269 - netdata_spinlock_unlock(&epdl_globals.protected.spinlock);
124 + aral_freez(pdc_globals.epdl.ar, epdl);
125 }
126
127 size_t epdl_cache_size(void) {
273 - return __atomic_load_n(&epdl_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(EPDL);
128 + return aral_overhead(pdc_globals.epdl.ar) + aral_structures(pdc_globals.epdl.ar);
129 }
130
131 // ----------------------------------------------------------------------------
132 // deol cache
133
279 -static struct {
280 - struct {
281 - SPINLOCK spinlock;
282 - DEOL *available_items;
283 - size_t available;
284 - } protected;
285 -
286 - struct {
287 - size_t allocated;
288 - } atomics;
289 -} deol_globals = {
290 - .protected = {
291 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
292 - .available_items = NULL,
293 - .available = 0,
294 - },
295 - .atomics = {
296 - .allocated = 0,
297 - },
298 -};
299 -
300 -void deol_cleanup1(void) {
301 - DEOL *item = NULL;
302 -
303 - if(!netdata_spinlock_trylock(&deol_globals.protected.spinlock))
304 - return;
305 -
306 - if(deol_globals.protected.available_items && deol_globals.protected.available > 100) {
307 - item = deol_globals.protected.available_items;
308 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(deol_globals.protected.available_items, item, cache.prev, cache.next);
309 - deol_globals.protected.available--;
310 - }
311 -
312 - netdata_spinlock_unlock(&deol_globals.protected.spinlock);
313 -
314 - if(item) {
315 - freez(item);
316 - __atomic_sub_fetch(&deol_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
317 - }
134 +void deol_init(void) {
135 + pdc_globals.deol.ar = aral_create(
136 + "dbengine-deol",
137 + sizeof(DEOL),
138 + 0,
139 + 65536,
140 + NULL,
141 + NULL, NULL, false, false
142 + );
143 }
144
145 static DEOL *deol_get(void) {
321 - DEOL *deol = NULL;
322 -
323 - netdata_spinlock_lock(&deol_globals.protected.spinlock);
324 -
325 - if(likely(deol_globals.protected.available_items)) {
326 - deol = deol_globals.protected.available_items;
327 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(deol_globals.protected.available_items, deol, cache.prev, cache.next);
328 - deol_globals.protected.available--;
329 - }
330 -
331 - netdata_spinlock_unlock(&deol_globals.protected.spinlock);
332 -
333 - if(unlikely(!deol)) {
334 - deol = mallocz(sizeof(DEOL));
335 - __atomic_add_fetch(&deol_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
336 - }
337 -
146 + DEOL *deol = aral_mallocz(pdc_globals.deol.ar);
147 memset(deol, 0, sizeof(DEOL));
148 return deol;
149 }
150
151 static void deol_release(DEOL *deol) {
343 - if(unlikely(!deol)) return;
344 -
345 - netdata_spinlock_lock(&deol_globals.protected.spinlock);
346 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(deol_globals.protected.available_items, deol, cache.prev, cache.next);
347 - deol_globals.protected.available++;
348 - netdata_spinlock_unlock(&deol_globals.protected.spinlock);
152 + aral_freez(pdc_globals.deol.ar, deol);
153 }
154
155 size_t deol_cache_size(void) {
352 - return __atomic_load_n(&deol_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(DEOL);
156 + return aral_overhead(pdc_globals.deol.ar) + aral_structures(pdc_globals.deol.ar);
157 }
158
159 // ----------------------------------------------------------------------------
@@ -1160,7 +964,7 @@ static bool epdl_populate_pages_from_extent_data(
964 have_read_error);
965
966 if(worker)
1163 - worker_is_busy(UV_EVENT_DBENGINE_EXTENT_PAGE_POPULATION);
967 + worker_is_busy(UV_EVENT_DBENGINE_EXTENT_PAGE_ALLOCATION);
968
969 void *page_data;
970
@@ -1192,6 +996,9 @@ static bool epdl_populate_pages_from_extent_data(
996 }
997 }
998
999 + if(worker)
1000 + worker_is_busy(UV_EVENT_DBENGINE_EXTENT_PAGE_POPULATION);
1001 +
1002 PGC_ENTRY page_entry = {
1003 .hot = false,
1004 .section = (Word_t)ctx,
database/engine/pdc.h
+4 -4
@@ -40,10 +40,10 @@ size_t epdl_cache_size(void);
40 size_t deol_cache_size(void);
41 size_t extent_buffer_cache_size(void);
42
43 -void pdc_cleanup1(void);
44 -void page_details_cleanup1(void);
45 -void epdl_cleanup1(void);
46 -void deol_cleanup1(void);
43 +void pdc_init(void);
44 +void page_details_init(void);
45 +void epdl_init(void);
46 +void deol_init(void);
47 void extent_buffer_cleanup1(void);
48
49 void epdl_cmd_dequeued(void *epdl_ptr);
database/engine/rrdengine.c
+171 -495
@@ -26,6 +26,41 @@ struct rrdeng_main {
26 size_t flushes_running;
27 size_t evictions_running;
28 size_t cleanup_running;
29 +
30 + struct {
31 + ARAL *ar;
32 +
33 + struct {
34 + SPINLOCK spinlock;
35 +
36 + size_t waiting;
37 + struct rrdeng_cmd *waiting_items_by_priority[STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE];
38 + size_t executed_by_priority[STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE];
39 + } unsafe;
40 + } cmd_queue;
41 +
42 + struct {
43 + ARAL *ar;
44 +
45 + struct {
46 + size_t dispatched;
47 + size_t executing;
48 + size_t pending_cb;
49 + } atomics;
50 + } work_cmd;
51 +
52 + struct {
53 + ARAL *ar;
54 + } handles;
55 +
56 + struct {
57 + ARAL *ar;
58 + } descriptors;
59 +
60 + struct {
61 + ARAL *ar;
62 + } xt_io_descr;
63 +
64 } rrdeng_main = {
65 .thread = 0,
66 .loop = {},
@@ -34,6 +69,12 @@ struct rrdeng_main {
69 .flushes_running = 0,
70 .evictions_running = 0,
71 .cleanup_running = 0,
72 +
73 + .cmd_queue = {
74 + .unsafe = {
75 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
76 + },
77 + }
78 };
79
80 static void sanity_check(void)
@@ -79,71 +120,28 @@ struct rrdeng_work {
120 work_cb work_cb;
121 after_work_cb after_work_cb;
122 enum rrdeng_opcode opcode;
82 -
83 - struct {
84 - struct rrdeng_work *prev;
85 - struct rrdeng_work *next;
86 - } cache;
123 };
124
89 -static struct {
90 - struct {
91 - SPINLOCK spinlock;
92 - struct rrdeng_work *available_items;
93 - size_t available;
94 - } protected;
95 -
96 - struct {
97 - size_t allocated;
98 - size_t dispatched;
99 - size_t executing;
100 - size_t pending_cb;
101 - } atomics;
102 -} work_request_globals = {
103 - .protected = {
104 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
105 - .available_items = NULL,
106 - .available = 0,
107 - },
108 - .atomics = {
109 - .allocated = 0,
110 - .dispatched = 0,
111 - .executing = 0,
112 - },
113 -};
114 -
115 -static inline bool work_request_full(void) {
116 - return __atomic_load_n(&work_request_globals.atomics.dispatched, __ATOMIC_RELAXED) >= (size_t)(libuv_worker_threads - RESERVED_LIBUV_WORKER_THREADS);
125 +static void work_request_init(void) {
126 + rrdeng_main.work_cmd.ar = aral_create(
127 + "dbengine-work-cmd",
128 + sizeof(struct rrdeng_work),
129 + 0,
130 + 65536, NULL,
131 + NULL, NULL, false, false
132 + );
133 }
134
119 -static void work_request_cleanup1(void) {
120 - struct rrdeng_work *item = NULL;
121 -
122 - if(!netdata_spinlock_trylock(&work_request_globals.protected.spinlock))
123 - return;
124 -
125 - if(work_request_globals.protected.available_items && work_request_globals.protected.available > (size_t)libuv_worker_threads) {
126 - item = work_request_globals.protected.available_items;
127 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(work_request_globals.protected.available_items, item, cache.prev, cache.next);
128 - work_request_globals.protected.available--;
129 - }
130 - netdata_spinlock_unlock(&work_request_globals.protected.spinlock);
131 -
132 - if(item) {
133 - freez(item);
134 - __atomic_sub_fetch(&work_request_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
135 - }
135 +static inline bool work_request_full(void) {
136 + return __atomic_load_n(&rrdeng_main.work_cmd.atomics.dispatched, __ATOMIC_RELAXED) >= (size_t)(libuv_worker_threads - RESERVED_LIBUV_WORKER_THREADS);
137 }
138
139 static inline void work_done(struct rrdeng_work *work_request) {
139 - netdata_spinlock_lock(&work_request_globals.protected.spinlock);
140 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(work_request_globals.protected.available_items, work_request, cache.prev, cache.next);
141 - work_request_globals.protected.available++;
142 - netdata_spinlock_unlock(&work_request_globals.protected.spinlock);
140 + aral_freez(rrdeng_main.work_cmd.ar, work_request);
141 }
142
143 static void work_standard_worker(uv_work_t *req) {
146 - __atomic_add_fetch(&work_request_globals.atomics.executing, 1, __ATOMIC_RELAXED);
144 + __atomic_add_fetch(&rrdeng_main.work_cmd.atomics.executing, 1, __ATOMIC_RELAXED);
145
146 register_libuv_worker_jobs();
147 worker_is_busy(UV_EVENT_WORKER_INIT);
@@ -152,9 +150,9 @@ static void work_standard_worker(uv_work_t *req) {
150 work_request->data = work_request->work_cb(work_request->ctx, work_request->data, work_request->completion, req);
151 worker_is_idle();
152
155 - __atomic_sub_fetch(&work_request_globals.atomics.dispatched, 1, __ATOMIC_RELAXED);
156 - __atomic_sub_fetch(&work_request_globals.atomics.executing, 1, __ATOMIC_RELAXED);
157 - __atomic_add_fetch(&work_request_globals.atomics.pending_cb, 1, __ATOMIC_RELAXED);
153 + __atomic_sub_fetch(&rrdeng_main.work_cmd.atomics.dispatched, 1, __ATOMIC_RELAXED);
154 + __atomic_sub_fetch(&rrdeng_main.work_cmd.atomics.executing, 1, __ATOMIC_RELAXED);
155 + __atomic_add_fetch(&rrdeng_main.work_cmd.atomics.pending_cb, 1, __ATOMIC_RELAXED);
156
157 // signal the event loop a worker is available
158 fatal_assert(0 == uv_async_send(&rrdeng_main.async));
@@ -169,7 +167,7 @@ static void after_work_standard_callback(uv_work_t* req, int status) {
167 work_request->after_work_cb(work_request->ctx, work_request->data, work_request->completion, req, status);
168
169 work_done(work_request);
172 - __atomic_sub_fetch(&work_request_globals.atomics.pending_cb, 1, __ATOMIC_RELAXED);
170 + __atomic_sub_fetch(&rrdeng_main.work_cmd.atomics.pending_cb, 1, __ATOMIC_RELAXED);
171
172 worker_is_idle();
173 }
@@ -179,21 +177,7 @@ static bool work_dispatch(struct rrdengine_instance *ctx, void *data, struct com
177
178 internal_fatal(rrdeng_main.tid != gettid(), "work_dispatch() can only be run from the event loop thread");
179
182 - netdata_spinlock_lock(&work_request_globals.protected.spinlock);
183 -
184 - if(likely(work_request_globals.protected.available_items)) {
185 - work_request = work_request_globals.protected.available_items;
186 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(work_request_globals.protected.available_items, work_request, cache.prev, cache.next);
187 - work_request_globals.protected.available--;
188 - }
189 -
190 - netdata_spinlock_unlock(&work_request_globals.protected.spinlock);
191 -
192 - if(unlikely(!work_request)) {
193 - work_request = mallocz(sizeof(struct rrdeng_work));
194 - __atomic_add_fetch(&work_request_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
195 - }
196 -
180 + work_request = aral_mallocz(rrdeng_main.work_cmd.ar);
181 memset(work_request, 0, sizeof(struct rrdeng_work));
182 work_request->req.data = work_request;
183 work_request->ctx = ctx;
@@ -209,7 +193,7 @@ static bool work_dispatch(struct rrdengine_instance *ctx, void *data, struct com
193 return false;
194 }
195
212 - __atomic_add_fetch(&work_request_globals.atomics.dispatched, 1, __ATOMIC_RELAXED);
196 + __atomic_add_fetch(&rrdeng_main.work_cmd.atomics.dispatched, 1, __ATOMIC_RELAXED);
197
198 return true;
199 }
@@ -217,226 +201,71 @@ static bool work_dispatch(struct rrdengine_instance *ctx, void *data, struct com
201 // ----------------------------------------------------------------------------
202 // page descriptor cache
203
220 -static struct {
221 - struct {
222 - SPINLOCK spinlock;
223 - struct page_descr_with_data *available_items;
224 - size_t available;
225 - } protected;
226 -
227 - struct {
228 - size_t allocated;
229 - } atomics;
230 -} page_descriptor_globals = {
231 - .protected = {
232 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
233 - .available_items = NULL,
234 - .available = 0,
235 - },
236 - .atomics = {
237 - .allocated = 0,
238 - },
239 -};
240 -
241 -static void page_descriptor_cleanup1(void) {
242 - struct page_descr_with_data *item = NULL;
243 -
244 - if(!netdata_spinlock_trylock(&page_descriptor_globals.protected.spinlock))
245 - return;
246 -
247 - if(page_descriptor_globals.protected.available_items && page_descriptor_globals.protected.available > MAX_PAGES_PER_EXTENT) {
248 - item = page_descriptor_globals.protected.available_items;
249 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_descriptor_globals.protected.available_items, item, cache.prev, cache.next);
250 - page_descriptor_globals.protected.available--;
251 - }
252 -
253 - netdata_spinlock_unlock(&page_descriptor_globals.protected.spinlock);
254 -
255 - if(item) {
256 - freez(item);
257 - __atomic_sub_fetch(&page_descriptor_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
258 - }
204 +void page_descriptors_init(void) {
205 + rrdeng_main.descriptors.ar = aral_create(
206 + "dbengine-descriptors",
207 + sizeof(struct page_descr_with_data),
208 + 0,
209 + 65536 * 4,
210 + NULL,
211 + NULL, NULL, false, false);
212 }
213
214 struct page_descr_with_data *page_descriptor_get(void) {
262 - struct page_descr_with_data *descr = NULL;
263 -
264 - netdata_spinlock_lock(&page_descriptor_globals.protected.spinlock);
265 -
266 - if(likely(page_descriptor_globals.protected.available_items)) {
267 - descr = page_descriptor_globals.protected.available_items;
268 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(page_descriptor_globals.protected.available_items, descr, cache.prev, cache.next);
269 - page_descriptor_globals.protected.available--;
270 - }
271 -
272 - netdata_spinlock_unlock(&page_descriptor_globals.protected.spinlock);
273 -
274 - if(unlikely(!descr)) {
275 - descr = mallocz(sizeof(struct page_descr_with_data));
276 - __atomic_add_fetch(&page_descriptor_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
277 - }
278 -
215 + struct page_descr_with_data *descr = aral_mallocz(rrdeng_main.descriptors.ar);
216 memset(descr, 0, sizeof(struct page_descr_with_data));
217 return descr;
218 }
219
220 static inline void page_descriptor_release(struct page_descr_with_data *descr) {
284 - if(unlikely(!descr)) return;
285 -
286 - netdata_spinlock_lock(&page_descriptor_globals.protected.spinlock);
287 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(page_descriptor_globals.protected.available_items, descr, cache.prev, cache.next);
288 - page_descriptor_globals.protected.available++;
289 - netdata_spinlock_unlock(&page_descriptor_globals.protected.spinlock);
221 + aral_freez(rrdeng_main.descriptors.ar, descr);
222 }
223
224 // ----------------------------------------------------------------------------
225 // extent io descriptor cache
226
295 -static struct {
296 - struct {
297 - SPINLOCK spinlock;
298 - struct extent_io_descriptor *available_items;
299 - size_t available;
300 - } protected;
301 -
302 - struct {
303 - size_t allocated;
304 - } atomics;
305 -
306 -} extent_io_descriptor_globals = {
307 - .protected = {
308 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
309 - .available_items = NULL,
310 - .available = 0,
311 - },
312 - .atomics = {
313 - .allocated = 0,
314 - },
315 -};
316 -
317 -static void extent_io_descriptor_cleanup1(void) {
318 - struct extent_io_descriptor *item = NULL;
319 -
320 - if(!netdata_spinlock_trylock(&extent_io_descriptor_globals.protected.spinlock))
321 - return;
322 -
323 - if(extent_io_descriptor_globals.protected.available_items && extent_io_descriptor_globals.protected.available > (size_t)libuv_worker_threads) {
324 - item = extent_io_descriptor_globals.protected.available_items;
325 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(extent_io_descriptor_globals.protected.available_items, item, cache.prev, cache.next);
326 - extent_io_descriptor_globals.protected.available--;
327 - }
328 - netdata_spinlock_unlock(&extent_io_descriptor_globals.protected.spinlock);
329 -
330 - if(item) {
331 - freez(item);
332 - __atomic_sub_fetch(&extent_io_descriptor_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
333 - }
227 +static void extent_io_descriptor_init(void) {
228 + rrdeng_main.xt_io_descr.ar = aral_create(
229 + "dbengine-extent-io",
230 + sizeof(struct extent_io_descriptor),
231 + 0,
232 + 65536,
233 + NULL,
234 + NULL, NULL, false, false
235 + );
236 }
237
238 static struct extent_io_descriptor *extent_io_descriptor_get(void) {
337 - struct extent_io_descriptor *xt_io_descr = NULL;
338 -
339 - netdata_spinlock_lock(&extent_io_descriptor_globals.protected.spinlock);
340 -
341 - if(likely(extent_io_descriptor_globals.protected.available_items)) {
342 - xt_io_descr = extent_io_descriptor_globals.protected.available_items;
343 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(extent_io_descriptor_globals.protected.available_items, xt_io_descr, cache.prev, cache.next);
344 - extent_io_descriptor_globals.protected.available--;
345 - }
346 -
347 - netdata_spinlock_unlock(&extent_io_descriptor_globals.protected.spinlock);
348 -
349 - if(unlikely(!xt_io_descr)) {
350 - xt_io_descr = mallocz(sizeof(struct extent_io_descriptor));
351 - __atomic_add_fetch(&extent_io_descriptor_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
352 - }
353 -
239 + struct extent_io_descriptor *xt_io_descr = aral_mallocz(rrdeng_main.xt_io_descr.ar);
240 memset(xt_io_descr, 0, sizeof(struct extent_io_descriptor));
241 return xt_io_descr;
242 }
243
244 static inline void extent_io_descriptor_release(struct extent_io_descriptor *xt_io_descr) {
359 - if(unlikely(!xt_io_descr)) return;
360 -
361 - netdata_spinlock_lock(&extent_io_descriptor_globals.protected.spinlock);
362 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(extent_io_descriptor_globals.protected.available_items, xt_io_descr, cache.prev, cache.next);
363 - extent_io_descriptor_globals.protected.available++;
364 - netdata_spinlock_unlock(&extent_io_descriptor_globals.protected.spinlock);
245 + aral_freez(rrdeng_main.xt_io_descr.ar, xt_io_descr);
246 }
247
248 // ----------------------------------------------------------------------------
249 // query handle cache
250
370 -static struct {
371 - struct {
372 - SPINLOCK spinlock;
373 - struct rrdeng_query_handle *available_items;
374 - size_t available;
375 - } protected;
376 -
377 - struct {
378 - size_t allocated;
379 - } atomics;
380 -} rrdeng_query_handle_globals = {
381 - .protected = {
382 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
383 - .available_items = NULL,
384 - .available = 0,
385 - },
386 - .atomics = {
387 - .allocated = 0,
388 - },
389 -};
390 -
391 -static void rrdeng_query_handle_cleanup1(void) {
392 - struct rrdeng_query_handle *item = NULL;
393 -
394 - if(!netdata_spinlock_trylock(&rrdeng_query_handle_globals.protected.spinlock))
395 - return;
396 -
397 - if(rrdeng_query_handle_globals.protected.available_items && rrdeng_query_handle_globals.protected.available > 10) {
398 - item = rrdeng_query_handle_globals.protected.available_items;
399 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_query_handle_globals.protected.available_items, item, cache.prev, cache.next);
400 - rrdeng_query_handle_globals.protected.available--;
401 - }
402 -
403 - netdata_spinlock_unlock(&rrdeng_query_handle_globals.protected.spinlock);
404 -
405 - if(item) {
406 - freez(item);
407 - __atomic_sub_fetch(&rrdeng_query_handle_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
408 - }
251 +void rrdeng_query_handle_init(void) {
252 + rrdeng_main.handles.ar = aral_create(
253 + "dbengine-query-handles",
254 + sizeof(struct rrdeng_query_handle),
255 + 0,
256 + 65536,
257 + NULL,
258 + NULL, NULL, false, false);
259 }
260
261 struct rrdeng_query_handle *rrdeng_query_handle_get(void) {
412 - struct rrdeng_query_handle *handle = NULL;
413 -
414 - netdata_spinlock_lock(&rrdeng_query_handle_globals.protected.spinlock);
415 -
416 - if(likely(rrdeng_query_handle_globals.protected.available_items)) {
417 - handle = rrdeng_query_handle_globals.protected.available_items;
418 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_query_handle_globals.protected.available_items, handle, cache.prev, cache.next);
419 - rrdeng_query_handle_globals.protected.available--;
420 - }
421 -
422 - netdata_spinlock_unlock(&rrdeng_query_handle_globals.protected.spinlock);
423 -
424 - if(unlikely(!handle)) {
425 - handle = mallocz(sizeof(struct rrdeng_query_handle));
426 - __atomic_add_fetch(&rrdeng_query_handle_globals.atomics.allocated, 1, __ATOMIC_RELAXED);
427 - }
428 -
262 + struct rrdeng_query_handle *handle = aral_mallocz(rrdeng_main.handles.ar);
263 memset(handle, 0, sizeof(struct rrdeng_query_handle));
264 return handle;
265 }
266
267 void rrdeng_query_handle_release(struct rrdeng_query_handle *handle) {
434 - if(unlikely(!handle)) return;
435 -
436 - netdata_spinlock_lock(&rrdeng_query_handle_globals.protected.spinlock);
437 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_query_handle_globals.protected.available_items, handle, cache.prev, cache.next);
438 - rrdeng_query_handle_globals.protected.available++;
439 - netdata_spinlock_unlock(&rrdeng_query_handle_globals.protected.spinlock);
268 + aral_freez(rrdeng_main.handles.ar, handle);
269 }
270
271 // ----------------------------------------------------------------------------
@@ -551,60 +380,16 @@ struct rrdeng_cmd {
380 struct {
381 struct rrdeng_cmd *prev;
382 struct rrdeng_cmd *next;
554 - } cache;
555 -};
556 -
557 -static struct {
558 - struct {
559 - SPINLOCK spinlock;
560 - struct rrdeng_cmd *available_items;
561 - size_t available;
562 -
563 - struct {
564 - size_t allocated;
565 - } atomics;
566 - } cache;
567 -
568 - struct {
569 - SPINLOCK spinlock;
570 - size_t waiting;
571 - struct rrdeng_cmd *waiting_items_by_priority[STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE];
572 - size_t executed_by_priority[STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE];
383 } queue;
574 -
575 -
576 -} rrdeng_cmd_globals = {
577 - .cache = {
578 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
579 - .available_items = NULL,
580 - .available = 0,
581 - .atomics = {
582 - .allocated = 0,
583 - },
584 - },
585 - .queue = {
586 - .spinlock = NETDATA_SPINLOCK_INITIALIZER,
587 - .waiting = 0,
588 - },
384 };
385
591 -static void rrdeng_cmd_cleanup1(void) {
592 - struct rrdeng_cmd *item = NULL;
593 -
594 - if(!netdata_spinlock_trylock(&rrdeng_cmd_globals.cache.spinlock))
595 - return;
596 -
597 - if(rrdeng_cmd_globals.cache.available_items && rrdeng_cmd_globals.cache.available > 100) {
598 - item = rrdeng_cmd_globals.cache.available_items;
599 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.cache.available_items, item, cache.prev, cache.next);
600 - rrdeng_cmd_globals.cache.available--;
601 - }
602 - netdata_spinlock_unlock(&rrdeng_cmd_globals.cache.spinlock);
603 -
604 - if(item) {
605 - freez(item);
606 - __atomic_sub_fetch(&rrdeng_cmd_globals.cache.atomics.allocated, 1, __ATOMIC_RELAXED);
607 - }
386 +static void rrdeng_cmd_queue_init(void) {
387 + rrdeng_main.cmd_queue.ar = aral_create("dbengine-opcodes",
388 + sizeof(struct rrdeng_cmd),
389 + 0,
390 + 65536,
391 + NULL,
392 + NULL, NULL, false, false);
393 }
394
395 static inline STORAGE_PRIORITY rrdeng_enq_cmd_map_opcode_to_priority(enum rrdeng_opcode opcode, STORAGE_PRIORITY priority) {
@@ -632,41 +417,28 @@ void rrdeng_dequeue_epdl_cmd(struct rrdeng_cmd *cmd) {
417 }
418
419 void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY priority) {
635 - netdata_spinlock_lock(&rrdeng_cmd_globals.queue.spinlock);
420 + netdata_spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);
421
422 struct rrdeng_cmd *cmd = get_cmd_cb(data);
423 if(cmd) {
424 priority = rrdeng_enq_cmd_map_opcode_to_priority(cmd->opcode, priority);
425
426 if (cmd->priority > priority) {
642 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[cmd->priority], cmd, cache.prev, cache.next);
643 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
427 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[cmd->priority], cmd, queue.prev, queue.next);
428 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority], cmd, queue.prev, queue.next);
429 cmd->priority = priority;
430 }
431 }
432
648 - netdata_spinlock_unlock(&rrdeng_cmd_globals.queue.spinlock);
433 + netdata_spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);
434 }
435
436 void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, void *data, struct completion *completion,
437 enum storage_priority priority, enqueue_callback_t enqueue_cb, dequeue_callback_t dequeue_cb) {
653 - struct rrdeng_cmd *cmd = NULL;
438
439 priority = rrdeng_enq_cmd_map_opcode_to_priority(opcode, priority);
440
657 - netdata_spinlock_lock(&rrdeng_cmd_globals.cache.spinlock);
658 - if(likely(rrdeng_cmd_globals.cache.available_items)) {
659 - cmd = rrdeng_cmd_globals.cache.available_items;
660 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.cache.available_items, cmd, cache.prev, cache.next);
661 - rrdeng_cmd_globals.cache.available--;
662 - }
663 - netdata_spinlock_unlock(&rrdeng_cmd_globals.cache.spinlock);
664 -
665 - if(unlikely(!cmd)) {
666 - cmd = mallocz(sizeof(struct rrdeng_cmd));
667 - __atomic_add_fetch(&rrdeng_cmd_globals.cache.atomics.allocated, 1, __ATOMIC_RELAXED);
668 - }
669 -
441 + struct rrdeng_cmd *cmd = aral_mallocz(rrdeng_main.cmd_queue.ar);
442 memset(cmd, 0, sizeof(struct rrdeng_cmd));
443 cmd->ctx = ctx;
444 cmd->opcode = opcode;
@@ -675,19 +447,19 @@ void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, v
447 cmd->priority = priority;
448 cmd->dequeue_cb = dequeue_cb;
449
678 - netdata_spinlock_lock(&rrdeng_cmd_globals.queue.spinlock);
679 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
680 - rrdeng_cmd_globals.queue.waiting++;
450 + netdata_spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);
451 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority], cmd, queue.prev, queue.next);
452 + rrdeng_main.cmd_queue.unsafe.waiting++;
453 if(enqueue_cb)
454 enqueue_cb(cmd);
683 - netdata_spinlock_unlock(&rrdeng_cmd_globals.queue.spinlock);
455 + netdata_spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);
456
457 fatal_assert(0 == uv_async_send(&rrdeng_main.async));
458 }
459
460 static inline bool rrdeng_cmd_has_waiting_opcodes_in_lower_priorities(STORAGE_PRIORITY priority, STORAGE_PRIORITY max_priority) {
461 for(; priority <= max_priority ; priority++)
690 - if(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority])
462 + if(rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority])
463 return true;
464
465 return false;
@@ -699,15 +471,15 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
471 STORAGE_PRIORITY max_priority = work_request_full() ? STORAGE_PRIORITY_INTERNAL_DBENGINE : STORAGE_PRIORITY_BEST_EFFORT;
472
473 // find an opcode to execute from the queue
702 - netdata_spinlock_lock(&rrdeng_cmd_globals.queue.spinlock);
474 + netdata_spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);
475 for(STORAGE_PRIORITY priority = STORAGE_PRIORITY_INTERNAL_DBENGINE; priority <= max_priority ; priority++) {
704 - cmd = rrdeng_cmd_globals.queue.waiting_items_by_priority[priority];
476 + cmd = rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority];
477 if(cmd) {
478
479 // avoid starvation of lower priorities
480 if(unlikely(priority >= STORAGE_PRIORITY_HIGH &&
481 priority < STORAGE_PRIORITY_BEST_EFFORT &&
710 - ++rrdeng_cmd_globals.queue.executed_by_priority[priority] % 50 == 0 &&
482 + ++rrdeng_main.cmd_queue.unsafe.executed_by_priority[priority] % 50 == 0 &&
483 rrdeng_cmd_has_waiting_opcodes_in_lower_priorities(priority + 1, max_priority))) {
484 // let the others run 2% of the requests
485 cmd = NULL;
@@ -715,8 +487,8 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
487 }
488
489 // remove it from the queue
718 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_cmd_globals.queue.waiting_items_by_priority[priority], cmd, cache.prev, cache.next);
719 - rrdeng_cmd_globals.queue.waiting--;
490 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority], cmd, queue.prev, queue.next);
491 + rrdeng_main.cmd_queue.unsafe.waiting--;
492 break;
493 }
494 }
@@ -726,18 +498,14 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
498 cmd->dequeue_cb = NULL;
499 }
500
729 - netdata_spinlock_unlock(&rrdeng_cmd_globals.queue.spinlock);
501 + netdata_spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);
502
503 struct rrdeng_cmd ret;
504 if(cmd) {
505 // copy it, to return it
506 ret = *cmd;
507
736 - // put it in the cache
737 - netdata_spinlock_lock(&rrdeng_cmd_globals.cache.spinlock);
738 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_cmd_globals.cache.available_items, cmd, cache.prev, cache.next);
739 - rrdeng_cmd_globals.cache.available++;
740 - netdata_spinlock_unlock(&rrdeng_cmd_globals.cache.spinlock);
508 + aral_freez(rrdeng_main.cmd_queue.ar, cmd);
509 }
510 else
511 ret = (struct rrdeng_cmd) {
@@ -754,152 +522,50 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
522
523 // ----------------------------------------------------------------------------
524
757 -#define MAX_PAGE_SIZES_TO_KEEP 3
758 -#define MIN_PAGES_PER_SIZE_TO_KEEP 100
759 -
760 -struct dbengine_page_size {
761 - SPINLOCK spinlock;
762 - size_t page_size; // read-only, no lock required to read it
763 -
764 - size_t demand;
765 - size_t supply;
766 -
767 - size_t hit;
768 - size_t miss;
769 -
770 - size_t used;
771 - size_t array_size;
772 - void **array;
773 -};
774 -
525 struct {
776 - struct {
777 - size_t hit;
778 - size_t miss_wrong_size;
779 - size_t miss_short_supply;
780 - size_t cached_size;
781 - size_t struct_size;
782 - } atomic;
783 -
784 - struct dbengine_page_size slots[MAX_PAGE_SIZES_TO_KEEP];
785 -} dbengine_page_alloc_globals = {
786 - .atomic = {
787 - .struct_size = sizeof(dbengine_page_alloc_globals),
788 - }
789 -};
526 + ARAL *aral[RRD_STORAGE_TIERS];
527 +} dbengine_page_alloc_globals = {};
528
791 -__attribute__((constructor)) void initialize_sizes_to_slots(void) {
792 - uint8_t found[RRDENG_BLOCK_SIZE + 1];
793 - memset(found, 0, RRDENG_BLOCK_SIZE + 1);
794 -
795 - for(int i = 0; i < MAX_PAGE_SIZES_TO_KEEP ; i++) {
796 - struct dbengine_page_size *dps = &dbengine_page_alloc_globals.slots[i];
797 - memset(dps, 0, sizeof(struct dbengine_page_size));
798 - netdata_spinlock_init(&dps->spinlock);
799 - }
800 -
801 - for(int tier = 0; tier < MAX_PAGE_SIZES_TO_KEEP && tier < RRD_STORAGE_TIERS ; tier++) {
802 - size_t size = tier_page_size[tier];
803 -
804 - if(size <= RRDENG_BLOCK_SIZE && !found[size]) {
805 - struct dbengine_page_size *dps = &dbengine_page_alloc_globals.slots[tier];
806 - dps->page_size = size;
807 - found[size] = 1;
808 - }
809 - }
810 -}
529 +static inline ARAL *page_size_lookup(size_t size) {
530 + for(size_t tier = 0; tier < storage_tiers ;tier++)
531 + if(size == tier_page_size[tier])
532 + return dbengine_page_alloc_globals.aral[tier];
533
812 -static inline struct dbengine_page_size *page_size_lookup(size_t size) {
813 - for(int i = 0; i < MAX_PAGE_SIZES_TO_KEEP ; i++) {
814 - if(size == dbengine_page_alloc_globals.slots[i].page_size)
815 - return &dbengine_page_alloc_globals.slots[i];
816 - }
534 return NULL;
535 }
536
820 -static void dbengine_page_alloc_cleanup1(void) {
821 - for(int i = 0; i < MAX_PAGE_SIZES_TO_KEEP ; i++) {
822 - void *page = NULL;
537 +static void dbengine_page_alloc_init(void) {
538 + for(size_t i = storage_tiers; i > 0 ;i--) {
539 + size_t tier = storage_tiers - i;
540
824 - struct dbengine_page_size *dps = &dbengine_page_alloc_globals.slots[i];
825 - netdata_spinlock_lock(&dps->spinlock);
826 - if(dps->used > MIN_PAGES_PER_SIZE_TO_KEEP) {
827 - dps->used--;
828 - internal_fatal(!dps->array[dps->used], "DBENGINE: slot should have a page but is empty");
829 - page = dps->array[dps->used];
830 - dps->array[dps->used] = NULL;
831 - __atomic_sub_fetch(&dbengine_page_alloc_globals.atomic.cached_size, dps->page_size, __ATOMIC_RELAXED);
832 - }
833 - netdata_spinlock_unlock(&dps->spinlock);
541 + char buf[20 + 1];
542 + snprintfz(buf, 20, "tier%zu-pages", tier);
543
835 - if(page)
836 - freez(page);
544 + dbengine_page_alloc_globals.aral[tier] = aral_create(
545 + buf,
546 + tier_page_size[tier],
547 + 64,
548 + 512 * tier_page_size[tier],
549 + pgc_aral_statistics(),
550 + NULL, NULL, false, false);
551 }
552 }
553
554 void *dbengine_page_alloc(size_t size) {
841 - void *page = NULL;
842 -
843 - struct dbengine_page_size *dps = page_size_lookup(size);
844 - if(dps) {
845 - netdata_spinlock_lock(&dps->spinlock);
846 - dps->demand++;
847 -
848 - if(dps->used > 0) {
849 - dps->hit++;
850 - dps->used--;
851 - internal_fatal(!dps->array[dps->used], "DBENGINE: slot should have a page but is empty");
852 - page = dps->array[dps->used];
853 - dps->array[dps->used] = NULL;
854 - __atomic_add_fetch(&dbengine_page_alloc_globals.atomic.hit, 1, __ATOMIC_RELAXED);
855 - __atomic_sub_fetch(&dbengine_page_alloc_globals.atomic.cached_size, dps->page_size, __ATOMIC_RELAXED);
856 - }
857 - else {
858 - dps->miss++;
859 - __atomic_add_fetch(&dbengine_page_alloc_globals.atomic.miss_short_supply, 1, __ATOMIC_RELAXED);
860 - }
555 + ARAL *ar = page_size_lookup(size);
556 + if(ar) return aral_mallocz(ar);
557
862 - netdata_spinlock_unlock(&dps->spinlock);
863 - }
864 - else
865 - __atomic_add_fetch(&dbengine_page_alloc_globals.atomic.miss_wrong_size, 1, __ATOMIC_RELAXED);
866 -
867 - if(!page)
868 - page = mallocz(size);
869 -
870 - return page;
558 + return mallocz(size);
559 }
560
561 void dbengine_page_free(void *page, size_t size __maybe_unused) {
562 if(unlikely(!page || page == DBENGINE_EMPTY_PAGE))
563 return;
564
877 - struct dbengine_page_size *dps = page_size_lookup(size);
878 - if(dps) {
879 - netdata_spinlock_lock(&dps->spinlock);
880 - dps->supply++;
881 -
882 - if(dps->used == dps->array_size) {
883 - size_t new_array_size = dps->array_size ? dps->array_size * 2 : MIN_PAGES_PER_SIZE_TO_KEEP;
884 - dps->array = reallocz(dps->array, new_array_size * sizeof(void *));
885 -
886 - __atomic_add_fetch(&dbengine_page_alloc_globals.atomic.struct_size,
887 - (new_array_size - dps->array_size) * sizeof(void *), __ATOMIC_RELAXED);
888 -
889 - dps->array_size = new_array_size;
890 - }
891 -
892 - if(dps->used < dps->array_size) {
893 - dps->array[dps->used] = page;
894 - dps->used++;
895 - page = NULL;
896 - __atomic_add_fetch(&dbengine_page_alloc_globals.atomic.cached_size, dps->page_size, __ATOMIC_RELAXED);
897 - }
898 -
899 - netdata_spinlock_unlock(&dps->spinlock);
900 - }
901 -
902 - if(page)
565 + ARAL *ar = page_size_lookup(size);
566 + if(ar)
567 + aral_freez(ar, page);
568 + else
569 freez(page);
570 }
571
@@ -1817,19 +1483,19 @@ static void after_journal_v2_indexing(struct rrdengine_instance *ctx __maybe_unu
1483
1484 struct rrdeng_buffer_sizes rrdeng_get_buffer_sizes(void) {
1485 return (struct rrdeng_buffer_sizes) {
1820 - .opcodes = __atomic_load_n(&rrdeng_cmd_globals.cache.atomics.allocated, __ATOMIC_RELAXED) * sizeof(struct rrdeng_cmd),
1821 - .handles = __atomic_load_n(&rrdeng_query_handle_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(struct rrdeng_query_handle),
1822 - .descriptors = __atomic_load_n(&page_descriptor_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(struct page_descr_with_data),
1486 + .pgc = pgc_aral_overhead() + pgc_aral_structures(),
1487 + .mrg = mrg_aral_overhead() + mrg_aral_structures(),
1488 + .opcodes = aral_overhead(rrdeng_main.cmd_queue.ar) + aral_structures(rrdeng_main.cmd_queue.ar),
1489 + .handles = aral_overhead(rrdeng_main.handles.ar) + aral_structures(rrdeng_main.handles.ar),
1490 + .descriptors = aral_overhead(rrdeng_main.descriptors.ar) + aral_structures(rrdeng_main.descriptors.ar),
1491 .wal = __atomic_load_n(&wal_globals.atomics.allocated, __ATOMIC_RELAXED) * (sizeof(WAL) + RRDENG_BLOCK_SIZE),
1824 - .workers = __atomic_load_n(&work_request_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(struct rrdeng_work),
1492 + .workers = aral_overhead(rrdeng_main.work_cmd.ar),
1493 .pdc = pdc_cache_size(),
1826 - .xt_io = __atomic_load_n(&extent_io_descriptor_globals.atomics.allocated, __ATOMIC_RELAXED) * sizeof(struct extent_io_descriptor),
1494 + .xt_io = aral_overhead(rrdeng_main.xt_io_descr.ar) + aral_structures(rrdeng_main.xt_io_descr.ar),
1495 .xt_buf = extent_buffer_cache_size(),
1496 .epdl = epdl_cache_size(),
1497 .deol = deol_cache_size(),
1498 .pd = pd_cache_size(),
1831 - .pages = __atomic_load_n(&dbengine_page_alloc_globals.atomic.cached_size, __ATOMIC_RELAXED) +
1832 - __atomic_load_n(&dbengine_page_alloc_globals.atomic.struct_size, __ATOMIC_RELAXED),
1499
1500 #ifdef PDC_USE_JULYL
1501 .julyl = julyl_cache_size(),
@@ -1844,18 +1510,8 @@ static void after_cleanup(struct rrdengine_instance *ctx __maybe_unused, void *d
1510 static void *cleanup_tp_worker(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t *uv_work_req __maybe_unused) {
1511 worker_is_busy(UV_EVENT_DBENGINE_BUFFERS_CLEANUP);
1512
1847 - rrdeng_cmd_cleanup1();
1848 - work_request_cleanup1();
1849 - page_descriptor_cleanup1();
1850 - extent_io_descriptor_cleanup1();
1851 - pdc_cleanup1();
1852 - page_details_cleanup1();
1853 - rrdeng_query_handle_cleanup1();
1513 wal_cleanup1();
1514 extent_buffer_cleanup1();
1856 - epdl_cleanup1();
1857 - deol_cleanup1();
1858 - dbengine_page_alloc_cleanup1();
1515
1516 {
1517 static time_t last_run_s = 0;
@@ -1878,9 +1534,9 @@ void timer_cb(uv_timer_t* handle) {
1534 uv_stop(handle->loop);
1535 uv_update_time(handle->loop);
1536
1881 - worker_set_metric(RRDENG_OPCODES_WAITING, (NETDATA_DOUBLE)rrdeng_cmd_globals.queue.waiting);
1882 - worker_set_metric(RRDENG_WORKS_DISPATCHED, (NETDATA_DOUBLE)__atomic_load_n(&work_request_globals.atomics.dispatched, __ATOMIC_RELAXED));
1883 - worker_set_metric(RRDENG_WORKS_EXECUTING, (NETDATA_DOUBLE)__atomic_load_n(&work_request_globals.atomics.executing, __ATOMIC_RELAXED));
1537 + worker_set_metric(RRDENG_OPCODES_WAITING, (NETDATA_DOUBLE)rrdeng_main.cmd_queue.unsafe.waiting);
1538 + worker_set_metric(RRDENG_WORKS_DISPATCHED, (NETDATA_DOUBLE)__atomic_load_n(&rrdeng_main.work_cmd.atomics.dispatched, __ATOMIC_RELAXED));
1539 + worker_set_metric(RRDENG_WORKS_EXECUTING, (NETDATA_DOUBLE)__atomic_load_n(&rrdeng_main.work_cmd.atomics.executing, __ATOMIC_RELAXED));
1540
1541 rrdeng_enq_cmd(NULL, RRDENG_OPCODE_FLUSH_INIT, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);
1542 rrdeng_enq_cmd(NULL, RRDENG_OPCODE_EVICT_INIT, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);
@@ -1889,8 +1545,27 @@ void timer_cb(uv_timer_t* handle) {
1545 worker_is_idle();
1546 }
1547
1548 +static void dbengine_initialize_structures(void) {
1549 + pgc_and_mrg_initialize();
1550 +
1551 + pdc_init();
1552 + page_details_init();
1553 + epdl_init();
1554 + deol_init();
1555 + rrdeng_cmd_queue_init();
1556 + work_request_init();
1557 + rrdeng_query_handle_init();
1558 + page_descriptors_init();
1559 + extent_buffer_init();
1560 + dbengine_page_alloc_init();
1561 + extent_io_descriptor_init();
1562 +}
1563 +
1564 bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx __maybe_unused) {
1565 static bool spawned = false;
1566 + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
1567 +
1568 + netdata_spinlock_lock(&spinlock);
1569
1570 if(!spawned) {
1571 int ret;
@@ -1919,10 +1594,13 @@ bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx __maybe_unused) {
1594 }
1595 rrdeng_main.timer.data = &rrdeng_main;
1596
1597 + dbengine_initialize_structures();
1598 +
1599 fatal_assert(0 == uv_thread_create(&rrdeng_main.thread, dbengine_event_loop, &rrdeng_main));
1600 spawned = true;
1601 }
1602
1603 + netdata_spinlock_unlock(&spinlock);
1604 return true;
1605 }
1606
@@ -1968,8 +1646,6 @@ void dbengine_event_loop(void* arg) {
1646 worker_register_job_custom_metric(RRDENG_WORKS_DISPATCHED, "works dispatched", "works", WORKER_METRIC_ABSOLUTE);
1647 worker_register_job_custom_metric(RRDENG_WORKS_EXECUTING, "works executing", "works", WORKER_METRIC_ABSOLUTE);
1648
1971 - extent_buffer_init();
1972 -
1649 struct rrdeng_main *main = arg;
1650 enum rrdeng_opcode opcode;
1651 struct rrdeng_cmd cmd;
database/engine/rrdengine.h
+1 -20
@@ -55,11 +55,6 @@ typedef struct page_details_control {
55 STORAGE_PRIORITY priority;
56
57 time_t optimal_end_time_s;
58 -
59 - struct {
60 - struct page_details_control *prev;
61 - struct page_details_control *next;
62 - } cache;
58 } PDC;
59
60 PDC *pdc_get(void);
@@ -128,11 +123,6 @@ struct page_details {
123 struct page_details *prev;
124 struct page_details *next;
125 } load;
131 -
132 - struct {
133 - struct page_details *prev;
134 - struct page_details *next;
135 - } cache;
126 };
127
128 struct page_details *page_details_get(void);
@@ -225,11 +215,6 @@ struct rrdeng_query_handle {
215 unsigned position;
216 unsigned entries;
217
228 - struct {
229 - struct rrdeng_query_handle *prev;
230 - struct rrdeng_query_handle *next;
231 - } cache;
232 -
218 #ifdef NETDATA_INTERNAL_CHECKS
219 usec_t started_time_s;
220 pid_t query_pid;
@@ -292,11 +277,6 @@ struct extent_io_descriptor {
277 struct page_descr_with_data *descr_array[MAX_PAGES_PER_EXTENT];
278 struct rrdengine_datafile *datafile;
279 struct extent_io_descriptor *next; /* multiple requests to be served by the same cached extent */
295 -
296 - struct {
297 - struct extent_io_descriptor *prev;
298 - struct extent_io_descriptor *next;
299 - } cache;
280 };
281
282 struct generic_io_descriptor {
@@ -463,6 +443,7 @@ int init_rrd_files(struct rrdengine_instance *ctx);
443 void finalize_rrd_files(struct rrdengine_instance *ctx);
444 bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx);
445 void dbengine_event_loop(void *arg);
446 +
447 typedef void (*enqueue_callback_t)(struct rrdeng_cmd *cmd);
448 typedef void (*dequeue_callback_t)(struct rrdeng_cmd *cmd);
449
database/engine/rrdengineapi.c
+8 -15
@@ -1090,12 +1090,12 @@ static void rrdeng_populate_mrg(struct rrdengine_instance *ctx) {
1090 datafiles++;
1091 uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
1092
1093 - size_t cpus = get_system_cpus() / 2;
1093 + size_t cpus = get_netdata_cpus() / storage_tiers;
1094 if(cpus > datafiles)
1095 cpus = datafiles;
1096
1097 - if(cpus < 2)
1098 - cpus = 2;
1097 + if(cpus < 1)
1098 + cpus = 1;
1099
1100 if(cpus > (size_t)libuv_worker_threads)
1101 cpus = (size_t)libuv_worker_threads;
@@ -1158,7 +1158,7 @@ void rrdeng_exit_mode(struct rrdengine_instance *ctx) {
1158 /*
1159 * Returns 0 on success, negative on error
1160 */
1161 -int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
1161 +int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
1162 unsigned disk_space_mb, size_t tier) {
1163 struct rrdengine_instance *ctx;
1164 uint32_t max_open_files;
@@ -1191,8 +1191,6 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
1191 ctx->config.tier = (int)tier;
1192 ctx->config.page_type = tier_page_type[tier];
1193 ctx->config.global_compress_alg = RRD_LZ4;
1194 - if (page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB)
1195 - page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
1194 if (disk_space_mb < RRDENG_MIN_DISK_SPACE_MB)
1195 disk_space_mb = RRDENG_MIN_DISK_SPACE_MB;
1196 ctx->config.max_disk_space = disk_space_mb * 1048576LLU;
@@ -1202,15 +1200,10 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
1200 ctx->atomic.transaction_id = 1;
1201 ctx->quiesce.enabled = false;
1202
1205 - init_page_cache();
1206 - if (!init_rrd_files(ctx)) {
1207 - if(rrdeng_dbengine_spawn(ctx)) {
1208 - // success - we run this ctx too
1209 - rrdeng_populate_mrg(ctx);
1210 - return 0;
1211 - }
1212 -
1213 - finalize_rrd_files(ctx);
1203 + if (rrdeng_dbengine_spawn(ctx) && !init_rrd_files(ctx)) {
1204 + // success - we run this ctx too
1205 + rrdeng_populate_mrg(ctx);
1206 + return 0;
1207 }
1208
1209 if (ctx->config.legacy) {
database/engine/rrdengineapi.h
+3 -2
@@ -61,7 +61,7 @@ time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *r
61 void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
62
63 /* must call once before using anything */
64 -int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
64 +int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
65 unsigned disk_space_mb, size_t tier);
66
67 void rrdeng_readiness_wait(struct rrdengine_instance *ctx);
@@ -214,7 +214,8 @@ struct rrdeng_buffer_sizes {
214 size_t epdl;
215 size_t deol;
216 size_t pd;
217 - size_t pages;
217 + size_t pgc;
218 + size_t mrg;
219 #ifdef PDC_USE_JULYL
220 size_t julyl;
221 #endif
database/rrd.h
+1
@@ -134,6 +134,7 @@ typedef struct storage_point {
134 extern bool unittest_running;
135 extern bool dbengine_enabled;
136 extern size_t storage_tiers;
137 +extern bool use_direct_io;
138 extern size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
139
140 typedef enum __attribute__ ((__packed__)) {
database/rrdcalc.c
+2 -1
@@ -627,7 +627,8 @@ static void rrdcalc_rrdhost_delete_callback(const DICTIONARY_ITEM *item __maybe_
627
628 void rrdcalc_rrdhost_index_init(RRDHOST *host) {
629 if(!host->rrdcalc_root_index) {
630 - host->rrdcalc_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdhealth);
630 + host->rrdcalc_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
631 + &dictionary_stats_category_rrdhealth, sizeof(RRDCALC));
632
633 dictionary_register_insert_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_insert_callback, NULL);
634 dictionary_register_conflict_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_conflict_callback, NULL);
database/rrdcalctemplate.c
+2 -1
@@ -189,7 +189,8 @@ static void rrdcalctemplate_delete_callback(const DICTIONARY_ITEM *item __maybe_
189
190 void rrdcalctemplate_index_init(RRDHOST *host) {
191 if(!host->rrdcalctemplate_root_index) {
192 - host->rrdcalctemplate_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdhealth);
192 + host->rrdcalctemplate_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
193 + &dictionary_stats_category_rrdhealth, sizeof(RRDCALCTEMPLATE));
194
195 dictionary_register_insert_callback(host->rrdcalctemplate_root_index, rrdcalctemplate_insert_callback, NULL);
196 dictionary_register_delete_callback(host->rrdcalctemplate_root_index, rrdcalctemplate_delete_callback, host);
database/rrdcontext.c
+12 -6
@@ -611,7 +611,9 @@ static void rrdmetrics_create_in_rrdinstance(RRDINSTANCE *ri) {
611 if(unlikely(!ri)) return;
612 if(likely(ri->rrdmetrics)) return;
613
614 - ri->rrdmetrics = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext);
614 + ri->rrdmetrics = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
615 + &dictionary_stats_category_rrdcontext, sizeof(RRDMETRIC));
616 +
617 dictionary_register_insert_callback(ri->rrdmetrics, rrdmetric_insert_callback, ri);
618 dictionary_register_delete_callback(ri->rrdmetrics, rrdmetric_delete_callback, ri);
619 dictionary_register_conflict_callback(ri->rrdmetrics, rrdmetric_conflict_callback, ri);
@@ -914,7 +916,9 @@ static void rrdinstance_react_callback(const DICTIONARY_ITEM *item __maybe_unuse
916 void rrdinstances_create_in_rrdcontext(RRDCONTEXT *rc) {
917 if(unlikely(!rc || rc->rrdinstances)) return;
918
917 - rc->rrdinstances = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext);
919 + rc->rrdinstances = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
920 + &dictionary_stats_category_rrdcontext, sizeof(RRDINSTANCE));
921 +
922 dictionary_register_insert_callback(rc->rrdinstances, rrdinstance_insert_callback, rc);
923 dictionary_register_delete_callback(rc->rrdinstances, rrdinstance_delete_callback, rc);
924 dictionary_register_conflict_callback(rc->rrdinstances, rrdinstance_conflict_callback, rc);
@@ -1392,18 +1396,20 @@ void rrdhost_create_rrdcontexts(RRDHOST *host) {
1396 if(unlikely(!host)) return;
1397 if(likely(host->rrdctx)) return;
1398
1395 - host->rrdctx = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext);
1399 + host->rrdctx = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
1400 + &dictionary_stats_category_rrdcontext, sizeof(RRDCONTEXT));
1401 +
1402 dictionary_register_insert_callback((DICTIONARY *)host->rrdctx, rrdcontext_insert_callback, host);
1403 dictionary_register_delete_callback((DICTIONARY *)host->rrdctx, rrdcontext_delete_callback, host);
1404 dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx, rrdcontext_conflict_callback, host);
1405 dictionary_register_react_callback((DICTIONARY *)host->rrdctx, rrdcontext_react_callback, host);
1406
1401 - host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext);
1407 + host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext, 0);
1408 dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_insert_callback, NULL);
1409 dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_delete_callback, NULL);
1410 dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_conflict_callback, NULL);
1411
1406 - host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext);
1412 + host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext, 0);
1413 dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_post_processing_queue, rrdcontext_post_processing_queue_insert_callback, NULL);
1414 dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_post_processing_queue, rrdcontext_post_processing_queue_delete_callback, NULL);
1415 dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx_post_processing_queue, rrdcontext_post_processing_queue_conflict_callback, NULL);
@@ -2211,7 +2217,7 @@ DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contex
2217 if(!host || !host->rrdctx)
2218 return NULL;
2219
2214 - DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext);
2220 + DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext, 0);
2221 dictionary_register_insert_callback(dict, metric_entry_insert_callback, NULL);
2222 dictionary_register_delete_callback(dict, metric_entry_delete_callback, NULL);
2223 dictionary_register_conflict_callback(dict, metric_entry_conflict_callback, NULL);
database/rrddim.c
+2 -1
@@ -291,7 +291,8 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
291
292 void rrddim_index_init(RRDSET *st) {
293 if(!st->rrddim_root_index) {
294 - st->rrddim_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdset_rrddim);
294 + st->rrddim_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
295 + &dictionary_stats_category_rrdset_rrddim, sizeof(RRDDIM));
296
297 dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL);
298 dictionary_register_conflict_callback(st->rrddim_root_index, rrddim_conflict_callback, NULL);
database/rrddimvar.c
+2 -1
@@ -214,7 +214,8 @@ static void rrddimvar_delete_callback(const DICTIONARY_ITEM *item __maybe_unused
214
215 void rrddimvar_index_init(RRDSET *st) {
216 if(!st->rrddimvar_root_index) {
217 - st->rrddimvar_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdhealth);
217 + st->rrddimvar_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
218 + &dictionary_stats_category_rrdhealth, sizeof(RRDDIMVAR));
219
220 dictionary_register_insert_callback(st->rrddimvar_root_index, rrddimvar_insert_callback, NULL);
221 dictionary_register_conflict_callback(st->rrddimvar_root_index, rrddimvar_conflict_callback, NULL);
database/rrdfamily.c
+2 -1
@@ -33,7 +33,8 @@ static void rrdfamily_delete_callback(const DICTIONARY_ITEM *item __maybe_unused
33
34 void rrdfamily_index_init(RRDHOST *host) {
35 if(!host->rrdfamily_root_index) {
36 - host->rrdfamily_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdhealth);
36 + host->rrdfamily_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
37 + &dictionary_stats_category_rrdhealth, sizeof(RRDFAMILY));
38
39 dictionary_register_insert_callback(host->rrdfamily_root_index, rrdfamily_insert_callback, NULL);
40 dictionary_register_delete_callback(host->rrdfamily_root_index, rrdfamily_delete_callback, host);
database/rrdfunctions.c
+3 -1
@@ -424,7 +424,9 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_
424 void rrdfunctions_init(RRDHOST *host) {
425 if(host->functions) return;
426
427 - host->functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions);
427 + host->functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
428 + &dictionary_stats_category_functions, sizeof(struct rrd_collector_function));
429 +
430 dictionary_register_insert_callback(host->functions, rrd_functions_insert_callback, host);
431 dictionary_register_delete_callback(host->functions, rrd_functions_delete_callback, host);
432 dictionary_register_conflict_callback(host->functions, rrd_functions_conflict_callback, host);
database/rrdhost.c
+48 -15
@@ -7,6 +7,7 @@ static void rrdhost_streaming_sender_structures_init(RRDHOST *host);
7
8 bool dbengine_enabled = false; // will become true if and when dbengine is initialized
9 size_t storage_tiers = 3;
10 +bool use_direct_io = true;
11 size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
12 RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
13
@@ -51,13 +52,13 @@ static inline void rrdhost_init() {
52 if(unlikely(!rrdhost_root_index)) {
53 rrdhost_root_index = dictionary_create_advanced(
54 DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
54 - &dictionary_stats_category_rrdhost);
55 + &dictionary_stats_category_rrdhost, 0);
56 }
57
58 if(unlikely(!rrdhost_root_index_hostname)) {
59 rrdhost_root_index_hostname = dictionary_create_advanced(
60 DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
60 - &dictionary_stats_category_rrdhost);
61 + &dictionary_stats_category_rrdhost, 0);
62 }
63 }
64
@@ -390,7 +391,6 @@ int is_legacy = 1;
391 ret = rrdeng_init(
392 (struct rrdengine_instance **)&host->db[0].instance,
393 dbenginepath,
393 - default_rrdeng_page_cache_mb,
394 default_rrdeng_disk_quota_mb,
395 0); // may fail here for legacy dbengine initialization
396
@@ -768,8 +768,26 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim
768 // ----------------------------------------------------------------------------
769 // RRDHOST global / startup initialization
770
771 +#ifdef ENABLE_DBENGINE
772 +struct dbengine_initialization {
773 + netdata_thread_t thread;
774 + char path[FILENAME_MAX + 1];
775 + int disk_space_mb;
776 + size_t tier;
777 + int ret;
778 +};
779 +
780 +void *dbengine_tier_init(void *ptr) {
781 + struct dbengine_initialization *dbi = ptr;
782 + dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier);
783 + return ptr;
784 +}
785 +#endif
786 +
787 void dbengine_init(char *hostname) {
788 #ifdef ENABLE_DBENGINE
789 + use_direct_io = config_get_boolean(CONFIG_SECTION_DB, "dbengine use direct io", use_direct_io);
790 +
791 unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
792 if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
793 rrdeng_pages_per_extent = read_num;
@@ -790,6 +808,9 @@ void dbengine_init(char *hostname) {
808 config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
809 }
810
811 + bool parallel_initialization = (storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;
812 + parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);
813 +
814 default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
815 if (default_rrdeng_page_fetch_timeout < 1) {
816 info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout);
@@ -804,6 +825,8 @@ void dbengine_init(char *hostname) {
825 config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries);
826 }
827
828 + struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};
829 +
830 size_t created_tiers = 0;
831 char dbenginepath[FILENAME_MAX + 1];
832 char dbengineconfig[200 + 1];
@@ -823,15 +846,11 @@ void dbengine_init(char *hostname) {
846 if(tier > 0)
847 divisor *= 2;
848
826 - int page_cache_mb = default_rrdeng_page_cache_mb / divisor;
849 int disk_space_mb = default_multidb_disk_quota_mb / divisor;
850 size_t grouping_iterations = storage_tiers_grouping_iterations[tier];
851 RRD_BACKFILL backfill = storage_tiers_backfill[tier];
852
853 if(tier > 0) {
832 - snprintfz(dbengineconfig, 200, "dbengine tier %zu page cache size MB", tier);
833 - page_cache_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, page_cache_mb);
834 -
854 snprintfz(dbengineconfig, 200, "dbengine tier %zu multihost disk space MB", tier);
855 disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
856
@@ -865,17 +884,31 @@ void dbengine_init(char *hostname) {
884 }
885
886 internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, storage_tiers_grouping_iterations[tier]);
868 - ret = rrdeng_init(NULL, dbenginepath, page_cache_mb, disk_space_mb, tier);
869 - if(ret != 0) {
887 +
888 + tiers_init[tier].disk_space_mb = disk_space_mb;
889 + tiers_init[tier].tier = tier;
890 + strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);
891 + tiers_init[tier].ret = 0;
892 +
893 + if(parallel_initialization)
894 + netdata_thread_create(&tiers_init[tier].thread, "DBENGINE_INIT", NETDATA_THREAD_OPTION_JOINABLE,
895 + dbengine_tier_init, &tiers_init[tier]);
896 + else
897 + dbengine_tier_init(&tiers_init[tier]);
898 + }
899 +
900 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
901 + void *ptr;
902 +
903 + if(parallel_initialization)
904 + netdata_thread_join(tiers_init[tier].thread, &ptr);
905 +
906 + if(tiers_init[tier].ret != 0) {
907 error("DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'",
871 - hostname, tier, dbenginepath);
872 - break;
908 + hostname, tiers_init[tier].tier, tiers_init[tier].path);
909 }
874 - else {
875 - if (rrdeng_ctx_exceeded_disk_quota(multidb_ctx[created_tiers]))
876 - rrdeng_enq_cmd(multidb_ctx[created_tiers], RRDENG_OPCODE_DATABASE_ROTATE, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);
910 + else if(created_tiers == tier)
911 created_tiers++;
878 - }
912 }
913
914 if(created_tiers && created_tiers < storage_tiers) {
database/rrdlabels.c
+3 -1
@@ -533,7 +533,9 @@ static bool rrdlabel_conflict_callback(const DICTIONARY_ITEM *item __maybe_unuse
533 }
534
535 DICTIONARY *rrdlabels_create(void) {
536 - DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdlabels);
536 + DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
537 + &dictionary_stats_category_rrdlabels, sizeof(RRDLABEL));
538 +
539 dictionary_register_insert_callback(dict, rrdlabel_insert_callback, dict);
540 dictionary_register_delete_callback(dict, rrdlabel_delete_callback, dict);
541 dictionary_register_conflict_callback(dict, rrdlabel_conflict_callback, dict);
database/rrdset.c
+3 -2
@@ -399,7 +399,8 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
399
400 void rrdset_index_init(RRDHOST *host) {
401 if(!host->rrdset_root_index) {
402 - host->rrdset_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdset_rrddim);
402 + host->rrdset_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
403 + &dictionary_stats_category_rrdset_rrddim, sizeof(RRDSET));
404
405 dictionary_register_insert_callback(host->rrdset_root_index, rrdset_insert_callback, NULL);
406 dictionary_register_conflict_callback(host->rrdset_root_index, rrdset_conflict_callback, NULL);
@@ -410,7 +411,7 @@ void rrdset_index_init(RRDHOST *host) {
411 if(!host->rrdset_root_index_name) {
412 host->rrdset_root_index_name = dictionary_create_advanced(
413 DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
413 - &dictionary_stats_category_rrdset_rrddim);
414 + &dictionary_stats_category_rrdset_rrddim, 0);
415
416 dictionary_register_insert_callback(host->rrdset_root_index_name, rrdset_name_insert_callback, host);
417 dictionary_register_delete_callback(host->rrdset_root_index_name, rrdset_name_delete_callback, host);
database/rrdsetvar.c
+2 -1
@@ -189,7 +189,8 @@ static void rrdsetvar_delete_callback(const DICTIONARY_ITEM *item __maybe_unused
189
190 void rrdsetvar_index_init(RRDSET *st) {
191 if(!st->rrdsetvar_root_index) {
192 - st->rrdsetvar_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdhealth);
192 + st->rrdsetvar_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
193 + &dictionary_stats_category_rrdhealth, sizeof(RRDSETVAR));
194
195 dictionary_register_insert_callback(st->rrdsetvar_root_index, rrdsetvar_insert_callback, NULL);
196 dictionary_register_conflict_callback(st->rrdsetvar_root_index, rrdsetvar_conflict_callback, NULL);
database/rrdvar.c
+2 -1
@@ -84,7 +84,8 @@ static void rrdvar_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
84 }
85
86 DICTIONARY *rrdvariables_create(void) {
87 - DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdhealth);
87 + DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
88 + &dictionary_stats_category_rrdhealth, sizeof(RRDVAR));
89
90 dictionary_register_insert_callback(dict, rrdvar_insert_callback, NULL);
91 dictionary_register_delete_callback(dict, rrdvar_delete_callback, NULL);
database/sqlite/sqlite3.c
+3 -3
@@ -51886,9 +51886,9 @@ end_deserialize:
51886 /*
51887 ** Return true if the VFS is the memvfs.
51888 */
51889 -SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs *pVfs){
51890 - return pVfs==&memdb_vfs;
51891 -}
51889 +//SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs *pVfs){
51890 +// return pVfs==&memdb_vfs;
51891 +//}
51892
51893 /*
51894 ** This routine is called when the extension is loaded.
libnetdata/aral/aral.c
+285 -122
@@ -35,6 +35,9 @@ typedef struct aral_page {
35 struct {
36 uint32_t used_elements; // the number of used elements on this page
37 uint32_t free_elements; // the number of free elements on this page
38 +
39 + struct aral_page *prev; // the prev page on the list
40 + struct aral_page *next; // the next page on the list
41 } aral_lock;
42
43 struct {
@@ -42,25 +45,29 @@ typedef struct aral_page {
45 ARAL_FREE *list;
46 } free;
47
45 - struct aral_page *prev; // the prev page on the list
46 - struct aral_page *next; // the next page on the list
48 } ARAL_PAGE;
49
50 +typedef enum {
51 + ARAL_LOCKLESS = (1 << 0),
52 + ARAL_DEFRAGMENT = (1 << 1),
53 + ARAL_ALLOCATED_STATS = (1 << 2),
54 +} ARAL_OPTIONS;
55 +
56 struct aral {
57 struct {
58 char name[ARAL_MAX_NAME + 1];
59
53 - bool lockless;
54 - bool defragment;
60 + ARAL_OPTIONS options;
61
62 size_t element_size; // calculated to take into account ARAL overheads
57 - size_t max_allocation_size; // calculated in bytes
63 + size_t max_allocation_size; // calculated in bytes
64 + size_t max_page_elements; // calculated
65 size_t page_ptr_offset; // calculated
66 size_t natural_page_size; // calculated
67
61 - size_t requested_element_size;
68 size_t initial_page_elements;
63 - size_t max_page_elements;
69 + size_t requested_element_size;
70 + size_t requested_max_page_size;
71
72 struct {
73 bool enabled;
@@ -82,40 +89,36 @@ struct aral {
89
90 struct {
91 SPINLOCK spinlock;
85 - size_t allocation_size; // current allocation size
92 + size_t allocating_elements; // currently allocating elements
93 + size_t allocation_size; // current / next allocation size
94 } adders;
95
96 struct {
97 + size_t allocators; // the number of threads currently trying to allocate memory
98 } atomic;
99 +
100 + struct aral_statistics *stats;
101 };
102
92 -struct {
93 - struct {
94 - struct {
95 - size_t allocations;
96 - size_t allocated;
97 - } structures;
103 +size_t aral_structures_from_stats(struct aral_statistics *stats) {
104 + return __atomic_load_n(&stats->structures.allocated_bytes, __ATOMIC_RELAXED);
105 +}
106
99 - struct {
100 - size_t allocations;
101 - size_t allocated;
102 - size_t used;
103 - } malloc;
107 +size_t aral_overhead_from_stats(struct aral_statistics *stats) {
108 + return __atomic_load_n(&stats->malloc.allocated_bytes, __ATOMIC_RELAXED) -
109 + __atomic_load_n(&stats->malloc.used_bytes, __ATOMIC_RELAXED);
110 +}
111
105 - struct {
106 - size_t allocations;
107 - size_t allocated;
108 - size_t used;
109 - } mmap;
110 - } atomic;
111 -} aral_globals = {};
112 +size_t aral_overhead(ARAL *ar) {
113 + return aral_overhead_from_stats(ar->stats);
114 +}
115 +
116 +size_t aral_structures(ARAL *ar) {
117 + return aral_structures_from_stats(ar->stats);
118 +}
119
113 -void aral_get_size_statistics(size_t *structures, size_t *malloc_allocated, size_t *malloc_used, size_t *mmap_allocated, size_t *mmap_used) {
114 - *structures = __atomic_load_n(&aral_globals.atomic.structures.allocated, __ATOMIC_RELAXED);
115 - *malloc_allocated = __atomic_load_n(&aral_globals.atomic.malloc.allocated, __ATOMIC_RELAXED);
116 - *malloc_used = __atomic_load_n(&aral_globals.atomic.malloc.used, __ATOMIC_RELAXED);
117 - *mmap_allocated = __atomic_load_n(&aral_globals.atomic.mmap.allocated, __ATOMIC_RELAXED);
118 - *mmap_used = __atomic_load_n(&aral_globals.atomic.mmap.used, __ATOMIC_RELAXED);
120 +struct aral_statistics *aral_statistics(ARAL *ar) {
121 + return ar->stats;
122 }
123
124 #define ARAL_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
@@ -137,15 +140,42 @@ static size_t aral_align_alloc_size(ARAL *ar, uint64_t size) {
140 }
141
142 static inline void aral_lock(ARAL *ar) {
140 - if(likely(!ar->config.lockless))
143 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))
144 netdata_spinlock_lock(&ar->aral_lock.spinlock);
145 }
146
147 static inline void aral_unlock(ARAL *ar) {
145 - if(likely(!ar->config.lockless))
148 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))
149 netdata_spinlock_unlock(&ar->aral_lock.spinlock);
150 }
151
152 +static inline void aral_page_free_lock(ARAL *ar, ARAL_PAGE *page) {
153 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))
154 + netdata_spinlock_lock(&page->free.spinlock);
155 +}
156 +
157 +static inline void aral_page_free_unlock(ARAL *ar, ARAL_PAGE *page) {
158 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))
159 + netdata_spinlock_unlock(&page->free.spinlock);
160 +}
161 +
162 +static inline bool aral_adders_trylock(ARAL *ar) {
163 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))
164 + return netdata_spinlock_trylock(&ar->adders.spinlock);
165 +
166 + return true;
167 +}
168 +
169 +static inline void aral_adders_lock(ARAL *ar) {
170 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))
171 + netdata_spinlock_lock(&ar->adders.spinlock);
172 +}
173 +
174 +static inline void aral_adders_unlock(ARAL *ar) {
175 + if(likely(!(ar->config.options & ARAL_LOCKLESS)))
176 + netdata_spinlock_unlock(&ar->adders.spinlock);
177 +}
178 +
179 static void aral_delete_leftover_files(const char *name, const char *path, const char *required_prefix) {
180 DIR *dir = opendir(path);
181 if(!dir) return;
@@ -197,7 +227,7 @@ static inline ARAL_PAGE *find_page_with_allocation_internal_check(ARAL *ar, void
227 uintptr_t seeking = (uintptr_t)ptr;
228 ARAL_PAGE *page;
229
200 - for(page = ar->aral_lock.pages; page ; page = page->next) {
230 + for(page = ar->aral_lock.pages; page ; page = page->aral_lock.next) {
231 if(unlikely(seeking >= (uintptr_t)page->data && seeking < (uintptr_t)page->data + page->size))
232 break;
233 }
@@ -230,23 +260,29 @@ static inline ARAL_PAGE *find_page_with_free_slots_internal_check___with_aral_lo
260 }
261 #endif
262
233 -static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
234 - ARAL_PAGE *page = callocz(1, sizeof(ARAL_PAGE));
235 - netdata_spinlock_init(&page->free.spinlock);
236 - page->size = ar->adders.allocation_size;
263 +size_t aral_next_allocation_size___adders_lock_needed(ARAL *ar) {
264 + size_t size = ar->adders.allocation_size;
265
238 - if(page->size > ar->config.max_allocation_size)
239 - page->size = ar->config.max_allocation_size;
266 + if(size > ar->config.max_allocation_size)
267 + size = ar->config.max_allocation_size;
268 else
241 - ar->adders.allocation_size = aral_align_alloc_size(ar, (uint64_t)ar->adders.allocation_size * 4 / 3);
269 + ar->adders.allocation_size = aral_align_alloc_size(ar, (uint64_t)ar->adders.allocation_size * 2);
270
243 - page->max_elements = page->aral_lock.free_elements = page->size / ar->config.element_size;
271 + return size;
272 +}
273 +
274 +static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
275 + ARAL_PAGE *page = callocz(1, sizeof(ARAL_PAGE));
276 + netdata_spinlock_init(&page->free.spinlock);
277 + page->size = size;
278 + page->max_elements = page->size / ar->config.element_size;
279 + page->aral_lock.free_elements = page->max_elements;
280 page->free_elements_to_move_first = page->max_elements / 4;
281 if(unlikely(page->free_elements_to_move_first < 1))
282 page->free_elements_to_move_first = 1;
283
248 - __atomic_add_fetch(&aral_globals.atomic.structures.allocations, 1, __ATOMIC_RELAXED);
249 - __atomic_add_fetch(&aral_globals.atomic.structures.allocated, sizeof(ARAL_PAGE), __ATOMIC_RELAXED);
284 + __atomic_add_fetch(&ar->stats->structures.allocations, 1, __ATOMIC_RELAXED);
285 + __atomic_add_fetch(&ar->stats->structures.allocated_bytes, sizeof(ARAL_PAGE), __ATOMIC_RELAXED);
286
287 if(unlikely(ar->config.mmap.enabled)) {
288 ar->aral_lock.file_number++;
@@ -257,8 +293,8 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar TRACE_ALLOCATIONS_F
293 if (unlikely(!page->data))
294 fatal("ARAL: '%s' cannot allocate aral buffer of size %zu on filename '%s'",
295 ar->config.name, page->size, page->filename);
260 - __atomic_add_fetch(&aral_globals.atomic.mmap.allocations, 1, __ATOMIC_RELAXED);
261 - __atomic_add_fetch(&aral_globals.atomic.mmap.allocated, page->size, __ATOMIC_RELAXED);
296 + __atomic_add_fetch(&ar->stats->mmap.allocations, 1, __ATOMIC_RELAXED);
297 + __atomic_add_fetch(&ar->stats->mmap.allocated_bytes, page->size, __ATOMIC_RELAXED);
298 }
299 else {
300 #ifdef NETDATA_TRACE_ALLOCATIONS
@@ -266,8 +302,8 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar TRACE_ALLOCATIONS_F
302 #else
303 page->data = mallocz(page->size);
304 #endif
269 - __atomic_add_fetch(&aral_globals.atomic.malloc.allocations, 1, __ATOMIC_RELAXED);
270 - __atomic_add_fetch(&aral_globals.atomic.malloc.allocated, page->size, __ATOMIC_RELAXED);
305 + __atomic_add_fetch(&ar->stats->malloc.allocations, 1, __ATOMIC_RELAXED);
306 + __atomic_add_fetch(&ar->stats->malloc.allocated_bytes, page->size, __ATOMIC_RELAXED);
307 }
308
309 // link the free space to its page
@@ -292,8 +328,8 @@ void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_
328
329 freez((void *)page->filename);
330
295 - __atomic_sub_fetch(&aral_globals.atomic.mmap.allocations, 1, __ATOMIC_RELAXED);
296 - __atomic_sub_fetch(&aral_globals.atomic.mmap.allocated, page->size, __ATOMIC_RELAXED);
331 + __atomic_sub_fetch(&ar->stats->mmap.allocations, 1, __ATOMIC_RELAXED);
332 + __atomic_sub_fetch(&ar->stats->mmap.allocated_bytes, page->size, __ATOMIC_RELAXED);
333 }
334 else {
335 #ifdef NETDATA_TRACE_ALLOCATIONS
@@ -301,14 +337,14 @@ void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_
337 #else
338 freez(page->data);
339 #endif
304 - __atomic_sub_fetch(&aral_globals.atomic.malloc.allocations, 1, __ATOMIC_RELAXED);
305 - __atomic_sub_fetch(&aral_globals.atomic.malloc.allocated, page->size, __ATOMIC_RELAXED);
340 + __atomic_sub_fetch(&ar->stats->malloc.allocations, 1, __ATOMIC_RELAXED);
341 + __atomic_sub_fetch(&ar->stats->malloc.allocated_bytes, page->size, __ATOMIC_RELAXED);
342 }
343
344 freez(page);
345
310 - __atomic_sub_fetch(&aral_globals.atomic.structures.allocations, 1, __ATOMIC_RELAXED);
311 - __atomic_sub_fetch(&aral_globals.atomic.structures.allocated, sizeof(ARAL_PAGE), __ATOMIC_RELAXED);
346 + __atomic_sub_fetch(&ar->stats->structures.allocations, 1, __ATOMIC_RELAXED);
347 + __atomic_sub_fetch(&ar->stats->structures.allocated_bytes, sizeof(ARAL_PAGE), __ATOMIC_RELAXED);
348 }
349
350 static inline void aral_insert_not_linked_page_with_free_items_to_proper_position___aral_lock_needed(ARAL *ar, ARAL_PAGE *page) {
@@ -319,23 +355,24 @@ static inline void aral_insert_not_linked_page_with_free_items_to_proper_positio
355 !first->aral_lock.free_elements ||
356 page->aral_lock.free_elements <= first->aral_lock.free_elements + ARAL_FREE_PAGES_DELTA_TO_REARRANGE_LIST) {
357 // first position
322 - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
358 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
359 }
360 else {
325 - ARAL_PAGE *second = first->next;
361 + ARAL_PAGE *second = first->aral_lock.next;
362
363 if (!second ||
364 !second->aral_lock.free_elements ||
365 page->aral_lock.free_elements <= second->aral_lock.free_elements)
366 // second position
331 - DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, first, page, prev, next);
367 + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, first, page, aral_lock.prev, aral_lock.next);
368 else
369 // third position
334 - DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, second, page, prev, next);
370 + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, second, page, aral_lock.prev, aral_lock.next);
371 }
372 }
373
374 static inline ARAL_PAGE *aral_acquire_a_free_slot(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
375 + __atomic_add_fetch(&ar->atomic.allocators, 1, __ATOMIC_RELAXED);
376 aral_lock(ar);
377
378 ARAL_PAGE *page = ar->aral_lock.pages;
@@ -346,34 +383,51 @@ static inline ARAL_PAGE *aral_acquire_a_free_slot(ARAL *ar TRACE_ALLOCATIONS_FUN
383 #endif
384 aral_unlock(ar);
385
349 - if(netdata_spinlock_trylock(&ar->adders.spinlock)) {
350 - page = aral_create_page___no_lock_needed(ar TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
386 + if(aral_adders_trylock(ar)) {
387 + if(ar->adders.allocating_elements < __atomic_load_n(&ar->atomic.allocators, __ATOMIC_RELAXED)) {
388
352 - aral_lock(ar);
353 - aral_insert_not_linked_page_with_free_items_to_proper_position___aral_lock_needed(ar, page);
354 - netdata_spinlock_unlock(&ar->adders.spinlock);
355 - break;
356 - }
357 - else {
358 - aral_lock(ar);
359 - page = ar->aral_lock.pages;
389 + size_t size = aral_next_allocation_size___adders_lock_needed(ar);
390 + ar->adders.allocating_elements += size / ar->config.element_size;
391 + aral_adders_unlock(ar);
392 +
393 + page = aral_create_page___no_lock_needed(ar, size TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
394 +
395 + aral_lock(ar);
396 + aral_insert_not_linked_page_with_free_items_to_proper_position___aral_lock_needed(ar, page);
397 +
398 + aral_adders_lock(ar);
399 + ar->adders.allocating_elements -= size / ar->config.element_size;
400 + aral_adders_unlock(ar);
401 +
402 + // we have a page that is all empty
403 + // and only aral_lock() is held, so
404 + // break the loop
405 + break;
406 + }
407 +
408 + aral_adders_unlock(ar);
409 }
410 +
411 + aral_lock(ar);
412 + page = ar->aral_lock.pages;
413 }
414
415 + __atomic_sub_fetch(&ar->atomic.allocators, 1, __ATOMIC_RELAXED);
416 +
417 // we have a page
418 // and aral locked
419
420 {
421 ARAL_PAGE *first = ar->aral_lock.pages;
368 - ARAL_PAGE *second = first->next;
422 + ARAL_PAGE *second = first->aral_lock.next;
423
424 if (!second ||
425 !second->aral_lock.free_elements ||
426 first->aral_lock.free_elements <= second->aral_lock.free_elements + ARAL_FREE_PAGES_DELTA_TO_REARRANGE_LIST)
427 page = first;
428 else {
375 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, second, prev, next);
376 - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, second, prev, next);
429 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, second, aral_lock.prev, aral_lock.next);
430 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, second, aral_lock.prev, aral_lock.next);
431 page = second;
432 }
433 }
@@ -401,8 +455,8 @@ static inline ARAL_PAGE *aral_acquire_a_free_slot(ARAL *ar TRACE_ALLOCATIONS_FUN
455 // we are done with this page
456 // move the full page last
457 // so that pages with free items remain first in the list
404 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
405 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
458 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
459 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
460 }
461
462 aral_unlock(ar);
@@ -414,7 +468,7 @@ void *aral_mallocz_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAM
468
469 ARAL_PAGE *page = aral_acquire_a_free_slot(ar TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
470
417 - netdata_spinlock_lock(&page->free.spinlock);
471 + aral_page_free_lock(ar, page);
472
473 internal_fatal(!page->free.list,
474 "ARAL: '%s' free item to use, cannot be NULL.", ar->config.name);
@@ -445,7 +499,7 @@ void *aral_mallocz_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAM
499 aral_free_validate_internal_check(ar, fr);
500 }
501
448 - netdata_spinlock_unlock(&page->free.spinlock);
502 + aral_page_free_unlock(ar, page);
503
504 // put the page pointer after the element
505 uint8_t *data = (uint8_t *)found_fr;
@@ -453,9 +507,9 @@ void *aral_mallocz_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAM
507 *page_ptr = page;
508
509 if(unlikely(ar->config.mmap.enabled))
456 - __atomic_add_fetch(&aral_globals.atomic.mmap.used, ar->config.element_size, __ATOMIC_RELAXED);
510 + __atomic_add_fetch(&ar->stats->mmap.used_bytes, ar->config.element_size, __ATOMIC_RELAXED);
511 else
458 - __atomic_add_fetch(&aral_globals.atomic.malloc.used, ar->config.element_size, __ATOMIC_RELAXED);
512 + __atomic_add_fetch(&ar->stats->malloc.used_bytes, ar->config.element_size, __ATOMIC_RELAXED);
513
514 return (void *)found_fr;
515 }
@@ -503,35 +557,35 @@ static void aral_defrag_sorted_page_position___aral_lock_needed(ARAL *ar, ARAL_P
557 int action = 0; (void)action;
558 size_t move_later = 0, move_earlier = 0;
559
506 - for(tmp = page->next ;
560 + for(tmp = page->aral_lock.next ;
561 tmp && tmp->aral_lock.free_elements && tmp->aral_lock.free_elements < page->aral_lock.free_elements ;
508 - tmp = tmp->next)
562 + tmp = tmp->aral_lock.next)
563 move_later++;
564
511 - if(!tmp && page->next) {
512 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
513 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
565 + if(!tmp && page->aral_lock.next) {
566 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
567 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
568 action = 1;
569 }
516 - else if(tmp != page->next) {
517 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
518 - DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(ar->aral_lock.pages, tmp, page, prev, next);
570 + else if(tmp != page->aral_lock.next) {
571 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
572 + DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(ar->aral_lock.pages, tmp, page, aral_lock.prev, aral_lock.next);
573 action = 2;
574 }
575 else {
522 - for(tmp = (page == ar->aral_lock.pages) ? NULL : page->prev ;
576 + for(tmp = (page == ar->aral_lock.pages) ? NULL : page->aral_lock.prev ;
577 tmp && (!tmp->aral_lock.free_elements || tmp->aral_lock.free_elements > page->aral_lock.free_elements);
524 - tmp = (tmp == ar->aral_lock.pages) ? NULL : tmp->prev)
578 + tmp = (tmp == ar->aral_lock.pages) ? NULL : tmp->aral_lock.prev)
579 move_earlier++;
580
581 if(!tmp) {
528 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
529 - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
582 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
583 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
584 action = 3;
585 }
532 - else if(tmp != page->prev){
533 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
534 - DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, tmp, page, prev, next);
586 + else if(tmp != page->aral_lock.prev){
587 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
588 + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(ar->aral_lock.pages, tmp, page, aral_lock.prev, aral_lock.next);
589 action = 4;
590 }
591 }
@@ -539,10 +593,10 @@ static void aral_defrag_sorted_page_position___aral_lock_needed(ARAL *ar, ARAL_P
593 ar->aral_lock.defragment_operations++;
594 ar->aral_lock.defragment_linked_list_traversals += move_earlier + move_later;
595
542 - internal_fatal(page->next && page->next->aral_lock.free_elements && page->next->aral_lock.free_elements < page->aral_lock.free_elements,
596 + internal_fatal(page->aral_lock.next && page->aral_lock.next->aral_lock.free_elements && page->aral_lock.next->aral_lock.free_elements < page->aral_lock.free_elements,
597 "ARAL: '%s' item should be later in the list", ar->config.name);
598
545 - internal_fatal(page != ar->aral_lock.pages && (!page->prev->aral_lock.free_elements || page->prev->aral_lock.free_elements > page->aral_lock.free_elements),
599 + internal_fatal(page != ar->aral_lock.pages && (!page->aral_lock.prev->aral_lock.free_elements || page->aral_lock.prev->aral_lock.free_elements > page->aral_lock.free_elements),
600 "ARAL: '%s' item should be earlier in the list", ar->config.name);
601 }
602
@@ -551,8 +605,8 @@ static inline void aral_move_page_with_free_list___aral_lock_needed(ARAL *ar, AR
605 // we are the first already
606 return;
607
554 - if(likely(!ar->config.defragment)) {
555 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
608 + if(likely(!(ar->config.options & ARAL_DEFRAGMENT))) {
609 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
610 aral_insert_not_linked_page_with_free_items_to_proper_position___aral_lock_needed(ar, page);
611 }
612 else
@@ -566,18 +620,18 @@ void aral_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITI
620 ARAL_PAGE *page = aral_ptr_to_page___must_NOT_have_aral_lock(ar, ptr);
621
622 if(unlikely(ar->config.mmap.enabled))
569 - __atomic_sub_fetch(&aral_globals.atomic.mmap.used, ar->config.element_size, __ATOMIC_RELAXED);
623 + __atomic_sub_fetch(&ar->stats->mmap.used_bytes, ar->config.element_size, __ATOMIC_RELAXED);
624 else
571 - __atomic_sub_fetch(&aral_globals.atomic.malloc.used, ar->config.element_size, __ATOMIC_RELAXED);
625 + __atomic_sub_fetch(&ar->stats->malloc.used_bytes, ar->config.element_size, __ATOMIC_RELAXED);
626
627 // make this element available
628 ARAL_FREE *fr = (ARAL_FREE *)ptr;
629 fr->size = ar->config.element_size;
630
577 - netdata_spinlock_lock(&page->free.spinlock);
631 + aral_page_free_lock(ar, page);
632 fr->next = page->free.list;
633 page->free.list = fr;
580 - netdata_spinlock_unlock(&page->free.spinlock);
634 + aral_page_free_unlock(ar, page);
635
636 aral_lock(ar);
637
@@ -603,9 +657,15 @@ void aral_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITI
657
658 // if the page is empty, release it
659 if(unlikely(!page->aral_lock.used_elements)) {
606 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
660 + bool is_this_page_the_last_one = ar->aral_lock.pages == page && !page->aral_lock.next;
661 +
662 + if(!is_this_page_the_last_one)
663 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
664 +
665 aral_unlock(ar);
608 - aral_del_page___no_lock_needed(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
666 +
667 + if(!is_this_page_the_last_one)
668 + aral_del_page___no_lock_needed(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
669 }
670 else {
671 aral_move_page_with_free_list___aral_lock_needed(ar, page);
@@ -618,27 +678,44 @@ void aral_destroy_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS
678
679 ARAL_PAGE *page;
680 while((page = ar->aral_lock.pages)) {
621 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, prev, next);
681 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ar->aral_lock.pages, page, aral_lock.prev, aral_lock.next);
682 aral_del_page___no_lock_needed(ar, page TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
683 }
684
685 aral_unlock(ar);
686 +
687 + if(ar->config.options & ARAL_ALLOCATED_STATS)
688 + freez(ar->stats);
689 +
690 freez(ar);
691 }
692
629 -ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_elements, const char *filename, char **cache_dir, bool mmap, bool lockless) {
693 +size_t aral_element_size(ARAL *ar) {
694 + return ar->config.requested_element_size;
695 +}
696 +
697 +ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_size,
698 + struct aral_statistics *stats, const char *filename, char **cache_dir, bool mmap, bool lockless) {
699 ARAL *ar = callocz(1, sizeof(ARAL));
700 + ar->config.options = (lockless) ? ARAL_LOCKLESS : 0;
701 ar->config.requested_element_size = element_size;
702 ar->config.initial_page_elements = initial_page_elements;
633 - ar->config.max_page_elements = max_page_elements;
703 + ar->config.requested_max_page_size = max_page_size;
704 ar->config.mmap.filename = filename;
705 ar->config.mmap.cache_dir = cache_dir;
706 ar->config.mmap.enabled = mmap;
637 - ar->config.lockless = lockless;
638 - ar->config.defragment = false;
707 strncpyz(ar->config.name, name, ARAL_MAX_NAME);
708 netdata_spinlock_init(&ar->aral_lock.spinlock);
709
710 + if(stats) {
711 + ar->stats = stats;
712 + ar->config.options &= ~ARAL_ALLOCATED_STATS;
713 + }
714 + else {
715 + ar->stats = callocz(1, sizeof(struct aral_statistics));
716 + ar->config.options |= ARAL_ALLOCATED_STATS;
717 + }
718 +
719 long int page_size = sysconf(_SC_PAGE_SIZE);
720 if (unlikely(page_size == -1))
721 ar->config.natural_page_size = 4096;
@@ -659,6 +736,8 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele
736 // and finally align it to the natural alignment
737 ar->config.element_size = natural_alignment(ar->config.element_size, ARAL_NATURAL_ALIGNMENT);
738
739 + ar->config.max_page_elements = ar->config.requested_max_page_size / ar->config.element_size;
740 +
741 // we write the page pointer just after each element
742 ar->config.page_ptr_offset = ar->config.element_size - sizeof(uintptr_t);
743
@@ -709,20 +788,106 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele
788 "ARAL: '%s' "
789 "element size %zu (requested %zu bytes), "
790 "min elements per page %zu (requested %zu), "
712 - "max elements per page %zu (requested %zu), "
713 - "max page size %zu bytes, "
791 + "max elements per page %zu, "
792 + "max page size %zu bytes (requested %zu) "
793 , ar->config.name
794 , ar->config.element_size, ar->config.requested_element_size
795 , ar->adders.allocation_size / ar->config.element_size, ar->config.initial_page_elements
717 - , ar->config.max_allocation_size / ar->config.element_size, ar->config.max_page_elements
718 - , ar->config.max_allocation_size
796 + , ar->config.max_allocation_size / ar->config.element_size
797 + , ar->config.max_allocation_size, ar->config.requested_max_page_size
798 );
799
721 - __atomic_add_fetch(&aral_globals.atomic.structures.allocations, 1, __ATOMIC_RELAXED);
722 - __atomic_add_fetch(&aral_globals.atomic.structures.allocated, sizeof(ARAL), __ATOMIC_RELAXED);
800 + __atomic_add_fetch(&ar->stats->structures.allocations, 1, __ATOMIC_RELAXED);
801 + __atomic_add_fetch(&ar->stats->structures.allocated_bytes, sizeof(ARAL), __ATOMIC_RELAXED);
802 + return ar;
803 +}
804 +
805 +// ----------------------------------------------------------------------------
806 +// global aral caching
807 +
808 +#define ARAL_BY_SIZE_MAX_SIZE 1024
809 +
810 +struct aral_by_size {
811 + ARAL *ar;
812 + int32_t refcount;
813 +};
814 +
815 +struct {
816 + struct aral_statistics shared_statistics;
817 + SPINLOCK spinlock;
818 + struct aral_by_size array[ARAL_BY_SIZE_MAX_SIZE + 1];
819 +} aral_by_size_globals = {};
820 +
821 +struct aral_statistics *aral_by_size_statistics(void) {
822 + return &aral_by_size_globals.shared_statistics;
823 +}
824 +
825 +size_t aral_by_size_structures(void) {
826 + return aral_structures_from_stats(&aral_by_size_globals.shared_statistics);
827 +}
828 +
829 +size_t aral_by_size_overhead(void) {
830 + return aral_overhead_from_stats(&aral_by_size_globals.shared_statistics);
831 +}
832 +
833 +ARAL *aral_by_size_acquire(size_t size) {
834 + netdata_spinlock_lock(&aral_by_size_globals.spinlock);
835 +
836 + ARAL *ar = NULL;
837 +
838 + if(size <= ARAL_BY_SIZE_MAX_SIZE && aral_by_size_globals.array[size].ar) {
839 + ar = aral_by_size_globals.array[size].ar;
840 + aral_by_size_globals.array[size].refcount++;
841 +
842 + internal_fatal(aral_element_size(ar) != size, "DICTIONARY: aral has size %zu but we want %zu",
843 + aral_element_size(ar), size);
844 + }
845 +
846 + if(!ar) {
847 + char buf[30 + 1];
848 + snprintf(buf, 30, "size-%zu", size);
849 + ar = aral_create(buf,
850 + size,
851 + 0,
852 + 65536 * ((size / 150) + 1),
853 + &aral_by_size_globals.shared_statistics,
854 + NULL, NULL, false, false);
855 +
856 + if(size <= ARAL_BY_SIZE_MAX_SIZE) {
857 + aral_by_size_globals.array[size].ar = ar;
858 + aral_by_size_globals.array[size].refcount = 1;
859 + }
860 + }
861 +
862 + netdata_spinlock_unlock(&aral_by_size_globals.spinlock);
863 +
864 return ar;
865 }
866
867 +void aral_by_size_release(ARAL *ar) {
868 + size_t size = aral_element_size(ar);
869 +
870 + if(size <= ARAL_BY_SIZE_MAX_SIZE) {
871 + netdata_spinlock_lock(&aral_by_size_globals.spinlock);
872 +
873 + internal_fatal(aral_by_size_globals.array[size].ar != ar,
874 + "ARAL BY SIZE: aral pointers do not match");
875 +
876 + if(aral_by_size_globals.array[size].refcount <= 0)
877 + fatal("ARAL BY SIZE: double release detected");
878 +
879 + aral_by_size_globals.array[size].refcount--;
880 + if(!aral_by_size_globals.array[size].refcount) {
881 + aral_destroy(aral_by_size_globals.array[size].ar);
882 + aral_by_size_globals.array[size].ar = NULL;
883 + }
884 +
885 + netdata_spinlock_unlock(&aral_by_size_globals.spinlock);
886 + }
887 + else
888 + aral_destroy(ar);
889 +}
890 +
891 // ----------------------------------------------------------------------------
892 // unittest
893
@@ -774,7 +939,7 @@ static void *aral_test_thread(void *ptr) {
939 pointers[i] = NULL;
940 }
941
777 - if (auc->single_threaded && ar->aral_lock.pages) {
942 + if (auc->single_threaded && ar->aral_lock.pages && ar->aral_lock.pages->aral_lock.used_elements) {
943 fprintf(stderr, "\n\nARAL leftovers detected (1)\n\n");
944 __atomic_add_fetch(&auc->errors, 1, __ATOMIC_RELAXED);
945 }
@@ -789,7 +954,7 @@ static void *aral_test_thread(void *ptr) {
954 size_t increment = elements / ar->config.max_page_elements;
955 for (size_t all = increment; all <= elements / 2; all += increment) {
956
792 - size_t to_free = all % ar->config.max_page_elements;
957 + size_t to_free = (all % ar->config.max_page_elements) + 1;
958 size_t step = elements / to_free;
959 if(!step) step = 1;
960
@@ -814,7 +979,7 @@ static void *aral_test_thread(void *ptr) {
979 pointers[i] = NULL;
980 }
981
817 - if (auc->single_threaded && ar->aral_lock.pages) {
982 + if (auc->single_threaded && ar->aral_lock.pages && ar->aral_lock.pages->aral_lock.used_elements) {
983 fprintf(stderr, "\n\nARAL leftovers detected (2)\n\n");
984 __atomic_add_fetch(&auc->errors, 1, __ATOMIC_RELAXED);
985 }
@@ -830,12 +995,10 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) {
995 fprintf(stderr, "Running stress test of %zu threads, with %zu elements each, for %zu seconds...\n",
996 threads, elements, seconds);
997
833 - memset(&aral_globals, 0, sizeof(aral_globals));
834 -
998 struct aral_unittest_config auc = {
999 .single_threaded = false,
1000 .threads = threads,
838 - .ar = aral_create("aral-test", 20, 10, 1024, "test-aral", NULL, false, false),
1001 + .ar = aral_create("aral-stress-test", 20, 0, 8192, NULL, "aral-stress-test", NULL, false, false),
1002 .elements = elements,
1003 .errors = 0,
1004 };
@@ -880,7 +1043,7 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) {
1043
1044 usec_t ended_ut = now_monotonic_usec();
1045
883 - if (auc.ar->aral_lock.pages) {
1046 + if (auc.ar->aral_lock.pages && auc.ar->aral_lock.pages->aral_lock.used_elements) {
1047 fprintf(stderr, "\n\nARAL leftovers detected (3)\n\n");
1048 __atomic_add_fetch(&auc.errors, 1, __ATOMIC_RELAXED);
1049 }
@@ -903,7 +1066,7 @@ int aral_unittest(size_t elements) {
1066 struct aral_unittest_config auc = {
1067 .single_threaded = true,
1068 .threads = 1,
906 - .ar = aral_create("aral-test", 20, 10, 1024, "test-aral", &cache_dir, false, false),
1069 + .ar = aral_create("aral-test", 20, 0, 8192, NULL, "aral-test", &cache_dir, false, false),
1070 .elements = elements,
1071 .errors = 0,
1072 };
libnetdata/aral/aral.h
+34 -2
@@ -8,9 +8,41 @@
8
9 typedef struct aral ARAL;
10
11 -ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_elements, const char *filename, char **cache_dir, bool mmap, bool lockless);
11 +struct aral_statistics {
12 + struct {
13 + size_t allocations;
14 + size_t allocated_bytes;
15 + } structures;
16 +
17 + struct {
18 + size_t allocations;
19 + size_t allocated_bytes;
20 + size_t used_bytes;
21 + } malloc;
22 +
23 + struct {
24 + size_t allocations;
25 + size_t allocated_bytes;
26 + size_t used_bytes;
27 + } mmap;
28 +};
29 +
30 +ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_size,
31 + struct aral_statistics *stats, const char *filename, char **cache_dir, bool mmap, bool lockless);
32 +size_t aral_element_size(ARAL *ar);
33 +size_t aral_overhead(ARAL *ar);
34 +size_t aral_structures(ARAL *ar);
35 +struct aral_statistics *aral_statistics(ARAL *ar);
36 +size_t aral_structures_from_stats(struct aral_statistics *stats);
37 +size_t aral_overhead_from_stats(struct aral_statistics *stats);
38 +
39 +ARAL *aral_by_size_acquire(size_t size);
40 +void aral_by_size_release(ARAL *ar);
41 +size_t aral_by_size_structures(void);
42 +size_t aral_by_size_overhead(void);
43 +struct aral_statistics *aral_by_size_statistics(void);
44 +
45 int aral_unittest(size_t elements);
13 -void aral_get_size_statistics(size_t *structures, size_t *malloc_allocated, size_t *malloc_used, size_t *mmap_allocated, size_t *mmap_used);
46
47 #ifdef NETDATA_TRACE_ALLOCATIONS
48
libnetdata/dictionary/dictionary.c
+60 -20
@@ -143,6 +143,8 @@ struct dictionary {
143 DICT_OPTIONS options; // the configuration flags of the dictionary (they never change - no atomics)
144 DICT_FLAGS flags; // run time flags for the dictionary (they change all the time - atomics needed)
145
146 + ARAL *value_aral;
147 +
148 struct { // support for multiple indexing engines
149 Pvoid_t JudyHSArray; // the hash table
150 netdata_rwlock_t rwlock; // protect the index
@@ -179,7 +181,9 @@ struct dictionary {
181 #endif
182 };
183
184 +// ----------------------------------------------------------------------------
185 // forward definitions of functions used in reverse order in the code
186 +
187 static void garbage_collect_pending_deletes(DICTIONARY *dict);
188 static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item);
189 static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item);
@@ -1249,7 +1253,8 @@ void dictionary_static_items_aral_init(void) {
1253 "dict-items",
1254 sizeof(DICTIONARY_ITEM),
1255 0,
1252 - 4096,
1256 + 65536,
1257 + aral_by_size_statistics(),
1258 NULL, NULL, false, false);
1259
1260 // we have to check again
@@ -1258,7 +1263,8 @@ void dictionary_static_items_aral_init(void) {
1263 "dict-shared-items",
1264 sizeof(DICTIONARY_ITEM_SHARED),
1265 0,
1261 - 4096,
1266 + 65536,
1267 + aral_by_size_statistics(),
1268 NULL, NULL, false, false);
1269
1270 netdata_spinlock_unlock(&spinlock);
@@ -1269,7 +1275,6 @@ static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t
1275 DICTIONARY_ITEM *item;
1276
1277 size_t size = sizeof(DICTIONARY_ITEM);
1272 -// item = callocz(1, size);
1278 item = aral_mallocz(dict_items_aral);
1279 memset(item, 0, sizeof(DICTIONARY_ITEM));
1280
@@ -1290,7 +1295,6 @@ static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t
1295 }
1296 else {
1297 size = sizeof(DICTIONARY_ITEM_SHARED);
1293 - // item->shared = callocz(1, size);
1298 item->shared = aral_mallocz(dict_shared_items_aral);
1299 memset(item->shared, 0, sizeof(DICTIONARY_ITEM_SHARED));
1300
@@ -1304,20 +1308,39 @@ static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t
1308 return item;
1309 }
1310
1307 -static void *dict_item_value_create(void *value, size_t value_len) {
1311 +static inline void *dict_item_value_mallocz(DICTIONARY *dict, size_t value_len) {
1312 + if(dict->value_aral) {
1313 + internal_fatal(aral_element_size(dict->value_aral) != value_len,
1314 + "DICTIONARY: item value size %zu does not match the configured fixed one %zu",
1315 + value_len, aral_element_size(dict->value_aral));
1316 + return aral_mallocz(dict->value_aral);
1317 + }
1318 + else
1319 + return mallocz(value_len);
1320 +}
1321 +
1322 +static inline void dict_item_value_freez(DICTIONARY *dict, void *ptr) {
1323 + if(dict->value_aral)
1324 + aral_freez(dict->value_aral, ptr);
1325 + else
1326 + freez(ptr);
1327 +}
1328 +
1329 +static void *dict_item_value_create(DICTIONARY *dict, void *value, size_t value_len) {
1330 void *ptr = NULL;
1331
1332 if(likely(value_len)) {
1333 if (likely(value)) {
1334 // a value has been supplied
1335 // copy it
1314 - ptr = mallocz(value_len);
1336 + ptr = dict_item_value_mallocz(dict, value_len);
1337 memcpy(ptr, value, value_len);
1338 }
1339 else {
1340 // no value has been supplied
1341 // allocate a clear memory block
1320 - ptr = callocz(1, value_len);
1342 + ptr = dict_item_value_mallocz(dict, value_len);
1343 + memset(ptr, 0, value_len);
1344 }
1345 }
1346 // else
@@ -1356,7 +1379,7 @@ static DICTIONARY_ITEM *dict_item_create_with_hooks(DICTIONARY *dict, const char
1379 if(unlikely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))
1380 item->shared->value = value;
1381 else
1359 - item->shared->value = dict_item_value_create(value, value_len);
1382 + item->shared->value = dict_item_value_create(dict, value, value_len);
1383
1384 item->shared->value_len = value_len;
1385 value_size += value_len;
@@ -1396,7 +1419,7 @@ static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *
1419 void *old_value = item->shared->value;
1420 void *new_value = NULL;
1421 if(value_len) {
1399 - new_value = mallocz(value_len);
1422 + new_value = dict_item_value_mallocz(dict, value_len);
1423 if(value) memcpy(new_value, value, value_len);
1424 else memset(new_value, 0, value_len);
1425 }
@@ -1404,7 +1427,7 @@ static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *
1427 item->shared->value_len = value_len;
1428
1429 debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", item_get_name(item));
1407 - freez(old_value);
1430 + dict_item_value_freez(dict, old_value);
1431 }
1432
1433 dictionary_execute_insert_callback(dict, item, constructor_data);
@@ -1427,18 +1450,16 @@ static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item)
1450
1451 if(unlikely(!(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))) {
1452 debug(D_DICTIONARY, "Dictionary freeing value of '%s'", item_get_name(item));
1430 - freez(item->shared->value);
1453 + dict_item_value_freez(dict, item->shared->value);
1454 item->shared->value = NULL;
1455 }
1456 value_size += item->shared->value_len;
1457
1435 - // freez(item->shared);
1458 aral_freez(dict_shared_items_aral, item->shared);
1459 item->shared = NULL;
1460 item_size += sizeof(DICTIONARY_ITEM_SHARED);
1461 }
1462
1441 - // freez(item);
1463 aral_freez(dict_items_aral, item);
1464
1465 item_size += sizeof(DICTIONARY_ITEM);
@@ -1788,6 +1809,9 @@ static bool dictionary_free_all_resources(DICTIONARY *dict, size_t *mem, bool fo
1809 dict_size += sizeof(DICTIONARY);
1810 DICTIONARY_STATS_MINUS_MEMORY(dict, 0, sizeof(DICTIONARY), 0);
1811
1812 + if(dict->value_aral)
1813 + aral_by_size_release(dict->value_aral);
1814 +
1815 freez(dict);
1816
1817 internal_error(
@@ -1973,13 +1997,27 @@ static bool api_is_name_good_with_trace(DICTIONARY *dict __maybe_unused, const c
1997 // ----------------------------------------------------------------------------
1998 // API - dictionary management
1999
1976 -static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dictionary_stats *stats) {
2000 +static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size) {
2001 cleanup_destroyed_dictionaries();
2002
2003 DICTIONARY *dict = callocz(1, sizeof(DICTIONARY));
2004 dict->options = options;
2005 dict->stats = stats;
2006
2007 + if((dict->options & DICT_OPTION_FIXED_SIZE) && !fixed_size) {
2008 + dict->options &= ~DICT_OPTION_FIXED_SIZE;
2009 + internal_fatal(true, "DICTIONARY: requested fixed size dictionary, without setting the size");
2010 + }
2011 + if(!(dict->options & DICT_OPTION_FIXED_SIZE) && fixed_size) {
2012 + dict->options |= DICT_OPTION_FIXED_SIZE;
2013 + internal_fatal(true, "DICTIONARY: set a fixed size for the items, without setting DICT_OPTION_FIXED_SIZE flag");
2014 + }
2015 +
2016 + if(dict->options & DICT_OPTION_FIXED_SIZE)
2017 + dict->value_aral = aral_by_size_acquire(fixed_size);
2018 + else
2019 + dict->value_aral = NULL;
2020 +
2021 size_t dict_size = 0;
2022 dict_size += sizeof(DICTIONARY);
2023 dict_size += dictionary_locks_init(dict);
@@ -1995,12 +2033,12 @@ static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dicti
2033 }
2034
2035 #ifdef NETDATA_INTERNAL_CHECKS
1998 -DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, const char *function, size_t line, const char *file) {
2036 +DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size, const char *function, size_t line, const char *file) {
2037 #else
2000 -DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats) {
2038 +DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size) {
2039 #endif
2040
2003 - DICTIONARY *dict = dictionary_create_internal(options, stats?stats:&dictionary_stats_category_other);
2041 + DICTIONARY *dict = dictionary_create_internal(options, stats?stats:&dictionary_stats_category_other, fixed_size);
2042
2043 #ifdef NETDATA_INTERNAL_CHECKS
2044 dict->creation_function = function;
@@ -2018,7 +2056,9 @@ DICTIONARY *dictionary_create_view_with_trace(DICTIONARY *master, const char *fu
2056 DICTIONARY *dictionary_create_view(DICTIONARY *master) {
2057 #endif
2058
2021 - DICTIONARY *dict = dictionary_create_internal(master->options, master->stats);
2059 + DICTIONARY *dict = dictionary_create_internal(master->options, master->stats,
2060 + master->value_aral ? aral_element_size(master->value_aral) : 0);
2061 +
2062 dict->master = master;
2063
2064 dictionary_hooks_allocate(master);
@@ -3335,7 +3375,7 @@ static int dictionary_unittest_view_threads() {
3375 // threads testing of dictionary
3376 struct dictionary_stats stats_master = {};
3377 struct dictionary_stats stats_view = {};
3338 - tv.master = dictionary_create_advanced(DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE, &stats_master);
3378 + tv.master = dictionary_create_advanced(DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE, &stats_master, 0);
3379 tv.view = dictionary_create_view(tv.master);
3380 tv.view->stats = &stats_view;
3381
@@ -3428,7 +3468,7 @@ static int dictionary_unittest_view_threads() {
3468 size_t dictionary_unittest_views(void) {
3469 size_t errors = 0;
3470 struct dictionary_stats stats = {};
3431 - DICTIONARY *master = dictionary_create_advanced(DICT_OPTION_NONE, &stats);
3471 + DICTIONARY *master = dictionary_create_advanced(DICT_OPTION_NONE, &stats, 0);
3472 DICTIONARY *view = dictionary_create_view(master);
3473
3474 fprintf(stderr, "\n\nChecking dictionary views...\n");
libnetdata/dictionary/dictionary.h
+6 -5
@@ -53,6 +53,7 @@ typedef enum dictionary_options {
53 DICT_OPTION_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy)
54 DICT_OPTION_DONT_OVERWRITE_VALUE = (1 << 3), // don't overwrite values of dictionary items (default: overwrite)
55 DICT_OPTION_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end)
56 + DICT_OPTION_FIXED_SIZE = (1 << 5), // the items of the dictionary have a fixed size
57 } DICT_OPTIONS;
58
59 struct dictionary_stats {
@@ -107,12 +108,12 @@ struct dictionary_stats {
108
109 // Create a dictionary
110 #ifdef NETDATA_INTERNAL_CHECKS
110 -#define dictionary_create(options) dictionary_create_advanced_with_trace(options, NULL, __FUNCTION__, __LINE__, __FILE__)
111 -#define dictionary_create_advanced(options, stats) dictionary_create_advanced_with_trace(options, stats, __FUNCTION__, __LINE__, __FILE__)
112 -DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, const char *function, size_t line, const char *file);
111 +#define dictionary_create(options) dictionary_create_advanced_with_trace(options, NULL, 0, __FUNCTION__, __LINE__, __FILE__)
112 +#define dictionary_create_advanced(options, stats, fixed_size) dictionary_create_advanced_with_trace(options, stats, fixed_size, __FUNCTION__, __LINE__, __FILE__)
113 +DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size, const char *function, size_t line, const char *file);
114 #else
114 -#define dictionary_create(options) dictionary_create_advanced(options, NULL);
115 -DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats);
115 +#define dictionary_create(options) dictionary_create_advanced(options, NULL, 0);
116 +DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size);
117 #endif
118
119 // Create a view on a dictionary
libnetdata/libnetdata.c
+75 -21
@@ -21,6 +21,81 @@ int enable_ksm = 0;
21 volatile sig_atomic_t netdata_exit = 0;
22 const char *program_version = VERSION;
23
24 +#define MAX_JUDY_SIZE_TO_ARAL 24
25 +static bool judy_sizes_config[MAX_JUDY_SIZE_TO_ARAL + 1] = {
26 + [3] = true,
27 + [4] = true,
28 + [5] = true,
29 + [6] = true,
30 + [7] = true,
31 + [8] = true,
32 + [10] = true,
33 + [11] = true,
34 + [15] = true,
35 + [23] = true,
36 +};
37 +static ARAL *judy_sizes_aral[MAX_JUDY_SIZE_TO_ARAL + 1] = {};
38 +
39 +struct aral_statistics judy_sizes_aral_statistics = {};
40 +
41 +void aral_judy_init(void) {
42 + for(size_t Words = 0; Words <= MAX_JUDY_SIZE_TO_ARAL; Words++)
43 + if(judy_sizes_config[Words]) {
44 + char buf[30+1];
45 + snprintfz(buf, 30, "judy-%zu", Words * sizeof(Word_t));
46 + judy_sizes_aral[Words] = aral_create(
47 + buf,
48 + Words * sizeof(Word_t),
49 + 0,
50 + 65536,
51 + &judy_sizes_aral_statistics,
52 + NULL, NULL, false, false);
53 + }
54 +}
55 +
56 +size_t judy_aral_overhead(void) {
57 + return aral_overhead_from_stats(&judy_sizes_aral_statistics);
58 +}
59 +
60 +size_t judy_aral_structures(void) {
61 + return aral_structures_from_stats(&judy_sizes_aral_statistics);
62 +}
63 +
64 +static ARAL *judy_size_aral(Word_t Words) {
65 + if(Words <= MAX_JUDY_SIZE_TO_ARAL && judy_sizes_aral[Words])
66 + return judy_sizes_aral[Words];
67 +
68 + return NULL;
69 +}
70 +
71 +inline Word_t JudyMalloc(Word_t Words) {
72 + Word_t Addr;
73 +
74 + ARAL *ar = judy_size_aral(Words);
75 + if(ar)
76 + Addr = (Word_t) aral_mallocz(ar);
77 + else
78 + Addr = (Word_t) mallocz(Words * sizeof(Word_t));
79 +
80 + return(Addr);
81 +}
82 +
83 +inline void JudyFree(void * PWord, Word_t Words) {
84 + ARAL *ar = judy_size_aral(Words);
85 + if(ar)
86 + aral_freez(ar, PWord);
87 + else
88 + freez(PWord);
89 +}
90 +
91 +Word_t JudyMallocVirtual(Word_t Words) {
92 + return JudyMalloc(Words);
93 +}
94 +
95 +void JudyFreeVirtual(void * PWord, Word_t Words) {
96 + JudyFree(PWord, Words);
97 +}
98 +
99 // ----------------------------------------------------------------------------
100 // memory allocation functions that handle failures
101
@@ -150,27 +225,6 @@ void posix_memfree(void *ptr) {
225 libc_free(ptr);
226 }
227
153 -Word_t JudyMalloc(Word_t Words) {
154 - Word_t Addr;
155 -
156 - Addr = (Word_t) mallocz(Words * sizeof(Word_t));
157 - return(Addr);
158 -}
159 -void JudyFree(void * PWord, Word_t Words) {
160 - (void)Words;
161 - freez(PWord);
162 -}
163 -Word_t JudyMallocVirtual(Word_t Words) {
164 - Word_t Addr;
165 -
166 - Addr = (Word_t) mallocz(Words * sizeof(Word_t));
167 - return(Addr);
168 -}
169 -void JudyFreeVirtual(void * PWord, Word_t Words) {
170 - (void)Words;
171 - freez(PWord);
172 -}
173 -
228 #define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2)
229 #define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED)
230 #define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED)
libnetdata/libnetdata.h
+4
@@ -225,6 +225,10 @@ extern "C" {
225 #define WARNUNUSED
226 #endif
227
228 +void aral_judy_init(void);
229 +size_t judy_aral_overhead(void);
230 +size_t judy_aral_structures(void);
231 +
232 #define ABS(x) (((x) < 0)? (-(x)) : (x))
233 #define MIN(a,b) (((a)<(b))?(a):(b))
234 #define MAX(a,b) (((a)>(b))?(a):(b))
libnetdata/log/log.c
+26 -18
@@ -631,7 +631,9 @@ int error_log_limit(int reset) {
631 static time_t start = 0;
632 static unsigned long counter = 0, prevented = 0;
633
634 - // fprintf(stderror, "FLOOD: counter=%lu, allowed=%lu, backup=%lu, period=%llu\n", counter, error_log_errors_per_period, error_log_errors_per_period_backup, (unsigned long long)error_log_throttle_period);
634 + FILE *fp = (!stderror) ? stderr : stderror;
635 +
636 + // fprintf(fp, "FLOOD: counter=%lu, allowed=%lu, backup=%lu, period=%llu\n", counter, error_log_errors_per_period, error_log_errors_per_period_backup, (unsigned long long)error_log_throttle_period);
637
638 // do not throttle if the period is 0
639 if(error_log_throttle_period == 0)
@@ -653,7 +655,7 @@ int error_log_limit(int reset) {
655 char date[LOG_DATE_LENGTH];
656 log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
657 fprintf(
656 - stderror,
658 + fp,
659 "%s: %s LOG FLOOD PROTECTION reset for process '%s' "
660 "(prevented %lu logs in the last %"PRId64" seconds).\n",
661 date,
@@ -676,7 +678,7 @@ int error_log_limit(int reset) {
678 char date[LOG_DATE_LENGTH];
679 log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
680 fprintf(
679 - stderror,
681 + fp,
682 "%s: %s LOG FLOOD PROTECTION resuming logging from process '%s' "
683 "(prevented %lu logs in the last %"PRId64" seconds).\n",
684 date,
@@ -700,7 +702,7 @@ int error_log_limit(int reset) {
702 char date[LOG_DATE_LENGTH];
703 log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
704 fprintf(
703 - stderror,
705 + fp,
706 "%s: %s LOG FLOOD PROTECTION too many logs (%lu logs in %"PRId64" seconds, threshold is set to %lu logs "
707 "in %"PRId64" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.\n",
708 date,
@@ -776,7 +778,7 @@ void debug_int( const char *file, const char *function, const unsigned long line
778 void info_int( int is_collector, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... )
779 {
780 va_list args;
779 - FILE *fp = (is_collector) ? stderr : stderror;
781 + FILE *fp = (is_collector || !stderror) ? stderr : stderror;
782
783 log_lock();
784
@@ -836,6 +838,8 @@ static const char *strerror_result_string(const char *a, const char *b) { (void)
838 #endif
839
840 void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
841 + FILE *fp = (!stderror) ? stderr : stderror;
842 +
843 if(erl->sleep_ut)
844 sleep_usec(erl->sleep_ut);
845
@@ -870,29 +874,29 @@ void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __ma
874
875 va_start( args, fmt );
876 #ifdef NETDATA_INTERNAL_CHECKS
873 - fprintf(stderror, "%s: %s %-5.5s : %s : (%04lu@%-20.20s:%-15.15s): ",
877 + fprintf(fp, "%s: %s %-5.5s : %s : (%04lu@%-20.20s:%-15.15s): ",
878 date, program_name, prefix, netdata_thread_tag(), line, file, function);
879 #else
876 - fprintf(stderror, "%s: %s %-5.5s : %s : ", date, program_name, prefix, netdata_thread_tag());
880 + fprintf(fp, "%s: %s %-5.5s : %s : ", date, program_name, prefix, netdata_thread_tag());
881 #endif
878 - vfprintf(stderror, fmt, args );
882 + vfprintf(fp, fmt, args );
883 va_end( args );
884
885 if(erl->count > 1)
882 - fprintf(stderror, " (similar messages repeated %zu times in the last %llu secs)",
886 + fprintf(fp, " (similar messages repeated %zu times in the last %llu secs)",
887 erl->count, (unsigned long long)(erl->last_logged ? now - erl->last_logged : 0));
888
889 if(erl->sleep_ut)
886 - fprintf(stderror, " (sleeping for %llu microseconds every time this happens)", erl->sleep_ut);
890 + fprintf(fp, " (sleeping for %llu microseconds every time this happens)", erl->sleep_ut);
891
892 if(__errno) {
893 char buf[1024];
890 - fprintf(stderror,
894 + fprintf(fp,
895 " (errno %d, %s)\n", __errno, strerror_result(strerror_r(__errno, buf, 1023), buf));
896 errno = 0;
897 }
898 else
895 - fputc('\n', stderror);
899 + fputc('\n', fp);
900
901 erl->last_logged = now;
902 erl->count = 0;
@@ -903,7 +907,7 @@ void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __ma
907 void error_int(int is_collector, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
908 // save a copy of errno - just in case this function generates a new error
909 int __errno = errno;
906 - FILE *fp = (is_collector) ? stderr : stderror;
910 + FILE *fp = (is_collector || !stderror) ? stderr : stderror;
911
912 va_list args;
913
@@ -956,16 +960,20 @@ static void crash_netdata(void) {
960 #ifdef HAVE_BACKTRACE
961 #define BT_BUF_SIZE 100
962 static void print_call_stack(void) {
963 + FILE *fp = (!stderror) ? stderr : stderror;
964 +
965 int nptrs;
966 void *buffer[BT_BUF_SIZE];
967
968 nptrs = backtrace(buffer, BT_BUF_SIZE);
969 if(nptrs)
964 - backtrace_symbols_fd(buffer, nptrs, fileno(stderror));
970 + backtrace_symbols_fd(buffer, nptrs, fileno(fp));
971 }
972 #endif
973
974 void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
975 + FILE *fp = (!stderror) ? stderr : stderror;
976 +
977 // save a copy of errno - just in case this function generates a new error
978 int __errno = errno;
979 va_list args;
@@ -993,16 +1001,16 @@ void fatal_int( const char *file, const char *function, const unsigned long line
1001
1002 va_start( args, fmt );
1003 #ifdef NETDATA_INTERNAL_CHECKS
996 - fprintf(stderror,
1004 + fprintf(fp,
1005 "%s: %s FATAL : %s : (%04lu@%-20.20s:%-15.15s): ", date, program_name, thread_tag, line, file, function);
1006 #else
999 - fprintf(stderror, "%s: %s FATAL : %s : ", date, program_name, thread_tag);
1007 + fprintf(fp, "%s: %s FATAL : %s : ", date, program_name, thread_tag);
1008 #endif
1001 - vfprintf(stderror, fmt, args );
1009 + vfprintf(fp, fmt, args );
1010 va_end( args );
1011
1012 perror(" # ");
1005 - fputc('\n', stderror);
1013 + fputc('\n', fp);
1014
1015 log_unlock();
1016
libnetdata/os.c
+87 -32
@@ -6,53 +6,58 @@
6 // system functions
7 // to retrieve settings of the system
8
9 -long get_system_cpus_with_cache(bool cache) {
10 - static long processors = 0;
9 +#define CPUS_FOR_COLLECTORS 0
10 +#define CPUS_FOR_NETDATA 1
11
12 - if(likely(cache && processors > 0))
13 - return processors;
12 +long get_system_cpus_with_cache(bool cache, bool for_netdata) {
13 + static long processors[2] = { 0, 0 };
14
15 -#ifdef __APPLE__
16 - int32_t tmp_processors;
15 + int index = for_netdata ? CPUS_FOR_NETDATA : CPUS_FOR_COLLECTORS;
16
18 - if (unlikely(GETSYSCTL_BY_NAME("hw.logicalcpu", tmp_processors)))
19 - error("Assuming system has %d processors.", processors);
20 - else
21 - processors = tmp_processors;
17 + if(likely(cache && processors[index] > 0))
18 + return processors[index];
19
23 - if(processors < 1)
24 - processors = 1;
20 +#if defined(__APPLE__) || defined(__FreeBSD__)
21 +#if defined(__APPLE__)
22 +#define HW_CPU_NAME "hw.logicalcpu"
23 +#else
24 +#define HW_CPU_NAME "hw.ncpu"
25 +#endif
26
26 - return processors;
27 -#elif __FreeBSD__
27 int32_t tmp_processors;
28 + bool error = false;
29
30 - if (unlikely(GETSYSCTL_BY_NAME("hw.ncpu", tmp_processors)))
31 - error("Assuming system has %d processors.", processors);
30 + if (unlikely(GETSYSCTL_BY_NAME(HW_CPU_NAME, tmp_processors)))
31 + error = true;
32 else
33 - processors = tmp_processors;
33 + processors[index] = tmp_processors;
34
35 - if(processors < 1)
36 - processors = 1;
35 + if(processors[index] < 1) {
36 + processors[index] = 1;
37
38 - return processors;
38 + if(error)
39 + error("Assuming system has %d processors.", processors[index]);
40 + }
41 +
42 + return processors[index];
43 #else
44
45 char filename[FILENAME_MAX + 1];
42 - snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix?netdata_configured_host_prefix:"");
46 + snprintfz(filename, FILENAME_MAX, "%s/proc/stat",
47 + (!for_netdata && netdata_configured_host_prefix) ? netdata_configured_host_prefix : "");
48
49 procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
50 if(!ff) {
46 - processors = 1;
47 - error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors);
48 - return processors;
51 + processors[index] = 1;
52 + error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]);
53 + return processors[index];
54 }
55
56 ff = procfile_readall(ff);
57 if(!ff) {
53 - processors = 1;
54 - error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors);
55 - return processors;
58 + processors[index] = 1;
59 + error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]);
60 + return processors[index];
61 }
62
63 long tmp_processors = 0;
@@ -65,13 +70,13 @@ long get_system_cpus_with_cache(bool cache) {
70 }
71 procfile_close(ff);
72
68 - processors = --tmp_processors;
73 + processors[index] = --tmp_processors;
74
70 - if(processors < 1)
71 - processors = 1;
75 + if(processors[index] < 1)
76 + processors[index] = 1;
77
73 - debug(D_SYSTEM, "System has %ld processors.", processors);
74 - return processors;
78 + debug(D_SYSTEM, "System has %ld processors.", processors[index]);
79 + return processors[index];
80
81 #endif /* __APPLE__, __FreeBSD__ */
82 }
@@ -131,6 +136,56 @@ void get_system_HZ(void) {
136 system_hz = (unsigned int) ticks;
137 }
138
139 +static inline unsigned long cpuset_str2ul(char **s) {
140 + unsigned long n = 0;
141 + char c;
142 + for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) {
143 + n *= 10;
144 + n += c - '0';
145 + }
146 + return n;
147 +}
148 +
149 +unsigned long read_cpuset_cpus(const char *filename, long system_cpus) {
150 + static char *buf = NULL;
151 + static size_t buf_size = 0;
152 +
153 + if(!buf) {
154 + buf_size = 100U + 6 * system_cpus; // taken from kernel/cgroup/cpuset.c
155 + buf = mallocz(buf_size + 1);
156 + }
157 +
158 + int ret = read_file(filename, buf, buf_size);
159 +
160 + if(!ret) {
161 + char *s = buf;
162 + unsigned long ncpus = 0;
163 +
164 + // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use
165 + while(*s) {
166 + unsigned long n = cpuset_str2ul(&s);
167 + ncpus++;
168 + if(*s == ',') {
169 + s++;
170 + continue;
171 + }
172 + if(*s == '-') {
173 + s++;
174 + unsigned long m = cpuset_str2ul(&s);
175 + ncpus += m - n; // calculate the number of cpus in the region
176 + }
177 + s++;
178 + }
179 +
180 + if(!ncpus)
181 + return 0;
182 +
183 + return ncpus;
184 + }
185 +
186 + return 0;
187 +}
188 +
189 // =====================================================================================================================
190 // FreeBSD
191
libnetdata/os.h
+4 -3
@@ -48,9 +48,10 @@ int getsysctl_by_name(const char *name, void *ptr, size_t len);
48
49 extern const char *os_type;
50
51 -#define get_system_cpus() get_system_cpus_with_cache(true)
52 -#define get_system_cpus_uncached() get_system_cpus_with_cache(false)
53 -long get_system_cpus_with_cache(bool cache);
51 +#define get_system_cpus() get_system_cpus_with_cache(true, false)
52 +#define get_system_cpus_uncached() get_system_cpus_with_cache(false, false)
53 +long get_system_cpus_with_cache(bool cache, bool for_netdata);
54 +unsigned long read_cpuset_cpus(const char *filename, long system_cpus);
55
56 extern pid_t pid_max;
57 pid_t get_system_pid_max(void);
streaming/replication.c
+49 -3
@@ -311,7 +311,7 @@ static void replication_query_execute(BUFFER *wb, struct replication_query *q, s
311 time_t now = after + 1;
312 time_t last_end_time_in_buffer = 0;
313 while(now <= before) {
314 - time_t min_start_time = 0, min_end_time = 0;
314 + time_t min_start_time = 0, max_start_time = 0, min_end_time = 0, max_end_time = 0, min_update_every = 0, max_update_every = 0;
315 for (size_t i = 0; i < dimensions ;i++) {
316 struct replication_dimension *d = &q->data[i];
317 if(unlikely(!d->enabled || d->skip)) continue;
@@ -339,14 +339,58 @@ static void replication_query_execute(BUFFER *wb, struct replication_query *q, s
339 // this dimension does not provide any data
340 continue;
341
342 + time_t update_every = d->sp.end_time_s - d->sp.start_time_s;
343 + if(unlikely(!update_every))
344 + update_every = q->st->update_every;
345 +
346 + if(unlikely(!min_update_every))
347 + min_update_every = update_every;
348 +
349 if(unlikely(!min_start_time))
350 min_start_time = d->sp.start_time_s;
351
352 if(unlikely(!min_end_time))
353 min_end_time = d->sp.end_time_s;
354
355 + min_update_every = MIN(min_update_every, update_every);
356 + max_update_every = MAX(max_update_every, update_every);
357 +
358 min_start_time = MIN(min_start_time, d->sp.start_time_s);
359 + max_start_time = MAX(max_start_time, d->sp.start_time_s);
360 +
361 min_end_time = MIN(min_end_time, d->sp.end_time_s);
362 + max_end_time = MAX(max_end_time, d->sp.end_time_s);
363 + }
364 +
365 + if (unlikely(min_update_every != max_update_every ||
366 + min_start_time != max_start_time)) {
367 +
368 + time_t fix_min_start_time;
369 + if(last_end_time_in_buffer &&
370 + last_end_time_in_buffer >= min_start_time &&
371 + last_end_time_in_buffer <= max_start_time) {
372 + fix_min_start_time = last_end_time_in_buffer;
373 + }
374 + else
375 + fix_min_start_time = min_end_time - min_update_every;
376 +
377 + error_limit_static_global_var(erl, 1, 0);
378 + error_limit(&erl, "REPLAY WARNING: 'host:%s/chart:%s' "
379 + "misaligned dimensions "
380 + "update every (min: %ld, max: %ld), "
381 + "start time (min: %ld, max: %ld), "
382 + "end time (min %ld, max %ld), "
383 + "now %ld, last end time sent %ld, "
384 + "min start time is fixed to %ld",
385 + rrdhost_hostname(q->st->rrdhost), rrdset_id(q->st),
386 + min_update_every, max_update_every,
387 + min_start_time, max_start_time,
388 + min_end_time, max_end_time,
389 + now, last_end_time_in_buffer,
390 + fix_min_start_time
391 + );
392 +
393 + min_start_time = fix_min_start_time;
394 }
395
396 if(likely(min_start_time <= now && min_end_time >= now)) {
@@ -1375,7 +1419,9 @@ void replication_sender_delete_pending_requests(struct sender_state *sender) {
1419 }
1420
1421 void replication_init_sender(struct sender_state *sender) {
1378 - sender->replication.requests = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
1422 + sender->replication.requests = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
1423 + NULL, sizeof(struct replication_request));
1424 +
1425 dictionary_register_react_callback(sender->replication.requests, replication_request_react_callback, sender);
1426 dictionary_register_conflict_callback(sender->replication.requests, replication_request_conflict_callback, sender);
1427 dictionary_register_delete_callback(sender->replication.requests, replication_request_delete_callback, sender);
@@ -1560,7 +1606,7 @@ static int replication_execute_next_pending_request(bool cancel) {
1606 }
1607
1608 if(unlikely(!rqs)) {
1563 - max_requests_ahead = get_system_cpus() / 2;
1609 + max_requests_ahead = get_netdata_cpus() / 2;
1610
1611 if(max_requests_ahead > libuv_worker_threads * 2)
1612 max_requests_ahead = libuv_worker_threads * 2;
web/server/static/static-threaded.c
+1 -1
@@ -507,7 +507,7 @@ void *socket_listen_main_static_threaded(void *ptr) {
507 // 6 threads is the optimal value
508 // since 6 are the parallel connections browsers will do
509 // so, if the machine has more CPUs, avoid using resources unnecessarily
510 - int def_thread_count = (get_system_cpus() > 6) ? 6 : (int)get_system_cpus();
510 + int def_thread_count = MIN(get_netdata_cpus(), 6);
511
512 if (!strcmp(config_get(CONFIG_SECTION_WEB, "mode", ""),"single-threaded")) {
513 info("Running web server with one thread, because mode is single-threaded");