@cryptotaxi247 / netdata-1 / commits / 4c9d6a371

Stress test insertions into dbengine and bugfixes (#6814)

* Fix memory corruption during deallocation of page cache * Refactored dataset generator in order to support the upcoming self-validating stress test and multithreading. * Fix starvation in database engine loop when the command queues are continuously populated * Fixing disk quota limits for dbengine dataset generator

Markos Fountoulakis committed Sep 12, 2019 at 13:39 UTC 4c9d6a37138d95495e2dba20eed1931556d4252d

3 files changed +110 -32

daemon/unit_test.c

+85 -28

@@ -1928,42 +1928,54 @@ error_out:

1928 return errors;

1929 }

1930

1931 -void generate_dbengine_dataset(unsigned history_seconds)

1931 +struct dbengine_chart_thread {

1932 + uv_thread_t thread;

1933 + RRDHOST *host;

1934 + char *chartname; /* Will be prefixed by type, e.g. "example_local1.", "example_local2." etc */

1935 + int dset_charts; /* number of charts */

1936 + int dset_dims; /* dimensions per chart */

1937 + int chart_i; /* current chart offset */

1938 + time_t time_present; /* current virtual time of the benchmark */

1939 + unsigned history_seconds; /* how far back in the past to go */

1940 +};

1941 +

1942 +collected_number generate_dbengine_chart_value(struct dbengine_chart_thread *thread_info, int dim_i,

1943 + time_t time_current)

1944 {

1933 - const int DSET_DIMS = 128;

1934 - const uint64_t EXPECTED_COMPRESSION_RATIO = 94;

1935 - int j, update_every = 1;

1936 - RRDHOST *host = NULL;

1937 - RRDSET *st;

1938 - RRDDIM *rd[DSET_DIMS];

1939 - char name[101];

1940 - time_t time_current, time_present;

1945 + collected_number value;

1946

1942 - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;

1943 - default_rrdeng_page_cache_mb = 128;

1944 - // Worst case for uncompressible data

1945 - default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS) * sizeof(storage_number) * history_seconds) / (1024 * 1024);

1946 - default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;

1947 + value = ((collected_number)time_current) * thread_info->chart_i;

1948 + value += ((collected_number)time_current) * dim_i;

1949 + value %= 1024LLU;

1950

1948 - error_log_limit_unlimited();

1949 - debug(D_RRDHOST, "Initializing localhost with hostname 'dbengine-dataset'");

1951 + return value;

1952 +}

1953

1951 - host = dbengine_rrdhost_find_or_create("dbengine-dataset");

1952 - if (NULL == host)

1953 - return;

1954 +static void generate_dbengine_chart(void *arg)

1955 +{

1956 + struct dbengine_chart_thread *thread_info = (struct dbengine_chart_thread *)arg;

1957 + RRDHOST *host = thread_info->host;

1958 + char *chartname = thread_info->chartname;

1959 + const int DSET_DIMS = thread_info->dset_dims;

1960 + unsigned history_seconds = thread_info->history_seconds;

1961 + time_t time_present = thread_info->time_present;

1962

1955 - fprintf(stderr, "\nRunning DB-engine workload generator\n");

1963 + int j, update_every = 1;

1964 + RRDSET *st;

1965 + RRDDIM *rd[DSET_DIMS];

1966 + char name[RRD_ID_LENGTH_MAX + 1];

1967 + time_t time_current;

1968

1969 // create the chart

1958 - st = rrdset_create(host, "example", "random", "random", "example", NULL, "random", "random", "random",

1959 - NULL, 1, update_every, RRDSET_TYPE_LINE);

1970 + snprintfz(name, RRD_ID_LENGTH_MAX, "example_local%d", thread_info->chart_i + 1);

1971 + st = rrdset_create(host, name, chartname, chartname, "example", NULL, chartname, chartname, chartname, NULL, 1,

1972 + update_every, RRDSET_TYPE_LINE);

1973 for (j = 0 ; j < DSET_DIMS ; ++j) {

1961 - snprintfz(name, 100, "random%d", j);

1974 + snprintfz(name, RRD_ID_LENGTH_MAX, "%s%d", chartname, j);

1975

1976 rd[j] = rrddim_add(st, name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

1977 }

1978

1966 - time_present = now_realtime_sec();

1979 // feed it with the test data

1980 time_current = time_present - history_seconds;

1981 for (j = 0 ; j < DSET_DIMS ; ++j) {

@@ -1972,17 +1984,62 @@ void generate_dbengine_dataset(unsigned history_seconds)

1984 rd[j]->last_collected_time.tv_usec =

1985 st->last_collected_time.tv_usec = st->last_updated.tv_usec = 0;

1986 }

1975 - for( ; time_current < time_present; ++time_current) {

1987 + for( ; time_current < time_present ; ++time_current) {

1988 st->usec_since_last_update = USEC_PER_SEC;

1989

1990 for (j = 0; j < DSET_DIMS; ++j) {

1979 - rrddim_set_by_pointer_fake_time(rd[j], (time_current + j) % 128, time_current);

1991 + collected_number value;

1992 +

1993 + value = generate_dbengine_chart_value(thread_info, j, time_current);

1994 + rrddim_set_by_pointer_fake_time(rd[j], value, time_current);

1995 }

1996 rrdset_done(st);

1997 }

1998 +}

1999 +

2000 +void generate_dbengine_dataset(unsigned history_seconds)

2001 +{

2002 + const int DSET_CHARTS = 16;

2003 + const int DSET_DIMS = 128;

2004 + const uint64_t EXPECTED_COMPRESSION_RATIO = 20;

2005 + RRDHOST *host = NULL;

2006 + struct dbengine_chart_thread thread_info[DSET_CHARTS];

2007 + int i;

2008 + time_t time_present;

2009 +

2010 + default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;

2011 + default_rrdeng_page_cache_mb = 128;

2012 + // Worst case for uncompressible data

2013 + default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * history_seconds) /

2014 + (1024 * 1024);

2015 + default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100;

2016 +

2017 + error_log_limit_unlimited();

2018 + debug(D_RRDHOST, "Initializing localhost with hostname 'dbengine-dataset'");

2019 +

2020 + host = dbengine_rrdhost_find_or_create("dbengine-dataset");

2021 + if (NULL == host)

2022 + return;

2023 +

2024 + fprintf(stderr, "\nRunning DB-engine workload generator\n");

2025 +

2026 + time_present = now_realtime_sec();

2027 + for (i = 0 ; i < DSET_CHARTS ; ++i) {

2028 + thread_info[i].host = host;

2029 + thread_info[i].chartname = "random";

2030 + thread_info[i].dset_charts = DSET_CHARTS;

2031 + thread_info[i].chart_i = i;

2032 + thread_info[i].dset_dims = DSET_DIMS;

2033 + thread_info[i].history_seconds = history_seconds;

2034 + thread_info[i].time_present = time_present;

2035 + assert(0 == uv_thread_create(&thread_info[i].thread, generate_dbengine_chart, &thread_info[i]));

2036 + }

2037 + for (i = 0 ; i < DSET_CHARTS ; ++i) {

2038 + assert(0 == uv_thread_join(&thread_info[i].thread));

2039 + }

2040 +

2041 rrd_wrlock();

2042 rrdhost_free(host);

2043 rrd_unlock();

1986 -

2044 }

1988 -#endif

\ No newline at end of file

2045 +#endif

database/engine/pagecache.c

+13 -3

@@ -216,6 +216,9 @@ static void pg_cache_release_pages(struct rrdengine_instance *ctx, unsigned numb

216 static void pg_cache_reserve_pages(struct rrdengine_instance *ctx, unsigned number)

217 {

218 struct page_cache *pg_cache = &ctx->pg_cache;

219 + unsigned failures = 0;

220 + const unsigned FAILURES_CEILING = 10; /* truncates exponential backoff to (2^FAILURES_CEILING x slot) */

221 + unsigned long exp_backoff_slot_usec = USEC_PER_MS * 10;

222

223 assert(number < ctx->max_cache_pages);

224

@@ -224,11 +227,13 @@ static void pg_cache_reserve_pages(struct rrdengine_instance *ctx, unsigned numb

227 debug(D_RRDENGINE, "==Page cache full. Reserving %u pages.==",

228 number);

229 while (pg_cache->populated_pages + number >= ctx->max_cache_pages + 1) {

230 +

231 if (!pg_cache_try_evict_one_page_unsafe(ctx)) {

232 /* failed to evict */

233 struct completion compl;

234 struct rrdeng_cmd cmd;

235

236 + ++failures;

237 uv_rwlock_wrunlock(&pg_cache->pg_cache_rwlock);

238

239 init_completion(&compl);

@@ -240,6 +245,12 @@ static void pg_cache_reserve_pages(struct rrdengine_instance *ctx, unsigned numb

245 wait_for_completion(&compl);

246 destroy_completion(&compl);

247

248 + if (unlikely(failures > 1)) {

249 + unsigned long slots;

250 + /* exponential backoff */

251 + slots = random() % (2LU << MIN(failures, FAILURES_CEILING));

252 + (void)sleep_usec(slots * exp_backoff_slot_usec);

253 + }

254 uv_rwlock_wrlock(&pg_cache->pg_cache_rwlock);

255 }

256 }

@@ -1042,9 +1053,8 @@ void free_page_cache(struct rrdengine_instance *ctx)

1053 /* Find first page in range */

1054 Index = (Word_t) 0;

1055 PValue = JudyLFirst(page_index->JudyL_array, &Index, PJE0);

1045 - if (likely(NULL != PValue)) {

1046 - descr = *PValue;

1047 - }

1056 + descr = unlikely(NULL == PValue) ? NULL : *PValue;

1057 +

1058 while (descr != NULL) {

1059 /* Iterate all page descriptors of this metric */

1060

database/engine/rrdengine.c

+12 -1

@@ -676,7 +676,7 @@ void timer_cb(uv_timer_t* handle)

676 /* Flushes dirty pages when timer expires */

677 #define TIMER_PERIOD_MS (1000)

678

679 -#define CMD_BATCH_SIZE (256)

679 +#define MAX_CMD_BATCH_SIZE (256)

680

681 void rrdeng_worker(void* arg)

682 {

@@ -687,6 +687,7 @@ void rrdeng_worker(void* arg)

687 enum rrdeng_opcode opcode;

688 uv_timer_t timer_req;

689 struct rrdeng_cmd cmd;

690 + unsigned cmd_batch_size;

691

692 rrdeng_init_cmd_queue(wc);

693

@@ -723,10 +724,20 @@ void rrdeng_worker(void* arg)

724 shutdown = 0;

725 while (shutdown == 0 || uv_loop_alive(loop)) {

726 uv_run(loop, UV_RUN_DEFAULT);

727 +

728 /* wait for commands */

729 + cmd_batch_size = 0;

730 do {

731 + /*

732 + * Avoid starving the loop when there are too many commands coming in.

733 + * timer_cb will interrupt the loop again to allow serving more commands.

734 + */

735 + if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE))

736 + break;

737 +

738 cmd = rrdeng_deq_cmd(wc);

739 opcode = cmd.opcode;

740 + ++cmd_batch_size;

741

742 switch (opcode) {

743 case RRDENG_NOOP: