Initialize the metadata database when performing dbengine stress test (#12861)
* Remove error (no real value) * Add a parameter to create an in-memory database for stress testing * Add a new parameter to the stresstest command to set the number of deisred libuv worker threads
Stelios Fragkakis committed
May 10, 2022 at 13:33 UTC
6ad3e612e04cb8ad47839d71d4e9118c5498b907
6 files changed
+25
-13
daemon/main.c
+15
-7
@@ -340,12 +340,12 @@ int help(int exitcode) {
340
" -W sqlite-compact Reclaim metadata database unused space and exit.\n\n"
341
#ifdef ENABLE_DBENGINE
342
" -W createdataset=N Create a DB engine dataset of N seconds and exit.\n\n"
343
- " -W stresstest=A,B,C,D,E,F\n"
343
+ " -W stresstest=A,B,C,D,E,F,G\n"
344
" Run a DB engine stress test for A seconds,\n"
345
" with B writers and C readers, with a ramp up\n"
346
" time of D seconds for writers, a page cache\n"
347
" size of E MiB, an optional disk space limit\n"
348
- " of F MiB and exit.\n\n"
348
+ " of F MiB, G libuv workers (default 16) and exit.\n\n"
349
#endif
350
" -W set section option value\n"
351
" set netdata.conf option from the command line.\n\n"
@@ -567,7 +567,7 @@ static void get_netdata_configured_variables() {
567
568
}
569
570
-static int load_netdata_conf(char *filename, char overwrite_used) {
570
+int load_netdata_conf(char *filename, char overwrite_used) {
571
errno = 0;
572
573
int ret = 0;
@@ -777,17 +777,17 @@ int main(int argc, char **argv) {
777
char* stresstest_string = "stresstest=";
778
#endif
779
if(strcmp(optarg, "sqlite-check") == 0) {
780
- sql_init_database(DB_CHECK_INTEGRITY);
780
+ sql_init_database(DB_CHECK_INTEGRITY, 0);
781
return 0;
782
}
783
784
if(strcmp(optarg, "sqlite-fix") == 0) {
785
- sql_init_database(DB_CHECK_FIX_DB);
785
+ sql_init_database(DB_CHECK_FIX_DB, 0);
786
return 0;
787
}
788
789
if(strcmp(optarg, "sqlite-compact") == 0) {
790
- sql_init_database(DB_CHECK_RECLAIM_SPACE);
790
+ sql_init_database(DB_CHECK_RECLAIM_SPACE, 0);
791
return 0;
792
}
793
@@ -835,7 +835,7 @@ int main(int argc, char **argv) {
835
else if(strncmp(optarg, stresstest_string, strlen(stresstest_string)) == 0) {
836
char *endptr;
837
unsigned test_duration_sec = 0, dset_charts = 0, query_threads = 0, ramp_up_seconds = 0,
838
- page_cache_mb = 0, disk_space_mb = 0;
838
+ page_cache_mb = 0, disk_space_mb = 0, workers = 16;
839
840
optarg += strlen(stresstest_string);
841
test_duration_sec = (unsigned)strtoul(optarg, &endptr, 0);
@@ -849,7 +849,15 @@ int main(int argc, char **argv) {
849
page_cache_mb = (unsigned)strtoul(endptr + 1, &endptr, 0);
850
if (',' == *endptr)
851
disk_space_mb = (unsigned)strtoul(endptr + 1, &endptr, 0);
852
+ if (',' == *endptr)
853
+ workers = (unsigned)strtoul(endptr + 1, &endptr, 0);
854
+
855
+ if (workers > 1024)
856
+ workers = 1024;
857
858
+ char workers_str[16];
859
+ snprintf(workers_str, 15, "%u", workers);
860
+ setenv("UV_THREADPOOL_SIZE", workers_str, 1);
861
dbengine_stress_test(test_duration_sec, dset_charts, query_threads, ramp_up_seconds,
862
page_cache_mb, disk_space_mb);
863
return 0;
daemon/unit_test.c
+1
@@ -2190,6 +2190,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2190
2191
fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n");
2192
2193
+ (void) sql_init_database(DB_CHECK_NONE, 1);
2194
host = dbengine_rrdhost_find_or_create("dbengine-stress-test");
2195
if (NULL == host)
2196
return;
database/engine/pagecache.c
-1
@@ -437,7 +437,6 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
437
ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
438
if (unlikely(0 == ret)) {
439
uv_rwlock_wrunlock(&page_index->lock);
440
- error("Page under deletion was not in index.");
440
if (unlikely(debug_flags & D_RRDENGINE)) {
441
print_page_descr(descr);
442
}
database/rrdhost.c
+1
-1
@@ -698,7 +698,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
698
if (gap_when_lost_iterations_above < 1)
699
gap_when_lost_iterations_above = 1;
700
701
- if (unlikely(sql_init_database(DB_CHECK_NONE))) {
701
+ if (unlikely(sql_init_database(DB_CHECK_NONE, 0))) {
702
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
703
fatal("Failed to initialize SQLite");
704
info("Skipping SQLITE metadata initialization since memory mode is not db engine");
database/sqlite/sqlite_functions.c
+7
-3
@@ -303,7 +303,7 @@ static int attempt_database_fix()
303
error_report("Failed to close database, rc = %d", rc);
304
info("Attempting to fix database");
305
db_meta = NULL;
306
- return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT);
306
+ return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT, 0);
307
}
308
309
static int init_database_batch(int rebuild, int init_type, const char *batch[])
@@ -334,13 +334,17 @@ static int init_database_batch(int rebuild, int init_type, const char *batch[])
334
* Initialize the SQLite database
335
* Return 0 on success
336
*/
337
-int sql_init_database(db_check_action_type_t rebuild)
337
+int sql_init_database(db_check_action_type_t rebuild, int memory)
338
{
339
char *err_msg = NULL;
340
char sqlite_database[FILENAME_MAX + 1];
341
int rc;
342
343
- snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
343
+ if (likely(!memory))
344
+ snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
345
+ else
346
+ strcpy(sqlite_database, ":memory:");
347
+
348
rc = sqlite3_open(sqlite_database, &db_meta);
349
if (rc != SQLITE_OK) {
350
error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
database/sqlite/sqlite_functions.h
+1
-1
@@ -56,7 +56,7 @@ typedef enum db_check_action_type {
56
return 1; \
57
}
58
59
-extern int sql_init_database(db_check_action_type_t rebuild);
59
+extern int sql_init_database(db_check_action_type_t rebuild, int memory);
60
extern void sql_close_database(void);
61
62
extern int sql_store_host(uuid_t *guid, const char *hostname, const char *registry_hostname, int update_every, const char *os, const char *timezone, const char *tags);