@cryptotaxi247 / netdata-1 / commits / 0586829ee

Add commands to check and fix database corruption (#11828)

* Set a flag to do aclk sync thread shutdown Attempt to dequeue a cmd in case the queue is full and someone is blocked * Drop tables and recreate instead of deleting * Add commands to check the database -W check-database, fix-database, compact-database * Split the database setup to config and cleanup part * Add checks during database setup and cleanup to detect corruption to the dimension and chart tables * Add full database check and refactor code * Change commands to better indicate that the operations refer to the sqlite metadata database (not the metrics dbengine database) * Add check for table being null (request for entire database check) * Rename command for better clarity

Stelios Fragkakis committed Nov 26, 2021 at 20:36 UTC 0586829ee645a672a032b2346d7ea9718e4142af

6 files changed +265 -25

daemon/main.c

+21 -1

@@ -44,6 +44,9 @@ void netdata_cleanup_and_exit(int ret) {

45 // stop everything

46 info("EXIT: stopping static threads...");

47 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL

48 + aclk_sync_exit_all();

49 +#endif

50 cancel_main_threads();

52 // free the database

@@ -360,13 +363,16 @@ int help(int exitcode) {

363 " -W stacksize=N Set the stacksize (in bytes).\n\n"

364 " -W debug_flags=N Set runtime tracing to debug.log.\n\n"

365 " -W unittest Run internal unittests and exit.\n\n"

366 + " -W sqlite-check Check metadata database integrity and exit.\n\n"

367 + " -W sqlite-fix Check metadata database integrity, fix if needed and exit.\n\n"

368 + " -W sqlite-compact Reclaim metadata database unused space and exit.\n\n"

369 #ifdef ENABLE_DBENGINE

370 " -W createdataset=N Create a DB engine dataset of N seconds and exit.\n\n"

371 " -W stresstest=A,B,C,D,E,F\n"

372 " Run a DB engine stress test for A seconds,\n"

373 " with B writers and C readers, with a ramp up\n"

374 " time of D seconds for writers, a page cache\n"

369 - " size of E MiB, an optional disk space limit"

375 + " size of E MiB, an optional disk space limit\n"

376 " of F MiB and exit.\n\n"

377 #endif

378 " -W set section option value\n"

@@ -801,6 +807,20 @@ int main(int argc, char **argv) {

807 char* createdataset_string = "createdataset=";

808 char* stresstest_string = "stresstest=";

809 #endif

810 + if(strcmp(optarg, "sqlite-check") == 0) {

811 + sql_init_database(DB_CHECK_INTEGRITY);

812 + return 0;

813 + }

814 +

815 + if(strcmp(optarg, "sqlite-fix") == 0) {

816 + sql_init_database(DB_CHECK_FIX_DB);

817 + return 0;

818 + }

819 +

820 + if(strcmp(optarg, "sqlite-compact") == 0) {

821 + sql_init_database(DB_CHECK_RECLAIM_SPACE);

822 + return 0;

823 + }

824

825 if(strcmp(optarg, "unittest") == 0) {

826 if(unit_test_buffer()) return 1;

database/rrdhost.c

+1 -1

@@ -690,7 +690,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {

690 if (gap_when_lost_iterations_above < 1)

691 gap_when_lost_iterations_above = 1;

692

693 - if (unlikely(sql_init_database())) {

693 + if (unlikely(sql_init_database(DB_CHECK_NONE))) {

694 if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

695 fatal("Failed to initialize SQLite");

696 info("Skipping SQLITE metadata initialization since memory mode is not db engine");

database/sqlite/sqlite_aclk.c

+43 -10

@@ -97,7 +97,7 @@ int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct

98 /* wait for free space in queue */

99 uv_mutex_lock(&wc->cmd_mutex);

100 - if ((queue_size = wc->queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE) {

100 + if ((queue_size = wc->queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE || wc->is_shutting_down) {

101 uv_mutex_unlock(&wc->cmd_mutex);

102 return 1;

103 }

@@ -118,6 +118,10 @@ void aclk_database_enq_cmd(struct aclk_database_worker_config *wc, struct aclk_d

118

119 /* wait for free space in queue */

120 uv_mutex_lock(&wc->cmd_mutex);

121 + if (wc->is_shutting_down) {

122 + uv_mutex_unlock(&wc->cmd_mutex);

123 + return;

124 + }

125 while ((queue_size = wc->queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE) {

126 uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);

127 }

@@ -142,10 +146,12 @@ struct aclk_database_cmd aclk_database_deq_cmd(struct aclk_database_worker_confi

146

147 uv_mutex_lock(&wc->cmd_mutex);

148 queue_size = wc->queue_size;

145 - if (queue_size == 0) {

149 + if (queue_size == 0 || wc->is_shutting_down) {

150 memset(&ret, 0, sizeof(ret));

151 ret.opcode = ACLK_DATABASE_NOOP;

152 ret.completion = NULL;

153 + if (wc->is_shutting_down)

154 + uv_cond_signal(&wc->cmd_cond);

155 } else {

156 /* dequeue command */

157 ret = wc->cmd_queue.cmd_array[wc->cmd_queue.head];

@@ -156,7 +162,6 @@ struct aclk_database_cmd aclk_database_deq_cmd(struct aclk_database_worker_confi

162 wc->cmd_queue.head + 1 : 0;

163 }

164 wc->queue_size = queue_size - 1;

159 -

165 /* wake up producers */

166 uv_cond_signal(&wc->cmd_cond);

167 }

@@ -184,6 +189,30 @@ int aclk_worker_enq_cmd(char *node_id, struct aclk_database_cmd *cmd)

189 return (wc == NULL);

190 }

191

192 +void aclk_sync_exit_all()

193 +{

194 + rrd_wrlock();

195 + RRDHOST *host = localhost;

196 + while(host) {

197 + struct aclk_database_worker_config *wc = host->dbsync_worker;

198 + if (wc) {

199 + wc->is_shutting_down = 1;

200 + (void) aclk_database_deq_cmd(wc);

201 + uv_cond_signal(&wc->cmd_cond);

202 + }

203 + host = host->next;

204 + }

205 + rrd_unlock();

206 +

207 + uv_mutex_lock(&aclk_async_lock);

208 + struct aclk_database_worker_config *wc = aclk_thread_head;

209 + while (wc) {

210 + wc->is_shutting_down = 1;

211 + wc = wc->next;

212 + }

213 + uv_mutex_unlock(&aclk_async_lock);

214 +}

215 +

216 int aclk_start_sync_thread(void *data, int argc, char **argv, char **column)

217 {

218 char uuid_str[GUID_LEN + 1];

@@ -313,7 +342,7 @@ void aclk_database_worker(void *arg)

342 struct aclk_database_cmd cmd;

343 unsigned cmd_batch_size;

344

316 - aclk_database_init_cmd_queue(wc);

345 + //aclk_database_init_cmd_queue(wc);

346

347 char threadname[NETDATA_THREAD_NAME_MAX+1];

348 if (wc->host)

@@ -347,9 +376,9 @@ void aclk_database_worker(void *arg)

376 timer_req.data = wc;

377 fatal_assert(0 == uv_timer_start(&timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));

378

350 - wc->retry_count = 0;

379 +// wc->retry_count = 0;

380 wc->node_info_send = (wc->host && !localhost);

352 - aclk_add_worker_thread(wc);

381 +// aclk_add_worker_thread(wc);

382 info("Starting ACLK sync thread for host %s -- scratch area %lu bytes", wc->host_guid, sizeof(*wc));

383

384 memset(&cmd, 0, sizeof(cmd));

@@ -359,11 +388,10 @@ void aclk_database_worker(void *arg)

388 if (!wc->chart_payload_count)

389 info("%s: No pending charts and dimensions detected during startup", wc->host_guid);

390 #endif

362 - wc->chart_updates = 0;

391 +

392 wc->startup_time = now_realtime_sec();

393 wc->cleanup_after = wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST;

394 wc->rotation_after = wc->startup_time + ACLK_DATABASE_ROTATION_DELAY;

366 - wc->alert_updates = 0;

395

396 debug(D_ACLK_SYNC,"Node %s reports pending message count = %u", wc->node_id, wc->chart_payload_count);

397 while (likely(!netdata_exit)) {

@@ -498,7 +526,7 @@ void aclk_database_worker(void *arg)

526 uv_close((uv_handle_t *)&timer_req, NULL);

527

528 /* cleanup operations of the event loop */

501 - info("Shutting down ACLK sync event loop.");

529 + //info("Shutting down ACLK sync event loop for %s", wc->host_guid);

530

531 /*

532 * uv_async_send after uv_close does not seem to crash in linux at the moment,

@@ -508,7 +536,7 @@ void aclk_database_worker(void *arg)

536 uv_close((uv_handle_t *)&wc->async, NULL);

537 uv_run(loop, UV_RUN_DEFAULT);

538

511 - info("Shutting down ACLK sync event loop complete.");

539 + info("Shutting down ACLK sync event loop complete for host %s", wc->host_guid);

540 /* TODO: don't let the API block by waiting to enqueue commands */

541 uv_cond_destroy(&wc->cmd_cond);

542 /* uv_mutex_destroy(&wc->cmd_mutex); */

@@ -597,6 +625,11 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)

625 strcpy(wc->host_guid, host_guid);

626 if (node_id && !uuid_is_null(*node_id))

627 uuid_unparse_lower(*node_id, wc->node_id);

628 + wc->chart_updates = 0;

629 + wc->alert_updates = 0;

630 + wc->retry_count = 0;

631 + aclk_database_init_cmd_queue(wc);

632 + aclk_add_worker_thread(wc);

633 fatal_assert(0 == uv_thread_create(&(wc->thread), aclk_database_worker, wc));

634 #else

635 UNUSED(host);

database/sqlite/sqlite_aclk.h

@@ -193,6 +193,7 @@ struct aclk_database_worker_config {

193 int node_info_send;

194 int chart_pending;

195 int chart_reset_count;

196 + volatile unsigned is_shutting_down;

197 struct aclk_database_worker_config *next;

198 };

199

@@ -227,4 +228,5 @@ void sql_check_aclk_table_list(struct aclk_database_worker_config *wc);

228 void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);

229 void sql_maint_aclk_sync_database(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);

230 int claimed();

231 +void aclk_sync_exit_all();

232 #endif //NETDATA_SQLITE_ACLK_H

database/sqlite/sqlite_functions.c

+190 -12

@@ -12,6 +12,10 @@ const char *database_config[] = {

12 "chart_type int, memory_mode int, history_entries);",

13 "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "

14 "multiplier int, divisor int , algorithm int, options text);",

15 +

16 + "DROP TABLE IF EXISTS chart_active;",

17 + "DROP TABLE IF EXISTS dimension_active;",

18 +

19 "CREATE TABLE IF NOT EXISTS chart_active(chart_id blob PRIMARY KEY, date_created int);",

20 "CREATE TABLE IF NOT EXISTS dimension_active(dim_id blob primary key, date_created int);",

21 "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int);",

@@ -45,8 +49,10 @@ const char *database_config[] = {

49 "INSERT INTO chart_hash_map (chart_id, hash_id) values (new.chart_id, new.hash_id) "

50 "on conflict (chart_id, hash_id) do nothing; END; ",

48 - "delete from chart_active;",

49 - "delete from dimension_active;",

52 + NULL

53 +};

54 +

55 +const char *database_cleanup[] = {

56 "delete from chart where chart_id not in (select chart_id from dimension);",

57 "delete from host where host_id not in (select host_id from chart);",

58 "delete from chart_label where chart_id not in (select chart_id from chart);",

@@ -180,18 +186,153 @@ void store_active_dimension(uuid_t *dimension_uuid)

186 return;

187 }

188

189 +static int check_table_integrity_cb(void *data, int argc, char **argv, char **column)

190 +{

191 + int *status = data;

192 + UNUSED(argc);

193 + UNUSED(column);

194 + info("---> %s", argv[0]);

195 + *status = (strcmp(argv[0], "ok") != 0);

196 + return 0;

197 +}

198 +

199 +

200 +static int check_table_integrity(char *table)

201 +{

202 + int status = 0;

203 + char *err_msg = NULL;

204 + char wstr[255];

205 +

206 + if (table) {

207 + info("Checking table %s", table);

208 + snprintfz(wstr, 254, "PRAGMA integrity_check(%s);", table);

209 + }

210 + else {

211 + info("Checking entire database");

212 + strcpy(wstr,"PRAGMA integrity_check;");

213 + }

214 +

215 + int rc = sqlite3_exec(db_meta, wstr, check_table_integrity_cb, (void *) &status, &err_msg);

216 + if (rc != SQLITE_OK) {

217 + error_report("SQLite error during database integrity check for %s, rc = %d (%s)",

218 + table ? table : "the entire database", rc, err_msg);

219 + sqlite3_free(err_msg);

220 + }

221 +

222 + return status;

223 +}

224 +

225 +const char *rebuild_chart_commands[] = {

226 + "BEGIN TRANSACTION; ",

227 + "DROP INDEX IF EXISTS ind_c1;" ,

228 + "DROP TABLE IF EXISTS chart_backup; " ,

229 + "CREATE TABLE chart_backup AS SELECT * FROM chart; " ,

230 + "DROP TABLE chart; ",

231 + "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, "

232 + "name text, family text, context text, title text, unit text, plugin text, "

233 + "module text, priority int, update_every int, chart_type int, memory_mode int, history_entries); ",

234 + "INSERT INTO chart SELECT DISTINCT * FROM chart_backup; ",

235 + "DROP TABLE chart_backup; " ,

236 + "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);",

237 + "COMMIT TRANSACTION;",

238 + NULL

239 +};

240 +

241 +static void rebuild_chart()

242 +{

243 + int rc;

244 + char *err_msg = NULL;

245 + info("Rebuilding chart table");

246 + for (int i = 0; rebuild_chart_commands[i]; i++) {

247 + info("Executing %s", rebuild_chart_commands[i]);

248 + rc = sqlite3_exec(db_meta, rebuild_chart_commands[i], 0, 0, &err_msg);

249 + if (rc != SQLITE_OK) {

250 + error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);

251 + error_report("SQLite failed statement %s", rebuild_chart_commands[i]);

252 + sqlite3_free(err_msg);

253 + }

254 + }

255 + return;

256 +}

257 +

258 +const char *rebuild_dimension_commands[] = {

259 + "BEGIN TRANSACTION; ",

260 + "DROP INDEX IF EXISTS ind_d1;" ,

261 + "DROP TABLE IF EXISTS dimension_backup; " ,

262 + "CREATE TABLE dimension_backup AS SELECT * FROM dimension; " ,

263 + "DROP TABLE dimension; " ,

264 + "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "

265 + "multiplier int, divisor int , algorithm int, options text);" ,

266 + "INSERT INTO dimension SELECT distinct * FROM dimension_backup; " ,

267 + "DROP TABLE dimension_backup; " ,

268 + "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);",

269 + "COMMIT TRANSACTION;",

270 + NULL

271 +};

272 +

273 +void rebuild_dimension()

274 +{

275 + int rc;

276 + char *err_msg = NULL;

277 +

278 + info("Rebuilding dimension table");

279 + for (int i = 0; rebuild_dimension_commands[i]; i++) {

280 + info("Executing %s", rebuild_dimension_commands[i]);

281 + rc = sqlite3_exec(db_meta, rebuild_dimension_commands[i], 0, 0, &err_msg);

282 + if (rc != SQLITE_OK) {

283 + error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);

284 + error_report("SQLite failed statement %s", rebuild_dimension_commands[i]);

285 + sqlite3_free(err_msg);

286 + }

287 + }

288 + return;

289 +}

290 +

291 +static int attempt_database_fix()

292 +{

293 + info("Closing database and attempting to fix it");

294 + int rc = sqlite3_close(db_meta);

295 + if (rc != SQLITE_OK)

296 + error_report("Failed to close database, rc = %d", rc);

297 + info("Attempting to fix database");

298 + db_meta = NULL;

299 + return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT);

300 +}

301 +

302 +static int init_database_batch(int rebuild, int init_type, const char *batch[])

303 +{

304 + int rc;

305 + char *err_msg = NULL;

306 + for (int i = 0; batch[i]; i++) {

307 + debug(D_METADATALOG, "Executing %s", batch[i]);

308 + rc = sqlite3_exec(db_meta, batch[i], 0, 0, &err_msg);

309 + if (rc != SQLITE_OK) {

310 + error_report("SQLite error during database %s, rc = %d (%s)", init_type ? "cleanup" : "setup", rc, err_msg);

311 + error_report("SQLite failed statement %s", batch[i]);

312 + sqlite3_free(err_msg);

313 + if (SQLITE_CORRUPT == rc) {

314 + if (!rebuild)

315 + return attempt_database_fix();

316 + rc = check_table_integrity(NULL);

317 + if (rc)

318 + error_report("Databse integrity errors reported");

319 + }

320 + return 1;

321 + }

322 + }

323 + return 0;

324 +}

325 +

326 /*

327 * Initialize the SQLite database

328 * Return 0 on success

329 */

187 -int sql_init_database(void)

330 +int sql_init_database(db_check_action_type_t rebuild)

331 {

332 char *err_msg = NULL;

333 char sqlite_database[FILENAME_MAX + 1];

334 int rc;

335

193 - fatal_assert(0 == uv_mutex_init(&sqlite_transaction_lock));

194 -

336 snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);

337 rc = sqlite3_open(sqlite_database, &db_meta);

338 if (rc != SQLITE_OK) {

@@ -201,18 +342,55 @@ int sql_init_database(void)

342 return 1;

343 }

344

204 - info("SQLite database %s initialization", sqlite_database);

345 + if (rebuild & (DB_CHECK_INTEGRITY | DB_CHECK_FIX_DB)) {

346 + int errors_detected = 0;

347 + if (!(rebuild & DB_CHECK_CONT))

348 + info("Running database check on %s", sqlite_database);

349 +

350 + if (check_table_integrity("chart")) {

351 + errors_detected++;

352 + if (rebuild & DB_CHECK_FIX_DB)

353 + rebuild_chart();

354 + else

355 + error_report("Errors reported -- run with -W sqlite-fix");

356 + }

357

206 - for (int i = 0; database_config[i]; i++) {

207 - debug(D_METADATALOG, "Executing %s", database_config[i]);

208 - rc = sqlite3_exec(db_meta, database_config[i], 0, 0, &err_msg);

358 + if (check_table_integrity("dimension")) {

359 + errors_detected++;

360 + if (rebuild & DB_CHECK_FIX_DB)

361 + rebuild_dimension();

362 + else

363 + error_report("Errors reported -- run with -W sqlite-fix");

364 + }

365 +

366 + if (!errors_detected) {

367 + if (check_table_integrity(NULL))

368 + error_report("Errors reported");

369 + }

370 + }

371 +

372 + if (rebuild & DB_CHECK_RECLAIM_SPACE) {

373 + if (!(rebuild & DB_CHECK_CONT))

374 + info("Reclaiming space of %s", sqlite_database);

375 + rc = sqlite3_exec(db_meta, "VACUUM;", 0, 0, &err_msg);

376 if (rc != SQLITE_OK) {

210 - error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);

211 - error_report("SQLite failed statement %s", database_config[i]);

377 + error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);

378 sqlite3_free(err_msg);

213 - return 1;

379 }

380 }

381 +

382 + if (rebuild && !(rebuild & DB_CHECK_CONT))

383 + return 1;

384 +

385 + info("SQLite database %s initialization", sqlite_database);

386 +

387 + if (init_database_batch(rebuild, 0, &database_config[0]))

388 + return 1;

389 +

390 + if (init_database_batch(rebuild, 0, &database_cleanup[0]))

391 + return 1;

392 +

393 + fatal_assert(0 == uv_mutex_init(&sqlite_transaction_lock));

394 info("SQLite database initialization completed");

395 return 0;

396 }

database/sqlite/sqlite_functions.h

+8 -1

@@ -16,6 +16,13 @@ struct node_instance_list {

16 int hops;

17 };

19 +typedef enum db_check_action_type {

20 + DB_CHECK_NONE = 0x0000,

21 + DB_CHECK_INTEGRITY = 0x0001,

22 + DB_CHECK_FIX_DB = 0x0002,

23 + DB_CHECK_RECLAIM_SPACE = 0x0004,

24 + DB_CHECK_CONT = 0x00008

25 +} db_check_action_type_t;

27 #define SQLITE_INSERT_DELAY (50) // Insert delay in case of lock

@@ -49,7 +56,7 @@ struct node_instance_list {

56 return 1; \

57 }

52 -extern int sql_init_database(void);

59 +extern int sql_init_database(db_check_action_type_t rebuild);

60 extern void sql_close_database(void);

62 extern int sql_store_host(uuid_t *guid, const char *hostname, const char *registry_hostname, int update_every, const char *os, const char *timezone, const char *tags);