Rollback ML transaction on failure. (#15013)
vkalintiris committed
May 9, 2023 at 15:28 UTC
d631ee80ede879174be07ab945a4133b0b96cfc8
1 file changed
+40
-18
ml/ml.cc
+40
-18
@@ -490,12 +490,16 @@ ml_dimension_add_model(const uuid_t *metric_uuid, const ml_kmeans_t *km)
490
}
491
492
rc = execute_insert(res);
493
- if (unlikely(rc != SQLITE_DONE))
493
+ if (unlikely(rc != SQLITE_DONE)) {
494
error_report("Failed to store model, rc = %d", rc);
495
+ return rc;
496
+ }
497
498
rc = sqlite3_reset(res);
497
- if (unlikely(rc != SQLITE_OK))
499
+ if (unlikely(rc != SQLITE_OK)) {
500
error_report("Failed to reset statement when storing model, rc = %d", rc);
501
+ return rc;
502
+ }
503
504
return 0;
505
@@ -504,7 +508,7 @@ bind_fail:
508
rc = sqlite3_reset(res);
509
if (unlikely(rc != SQLITE_OK))
510
error_report("Failed to reset statement to store model, rc = %d", rc);
507
- return 1;
511
+ return rc;
512
}
513
514
static int
@@ -523,7 +527,7 @@ ml_dimension_delete_models(const uuid_t *metric_uuid, time_t before)
527
rc = prepare_statement(db, db_models_delete, &res);
528
if (unlikely(rc != SQLITE_OK)) {
529
error_report("Failed to prepare statement to delete models, rc = %d", rc);
526
- return 1;
530
+ return rc;
531
}
532
}
533
@@ -536,12 +540,16 @@ ml_dimension_delete_models(const uuid_t *metric_uuid, time_t before)
540
goto bind_fail;
541
542
rc = execute_insert(res);
539
- if (unlikely(rc != SQLITE_DONE))
543
+ if (unlikely(rc != SQLITE_DONE)) {
544
error_report("Failed to delete models, rc = %d", rc);
545
+ return rc;
546
+ }
547
548
rc = sqlite3_reset(res);
543
- if (unlikely(rc != SQLITE_OK))
549
+ if (unlikely(rc != SQLITE_OK)) {
550
error_report("Failed to reset statement when deleting models, rc = %d", rc);
551
+ return rc;
552
+ }
553
554
return 0;
555
@@ -550,7 +558,7 @@ bind_fail:
558
rc = sqlite3_reset(res);
559
if (unlikely(rc != SQLITE_OK))
560
error_report("Failed to reset statement to delete models, rc = %d", rc);
553
- return 1;
561
+ return rc;
562
}
563
564
int ml_dimension_load_models(RRDDIM *rd) {
@@ -1370,23 +1378,37 @@ bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool
1378
return is_anomalous;
1379
}
1380
1373
-static int ml_flush_pending_models(ml_training_thread_t *training_thread) {
1374
- (void) db_execute(db, "BEGIN TRANSACTION;");
1381
+static void ml_flush_pending_models(ml_training_thread_t *training_thread) {
1382
+ int rc = db_execute(db, "BEGIN TRANSACTION;");
1383
+ int op_no = 1;
1384
1376
- for (const auto &pending_model: training_thread->pending_model_info) {
1377
- int rc = ml_dimension_add_model(&pending_model.metric_uuid, &pending_model.kmeans);
1378
- if (rc)
1379
- return rc;
1385
+ if (!rc) {
1386
+ op_no++;
1387
1381
- rc = ml_dimension_delete_models(&pending_model.metric_uuid, pending_model.kmeans.before - (Cfg.num_models_to_use * Cfg.train_every));
1382
- if (rc)
1383
- return rc;
1388
+ for (const auto &pending_model: training_thread->pending_model_info) {
1389
+ if (!rc)
1390
+ rc = ml_dimension_add_model(&pending_model.metric_uuid, &pending_model.kmeans);
1391
+
1392
+ if (!rc)
1393
+ rc = ml_dimension_delete_models(&pending_model.metric_uuid, pending_model.kmeans.before - (Cfg.num_models_to_use * Cfg.train_every));
1394
+ }
1395
}
1396
1386
- (void) db_execute(db, "COMMIT TRANSACTION;");
1397
+ if (!rc) {
1398
+ op_no++;
1399
+ rc = db_execute(db, "COMMIT TRANSACTION;");
1400
+ }
1401
+
1402
+ // try to rollback transaction if we got any failures
1403
+ if (rc) {
1404
+ error("Trying to rollback ML transaction because it failed with rc=%d, op_no=%d", rc, op_no);
1405
+ op_no++;
1406
+ rc = db_execute(db, "ROLLBACK;");
1407
+ if (rc)
1408
+ error("ML transaction rollback failed with rc=%d", rc);
1409
+ }
1410
1411
training_thread->pending_model_info.clear();
1389
- return 0;
1412
}
1413
1414
static void *ml_train_main(void *arg) {