Reuse ML load prepared statement (#16240)
Reuse ML load prepared statement and release resources on each batch load Fix parameter to ML model load to be in seconds not usec
Stelios Fragkakis committed
Oct 18, 2023 at 17:20 UTC
9caea28bcda541558d0adf611a7f69aafed251cc
4 files changed
+21
-9
database/sqlite/sqlite_metadata.c
+8
-1
@@ -1539,13 +1539,20 @@ static void start_ml_model_load(uv_work_t *req __maybe_unused)
1539
RRDDIM *rd;
1540
RRDDIM_ACQUIRED *rda;
1541
internal_error(true, "Batch ML load loader, %zu items", ml_data->count);
1542
+
1543
+ sqlite3_stmt *ml_load_stmt = NULL;
1544
while((PValue = JudyLFirstThenNext(ml_data->JudyL, &Index, &first))) {
1545
UNUSED(PValue);
1546
rda = (RRDDIM_ACQUIRED *) Index;
1547
rd = rrddim_acquired_to_rrddim(rda);
1546
- ml_dimension_load_models(rd);
1548
+ ml_dimension_load_models(rd, &ml_load_stmt);
1549
rrddim_acquired_release(rda);
1550
}
1551
+
1552
+ if (ml_load_stmt) {
1553
+ sqlite3_finalize(ml_load_stmt);
1554
+ ml_load_stmt = NULL;
1555
+ }
1556
worker_is_idle();
1557
}
1558
ml/ml-dummy.c
+1
-1
@@ -100,7 +100,7 @@ bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool
100
return false;
101
}
102
103
-int ml_dimension_load_models(RRDDIM *rd) {
103
+int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **stmp) {
104
UNUSED(rd);
105
return 0;
106
}
ml/ml.cc
+11
-6
@@ -621,7 +621,7 @@ bind_fail:
621
return rc;
622
}
623
624
-int ml_dimension_load_models(RRDDIM *rd) {
624
+int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **active_stmt) {
625
ml_dimension_t *dim = (ml_dimension_t *) rd->ml_dimension;
626
if (!dim)
627
return 0;
@@ -635,7 +635,7 @@ int ml_dimension_load_models(RRDDIM *rd) {
635
636
std::vector<ml_kmeans_t> V;
637
638
- static __thread sqlite3_stmt *res = NULL;
638
+ sqlite3_stmt *res = active_stmt ? *active_stmt : NULL;
639
int rc = 0;
640
int param = 0;
641
@@ -645,18 +645,20 @@ int ml_dimension_load_models(RRDDIM *rd) {
645
}
646
647
if (unlikely(!res)) {
648
- rc = prepare_statement(db, db_models_load, &res);
648
+ rc = sqlite3_prepare_v2(db, db_models_load, -1, &res, NULL);
649
if (unlikely(rc != SQLITE_OK)) {
650
error_report("Failed to prepare statement to load models, rc = %d", rc);
651
return 1;
652
}
653
+ if (active_stmt)
654
+ *active_stmt = res;
655
}
656
657
rc = sqlite3_bind_blob(res, ++param, &dim->rd->metric_uuid, sizeof(dim->rd->metric_uuid), SQLITE_STATIC);
658
if (unlikely(rc != SQLITE_OK))
659
goto bind_fail;
660
659
- rc = sqlite3_bind_int(res, ++param, now_realtime_usec() - (Cfg.num_models_to_use * Cfg.max_train_samples));
661
+ rc = sqlite3_bind_int64(res, ++param, now_realtime_sec() - (Cfg.num_models_to_use * Cfg.max_train_samples));
662
if (unlikely(rc != SQLITE_OK))
663
goto bind_fail;
664
@@ -702,9 +704,12 @@ int ml_dimension_load_models(RRDDIM *rd) {
704
if (unlikely(rc != SQLITE_DONE))
705
error_report("Failed to load models, rc = %d", rc);
706
705
- rc = sqlite3_reset(res);
707
+ if (active_stmt)
708
+ rc = sqlite3_reset(res);
709
+ else
710
+ rc = sqlite3_finalize(res);
711
if (unlikely(rc != SQLITE_OK))
707
- error_report("Failed to reset statement when loading models, rc = %d", rc);
712
+ error_report("Failed to %s statement when loading models, rc = %d", active_stmt ? "reset" : "finalize", rc);
713
714
return 0;
715
ml/ml.h
+1
-1
@@ -40,7 +40,7 @@ void ml_dimension_new(RRDDIM *rd);
40
void ml_dimension_delete(RRDDIM *rd);
41
bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool exists);
42
43
-int ml_dimension_load_models(RRDDIM *rd);
43
+int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **stmt);
44
45
void ml_update_global_statistics_charts(uint64_t models_consulted);
46