@cryptotaxi247 / netdata-1 / commits / 941fff633

Unconditionally delete very old models. (#15720)

* Unconditionally delete very old models. * Rebase origin/master * Use the training threads to prune old models. To keep performance in check, we shedule the pruning whenever the number of successfully completed transactions is a multiple of 64.

vkalintiris committed Aug 23, 2023 at 14:53 UTC 941fff633212dc0034a08148622fd0e4023f07e2

3 files changed +86 -2

ml/Config.cc

@@ -28,7 +28,9 @@ void ml_config_load(ml_config_t *cfg) {

28 unsigned max_train_samples = config_get_number(config_section_ml, "maximum num samples to train", 6 * 3600);

29 unsigned min_train_samples = config_get_number(config_section_ml, "minimum num samples to train", 1 * 900);

30 unsigned train_every = config_get_number(config_section_ml, "train every", 3 * 3600);

31 +

32 unsigned num_models_to_use = config_get_number(config_section_ml, "number of models per dimension", 9);

33 + unsigned delete_models_older_than = config_get_number(config_section_ml, "delete models older than", 60 * 60 * 24 * 7);

35 unsigned diff_n = config_get_number(config_section_ml, "num samples to diff", 1);

36 unsigned smooth_n = config_get_number(config_section_ml, "num samples to smooth", 3);

@@ -58,7 +60,9 @@ void ml_config_load(ml_config_t *cfg) {

60 max_train_samples = clamp<unsigned>(max_train_samples, 1 * 3600, 24 * 3600);

61 min_train_samples = clamp<unsigned>(min_train_samples, 1 * 900, 6 * 3600);

62 train_every = clamp<unsigned>(train_every, 1 * 3600, 6 * 3600);

63 +

64 num_models_to_use = clamp<unsigned>(num_models_to_use, 1, 7 * 24);

65 + delete_models_older_than = clamp<unsigned>(delete_models_older_than, 60 * 60 * 24 * 1, 60 * 60 * 24 * 7);

67 diff_n = clamp(diff_n, 0u, 1u);

68 smooth_n = clamp(smooth_n, 0u, 5u);

@@ -100,6 +104,7 @@ void ml_config_load(ml_config_t *cfg) {

104 cfg->train_every = train_every;

105

106 cfg->num_models_to_use = num_models_to_use;

107 + cfg->delete_models_older_than = delete_models_older_than;

108

109 cfg->diff_n = diff_n;

110 cfg->smooth_n = smooth_n;

ml/ml-private.h

@@ -291,6 +291,9 @@ typedef struct {

291 RRDDIM *training_results_not_enough_collected_values_rd;

292 RRDDIM *training_results_null_acquired_dimension_rd;

293 RRDDIM *training_results_chart_under_replication_rd;

294 +

295 + size_t num_db_transactions;

296 + size_t num_models_to_prune;

297 } ml_training_thread_t;

298

299 typedef struct {

@@ -301,6 +304,7 @@ typedef struct {

304 unsigned train_every;

305

306 unsigned num_models_to_use;

307 + unsigned delete_models_older_than;

308

309 unsigned db_engine_anomaly_rate_every;

310

ml/ml.cc

+77 -2

@@ -436,6 +436,10 @@ const char *db_models_delete =

436 "DELETE FROM models "

437 "WHERE dim_id = @dim_id AND before < @before;";

438

439 +const char *db_models_prune =

440 + "DELETE FROM models "

441 + "WHERE after < @after LIMIT @n;";

442 +

443 static int

444 ml_dimension_add_model(const uuid_t *metric_uuid, const ml_kmeans_t *km)

445 {

@@ -563,6 +567,58 @@ bind_fail:

567 return rc;

568 }

569

570 +static int

571 +ml_prune_old_models(size_t num_models_to_prune)

572 +{

573 + static __thread sqlite3_stmt *res = NULL;

574 + int rc = 0;

575 + int param = 0;

576 +

577 + if (unlikely(!db)) {

578 + error_report("Database has not been initialized");

579 + return 1;

580 + }

581 +

582 + if (unlikely(!res)) {

583 + rc = prepare_statement(db, db_models_prune, &res);

584 + if (unlikely(rc != SQLITE_OK)) {

585 + error_report("Failed to prepare statement to prune models, rc = %d", rc);

586 + return rc;

587 + }

588 + }

589 +

590 + int after = (int) (now_realtime_sec() - Cfg.delete_models_older_than);

591 +

592 + rc = sqlite3_bind_int(res, ++param, after);

593 + if (unlikely(rc != SQLITE_OK))

594 + goto bind_fail;

595 +

596 + rc = sqlite3_bind_int(res, ++param, num_models_to_prune);

597 + if (unlikely(rc != SQLITE_OK))

598 + goto bind_fail;

599 +

600 + rc = execute_insert(res);

601 + if (unlikely(rc != SQLITE_DONE)) {

602 + error_report("Failed to prune old models, rc = %d", rc);

603 + return rc;

604 + }

605 +

606 + rc = sqlite3_reset(res);

607 + if (unlikely(rc != SQLITE_OK)) {

608 + error_report("Failed to reset statement when pruning old models, rc = %d", rc);

609 + return rc;

610 + }

611 +

612 + return 0;

613 +

614 +bind_fail:

615 + error_report("Failed to bind parameter %d to prune old models, rc = %d", param, rc);

616 + rc = sqlite3_reset(res);

617 + if (unlikely(rc != SQLITE_OK))

618 + error_report("Failed to reset statement to prune old models, rc = %d", rc);

619 + return rc;

620 +}

621 +

622 int ml_dimension_load_models(RRDDIM *rd) {

623 ml_dimension_t *dim = (ml_dimension_t *) rd->ml_dimension;

624 if (!dim)

@@ -1498,9 +1554,12 @@ bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool

1554 }

1555

1556 static void ml_flush_pending_models(ml_training_thread_t *training_thread) {

1501 - int rc = db_execute(db, "BEGIN TRANSACTION;");

1557 int op_no = 1;

1558

1559 + // begin transaction

1560 + int rc = db_execute(db, "BEGIN TRANSACTION;");

1561 +

1562 + // add/delete models

1563 if (!rc) {

1564 op_no++;

1565

@@ -1513,12 +1572,22 @@ static void ml_flush_pending_models(ml_training_thread_t *training_thread) {

1572 }

1573 }

1574

1575 + // prune old models

1576 + if (!rc) {

1577 + if ((training_thread->num_db_transactions % 64) == 0) {

1578 + rc = ml_prune_old_models(training_thread->num_models_to_prune);

1579 + if (!rc)

1580 + training_thread->num_models_to_prune = 0;

1581 + }

1582 + }

1583 +

1584 + // commit transaction

1585 if (!rc) {

1586 op_no++;

1587 rc = db_execute(db, "COMMIT TRANSACTION;");

1588 }

1589

1521 - // try to rollback transaction if we got any failures

1590 + // rollback transaction on failure

1591 if (rc) {

1592 netdata_log_error("Trying to rollback ML transaction because it failed with rc=%d, op_no=%d", rc, op_no);

1593 op_no++;

@@ -1527,6 +1596,11 @@ static void ml_flush_pending_models(ml_training_thread_t *training_thread) {

1596 netdata_log_error("ML transaction rollback failed with rc=%d", rc);

1597 }

1598

1599 + if (!rc) {

1600 + training_thread->num_db_transactions++;

1601 + training_thread->num_models_to_prune += training_thread->pending_model_info.size();

1602 + }

1603 +

1604 training_thread->pending_model_info.clear();

1605 }

1606

@@ -1677,6 +1751,7 @@ void ml_init()

1751 db = NULL;

1752 }

1753

1754 + // create table

1755 if (db) {

1756 char *err = NULL;

1757 int rc = sqlite3_exec(db, db_models_create_table, NULL, NULL, &err);