a334f9429 · cryptotaxi247/netdata · siGit Code & Deploy

@@ -1413,213 +1413,120 @@ static uint64_t get_rowid_from_statement(const char *sql)

1413 }

1414

1415

1416 -#define SQL_GET_MAX_DIM_ROW_ID "SELECT MAX(rowid) FROM dimension"

1417 -

1418 -static bool check_dimension_metadata(struct meta_config_s *wc)

1419 -{

1420 - static time_t next_execution_t = 0;

1421 - static uint64_t last_row_id = 0;

1422 - static uint64_t max_row_id = 0;

1423 -

1424 - time_t now = now_realtime_sec();

1425 -

1426 - if (!next_execution_t) {

1427 - next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;

1428 - max_row_id = get_rowid_from_statement(SQL_GET_MAX_DIM_ROW_ID);

1429 - nd_log(NDLS_DAEMON, NDLP_INFO, "Dimension metadata check has been scheduled to run (max id = %lu)", max_row_id);

1430 - }

1431 -

1432 - if (next_execution_t && next_execution_t > now)

1433 - return true;

1434 -

1435 - if (max_row_id && last_row_id >= max_row_id) {

1436 - nd_log_daemon(NDLP_INFO, "Dimension metadata check completed");

1437 - // For long running agents, check in a week

1438 - next_execution_t = now + 604800;

1439 - return true;

1440 - }

1441 -

1442 - sqlite3_stmt *res = NULL;

1443 -

1444 - if (!PREPARE_STATEMENT(db_meta, SELECT_DIMENSION_LIST, &res))

1445 - return true;

1446 -

1447 - uint32_t total_checked = 0;

1448 - uint32_t total_deleted = 0;

1449 -

1450 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking dimensions starting after row %" PRIu64, last_row_id);

1451 -

1452 - worker_is_busy(UV_EVENT_DIMENSION_CLEANUP);

1453 -

1454 - (void) run_cleanup_loop(

1455 - res,

1456 - wc,

1457 - dimension_can_be_deleted,

1458 - delete_dimension_uuid,

1459 - &total_checked,

1460 - &total_deleted,

1461 - &last_row_id,

1462 - NULL,

1463 - NULL,

1464 - false,

1465 - false);

1466 -

1467 - now = now_realtime_sec();

1468 - next_execution_t = now + METADATA_MAINTENANCE_REPEAT;

1469 - nd_log_daemon(

1470 - NDLP_DEBUG,

1471 - "Dimensions checked %u, deleted %u. Checks will resume in %d seconds",

1472 - total_checked,

1473 - total_deleted,

1474 - METADATA_MAINTENANCE_REPEAT);

1475 -

1476 - SQLITE_FINALIZE(res);

1477 -

1478 - worker_is_idle();

1479 - return false;

1480 -}

1481 -

1482 -#define SQL_GET_MAX_CHART_ROW_ID "SELECT MAX(rowid) FROM chart"

1416 +// Descriptor + state for one cleanup cycle (dimension / chart / chart_label).

1417 +// Replaces the three former check_*_metadata() functions with one driver:

1418 +// first call -> arm timer; snapshot_pending = true so the snapshot

1419 +// is taken at the *next* entry past the timer (NOT

1420 +// now, which would burn a SELECT for nothing — the

1421 +// scan can't run yet)

1422 +// timer not expired -> return true (yield)

1423 +// snapshot_pending true -> snapshot max(rowid) for the upcoming pass and log

1424 +// "scheduled to run"; using an explicit boolean

1425 +// (instead of max_row_id==0) disambiguates a

1426 +// legitimately empty table (MAX returns NULL→0)

1427 +// from "snapshot deferred"

1428 +// past max(rowid) -> log completion; one-shot cycles (chart, label)

1429 +// mark completed and never re-run; cycles with

1430 +// complete_repeat_after > 0 (dim) reset last_row_id

1431 +// and set snapshot_pending so the next firing takes

1432 +// a fresh snapshot

1433 +// else -> run one cleanup_loop slice and re-arm short timer

1434 +struct cleanup_cycle {

1435 + // descriptor (immutable)

1436 + const char *select_sql; // SELECT id, rowid FROM <table> WHERE rowid > ?

1437 + const char *max_rowid_sql; // SELECT MAX(rowid) FROM <table>

1438 + bool (*check_cb)(nd_uuid_t *, sqlite3_stmt **, bool);

1439 + void (*action_cb)(nd_uuid_t *, sqlite3_stmt **, bool);

1440 + bool check_flag;

1441 + bool action_flag;

1442 + int repeat_after; // seconds to next slice after a partial pass

1443 + int complete_repeat_after; // seconds to next pass after full completion;

1444 + // 0 means one-shot (don't re-run after first done)

1445 + size_t worker_event; // worker_is_busy id

1446 + const char *label_singular; // "Dimension" / "Chart" / "Chart label" (INFO logs)

1447 + const char *label_plural; // "Dimensions" / "Charts" / "Chart labels" (DEBUG summary)

1448 + const char *label_lower; // "dimensions" / "charts" / "chart labels" (DEBUG checking)

1449 +

1450 + // mutable state (per-cycle)

1451 + time_t next_execution_t;

1452 + uint64_t last_row_id;

1453 + uint64_t max_row_id;

1454 + bool snapshot_pending; // true => take a fresh max(rowid) snapshot at the next entry past the timer

1455 + bool completed; // for one-shot cycles only

1456 +};

1457

1484 -static bool check_chart_metadata(struct meta_config_s *wc)

1458 +static bool run_cleanup_cycle(struct cleanup_cycle *c, struct meta_config_s *wc)

1459 {

1486 - static time_t next_execution_t = 0;

1487 - static uint64_t last_row_id = 0;

1488 - static uint64_t max_row_id = 0;

1489 - static bool check_completed = false;

1490 -

1491 - if (check_completed)

1460 + if (c->complete_repeat_after == 0 && c->completed)

1461 return true;

1462

1463 time_t now = now_realtime_sec();

1464

1496 - if (!next_execution_t) {

1497 - next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;

1498 - max_row_id = get_rowid_from_statement(SQL_GET_MAX_CHART_ROW_ID);

1499 - nd_log(NDLS_DAEMON, NDLP_INFO, "Chart metadata check has been scheduled to run (max id = %lu)", max_row_id);

1465 + if (!c->next_execution_t) {

1466 + c->next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;

1467 + c->snapshot_pending = true;

1468 }

1469

1502 - if (next_execution_t && next_execution_t > now)

1470 + if (c->next_execution_t > now)

1471 return true;

1472

1505 - if (max_row_id && last_row_id >= max_row_id) {

1506 - nd_log(NDLS_DAEMON, NDLP_INFO, "Chart metadata check completed");

1507 - check_completed = true;

1473 + if (c->snapshot_pending) {

1474 + c->max_row_id = get_rowid_from_statement(c->max_rowid_sql);

1475 + c->snapshot_pending = false;

1476 + nd_log(NDLS_DAEMON, NDLP_INFO,

1477 + "%s metadata check has been scheduled to run (max id = %" PRIu64 ")",

1478 + c->label_singular, c->max_row_id);

1479 + }

1480 +

1481 + // No `c->max_row_id &&` guard: a legitimately empty table snapshots to 0,

1482 + // and `0 >= 0` correctly takes the completion branch (one-shot cycles

1483 + // mark completed; dim re-arms for the next pass).

1484 + if (c->last_row_id >= c->max_row_id) {

1485 + nd_log(NDLS_DAEMON, NDLP_INFO, "%s metadata check completed", c->label_singular);

1486 + if (c->complete_repeat_after) {

1487 + // Re-arm for another full pass; the snapshot is deferred to the

1488 + // next entry past the timer so it isn't stale by then.

1489 + c->next_execution_t = now + c->complete_repeat_after;

1490 + c->last_row_id = 0;

1491 + c->snapshot_pending = true;

1492 + }

1493 + else

1494 + c->completed = true;

1495 return true;

1496 }

1497

1498 sqlite3_stmt *res = NULL;

1512 -

1513 - if (!PREPARE_STATEMENT(db_meta, SELECT_CHART_LIST, &res))

1499 + if (!PREPARE_STATEMENT(db_meta, c->select_sql, &res))

1500 return true;

1501

1502 uint32_t total_checked = 0;

1503 uint32_t total_deleted = 0;

1504

1519 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts starting after row %" PRIu64, last_row_id);

1520 -

1521 - worker_is_busy(UV_EVENT_CHART_CLEANUP);

1522 - sqlite3_stmt *check_res = NULL;

1523 - sqlite3_stmt *action_res = NULL;

1524 - (void)run_cleanup_loop(

1525 - res,

1526 - wc,

1527 - chart_can_be_deleted,

1528 - delete_chart_uuid,

1529 - &total_checked,

1530 - &total_deleted,

1531 - &last_row_id,

1532 - &check_res,

1533 - &action_res,

1534 - true,

1535 - false);

1536 -

1537 - SQLITE_FINALIZE(check_res);

1538 - SQLITE_FINALIZE(action_res);

1539 -

1540 - now = now_realtime_sec();

1541 - next_execution_t = now + METADATA_MAINTENANCE_REPEAT;

1542 - nd_log_daemon(

1543 - NDLP_DEBUG,

1544 - "Charts checked %u, deleted %u. Checks will resume in %d seconds",

1545 - total_checked,

1546 - total_deleted,

1547 - METADATA_MAINTENANCE_REPEAT);

1548 -

1549 - SQLITE_FINALIZE(res);

1550 - worker_is_idle();

1551 - return false;

1552 -}

1553 -

1554 -#define SQL_GET_MAX_CHART_LABEL_ROW_ID "SELECT MAX(rowid) FROM chart_label"

1555 -

1556 -static bool check_label_metadata(struct meta_config_s *wc)

1557 -{

1558 - static time_t next_execution_t = 0;

1559 - static uint64_t last_row_id = 0;

1560 - static uint64_t max_row_id = 0;

1561 - static bool check_completed = false;

1562 -

1563 - if (check_completed)

1564 - return true;

1565 -

1566 - time_t now = now_realtime_sec();

1567 -

1568 - if (!next_execution_t) {

1569 - next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;

1570 - max_row_id = get_rowid_from_statement(SQL_GET_MAX_CHART_LABEL_ROW_ID);

1571 - nd_log(NDLS_DAEMON, NDLP_INFO, "Chart label metadata check has been scheduled to run (max id = %lu)", max_row_id);

1572 - }

1573 -

1574 - if (next_execution_t && next_execution_t > now)

1575 - return true;

1576 -

1577 - if (max_row_id && last_row_id >= max_row_id) {

1578 - nd_log(NDLS_DAEMON, NDLP_INFO, "Chart label metadata check completed");

1579 - check_completed = true;

1580 - return true;

1581 - }

1582 -

1583 - sqlite3_stmt *res = NULL;

1505 + nd_log(NDLS_DAEMON, NDLP_DEBUG,

1506 + "Checking %s starting after row %" PRIu64, c->label_lower, c->last_row_id);

1507

1585 - if (!PREPARE_STATEMENT(db_meta, SELECT_CHART_LABEL_LIST, &res))

1586 - return true;

1587 -

1588 - uint32_t total_checked = 0;

1589 - uint32_t total_deleted = 0;

1590 -

1591 - nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts labels starting after row %" PRIu64, last_row_id);

1508 + worker_is_busy(c->worker_event);

1509

1510 sqlite3_stmt *check_res = NULL;

1511 sqlite3_stmt *action_res = NULL;

1512

1596 - worker_is_busy(UV_EVENT_CHART_LABEL_CLEANUP);

1597 -

1598 - (void )run_cleanup_loop(

1599 - res,

1600 - wc,

1601 - chart_can_be_deleted,

1602 - delete_chart_uuid,

1603 - &total_checked,

1604 - &total_deleted,

1605 - &last_row_id,

1606 - &check_res,

1607 - &action_res,

1608 - false,

1609 - true);

1513 + (void) run_cleanup_loop(

1514 + res, wc,

1515 + c->check_cb, c->action_cb,

1516 + &total_checked, &total_deleted,

1517 + &c->last_row_id,

1518 + &check_res, &action_res,

1519 + c->check_flag, c->action_flag);

1520

1521 SQLITE_FINALIZE(check_res);

1522 SQLITE_FINALIZE(action_res);

1523

1524 now = now_realtime_sec();

1615 - next_execution_t = now + METADATA_LABEL_CHECK_INTERVAL;

1525 + c->next_execution_t = now + c->repeat_after;

1526

1617 - nd_log_daemon(

1618 - NDLP_DEBUG,

1619 - "Chart labels checked %u, deleted %u. Checks will resume in %d seconds",

1620 - total_checked,

1621 - total_deleted,

1622 - METADATA_LABEL_CHECK_INTERVAL);

1527 + nd_log_daemon(NDLP_DEBUG,

1528 + "%s checked %u, deleted %u. Checks will resume in %d seconds",

1529 + c->label_plural, total_checked, total_deleted, c->repeat_after);

1530

1531 SQLITE_FINALIZE(res);

1532

@@ -1627,6 +1534,51 @@ static bool check_label_metadata(struct meta_config_s *wc)

1534 return false;

1535 }

1536

1537 +static struct cleanup_cycle dim_cleanup_cycle = {

1538 + .select_sql = SELECT_DIMENSION_LIST,

1539 + .max_rowid_sql = "SELECT MAX(rowid) FROM dimension",

1540 + .check_cb = dimension_can_be_deleted,

1541 + .action_cb = delete_dimension_uuid,

1542 + .check_flag = false,

1543 + .action_flag = false,

1544 + .repeat_after = METADATA_MAINTENANCE_REPEAT,

1545 + .complete_repeat_after = 604800, // re-fire weekly: re-snapshots max(rowid) and rescans dimensions added since the previous pass

1546 + .worker_event = UV_EVENT_DIMENSION_CLEANUP,

1547 + .label_singular = "Dimension",

1548 + .label_plural = "Dimensions",

1549 + .label_lower = "dimensions",

1550 +};

1551 +

1552 +static struct cleanup_cycle chart_cleanup_cycle = {

1553 + .select_sql = SELECT_CHART_LIST,

1554 + .max_rowid_sql = "SELECT MAX(rowid) FROM chart",

1555 + .check_cb = chart_can_be_deleted,

1556 + .action_cb = delete_chart_uuid,

1557 + .check_flag = true,

1558 + .action_flag = false,

1559 + .repeat_after = METADATA_MAINTENANCE_REPEAT,

1560 + .complete_repeat_after = 0, // one-shot

1561 + .worker_event = UV_EVENT_CHART_CLEANUP,

1562 + .label_singular = "Chart",

1563 + .label_plural = "Charts",

1564 + .label_lower = "charts",

1565 +};

1566 +

1567 +static struct cleanup_cycle label_cleanup_cycle = {

1568 + .select_sql = SELECT_CHART_LABEL_LIST,

1569 + .max_rowid_sql = "SELECT MAX(rowid) FROM chart_label",

1570 + .check_cb = chart_can_be_deleted,

1571 + .action_cb = delete_chart_uuid,

1572 + .check_flag = false,

1573 + .action_flag = true,

1574 + .repeat_after = METADATA_LABEL_CHECK_INTERVAL,

1575 + .complete_repeat_after = 0, // one-shot

1576 + .worker_event = UV_EVENT_CHART_LABEL_CLEANUP,

1577 + .label_singular = "Chart label",

1578 + .label_plural = "Chart labels",

1579 + .label_lower = "chart labels",

1580 +};

1581 +

1582 static void cleanup_health_log(struct meta_config_s *config)

1583 {

1584 static time_t next_execution_t = 0;

@@ -1859,9 +1811,9 @@ void run_metadata_cleanup(struct meta_config_s *config)

1811 if (unlikely(SHUTDOWN_REQUESTED(config)))

1812 return;

1813

1862 - if (check_dimension_metadata(config))

1863 - if (check_chart_metadata(config))

1864 - check_label_metadata(config);

1814 + if (run_cleanup_cycle(&dim_cleanup_cycle, config))

1815 + if (run_cleanup_cycle(&chart_cleanup_cycle, config))

1816 + run_cleanup_cycle(&label_cleanup_cycle, config);

1817

1818 cleanup_health_log(config);

1819

Refactor chart/dim/label cleanup functions (#22435)