@cryptotaxi247 / netdata-1 / commits / 26b563cfa

Add an obsoletion time for statsd private charts (#16269)

add an obsoletion time for private charts

Emmanuel Vasilakis committed Nov 7, 2023 at 11:35 UTC 26b563cfad9f9222c948e502f0b87cdaa2663287

2 files changed +71 -17

collectors/statsd.plugin/README.md

+4 -6

@@ -173,8 +173,8 @@ You can find the configuration at `/etc/netdata/netdata.conf`:

173 # update every (flushInterval) = 1

174 # udp messages to process at once = 10

175 # create private charts for metrics matching = *

176 - # max private charts allowed = 200

176 # max private charts hard limit = 1000

177 + # cleanup obsolete charts after secs = 0

178 # private charts memory mode = save

179 # private charts history = 3996

180 # histograms and timers percentile (percentThreshold) = 95.00000

@@ -234,13 +234,11 @@ The default behavior is to use the same settings as the rest of the Netdata Agen

234 - `private charts memory mode`

235 - `private charts history`

236

237 -### Optimize private metric charts visualization and storage

237 +### Optimize private metric charts storage

238

239 -If you have thousands of metrics, each with its own private chart, you may notice that your web browser becomes slow when you view the Netdata dashboard (this is a web browser issue we need to address at the Netdata UI). So, Netdata has a protection to stop creating charts when `max private charts allowed = 200` (soft limit) is reached.

239 +For optimization reasons, Netdata imposes a hard limit on private metric charts. The limit is set via the `max private charts hard limit` setting (which defaults to 1000 charts). Metrics above this hard limit are still collected, but they can only be used in synthetic charts (once a metric is added to chart, it will be sent to backend servers too).

240

241 -The metrics above this soft limit are still processed by Netdata, can be used in synthetic charts and will be available to be sent to backend time-series databases, up to `max private charts hard limit = 1000`. So, between 200 and 1000 charts, Netdata will still generate charts, but they will automatically be created with `memory mode = none` (Netdata will not maintain a database for them). These metrics will be sent to backend time series databases, if the backend configuration is set to `as collected`.

242 -

243 -Metrics above the hard limit are still collected, but they can only be used in synthetic charts (once a metric is added to chart, it will be sent to backend servers too).

241 +If you have many ephemeral metrics collected (i.e. that you collect values for a certain amount of time), you can set the configuration option `set charts as obsolete after secs`. Setting a value in seconds here, means that Netdata will mark those metrics (and their private charts) as obsolete after the specified time has passed since the last sent metric value. Those charts will later be deleted according to the setting in `cleanup obsolete charts after secs`. Setting `set charts as obsolete after secs` to 0 (which is also the default value) will disable this functionality.

242

243 Example private charts (automatically generated without any configuration):

244

collectors/statsd.plugin/statsd.c

+67 -11

@@ -95,6 +95,7 @@ typedef enum __attribute__((packed)) statsd_metric_options {

95 STATSD_METRIC_OPTION_USEFUL = 0x00000080, // set when the charting thread finds the metric useful (i.e. used in a chart)

96 STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED = 0x00000100, // set when the collection is full for this metric

97 STATSD_METRIC_OPTION_UPDATED_CHART_METADATA = 0x00000200, // set when the private chart metadata have been updated via tags

98 + STATSD_METRIC_OPTION_OBSOLETE = 0x00004000, // set when the metric is obsoleted

99 } STATS_METRIC_OPTIONS;

100

101 typedef enum __attribute__((packed)) statsd_metric_type {

@@ -117,6 +118,7 @@ typedef struct statsd_metric {

118 // metadata about data collection

119 collected_number events; // the number of times this metric has been collected (never resets)

120 uint32_t count; // the number of times this metric has been collected since the last flush

121 + time_t last_collected; // timestamp of the last incoming value

122

123 // the actual collected data

124 union {

@@ -268,6 +270,7 @@ static struct statsd {

270 collected_number decimal_detail;

271 uint32_t private_charts;

272 uint32_t max_private_charts_hard;

273 + uint32_t set_obsolete_after;

274

275 STATSD_APP *apps;

276 uint32_t recvmmsg_size;

@@ -476,6 +479,16 @@ static inline int value_is_zinit(const char *value) {

479 #define is_metric_checked(m) ((m)->options & STATSD_METRIC_OPTION_CHECKED)

480 #define is_metric_useful_for_collection(m) (!is_metric_checked(m) || ((m)->options & STATSD_METRIC_OPTION_USEFUL))

481

482 +static inline void metric_update_counters_and_obsoletion(STATSD_METRIC *m) {

483 + m->events++;

484 + m->count++;

485 + m->last_collected = now_realtime_sec();

486 + if (m->st && unlikely(rrdset_flag_check(m->st, RRDSET_FLAG_OBSOLETE))) {

487 + rrdset_isnot_obsolete(m->st);

488 + m->options &= ~STATSD_METRIC_OPTION_OBSOLETE;

489 + }

490 +}

491 +

492 static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, const char *sampling) {

493 if(!is_metric_useful_for_collection(m)) return;

494

@@ -498,8 +511,7 @@ static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, con

511 else

512 m->gauge.value = statsd_parse_float(value, 1.0);

513

501 - m->events++;

502 - m->count++;

514 + metric_update_counters_and_obsoletion(m);

515 }

516 }

517

@@ -516,8 +528,7 @@ static inline void statsd_process_counter_or_meter(STATSD_METRIC *m, const char

528 else {

529 m->counter.value += llrintndd((NETDATA_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling));

530

519 - m->events++;

520 - m->count++;

531 + metric_update_counters_and_obsoletion(m);

532 }

533 }

534

@@ -559,8 +570,7 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha

570 m->histogram.ext->values[m->histogram.ext->used++] = v;

571 }

572

562 - m->events++;

563 - m->count++;

573 + metric_update_counters_and_obsoletion(m);

574 }

575 }

576

@@ -597,8 +607,7 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {

607 #else

608 dictionary_set(m->set.dict, value, NULL, 0);

609 #endif

600 - m->events++;

601 - m->count++;

610 + metric_update_counters_and_obsoletion(m);

611 }

612 }

613

@@ -630,8 +639,7 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value

639 }

640

641 t->count++;

633 - m->events++;

634 - m->count++;

642 + metric_update_counters_and_obsoletion(m);

643 }

644 }

645

@@ -1627,6 +1635,9 @@ static inline RRDSET *statsd_private_rrdset_create(

1635 static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {

1636 netdata_log_debug(D_STATSD, "updating private chart for gauge metric '%s'", m->name);

1637

1638 + if(m->st && unlikely(rrdset_flag_check(m->st, RRDSET_FLAG_OBSOLETE)))

1639 + return;

1640 +

1641 if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {

1642 m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;

1643

@@ -1667,6 +1678,9 @@ static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {

1678 static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) {

1679 netdata_log_debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);

1680

1681 + if(m->st && unlikely(rrdset_flag_check(m->st, RRDSET_FLAG_OBSOLETE)))

1682 + return;

1683 +

1684 if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {

1685 m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;

1686

@@ -1707,6 +1721,9 @@ static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const

1721 static inline void statsd_private_chart_set(STATSD_METRIC *m) {

1722 netdata_log_debug(D_STATSD, "updating private chart for set metric '%s'", m->name);

1723

1724 + if(m->st && unlikely(rrdset_flag_check(m->st, RRDSET_FLAG_OBSOLETE)))

1725 + return;

1726 +

1727 if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {

1728 m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;

1729

@@ -1747,6 +1764,9 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) {

1764 static inline void statsd_private_chart_dictionary(STATSD_METRIC *m) {

1765 netdata_log_debug(D_STATSD, "updating private chart for dictionary metric '%s'", m->name);

1766

1767 + if(m->st && unlikely(rrdset_flag_check(m->st, RRDSET_FLAG_OBSOLETE)))

1768 + return;

1769 +

1770 if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {

1771 m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;

1772

@@ -1790,6 +1810,9 @@ static inline void statsd_private_chart_dictionary(STATSD_METRIC *m) {

1810 static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {

1811 netdata_log_debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);

1812

1813 + if(m->st && unlikely(rrdset_flag_check(m->st, RRDSET_FLAG_OBSOLETE)))

1814 + return;

1815 +

1816 if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {

1817 m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;

1818

@@ -1842,6 +1865,16 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con

1865 // --------------------------------------------------------------------------------------------------------------------

1866 // statsd flush metrics

1867

1868 +static inline void metric_check_obsoletion(STATSD_METRIC *m) {

1869 + if(statsd.set_obsolete_after &&

1870 + !rrdset_flag_check(m->st, RRDSET_FLAG_OBSOLETE) &&

1871 + m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED &&

1872 + m->last_collected + statsd.set_obsolete_after < now_realtime_sec()) {

1873 + rrdset_is_obsolete(m->st);

1874 + m->options |= STATSD_METRIC_OPTION_OBSOLETE;

1875 + }

1876 +}

1877 +

1878 static inline void statsd_flush_gauge(STATSD_METRIC *m) {

1879 netdata_log_debug(D_STATSD, "flushing gauge metric '%s'", m->name);

1880

@@ -1855,6 +1888,8 @@ static inline void statsd_flush_gauge(STATSD_METRIC *m) {

1888

1889 if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))

1890 statsd_private_chart_gauge(m);

1891 +

1892 + metric_check_obsoletion(m);

1893 }

1894

1895 static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) {

@@ -1870,6 +1905,8 @@ static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *d

1905

1906 if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))

1907 statsd_private_chart_counter_or_meter(m, dim, family);

1908 +

1909 + metric_check_obsoletion(m);

1910 }

1911

1912 static inline void statsd_flush_counter(STATSD_METRIC *m) {

@@ -1896,6 +1933,8 @@ static inline void statsd_flush_set(STATSD_METRIC *m) {

1933

1934 if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))

1935 statsd_private_chart_set(m);

1936 +

1937 + metric_check_obsoletion(m);

1938 }

1939

1940 static inline void statsd_flush_dictionary(STATSD_METRIC *m) {

@@ -1924,6 +1963,8 @@ static inline void statsd_flush_dictionary(STATSD_METRIC *m) {

1963 dictionary_entries(m->dictionary.dict));

1964 }

1965 }

1966 +

1967 + metric_check_obsoletion(m);

1968 }

1969

1970 static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {

@@ -1977,6 +2018,8 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char

2018

2019 if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))

2020 statsd_private_chart_timer_or_histogram(m, dim, family, units);

2021 +

2022 + metric_check_obsoletion(m);

2023 }

2024

2025 static inline void statsd_flush_timer(STATSD_METRIC *m) {

@@ -2326,8 +2369,20 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_

2369 dfe_done(m);

2370

2371 // flush all the useful metrics

2329 - for(m = index->first_useful; m ; m = m->next_useful) {

2372 + STATSD_METRIC *m_prev;

2373 + for(m_prev = m = index->first_useful; m ; m = m->next_useful) {

2374 flush_metric(m);

2375 + if (m->options & STATSD_METRIC_OPTION_OBSOLETE) {

2376 + if (m == index->first_useful)

2377 + index->first_useful = m->next_useful;

2378 + else

2379 + m_prev->next_useful = m->next_useful;

2380 + dictionary_del(index->dict, m->name);

2381 + index->useful--;

2382 + index->metrics--;

2383 + statsd.private_charts--;

2384 + } else

2385 + m_prev = m;

2386 }

2387 }

2388

@@ -2447,6 +2502,7 @@ void *statsd_main(void *ptr) {

2502 config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL,

2503 SIMPLE_PATTERN_EXACT, true);

2504 statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts_hard);

2505 + statsd.set_obsolete_after = (size_t)config_get_number(CONFIG_SECTION_STATSD, "set charts as obsolete after secs", (long long)statsd.set_obsolete_after);

2506 statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);

2507 statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);

2508 statsd.private_charts_hidden = (unsigned int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);