move dbengine-retention chart to pulse (#19444)
Costa Tsaousis committed
Jan 19, 2025 at 19:37 UTC
c7a58fe37441b13a6a864b1d30df85b2cf94e774
12 files changed
+111
-95
CMakeLists.txt
+2
@@ -1170,6 +1170,8 @@ set(DAEMON_FILES
1170
src/daemon/pulse/pulse-daemon-memory-system.c
1171
src/daemon/pulse/pulse-network.c
1172
src/daemon/pulse/pulse-network.h
1173
+ src/daemon/pulse/pulse-db-dbengine-retention.c
1174
+ src/daemon/pulse/pulse-db-dbengine-retention.h
1175
)
1176
1177
set(H2O_FILES
src/daemon/config/netdata-conf-db.c
+1
-1
@@ -320,7 +320,7 @@ void netdata_conf_dbengine_init(const char *hostname) {
320
for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++)
321
rrdeng_readiness_wait(multidb_ctx[tier]);
322
323
- calculate_tier_disk_space_percentage();
323
+ rrdeng_calculate_tier_disk_space_percentage();
324
325
dbengine_enabled = true;
326
#else
src/daemon/pulse/pulse-db-dbengine-retention.c
new
+84
@@ -0,0 +1,84 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "pulse-db-dbengine-retention.h"
4
+#ifdef ENABLE_DBENGINE
5
+#include "database/engine/rrdengineapi.h"
6
+
7
+void dbengine_retention_statistics(bool extended __maybe_unused) {
8
+
9
+ static bool init = false;
10
+ static DBENGINE_TIER_STATS stats[RRD_STORAGE_TIERS];
11
+
12
+ if (!localhost)
13
+ return;
14
+
15
+ rrdeng_calculate_tier_disk_space_percentage();
16
+
17
+ for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
18
+ STORAGE_ENGINE *eng = localhost->db[tier].eng;
19
+ if (!eng || eng->seb != STORAGE_ENGINE_BACKEND_DBENGINE)
20
+ continue;
21
+
22
+ if (init == false) {
23
+ char id[200];
24
+ snprintfz(id, sizeof(id) - 1, "dbengine_retention_tier%zu", tier);
25
+ stats[tier].st = rrdset_create_localhost(
26
+ "netdata",
27
+ id,
28
+ NULL,
29
+ "dbengine retention",
30
+ "netdata.dbengine_tier_retention",
31
+ "dbengine space and time retention",
32
+ "%",
33
+ "netdata",
34
+ "stats",
35
+ 134900, // before "dbengine memory" (dbengine2_statistics_charts)
36
+ 10,
37
+ RRDSET_TYPE_LINE);
38
+
39
+ stats[tier].rd_space = rrddim_add(stats[tier].st, "space", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
40
+ stats[tier].rd_time = rrddim_add(stats[tier].st, "time", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
41
+
42
+ char tier_str[5];
43
+ snprintfz(tier_str, 4, "%zu", tier);
44
+ rrdlabels_add(stats[tier].st->rrdlabels, "tier", tier_str, RRDLABEL_SRC_AUTO);
45
+
46
+ rrdset_flag_set(stats[tier].st, RRDSET_FLAG_METADATA_UPDATE);
47
+ rrdhost_flag_set(stats[tier].st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
48
+ rrdset_metadata_updated(stats[tier].st);
49
+ }
50
+
51
+ time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
52
+ time_t retention = first_time_s ? now_realtime_sec() - first_time_s : 0;
53
+
54
+ //
55
+ // Note: storage_engine_disk_space_used is the exact diskspace (as reported by api/v2/node_instances
56
+ // get_used_disk_space is used to determine if database cleanup (file rotation should happen)
57
+ // and adds to the disk space used the desired file size of the active
58
+ // datafile
59
+ uint64_t disk_space = rrdeng_get_used_disk_space(multidb_ctx[tier]);
60
+ //uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
61
+
62
+ uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
63
+ if (!config_disk_space) {
64
+ config_disk_space = rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
65
+ config_disk_space += disk_space;
66
+ }
67
+
68
+ collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);
69
+
70
+ collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?
71
+ 100 * retention / multidb_ctx[tier]->config.max_retention_s :
72
+ 0;
73
+
74
+ if (retention_percentage > 100)
75
+ retention_percentage = 100;
76
+
77
+ rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_space, (collected_number) disk_percentage);
78
+ rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_time, (collected_number) retention_percentage);
79
+
80
+ rrdset_done(stats[tier].st);
81
+ }
82
+ init = true;
83
+}
84
+#endif
src/daemon/pulse/pulse-db-dbengine-retention.h
new
+12
@@ -0,0 +1,12 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_PULSE_DB_DBENGINE_RETENTION_H
4
+#define NETDATA_PULSE_DB_DBENGINE_RETENTION_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+
8
+#ifdef ENABLE_DBENGINE
9
+void dbengine_retention_statistics(bool extended __maybe_unused);
10
+#endif
11
+
12
+#endif //NETDATA_PULSE_DB_DBENGINE_RETENTION_H
src/daemon/pulse/pulse-db-dbengine.c
-1
@@ -686,7 +686,6 @@ void pulse_dbengine_do(bool extended) {
686
mrg_get_statistics(main_mrg, &mrg_stats);
687
688
size_t priority = 135000;
689
-
689
{
690
static RRDSET *st_pgc_memory = NULL;
691
static RRDDIM *rd_pgc_memory_main = NULL;
src/daemon/pulse/pulse-network.c
+1
-1
@@ -124,7 +124,7 @@ static void pulse_aclk_time_heatmap(void) {
124
125
rds[0] = rrddim_add(st, "instant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
126
for(size_t i = 1; i < _countof(rds) - 1 ;i++) {
127
- char buf[128];
127
+ char buf[350];
128
snprintf(buf, sizeof(buf), "%.2fs", (double)aclk_time_heatmap.array[i].upto / (double)USEC_PER_SEC);
129
// duration_snprintf(buf, sizeof(buf), aclk_time_heatmap.array[i].upto, "us", false);
130
rds[i] = rrddim_add(st, buf, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
src/daemon/pulse/pulse.c
+1
@@ -118,6 +118,7 @@ void *pulse_thread_main(void *ptr) {
118
if(dbengine_enabled) {
119
worker_is_busy(WORKER_JOB_DBENGINE);
120
pulse_dbengine_do(pulse_extended_enabled);
121
+ dbengine_retention_statistics(pulse_extended_enabled);
122
}
123
#endif
124
src/daemon/pulse/pulse.h
+1
@@ -17,6 +17,7 @@ extern bool pulse_extended_enabled;
17
#include "pulse-daemon-memory.h"
18
#include "pulse-sqlite3.h"
19
#include "pulse-db-dbengine.h"
20
+#include "pulse-db-dbengine-retention.h"
21
#include "pulse-db-rrd.h"
22
#include "pulse-string.h"
23
#include "pulse-heartbeat.h"
src/daemon/service.c
-4
@@ -312,10 +312,6 @@ void *service_main(void *ptr)
312
}
313
real_step = USEC_PER_SEC;
314
315
-#ifdef ENABLE_DBENGINE
316
- dbengine_retention_statistics();
317
-#endif
318
-
315
svc_rrd_cleanup_obsolete_charts_from_all_hosts();
316
317
if (service_running(SERVICE_MAINTENANCE))
src/database/contexts/api_v2_contexts_agents.c
+1
-1
@@ -113,7 +113,7 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
113
uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
114
#ifdef ENABLE_DBENGINE
115
if (!max && eng->seb == STORAGE_ENGINE_BACKEND_DBENGINE) {
116
- max = get_directory_free_bytes_space(multidb_ctx[tier]);
116
+ max = rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
117
max += used;
118
}
119
#endif
src/database/engine/rrdengine.c
+5
-83
@@ -1639,7 +1639,7 @@ static void *cleanup_tp_worker(struct rrdengine_instance *ctx __maybe_unused, vo
1639
return data;
1640
}
1641
1642
-uint64_t get_used_disk_space(struct rrdengine_instance *ctx)
1642
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx)
1643
{
1644
uint64_t active_space = 0;
1645
@@ -1680,7 +1680,7 @@ bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx)
1680
// only 1 datafile available
1681
return false;
1682
1683
- uint64_t estimated_disk_space = get_used_disk_space(ctx);
1683
+ uint64_t estimated_disk_space = rrdeng_get_used_disk_space(ctx);
1684
time_t retention = get_tier_retention(ctx);
1685
1686
if (ctx->config.max_retention_s && retention > ctx->config.max_retention_s)
@@ -1819,7 +1819,7 @@ static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_w
1819
work_dispatch(ctx, pdc, NULL, cmd.opcode, query_prep_tp_worker, NULL);
1820
}
1821
1822
-uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)
1822
+uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx)
1823
{
1824
uint64_t free_bytes = 0;
1825
struct statvfs buff_statvfs;
@@ -1829,7 +1829,7 @@ uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)
1829
return (free_bytes - (free_bytes * 5 / 100));
1830
}
1831
1832
-void calculate_tier_disk_space_percentage(void)
1832
+void rrdeng_calculate_tier_disk_space_percentage(void)
1833
{
1834
uint64_t tier_space[RRD_STORAGE_TIERS];
1835
@@ -1845,7 +1845,7 @@ void calculate_tier_disk_space_percentage(void)
1845
}
1846
uint64_t tier_disk_space = multidb_ctx[tier]->config.max_disk_space ?
1847
multidb_ctx[tier]->config.max_disk_space :
1848
- get_directory_free_bytes_space(multidb_ctx[tier]);
1848
+ rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
1849
total_diskspace += tier_disk_space;
1850
tier_space[tier] = tier_disk_space;
1851
}
@@ -1857,84 +1857,6 @@ void calculate_tier_disk_space_percentage(void)
1857
}
1858
}
1859
1860
-void dbengine_retention_statistics(void)
1861
-{
1862
- static bool init = false;
1863
- static DBENGINE_TIER_STATS stats[RRD_STORAGE_TIERS];
1864
-
1865
- if (!localhost)
1866
- return;
1867
-
1868
- calculate_tier_disk_space_percentage();
1869
-
1870
- for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
1871
- STORAGE_ENGINE *eng = localhost->db[tier].eng;
1872
- if (!eng || eng->seb != STORAGE_ENGINE_BACKEND_DBENGINE)
1873
- continue;
1874
-
1875
- if (init == false) {
1876
- char id[200];
1877
- snprintfz(id, sizeof(id) - 1, "dbengine_retention_tier%zu", tier);
1878
- stats[tier].st = rrdset_create_localhost(
1879
- "netdata",
1880
- id,
1881
- NULL,
1882
- "dbengine retention",
1883
- "netdata.dbengine_tier_retention",
1884
- "dbengine space and time retention",
1885
- "%",
1886
- "netdata",
1887
- "stats",
1888
- 134900, // before "dbengine memory" (dbengine2_statistics_charts)
1889
- 10,
1890
- RRDSET_TYPE_LINE);
1891
-
1892
- stats[tier].rd_space = rrddim_add(stats[tier].st, "space", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1893
- stats[tier].rd_time = rrddim_add(stats[tier].st, "time", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1894
-
1895
- char tier_str[5];
1896
- snprintfz(tier_str, 4, "%zu", tier);
1897
- rrdlabels_add(stats[tier].st->rrdlabels, "tier", tier_str, RRDLABEL_SRC_AUTO);
1898
-
1899
- rrdset_flag_set(stats[tier].st, RRDSET_FLAG_METADATA_UPDATE);
1900
- rrdhost_flag_set(stats[tier].st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
1901
- rrdset_metadata_updated(stats[tier].st);
1902
- }
1903
-
1904
- time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
1905
- time_t retention = first_time_s ? now_realtime_sec() - first_time_s : 0;
1906
-
1907
- //
1908
- // Note: storage_engine_disk_space_used is the exact diskspace (as reported by api/v2/node_instances
1909
- // get_used_disk_space is used to determine if database cleanup (file rotation should happen)
1910
- // and adds to the disk space used the desired file size of the active
1911
- // datafile
1912
- uint64_t disk_space = get_used_disk_space(multidb_ctx[tier]);
1913
- //uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
1914
-
1915
- uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
1916
- if (!config_disk_space) {
1917
- config_disk_space = get_directory_free_bytes_space(multidb_ctx[tier]);
1918
- config_disk_space += disk_space;
1919
- }
1920
-
1921
- collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);
1922
-
1923
- collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?
1924
- 100 * retention / multidb_ctx[tier]->config.max_retention_s :
1925
- 0;
1926
-
1927
- if (retention_percentage > 100)
1928
- retention_percentage = 100;
1929
-
1930
- rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_space, (collected_number) disk_percentage);
1931
- rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_time, (collected_number) retention_percentage);
1932
-
1933
- rrdset_done(stats[tier].st);
1934
- }
1935
- init = true;
1936
-}
1937
-
1860
void dbengine_event_loop(void* arg) {
1861
sanity_check();
1862
uv_thread_set_name_np("DBENGINE");
src/database/engine/rrdengine.h
+3
-4
@@ -563,9 +563,8 @@ static inline int journal_metric_uuid_compare(const void *key, const void *metri
563
}
564
565
// --------------------------------------------------------------------------------------------------------------------
566
-uint64_t get_used_disk_space(struct rrdengine_instance *ctx);
567
-void calculate_tier_disk_space_percentage(void);
568
-void dbengine_retention_statistics(void);
569
-uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx);
566
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx);
567
+void rrdeng_calculate_tier_disk_space_percentage(void);
568
+uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx);
569
570
#endif /* NETDATA_RRDENGINE_H */