Tiering statistics API endpoint (#13420)
* calculator statistics * added metrics and metrics_pages counters * implemented API * updates to match sheet * updates to match sheet No2 * fix update every calculation for single point pages * fix lgtm finding
Costa Tsaousis committed
Jul 26, 2022 at 12:05 UTC
e73df78a069b7fbf2547455491beb1f236d5807d
6 files changed
+264
-5
database/engine/rrdengine.c
+5
-5
@@ -9,7 +9,7 @@ rrdeng_stats_t rrdeng_reserved_file_descriptors = 0;
9
rrdeng_stats_t global_pg_cache_over_half_dirty_events = 0;
10
rrdeng_stats_t global_flushing_pressure_page_deletions = 0;
11
12
-static unsigned pages_per_extent = MAX_PAGES_PER_EXTENT;
12
+unsigned rrdeng_pages_per_extent = MAX_PAGES_PER_EXTENT;
13
14
#if WORKER_UTILIZATION_MAX_JOB_TYPES < (RRDENG_MAX_OPCODE + 2)
15
#error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least (RRDENG_MAX_OPCODE + 2)
@@ -741,7 +741,7 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct
741
PValue = JudyLFirst(pg_cache->committed_page_index.JudyL_array, &Index, PJE0),
742
descr = unlikely(NULL == PValue) ? NULL : *PValue ;
743
744
- descr != NULL && count != pages_per_extent ;
744
+ descr != NULL && count != rrdeng_pages_per_extent;
745
746
PValue = JudyLNext(pg_cache->committed_page_index.JudyL_array, &Index, PJE0),
747
descr = unlikely(NULL == PValue) ? NULL : *PValue) {
@@ -1113,10 +1113,10 @@ static void load_configuration_dynamic(void)
1113
{
1114
unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
1115
if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
1116
- pages_per_extent = read_num;
1116
+ rrdeng_pages_per_extent = read_num;
1117
else {
1118
- error("Invalid dbengine pages per extent %u given. Using %u.", read_num, pages_per_extent);
1119
- config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", pages_per_extent);
1118
+ error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent);
1119
+ config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent);
1120
}
1121
}
1122
database/engine/rrdengine.h
+2
@@ -26,6 +26,8 @@
26
27
#endif /* NETDATA_RRD_INTERNALS */
28
29
+extern unsigned rrdeng_pages_per_extent;
30
+
31
/* Forward declarations */
32
struct rrdengine_instance;
33
database/engine/rrdengineapi.c
+97
@@ -986,3 +986,100 @@ void rrdeng_prepare_exit(struct rrdengine_instance *ctx)
986
//metalog_prepare_exit(ctx->metalog_ctx);
987
}
988
989
+RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
990
+ RRDENG_SIZE_STATS stats = { 0 };
991
+
992
+ for(struct pg_cache_page_index *page_index = ctx->pg_cache.metrics_index.last_page_index;
993
+ page_index != NULL ;page_index = page_index->prev) {
994
+ stats.metrics++;
995
+ stats.metrics_pages += page_index->page_count;
996
+ }
997
+
998
+ for(struct rrdengine_datafile *df = ctx->datafiles.first; df ;df = df->next) {
999
+ stats.datafiles++;
1000
+
1001
+ for(struct extent_info *ei = df->extents.first; ei ; ei = ei->next) {
1002
+ stats.extents++;
1003
+ stats.extents_compressed_bytes += ei->size;
1004
+
1005
+ for(int p = 0; p < ei->number_of_pages ;p++) {
1006
+ struct rrdeng_page_descr *descr = ei->pages[p];
1007
+
1008
+ usec_t update_every_usec;
1009
+
1010
+ size_t points = descr->page_length / PAGE_POINT_SIZE_BYTES(descr);
1011
+
1012
+ if(likely(points > 1))
1013
+ update_every_usec = (descr->end_time - descr->start_time) / (points - 1);
1014
+ else {
1015
+ update_every_usec = default_rrd_update_every * get_tier_grouping(ctx->tier) * USEC_PER_SEC;
1016
+ stats.single_point_pages++;
1017
+ }
1018
+
1019
+ time_t duration_secs = (time_t)((descr->end_time - descr->start_time + update_every_usec)/USEC_PER_SEC);
1020
+
1021
+ stats.extents_pages++;
1022
+ stats.pages_uncompressed_bytes += descr->page_length;
1023
+ stats.pages_duration_secs += duration_secs;
1024
+ stats.points += points;
1025
+
1026
+ stats.page_types[descr->type].pages++;
1027
+ stats.page_types[descr->type].pages_uncompressed_bytes += descr->page_length;
1028
+ stats.page_types[descr->type].pages_duration_secs += duration_secs;
1029
+ stats.page_types[descr->type].points += points;
1030
+
1031
+ if(!stats.first_t || (descr->start_time - update_every_usec) < stats.first_t)
1032
+ stats.first_t = (descr->start_time - update_every_usec) / USEC_PER_SEC;
1033
+
1034
+ if(!stats.last_t || descr->end_time > stats.last_t)
1035
+ stats.last_t = descr->end_time / USEC_PER_SEC;
1036
+ }
1037
+ }
1038
+ }
1039
+
1040
+ internal_error(stats.metrics_pages != stats.extents_pages,
1041
+ "DBENGINE: metrics pages is %zu, but extents pages is %zu",
1042
+ stats.metrics_pages, stats.extents_pages);
1043
+
1044
+ stats.database_retention_secs = (time_t)(stats.last_t - stats.first_t);
1045
+
1046
+ if(stats.extents_pages)
1047
+ stats.average_page_size_bytes = (double)stats.pages_uncompressed_bytes / (double)stats.extents_pages;
1048
+
1049
+ if(stats.pages_uncompressed_bytes > 0)
1050
+ stats.average_compression_savings = 100.0 - ((double)stats.extents_compressed_bytes * 100.0 / (double)stats.pages_uncompressed_bytes);
1051
+
1052
+ if(stats.points)
1053
+ stats.average_point_duration_secs = (double)stats.pages_duration_secs / (double)stats.points;
1054
+
1055
+ if(stats.metrics) {
1056
+ stats.average_metric_retention_secs = (double)stats.pages_duration_secs / (double)stats.metrics;
1057
+
1058
+ if(stats.database_retention_secs) {
1059
+ double metric_coverage = stats.average_metric_retention_secs / (double)stats.database_retention_secs;
1060
+ double db_retention_days = (double)stats.database_retention_secs / 86400.0;
1061
+
1062
+ stats.estimated_concurrently_collected_metrics = stats.metrics * metric_coverage;
1063
+
1064
+ stats.ephemeral_metrics_per_day_percent = ((double)stats.metrics * 100.0 / (double)stats.estimated_concurrently_collected_metrics - 100.0) / (double)db_retention_days;
1065
+ }
1066
+ }
1067
+
1068
+ stats.sizeof_metric = struct_natural_alignment(sizeof(struct pg_cache_page_index));
1069
+ stats.sizeof_page = struct_natural_alignment(sizeof(struct rrdeng_page_descr));
1070
+ stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile));
1071
+ stats.sizeof_page_in_cache = struct_natural_alignment(sizeof(struct page_cache_descr));
1072
+ stats.sizeof_point_data = page_type_size[ctx->page_type];
1073
+ stats.sizeof_page_data = RRDENG_BLOCK_SIZE;
1074
+ stats.pages_per_extent = rrdeng_pages_per_extent;
1075
+
1076
+ stats.sizeof_extent = sizeof(struct extent_info);
1077
+ stats.sizeof_page_in_extent = sizeof(struct rrdeng_page_descr *);
1078
+
1079
+ stats.sizeof_metric_in_index = 40;
1080
+ stats.sizeof_page_in_index = 24;
1081
+
1082
+ stats.default_granularity_secs = (size_t)default_rrd_update_every * get_tier_grouping(ctx->tier);
1083
+
1084
+ return stats;
1085
+}
database/engine/rrdengineapi.h
+53
@@ -78,4 +78,57 @@ extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
78
extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier);
79
extern int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t);
80
81
+typedef struct rrdengine_size_statistics {
82
+ size_t default_granularity_secs;
83
+
84
+ size_t sizeof_metric;
85
+ size_t sizeof_metric_in_index;
86
+ size_t sizeof_page;
87
+ size_t sizeof_page_in_index;
88
+ size_t sizeof_extent;
89
+ size_t sizeof_page_in_extent;
90
+ size_t sizeof_datafile;
91
+ size_t sizeof_page_in_cache;
92
+ size_t sizeof_point_data;
93
+ size_t sizeof_page_data;
94
+
95
+ size_t pages_per_extent;
96
+
97
+ size_t datafiles;
98
+ size_t extents;
99
+ size_t extents_pages;
100
+ size_t points;
101
+ size_t metrics;
102
+ size_t metrics_pages;
103
+
104
+ size_t extents_compressed_bytes;
105
+ size_t pages_uncompressed_bytes;
106
+ time_t pages_duration_secs;
107
+
108
+ struct {
109
+ size_t pages;
110
+ size_t pages_uncompressed_bytes;
111
+ time_t pages_duration_secs;
112
+ size_t points;
113
+ } page_types[256];
114
+
115
+ size_t single_point_pages;
116
+
117
+ usec_t first_t;
118
+ usec_t last_t;
119
+
120
+ size_t estimated_concurrently_collected_metrics;
121
+
122
+ time_t database_retention_secs;
123
+ double average_compression_savings;
124
+ double average_point_duration_secs;
125
+ double average_metric_retention_secs;
126
+
127
+ double ephemeral_metrics_per_day_percent;
128
+
129
+ double average_page_size_bytes;
130
+} RRDENG_SIZE_STATS;
131
+
132
+extern RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
133
+
134
#endif /* NETDATA_RRDENGINEAPI_H */
libnetdata/libnetdata.h
+10
@@ -360,6 +360,16 @@ extern char *netdata_configured_host_prefix;
360
361
#define RRD_STORAGE_TIERS 5
362
363
+static inline size_t struct_natural_alignment(size_t size) __attribute__((const));
364
+
365
+#define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
366
+static inline size_t struct_natural_alignment(size_t size) {
367
+ if(unlikely(size % STRUCT_NATURAL_ALIGNMENT))
368
+ size = size + STRUCT_NATURAL_ALIGNMENT - (size % STRUCT_NATURAL_ALIGNMENT);
369
+
370
+ return size;
371
+}
372
+
373
# ifdef __cplusplus
374
}
375
# endif
web/api/web_api_v1.c
+97
@@ -1533,6 +1533,100 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie
1533
return metric_correlations(host, wb, method, group, group_options, baseline_after, baseline_before, after, before, points, options, timeout);
1534
}
1535
1536
+#ifndef ENABLE_DBENGINE
1537
+int web_client_api_request_v1_dbengine_stats(RRDHOST *host, struct web_client *w, char *url) {
1538
+ return HTTP_RESP_NOT_FOUND;
1539
+}
1540
+#else
1541
+static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, int tier) {
1542
+ RRDENG_SIZE_STATS stats = rrdeng_size_statistics(multidb_ctx[tier]);
1543
+
1544
+ buffer_sprintf(wb,
1545
+ "\n\t\t\"default_granularity_secs\":%zu"
1546
+ ",\n\t\t\"sizeof_metric\":%zu"
1547
+ ",\n\t\t\"sizeof_metric_in_index\":%zu"
1548
+ ",\n\t\t\"sizeof_page\":%zu"
1549
+ ",\n\t\t\"sizeof_page_in_index\":%zu"
1550
+ ",\n\t\t\"sizeof_extent\":%zu"
1551
+ ",\n\t\t\"sizeof_page_in_extent\":%zu"
1552
+ ",\n\t\t\"sizeof_datafile\":%zu"
1553
+ ",\n\t\t\"sizeof_page_in_cache\":%zu"
1554
+ ",\n\t\t\"sizeof_point_data\":%zu"
1555
+ ",\n\t\t\"sizeof_page_data\":%zu"
1556
+ ",\n\t\t\"pages_per_extent\":%zu"
1557
+ ",\n\t\t\"datafiles\":%zu"
1558
+ ",\n\t\t\"extents\":%zu"
1559
+ ",\n\t\t\"extents_pages\":%zu"
1560
+ ",\n\t\t\"points\":%zu"
1561
+ ",\n\t\t\"metrics\":%zu"
1562
+ ",\n\t\t\"metrics_pages\":%zu"
1563
+ ",\n\t\t\"extents_compressed_bytes\":%zu"
1564
+ ",\n\t\t\"pages_uncompressed_bytes\":%zu"
1565
+ ",\n\t\t\"pages_duration_secs\":%ld"
1566
+ ",\n\t\t\"single_point_pages\":%zu"
1567
+ ",\n\t\t\"first_t\":%llu"
1568
+ ",\n\t\t\"last_t\":%llu"
1569
+ ",\n\t\t\"database_retention_secs\":%ld"
1570
+ ",\n\t\t\"average_compression_savings\":%0.2f"
1571
+ ",\n\t\t\"average_point_duration_secs\":%0.2f"
1572
+ ",\n\t\t\"average_metric_retention_secs\":%0.2f"
1573
+ ",\n\t\t\"ephemeral_metrics_per_day_percent\":%0.2f"
1574
+ ",\n\t\t\"average_page_size_bytes\":%0.2f"
1575
+ ",\n\t\t\"estimated_concurrently_collected_metrics\":%zu"
1576
+ , stats.default_granularity_secs
1577
+ , stats.sizeof_metric
1578
+ , stats.sizeof_metric_in_index
1579
+ , stats.sizeof_page
1580
+ , stats.sizeof_page_in_index
1581
+ , stats.sizeof_extent
1582
+ , stats.sizeof_page_in_extent
1583
+ , stats.sizeof_datafile
1584
+ , stats.sizeof_page_in_cache
1585
+ , stats.sizeof_point_data
1586
+ , stats.sizeof_page_data
1587
+ , stats.pages_per_extent
1588
+ , stats.datafiles
1589
+ , stats.extents
1590
+ , stats.extents_pages
1591
+ , stats.points
1592
+ , stats.metrics
1593
+ , stats.metrics_pages
1594
+ , stats.extents_compressed_bytes
1595
+ , stats.pages_uncompressed_bytes
1596
+ , stats.pages_duration_secs
1597
+ , stats.single_point_pages
1598
+ , stats.first_t
1599
+ , stats.last_t
1600
+ , stats.database_retention_secs
1601
+ , stats.average_compression_savings
1602
+ , stats.average_point_duration_secs
1603
+ , stats.average_metric_retention_secs
1604
+ , stats.ephemeral_metrics_per_day_percent
1605
+ , stats.average_page_size_bytes
1606
+ , stats.estimated_concurrently_collected_metrics
1607
+ );
1608
+}
1609
+int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struct web_client *w, char *url __maybe_unused) {
1610
+ if (!netdata_ready)
1611
+ return HTTP_RESP_BACKEND_FETCH_FAILED;
1612
+
1613
+ BUFFER *wb = w->response.data;
1614
+ buffer_flush(wb);
1615
+ wb->contenttype = CT_APPLICATION_JSON;
1616
+ buffer_no_cacheable(wb);
1617
+
1618
+ buffer_strcat(wb, "{");
1619
+ for(int tier = 0; tier < storage_tiers ;tier++) {
1620
+ buffer_sprintf(wb, "%s\n\t\"tier%d\": {", tier?",":"", tier);
1621
+ web_client_api_v1_dbengine_stats_for_tier(wb, tier);
1622
+ buffer_strcat(wb, "\n\t}");
1623
+ }
1624
+ buffer_strcat(wb, "\n}");
1625
+
1626
+ return HTTP_RESP_OK;
1627
+}
1628
+#endif
1629
+
1630
static struct api_command {
1631
const char *command;
1632
uint32_t hash;
@@ -1569,6 +1663,9 @@ static struct api_command {
1663
{ "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health },
1664
{ "aclk", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state },
1665
{ "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_metric_correlations },
1666
+
1667
+ { "dbengine_stats", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_dbengine_stats },
1668
+
1669
// terminator
1670
{ NULL, 0, WEB_CLIENT_ACL_NONE, NULL },
1671
};