Add dbengine RAM usage statistics (#7038)
* Add dbengine RAM usage statistics * Fix code style * Aggregate dbengine statistics across all slave hosts and localhost
Markos Fountoulakis committed
Oct 14, 2019 at 14:41 UTC
9ab6a26acdb663c1119ee50215daabc82295df58
1 file changed
+83
-7
daemon/global_statistics.c
+83
-7
@@ -534,11 +534,30 @@ void global_statistics_charts(void) {
534
// ----------------------------------------------------------------
535
536
#ifdef ENABLE_DBENGINE
537
- if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
538
- unsigned long long stats_array[RRDENG_NR_STATS];
537
+ RRDHOST *host;
538
+ unsigned long long stats_array[RRDENG_NR_STATS] = {0};
539
+ unsigned long long local_stats_array[RRDENG_NR_STATS];
540
+ unsigned hosts_with_dbengine = 0, i;
541
+
542
+ rrd_rdlock();
543
+ rrdhost_foreach_read(host) {
544
+ if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
545
+ ++hosts_with_dbengine;
546
+ /* get localhost's DB engine's statistics */
547
+ rrdeng_get_33_statistics(host->rrdeng_ctx, local_stats_array);
548
+ for (i = 0 ; i < RRDENG_NR_STATS ; ++i) {
549
+ /* aggregate statistics across hosts */
550
+ stats_array[i] += local_stats_array[i];
551
+ }
552
+ }
553
+ }
554
+ rrd_unlock();
555
540
- /* get localhost's DB engine's statistics */
541
- rrdeng_get_33_statistics(localhost->rrdeng_ctx, stats_array);
556
+ if (hosts_with_dbengine) {
557
+ /* deduplicate global statistics by getting the ones from the last host */
558
+ stats_array[30] = local_stats_array[30];
559
+ stats_array[31] = local_stats_array[31];
560
+ stats_array[32] = local_stats_array[32];
561
562
// ----------------------------------------------------------------
563
@@ -639,7 +658,7 @@ void global_statistics_charts(void) {
658
static RRDSET *st_pg_cache_pages = NULL;
659
static RRDDIM *rd_descriptors = NULL;
660
static RRDDIM *rd_populated = NULL;
642
- static RRDDIM *rd_commited = NULL;
661
+ static RRDDIM *rd_committed = NULL;
662
static RRDDIM *rd_insertions = NULL;
663
static RRDDIM *rd_deletions = NULL;
664
static RRDDIM *rd_backfills = NULL;
@@ -663,7 +682,7 @@ void global_statistics_charts(void) {
682
683
rd_descriptors = rrddim_add(st_pg_cache_pages, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
684
rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
666
- rd_commited = rrddim_add(st_pg_cache_pages, "commited", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
685
+ rd_committed = rrddim_add(st_pg_cache_pages, "committed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
686
rd_insertions = rrddim_add(st_pg_cache_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
687
rd_deletions = rrddim_add(st_pg_cache_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
688
rd_backfills = rrddim_add(st_pg_cache_pages, "backfills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -674,7 +693,7 @@ void global_statistics_charts(void) {
693
694
rrddim_set_by_pointer(st_pg_cache_pages, rd_descriptors, (collected_number)stats_array[27]);
695
rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]);
677
- rrddim_set_by_pointer(st_pg_cache_pages, rd_commited, (collected_number)stats_array[4]);
696
+ rrddim_set_by_pointer(st_pg_cache_pages, rd_committed, (collected_number)stats_array[4]);
697
rrddim_set_by_pointer(st_pg_cache_pages, rd_insertions, (collected_number)stats_array[5]);
698
rrddim_set_by_pointer(st_pg_cache_pages, rd_deletions, (collected_number)stats_array[6]);
699
rrddim_set_by_pointer(st_pg_cache_pages, rd_backfills, (collected_number)stats_array[9]);
@@ -818,6 +837,63 @@ void global_statistics_charts(void) {
837
rrddim_set_by_pointer(st_fd, rd_fd_max, (collected_number)rlimit_nofile.rlim_cur / 4);
838
rrdset_done(st_fd);
839
}
840
+
841
+ // ----------------------------------------------------------------
842
+
843
+ {
844
+ static RRDSET *st_ram_usage = NULL;
845
+ static RRDDIM *rd_cached = NULL;
846
+ static RRDDIM *rd_pinned = NULL;
847
+ static RRDDIM *rd_metadata = NULL;
848
+
849
+ collected_number cached_pages, pinned_pages, API_producers, populated_pages, metadata, pages_on_disk,
850
+ page_cache_descriptors;
851
+
852
+ if (unlikely(!st_ram_usage)) {
853
+ st_ram_usage = rrdset_create_localhost(
854
+ "netdata"
855
+ , "dbengine_ram"
856
+ , NULL
857
+ , "dbengine"
858
+ , NULL
859
+ , "NetData DB engine RAM usage"
860
+ , "MiB"
861
+ , "netdata"
862
+ , "stats"
863
+ , 130509
864
+ , localhost->rrd_update_every
865
+ , RRDSET_TYPE_STACKED
866
+ );
867
+
868
+ rd_cached = rrddim_add(st_ram_usage, "cache", NULL, 1, 256, RRD_ALGORITHM_ABSOLUTE);
869
+ rd_pinned = rrddim_add(st_ram_usage, "collectors", NULL, 1, 256, RRD_ALGORITHM_ABSOLUTE);
870
+ rd_metadata = rrddim_add(st_ram_usage, "metadata", NULL, 1, 1048576, RRD_ALGORITHM_ABSOLUTE);
871
+ }
872
+ else
873
+ rrdset_next(st_ram_usage);
874
+
875
+ API_producers = (collected_number)stats_array[0];
876
+ pages_on_disk = (collected_number)stats_array[2];
877
+ populated_pages = (collected_number)stats_array[3];
878
+ page_cache_descriptors = (collected_number)stats_array[27];
879
+
880
+ if (API_producers * 2 > populated_pages) {
881
+ pinned_pages = API_producers;
882
+ } else{
883
+ pinned_pages = API_producers * 2;
884
+ }
885
+ cached_pages = populated_pages - pinned_pages;
886
+
887
+ metadata = page_cache_descriptors * sizeof(struct page_cache_descr);
888
+ metadata += pages_on_disk * sizeof(struct rrdeng_page_descr);
889
+ /* This is an empirical estimation for Judy array indexing and extent structures */
890
+ metadata += pages_on_disk * 58;
891
+
892
+ rrddim_set_by_pointer(st_ram_usage, rd_cached, cached_pages);
893
+ rrddim_set_by_pointer(st_ram_usage, rd_pinned, pinned_pages);
894
+ rrddim_set_by_pointer(st_ram_usage, rd_metadata, metadata);
895
+ rrdset_done(st_ram_usage);
896
+ }
897
}
898
#endif
899