Add Working Set charts to the cgroups plugin (#10712)
Vladimir Kobal committed
Mar 12, 2021 at 09:48 UTC
88fa66957558ccf8b30194d5fee68bdf9a94bcf2
2 files changed
+73
-33
collectors/cgroups.plugin/sys_fs_cgroup.c
+73
-20
@@ -31,7 +31,7 @@ static int cgroup_enable_pressure_memory_full = CONFIG_BOOLEAN_AUTO;
31
32
static int cgroup_enable_systemd_services = CONFIG_BOOLEAN_YES;
33
static int cgroup_enable_systemd_services_detailed_memory = CONFIG_BOOLEAN_NO;
34
-static int cgroup_used_memory_without_cache = CONFIG_BOOLEAN_YES;
34
+static int cgroup_used_memory = CONFIG_BOOLEAN_YES;
35
36
static int cgroup_use_unified_cgroups = CONFIG_BOOLEAN_NO;
37
static int cgroup_unified_exist = CONFIG_BOOLEAN_AUTO;
@@ -226,7 +226,7 @@ void read_cgroup_plugin_configuration() {
226
cgroup_enable_cpuacct_stat = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct stat (total CPU)", cgroup_enable_cpuacct_stat);
227
cgroup_enable_cpuacct_usage = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct usage (per core CPU)", cgroup_enable_cpuacct_usage);
228
229
- cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory (used mem including cache)", cgroup_enable_memory);
229
+ cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory", cgroup_enable_memory);
230
cgroup_enable_detailed_memory = config_get_boolean_ondemand("plugin:cgroups", "enable detailed memory", cgroup_enable_detailed_memory);
231
cgroup_enable_memory_failcnt = config_get_boolean_ondemand("plugin:cgroups", "enable memory limits fail count", cgroup_enable_memory_failcnt);
232
cgroup_enable_swap = config_get_boolean_ondemand("plugin:cgroups", "enable swap memory", cgroup_enable_swap);
@@ -250,7 +250,7 @@ void read_cgroup_plugin_configuration() {
250
251
cgroup_enable_systemd_services = config_get_boolean("plugin:cgroups", "enable systemd services", cgroup_enable_systemd_services);
252
cgroup_enable_systemd_services_detailed_memory = config_get_boolean("plugin:cgroups", "enable systemd services detailed memory", cgroup_enable_systemd_services_detailed_memory);
253
- cgroup_used_memory_without_cache = config_get_boolean("plugin:cgroups", "report used memory without cache", cgroup_used_memory_without_cache);
253
+ cgroup_used_memory = config_get_boolean("plugin:cgroups", "report used memory", cgroup_used_memory);
254
255
char filename[FILENAME_MAX + 1], *s;
256
struct mountinfo *mi, *root = mountinfo_read(0);
@@ -327,7 +327,7 @@ void read_cgroup_plugin_configuration() {
327
cgroup_enable_blkio_queued_ops = CONFIG_BOOLEAN_NO;
328
cgroup_search_in_devices = 0;
329
cgroup_enable_systemd_services_detailed_memory = CONFIG_BOOLEAN_NO;
330
- cgroup_used_memory_without_cache = CONFIG_BOOLEAN_NO; //unified cgroups use different values
330
+ cgroup_used_memory = CONFIG_BOOLEAN_NO; //unified cgroups use different values
331
332
//TODO: can there be more than 1 cgroup2 mount point?
333
mi = mountinfo_find_by_filesystem_super_option(root, "cgroup2", "rw"); //there is no cgroup2 specific super option - for now use 'rw' option
@@ -541,7 +541,11 @@ struct memory {
541
/*
542
unsigned long long total_inactive_anon;
543
unsigned long long total_active_anon;
544
+*/
545
+
546
unsigned long long total_inactive_file;
547
+
548
+/*
549
unsigned long long total_active_file;
550
unsigned long long total_unevictable;
551
*/
@@ -628,6 +632,7 @@ struct cgroup {
632
RRDSET *st_cpu_limit;
633
RRDSET *st_cpu_per_core;
634
RRDSET *st_mem;
635
+ RRDSET *st_mem_utilization;
636
RRDSET *st_writeback;
637
RRDSET *st_mem_activity;
638
RRDSET *st_pgfaults;
@@ -1069,6 +1074,7 @@ static inline void cgroup_read_memory(struct memory *mem, char parent_cg_is_unif
1074
arl_expect(mem->arl_base, "total_pgpgout", &mem->total_pgpgout);
1075
arl_expect(mem->arl_base, "total_pgfault", &mem->total_pgfault);
1076
arl_expect(mem->arl_base, "total_pgmajfault", &mem->total_pgmajfault);
1077
+ arl_expect(mem->arl_base, "total_inactive_file", &mem->total_inactive_file);
1078
} else {
1079
mem->arl_base = arl_create("cgroup/memory", NULL, 60);
1080
@@ -1082,6 +1088,7 @@ static inline void cgroup_read_memory(struct memory *mem, char parent_cg_is_unif
1088
mem->arl_dirty = arl_expect(mem->arl_base, "file_dirty", &mem->total_dirty);
1089
arl_expect(mem->arl_base, "pgfault", &mem->total_pgfault);
1090
arl_expect(mem->arl_base, "pgmajfault", &mem->total_pgmajfault);
1091
+ arl_expect(mem->arl_base, "inactive_file", &mem->total_inactive_file);
1092
}
1093
}
1094
@@ -1105,9 +1112,9 @@ static inline void cgroup_read_memory(struct memory *mem, char parent_cg_is_unif
1112
1113
if(unlikely(mem->enabled_detailed == CONFIG_BOOLEAN_AUTO)) {
1114
if(( (!parent_cg_is_unified) && ( mem->total_cache || mem->total_dirty || mem->total_rss || mem->total_rss_huge || mem->total_mapped_file || mem->total_writeback
1108
- || mem->total_swap || mem->total_pgpgin || mem->total_pgpgout || mem->total_pgfault || mem->total_pgmajfault))
1115
+ || mem->total_swap || mem->total_pgpgin || mem->total_pgpgout || mem->total_pgfault || mem->total_pgmajfault || mem->total_inactive_file))
1116
|| (parent_cg_is_unified && ( mem->anon || mem->total_dirty || mem->kernel_stack || mem->slab || mem->sock || mem->total_writeback
1110
- || mem->anon_thp || mem->total_pgfault || mem->total_pgmajfault))
1117
+ || mem->anon_thp || mem->total_pgfault || mem->total_pgmajfault || mem->total_inactive_file))
1118
|| netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)
1119
mem->enabled_detailed = CONFIG_BOOLEAN_YES;
1120
else
@@ -1125,6 +1132,11 @@ memory_next:
1132
mem->enabled_usage_in_bytes = CONFIG_BOOLEAN_YES;
1133
}
1134
1135
+ if (likely(mem->updated_usage_in_bytes && mem->updated_detailed)) {
1136
+ mem->usage_in_bytes =
1137
+ (mem->usage_in_bytes > mem->total_inactive_file) ? (mem->usage_in_bytes - mem->total_inactive_file) : 0;
1138
+ }
1139
+
1140
// read msw_usage_in_bytes
1141
if(likely(mem->filename_msw_usage_in_bytes)) {
1142
mem->updated_msw_usage_in_bytes = !read_single_number_file(mem->filename_msw_usage_in_bytes, &mem->msw_usage_in_bytes);
@@ -1515,6 +1527,7 @@ static inline void cgroup_free(struct cgroup *cg) {
1527
if(cg->st_pgfaults) rrdset_is_obsolete(cg->st_pgfaults);
1528
if(cg->st_mem_usage) rrdset_is_obsolete(cg->st_mem_usage);
1529
if(cg->st_mem_usage_limit) rrdset_is_obsolete(cg->st_mem_usage_limit);
1530
+ if(cg->st_mem_utilization) rrdset_is_obsolete(cg->st_mem_utilization);
1531
if(cg->st_mem_failcnt) rrdset_is_obsolete(cg->st_mem_failcnt);
1532
if(cg->st_io) rrdset_is_obsolete(cg->st_io);
1533
if(cg->st_serviced_ops) rrdset_is_obsolete(cg->st_serviced_ops);
@@ -1751,7 +1764,7 @@ static inline void update_filenames()
1764
debug(D_CGROUP, "cpuacct.usage_percpu file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1765
}
1766
1754
- if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory_without_cache) && !cg->memory.filename_detailed && (cgroup_used_memory_without_cache || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) {
1767
+ if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory) && !cg->memory.filename_detailed && (cgroup_used_memory || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) {
1768
snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_memory_base, cg->id);
1769
if(likely(stat(filename, &buf) != -1)) {
1770
cg->memory.filename_detailed = strdupz(filename);
@@ -1898,7 +1911,7 @@ static inline void update_filenames()
1911
else
1912
debug(D_CGROUP, "cpu.stat file for unified cgroup '%s': '%s' does not exist.", cg->id, filename);
1913
}
1901
- if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory_without_cache) && !cg->memory.filename_detailed && (cgroup_used_memory_without_cache || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) {
1914
+ if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory) && !cg->memory.filename_detailed && (cgroup_used_memory || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) {
1915
snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_unified_base, cg->id);
1916
if(likely(stat(filename, &buf) != -1)) {
1917
cg->memory.filename_detailed = strdupz(filename);
@@ -2187,8 +2200,7 @@ void update_systemd_services_charts(
2200
, NULL
2201
, "mem"
2202
, "services.mem_usage"
2190
- , (cgroup_used_memory_without_cache) ? "Systemd Services Used Memory without Cache"
2191
- : "Systemd Services Used Memory"
2203
+ , "Systemd Services Used Memory"
2204
, "MiB"
2205
, PLUGIN_CGROUPS_NAME
2206
, PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
@@ -2705,7 +2717,7 @@ void update_systemd_services_charts(
2717
if(unlikely(!cg->rd_mem_usage))
2718
cg->rd_mem_usage = rrddim_add(st_mem_usage, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2719
2708
- rrddim_set_by_pointer(st_mem_usage, cg->rd_mem_usage, cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.total_cache:0));
2720
+ rrddim_set_by_pointer(st_mem_usage, cg->rd_mem_usage, cg->memory.usage_in_bytes);
2721
}
2722
2723
if(likely(do_mem_detailed && cg->memory.updated_detailed)) {
@@ -3275,7 +3287,7 @@ void update_cgroup_charts(int update_every) {
3287
, "MiB"
3288
, PLUGIN_CGROUPS_NAME
3289
, PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
3278
- , cgroup_containers_chart_priority + 210
3290
+ , cgroup_containers_chart_priority + 220
3291
, update_every
3292
, RRDSET_TYPE_STACKED
3293
);
@@ -3421,7 +3433,7 @@ void update_cgroup_charts(int update_every) {
3433
3434
if(likely(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_BOOLEAN_YES)) {
3435
if(unlikely(!cg->st_mem_usage)) {
3424
- snprintfz(title, CHART_TITLE_MAX, "Used Memory %s", (cgroup_used_memory_without_cache && cg->memory.updated_detailed)?"without Cache ":"");
3436
+ snprintfz(title, CHART_TITLE_MAX, "Used Memory");
3437
3438
cg->st_mem_usage = rrdset_create_localhost(
3439
cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
@@ -3433,7 +3445,7 @@ void update_cgroup_charts(int update_every) {
3445
, "MiB"
3446
, PLUGIN_CGROUPS_NAME
3447
, PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
3436
- , cgroup_containers_chart_priority + 200
3448
+ , cgroup_containers_chart_priority + 210
3449
, update_every
3450
, RRDSET_TYPE_STACKED
3451
);
@@ -3446,9 +3458,13 @@ void update_cgroup_charts(int update_every) {
3458
else
3459
rrdset_next(cg->st_mem_usage);
3460
3449
- rrddim_set(cg->st_mem_usage, "ram", cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.total_cache:0));
3461
+ rrddim_set(cg->st_mem_usage, "ram", cg->memory.usage_in_bytes);
3462
if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
3451
- rrddim_set(cg->st_mem_usage, "swap", (cg->memory.msw_usage_in_bytes > cg->memory.usage_in_bytes)?cg->memory.msw_usage_in_bytes - cg->memory.usage_in_bytes:0);
3463
+ rrddim_set(
3464
+ cg->st_mem_usage,
3465
+ "swap",
3466
+ (cg->memory.msw_usage_in_bytes > cg->memory.usage_in_bytes) ?
3467
+ cg->memory.msw_usage_in_bytes - cg->memory.usage_in_bytes : 0);
3468
} else {
3469
rrddim_set(cg->st_mem_usage, "swap", cg->memory.msw_usage_in_bytes);
3470
}
@@ -3484,7 +3500,7 @@ void update_cgroup_charts(int update_every) {
3500
memory_limit = cg->memory_limit;
3501
3502
if(unlikely(!cg->st_mem_usage_limit)) {
3487
- snprintfz(title, CHART_TITLE_MAX, "Used RAM without Cache within the limits");
3503
+ snprintfz(title, CHART_TITLE_MAX, "Used RAM within the limits");
3504
3505
cg->st_mem_usage_limit = rrdset_create_localhost(
3506
cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
@@ -3496,7 +3512,7 @@ void update_cgroup_charts(int update_every) {
3512
, "MiB"
3513
, PLUGIN_CGROUPS_NAME
3514
, PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
3499
- , cgroup_containers_chart_priority + 199
3515
+ , cgroup_containers_chart_priority + 200
3516
, update_every
3517
, RRDSET_TYPE_STACKED
3518
);
@@ -3511,9 +3527,41 @@ void update_cgroup_charts(int update_every) {
3527
3528
rrdset_isnot_obsolete(cg->st_mem_usage_limit);
3529
3514
- rrddim_set(cg->st_mem_usage_limit, "available", memory_limit - (cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.total_cache:0)));
3515
- rrddim_set(cg->st_mem_usage_limit, "used", cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.total_cache:0));
3530
+ rrddim_set(cg->st_mem_usage_limit, "available", memory_limit - cg->memory.usage_in_bytes);
3531
+ rrddim_set(cg->st_mem_usage_limit, "used", cg->memory.usage_in_bytes);
3532
rrdset_done(cg->st_mem_usage_limit);
3533
+
3534
+ if (unlikely(!cg->st_mem_utilization)) {
3535
+ snprintfz(title, CHART_TITLE_MAX, "Memory Utilization");
3536
+
3537
+ cg->st_mem_utilization = rrdset_create_localhost(
3538
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
3539
+ , "mem_utilization"
3540
+ , NULL
3541
+ , "mem"
3542
+ , "cgroup.mem_utilization"
3543
+ , title
3544
+ , "percentage"
3545
+ , PLUGIN_CGROUPS_NAME
3546
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
3547
+ , cgroup_containers_chart_priority + 199
3548
+ , update_every
3549
+ , RRDSET_TYPE_AREA
3550
+ );
3551
+
3552
+ rrdset_update_labels(cg->st_mem_utilization, cg->chart_labels);
3553
+
3554
+ rrddim_add(cg->st_mem_utilization, "utilization", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
3555
+ } else
3556
+ rrdset_next(cg->st_mem_utilization);
3557
+
3558
+ if (memory_limit) {
3559
+ rrdset_isnot_obsolete(cg->st_mem_utilization);
3560
+
3561
+ rrddim_set(
3562
+ cg->st_mem_utilization, "utilization", cg->memory.usage_in_bytes * 100 / memory_limit);
3563
+ rrdset_done(cg->st_mem_utilization);
3564
+ }
3565
}
3566
}
3567
else {
@@ -3521,6 +3569,11 @@ void update_cgroup_charts(int update_every) {
3569
rrdset_is_obsolete(cg->st_mem_usage_limit);
3570
cg->st_mem_usage_limit = NULL;
3571
}
3572
+
3573
+ if(unlikely(cg->st_mem_utilization)) {
3574
+ rrdset_is_obsolete(cg->st_mem_utilization);
3575
+ cg->st_mem_utilization = NULL;
3576
+ }
3577
}
3578
3579
update_memory_limits(&cg->filename_memoryswap_limit, &cg->chart_var_memoryswap_limit, &cg->memoryswap_limit, "memory_and_swap_limit", cg);
health/health.d/cgroups.conf
-13
@@ -26,16 +26,3 @@ template: cgroup_ram_in_use
26
delay: down 15m multiplier 1.5 max 1h
27
info: RAM used by cgroup
28
to: sysadmin
29
-
30
-template: cgroup_ram_and_swap_in_use
31
- on: cgroup.mem_usage
32
- os: linux
33
- hosts: *
34
- calc: ($ram + $swap) * 100 / $memory_and_swap_limit
35
- units: %
36
- every: 10s
37
- warn: $this > (($status >= $WARNING) ? (80) : (90))
38
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
39
- delay: down 15m multiplier 1.5 max 1h
40
- info: RAM and Swap used by cgroup
41
- to: sysadmin