@cryptotaxi247 / netdata-1 / commits / 5bcc462f0

feat(cgroups.plugin): add CPU throttling charts (#12591)

Ilya Mashchenko committed Apr 4, 2022 at 18:38 UTC 5bcc462f05a013986dda01b080b334b7b67993c0
2 files changed +244 -28
collectors/cgroups.plugin/sys_fs_cgroup.c
+234 -28
@@ -13,6 +13,7 @@ static long system_page_size = 4096; // system will be queried via sysconf() in
13
14 static int cgroup_enable_cpuacct_stat = CONFIG_BOOLEAN_AUTO;
15 static int cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_AUTO;
16 +static int cgroup_enable_cpuacct_cpu_throttling = CONFIG_BOOLEAN_YES;
17 static int cgroup_enable_memory = CONFIG_BOOLEAN_AUTO;
18 static int cgroup_enable_detailed_memory = CONFIG_BOOLEAN_AUTO;
19 static int cgroup_enable_memory_failcnt = CONFIG_BOOLEAN_AUTO;
@@ -72,6 +73,12 @@ static uint32_t Read_hash = 0;
73 static uint32_t Write_hash = 0;
74 static uint32_t user_hash = 0;
75 static uint32_t system_hash = 0;
76 +static uint32_t user_usec_hash = 0;
77 +static uint32_t system_usec_hash = 0;
78 +static uint32_t nr_periods_hash = 0;
79 +static uint32_t nr_throttled_hash = 0;
80 +static uint32_t throttled_time_hash = 0;
81 +static uint32_t throttled_usec_hash = 0;
82
83 enum cgroups_type { CGROUPS_AUTODETECT_FAIL, CGROUPS_V1, CGROUPS_V2 };
84
@@ -248,6 +255,12 @@ void read_cgroup_plugin_configuration() {
255 Write_hash = simple_hash("Write");
256 user_hash = simple_hash("user");
257 system_hash = simple_hash("system");
258 + user_usec_hash = simple_hash("user_usec");
259 + system_usec_hash = simple_hash("system_usec");
260 + nr_periods_hash = simple_hash("nr_periods");
261 + nr_throttled_hash = simple_hash("nr_throttled");
262 + throttled_time_hash = simple_hash("throttled_time");
263 + throttled_usec_hash = simple_hash("throttled_usec");
264
265 cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", localhost->rrd_update_every);
266 if(cgroup_update_every < localhost->rrd_update_every)
@@ -269,6 +282,7 @@ void read_cgroup_plugin_configuration() {
282
283 cgroup_enable_cpuacct_stat = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct stat (total CPU)", cgroup_enable_cpuacct_stat);
284 cgroup_enable_cpuacct_usage = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct usage (per core CPU)", cgroup_enable_cpuacct_usage);
285 + cgroup_enable_cpuacct_cpu_throttling = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct cpu throttling", cgroup_enable_cpuacct_cpu_throttling);
286
287 cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory", cgroup_enable_memory);
288 cgroup_enable_detailed_memory = config_get_boolean_ondemand("plugin:cgroups", "enable detailed memory", cgroup_enable_detailed_memory);
@@ -658,12 +672,12 @@ struct memory {
672 // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
673 struct cpuacct_stat {
674 int updated;
661 - int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
675 + int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
676
677 char *filename;
678
665 - unsigned long long user;
666 - unsigned long long system;
679 + unsigned long long user; // v1, v2(user_usec)
680 + unsigned long long system; // v1, v2(system_usec)
681 };
682
683 // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
@@ -677,6 +691,20 @@ struct cpuacct_usage {
691 unsigned long long *cpu_percpu;
692 };
693
694 +// represents cpuacct/cpu.stat, for v2 'cpuacct_stat' is used for 'user_usec', 'system_usec'
695 +struct cpuacct_cpu_throttling {
696 + int updated;
697 + int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
698 +
699 + char *filename;
700 +
701 + unsigned long long nr_periods;
702 + unsigned long long nr_throttled;
703 + unsigned long long throttled_time;
704 +
705 + unsigned long long nr_throttled_perc;
706 +};
707 +
708 struct cgroup_network_interface {
709 const char *host_device;
710 const char *container_device;
@@ -705,6 +733,7 @@ struct cgroup {
733
734 struct cpuacct_stat cpuacct_stat;
735 struct cpuacct_usage cpuacct_usage;
736 + struct cpuacct_cpu_throttling cpuacct_cpu_throttling;
737
738 struct memory memory;
739
@@ -727,6 +756,9 @@ struct cgroup {
756 RRDSET *st_cpu;
757 RRDSET *st_cpu_limit;
758 RRDSET *st_cpu_per_core;
759 + RRDSET *st_cpu_nr_throttled;
760 + RRDSET *st_cpu_throttled_time;
761 +
762 RRDSET *st_mem;
763 RRDSET *st_mem_utilization;
764 RRDSET *st_writeback;
@@ -735,6 +767,7 @@ struct cgroup {
767 RRDSET *st_mem_usage;
768 RRDSET *st_mem_usage_limit;
769 RRDSET *st_mem_failcnt;
770 +
771 RRDSET *st_io;
772 RRDSET *st_serviced_ops;
773 RRDSET *st_throttle_io;
@@ -809,6 +842,22 @@ struct discovery_thread {
842 int exited;
843 } discovery_thread;
844
845 +// ----------------------------------------------------------------------------
846 +
847 +static unsigned long long calc_delta(unsigned long long curr, unsigned long long prev) {
848 + if (prev > curr) {
849 + return 0;
850 + }
851 + return curr - prev;
852 +}
853 +
854 +static unsigned long long calc_percentage(unsigned long long value, unsigned long long total) {
855 + if (total == 0) {
856 + return 0;
857 + }
858 + return (calculated_number)value / (calculated_number)total * 100;
859 +}
860 +
861 // ----------------------------------------------------------------------------
862 // read values from /sys
863
@@ -857,40 +906,126 @@ static inline void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) {
906 }
907 }
908
860 -static inline void cgroup2_read_cpuacct_stat(struct cpuacct_stat *cp) {
909 +static inline void cgroup_read_cpuacct_cpu_stat(struct cpuacct_cpu_throttling *cp) {
910 + if (unlikely(!cp->filename)) {
911 + return;
912 + }
913 +
914 static procfile *ff = NULL;
915 + ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT);
916 + if (unlikely(!ff)) {
917 + cp->updated = 0;
918 + cgroups_check = 1;
919 + return;
920 + }
921
863 - if(likely(cp->filename)) {
864 - ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT);
865 - if(unlikely(!ff)) {
866 - cp->updated = 0;
867 - cgroups_check = 1;
868 - return;
869 - }
922 + ff = procfile_readall(ff);
923 + if (unlikely(!ff)) {
924 + cp->updated = 0;
925 + cgroups_check = 1;
926 + return;
927 + }
928
871 - ff = procfile_readall(ff);
872 - if(unlikely(!ff)) {
873 - cp->updated = 0;
874 - cgroups_check = 1;
875 - return;
929 + unsigned long lines = procfile_lines(ff);
930 + if (unlikely(lines < 3)) {
931 + error("CGROUP: file '%s' should have 3 lines.", cp->filename);
932 + cp->updated = 0;
933 + return;
934 + }
935 +
936 + unsigned long long nr_periods_last = cp->nr_periods;
937 + unsigned long long nr_throttled_last = cp->nr_throttled;
938 +
939 + for (unsigned long i = 0; i < lines; i++) {
940 + char *s = procfile_lineword(ff, i, 0);
941 + uint32_t hash = simple_hash(s);
942 +
943 + if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
944 + cp->nr_periods = str2ull(procfile_lineword(ff, i, 1));
945 + } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
946 + cp->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
947 + } else if (unlikely(hash == throttled_time_hash && !strcmp(s, "throttled_time"))) {
948 + cp->throttled_time = str2ull(procfile_lineword(ff, i, 1));
949 }
950 + }
951 + cp->nr_throttled_perc =
952 + calc_percentage(calc_delta(cp->nr_throttled, nr_throttled_last), calc_delta(cp->nr_periods, nr_periods_last));
953
878 - unsigned long lines = procfile_lines(ff);
954 + cp->updated = 1;
955
880 - if(unlikely(lines < 3)) {
881 - error("CGROUP: file '%s' should have 3+ lines.", cp->filename);
882 - cp->updated = 0;
883 - return;
956 + if (unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO)) {
957 + if (likely(
958 + cp->nr_periods || cp->nr_throttled || cp->throttled_time ||
959 + netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
960 + cp->enabled = CONFIG_BOOLEAN_YES;
961 }
962 + }
963 +}
964
886 - cp->user = str2ull(procfile_lineword(ff, 1, 1));
887 - cp->system = str2ull(procfile_lineword(ff, 2, 1));
965 +static inline void cgroup2_read_cpuacct_cpu_stat(struct cpuacct_stat *cp, struct cpuacct_cpu_throttling *cpt) {
966 + static procfile *ff = NULL;
967 + if (unlikely(!cp->filename)) {
968 + return;
969 + }
970
889 - cp->updated = 1;
971 + ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT);
972 + if (unlikely(!ff)) {
973 + cp->updated = 0;
974 + cgroups_check = 1;
975 + return;
976 + }
977
891 - if(unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO &&
892 - (cp->user || cp->system || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
978 + ff = procfile_readall(ff);
979 + if (unlikely(!ff)) {
980 + cp->updated = 0;
981 + cgroups_check = 1;
982 + return;
983 + }
984 +
985 + unsigned long lines = procfile_lines(ff);
986 +
987 + if (unlikely(lines < 3)) {
988 + error("CGROUP: file '%s' should have at least 3 lines.", cp->filename);
989 + cp->updated = 0;
990 + return;
991 + }
992 +
993 + unsigned long long nr_periods_last = cpt->nr_periods;
994 + unsigned long long nr_throttled_last = cpt->nr_throttled;
995 +
996 + for (unsigned long i = 0; i < lines; i++) {
997 + char *s = procfile_lineword(ff, i, 0);
998 + uint32_t hash = simple_hash(s);
999 +
1000 + if (unlikely(hash == user_usec_hash && !strcmp(s, "user_usec"))) {
1001 + cp->user = str2ull(procfile_lineword(ff, i, 1));
1002 + } else if (unlikely(hash == system_usec_hash && !strcmp(s, "system_usec"))) {
1003 + cp->system = str2ull(procfile_lineword(ff, i, 1));
1004 + } else if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
1005 + cpt->nr_periods = str2ull(procfile_lineword(ff, i, 1));
1006 + } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
1007 + cpt->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
1008 + } else if (unlikely(hash == throttled_usec_hash && !strcmp(s, "throttled_usec"))) {
1009 + cpt->throttled_time = str2ull(procfile_lineword(ff, i, 1)) * 1000; // usec -> ns
1010 + }
1011 + }
1012 + cpt->nr_throttled_perc =
1013 + calc_percentage(calc_delta(cpt->nr_throttled, nr_throttled_last), calc_delta(cpt->nr_periods, nr_periods_last));
1014 +
1015 + cp->updated = 1;
1016 + cpt->updated = 1;
1017 +
1018 + if (unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO)) {
1019 + if (likely(cp->user || cp->system || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
1020 cp->enabled = CONFIG_BOOLEAN_YES;
1021 + }
1022 + }
1023 + if (unlikely(cpt->enabled == CONFIG_BOOLEAN_AUTO)) {
1024 + if (likely(
1025 + cpt->nr_periods || cpt->nr_throttled || cpt->throttled_time ||
1026 + netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
1027 + cpt->enabled = CONFIG_BOOLEAN_YES;
1028 + }
1029 }
1030 }
1031
@@ -1264,6 +1399,7 @@ static inline void cgroup_read(struct cgroup *cg) {
1399 if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
1400 cgroup_read_cpuacct_stat(&cg->cpuacct_stat);
1401 cgroup_read_cpuacct_usage(&cg->cpuacct_usage);
1402 + cgroup_read_cpuacct_cpu_stat(&cg->cpuacct_cpu_throttling);
1403 cgroup_read_memory(&cg->memory, 0);
1404 cgroup_read_blkio(&cg->io_service_bytes);
1405 cgroup_read_blkio(&cg->io_serviced);
@@ -1276,7 +1412,7 @@ static inline void cgroup_read(struct cgroup *cg) {
1412 //TODO: io_service_bytes and io_serviced use same file merge into 1 function
1413 cgroup2_read_blkio(&cg->io_service_bytes, 0);
1414 cgroup2_read_blkio(&cg->io_serviced, 4);
1279 - cgroup2_read_cpuacct_stat(&cg->cpuacct_stat);
1415 + cgroup2_read_cpuacct_cpu_stat(&cg->cpuacct_stat, &cg->cpuacct_cpu_throttling);
1416 cgroup2_read_pressure(&cg->cpu_pressure);
1417 cgroup2_read_pressure(&cg->io_pressure);
1418 cgroup2_read_pressure(&cg->memory_pressure);
@@ -1627,6 +1763,8 @@ static inline void cgroup_free(struct cgroup *cg) {
1763 if(cg->st_cpu) rrdset_is_obsolete(cg->st_cpu);
1764 if(cg->st_cpu_limit) rrdset_is_obsolete(cg->st_cpu_limit);
1765 if(cg->st_cpu_per_core) rrdset_is_obsolete(cg->st_cpu_per_core);
1766 + if(cg->st_cpu_nr_throttled) rrdset_is_obsolete(cg->st_cpu_nr_throttled);
1767 + if(cg->st_cpu_throttled_time) rrdset_is_obsolete(cg->st_cpu_throttled_time);
1768 if(cg->st_mem) rrdset_is_obsolete(cg->st_mem);
1769 if(cg->st_writeback) rrdset_is_obsolete(cg->st_writeback);
1770 if(cg->st_mem_activity) rrdset_is_obsolete(cg->st_mem_activity);
@@ -1654,6 +1792,7 @@ static inline void cgroup_free(struct cgroup *cg) {
1792
1793 freez(cg->cpuacct_stat.filename);
1794 freez(cg->cpuacct_usage.filename);
1795 + freez(cg->cpuacct_cpu_throttling.filename);
1796
1797 arl_free(cg->memory.arl_base);
1798 freez(cg->memory.filename_detailed);
@@ -1870,6 +2009,16 @@ static inline void update_filenames()
2009 else
2010 debug(D_CGROUP, "cpuacct.usage_percpu file for cgroup '%s': '%s' does not exist.", cg->id, filename);
2011 }
2012 + if(unlikely(cgroup_enable_cpuacct_cpu_throttling && !cg->cpuacct_cpu_throttling.filename && !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))) {
2013 + snprintfz(filename, FILENAME_MAX, "%s%s/cpu.stat", cgroup_cpuacct_base, cg->id);
2014 + if(likely(stat(filename, &buf) != -1)) {
2015 + cg->cpuacct_cpu_throttling.filename = strdupz(filename);
2016 + cg->cpuacct_cpu_throttling.enabled = cgroup_enable_cpuacct_cpu_throttling;
2017 + debug(D_CGROUP, "cpu.stat filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_cpu_throttling.filename);
2018 + }
2019 + else
2020 + debug(D_CGROUP, "cpu.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
2021 + }
2022
2023 if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory) && !cg->memory.filename_detailed && (cgroup_used_memory || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) {
2024 snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_memory_base, cg->id);
@@ -2053,11 +2202,14 @@ static inline void update_filenames()
2202 } else
2203 debug(D_CGROUP, "io.stat file for unified cgroup '%s': '%s' does not exist.", cg->id, filename);
2204 }
2056 - if(unlikely(cgroup_enable_cpuacct_stat && !cg->cpuacct_stat.filename)) {
2205 + if (unlikely(
2206 + (cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_cpu_throttling) &&
2207 + !cg->cpuacct_stat.filename)) {
2208 snprintfz(filename, FILENAME_MAX, "%s%s/cpu.stat", cgroup_unified_base, cg->id);
2209 if(likely(stat(filename, &buf) != -1)) {
2210 cg->cpuacct_stat.filename = strdupz(filename);
2211 cg->cpuacct_stat.enabled = cgroup_enable_cpuacct_stat;
2212 + cg->cpuacct_cpu_throttling.enabled = cgroup_enable_cpuacct_cpu_throttling;
2213 cg->filename_cpuset_cpus = NULL;
2214 cg->filename_cpu_cfs_period = NULL;
2215 snprintfz(filename, FILENAME_MAX, "%s%s/cpu.max", cgroup_unified_base, cg->id);
@@ -3463,6 +3615,60 @@ void update_cgroup_charts(int update_every) {
3615 }
3616 }
3617
3618 + if (likely(cg->cpuacct_cpu_throttling.updated && cg->cpuacct_cpu_throttling.enabled == CONFIG_BOOLEAN_YES)) {
3619 + if (unlikely(!cg->st_cpu_nr_throttled)) {
3620 + snprintfz(title, CHART_TITLE_MAX, "CPU Throttled Runnable Periods");
3621 +
3622 + cg->st_cpu_nr_throttled = rrdset_create_localhost(
3623 + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
3624 + , "throttled"
3625 + , NULL
3626 + , "cpu"
3627 + , "cgroup.throttled"
3628 + , title
3629 + , "percentage"
3630 + , PLUGIN_CGROUPS_NAME
3631 + , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
3632 + , cgroup_containers_chart_priority + 10
3633 + , update_every
3634 + , RRDSET_TYPE_LINE
3635 + );
3636 +
3637 + rrdset_update_labels(cg->st_cpu_nr_throttled, cg->chart_labels);
3638 + rrddim_add(cg->st_cpu_nr_throttled, "throttled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
3639 + } else {
3640 + rrdset_next(cg->st_cpu_nr_throttled);
3641 + rrddim_set(cg->st_cpu_nr_throttled, "throttled", cg->cpuacct_cpu_throttling.nr_throttled_perc);
3642 + rrdset_done(cg->st_cpu_nr_throttled);
3643 + }
3644 +
3645 + if (unlikely(!cg->st_cpu_throttled_time)) {
3646 + snprintfz(title, CHART_TITLE_MAX, "CPU Throttled Time Duration");
3647 +
3648 + cg->st_cpu_throttled_time = rrdset_create_localhost(
3649 + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
3650 + , "throttled_duration"
3651 + , NULL
3652 + , "cpu"
3653 + , "cgroup.throttled_duration"
3654 + , title
3655 + , "ms"
3656 + , PLUGIN_CGROUPS_NAME
3657 + , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
3658 + , cgroup_containers_chart_priority + 15
3659 + , update_every
3660 + , RRDSET_TYPE_LINE
3661 + );
3662 +
3663 + rrdset_update_labels(cg->st_cpu_throttled_time, cg->chart_labels);
3664 + rrddim_add(cg->st_cpu_throttled_time, "duration", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL);
3665 + } else {
3666 + rrdset_next(cg->st_cpu_throttled_time);
3667 + rrddim_set(cg->st_cpu_throttled_time, "duration", cg->cpuacct_cpu_throttling.throttled_time);
3668 + rrdset_done(cg->st_cpu_throttled_time);
3669 + }
3670 + }
3671 +
3672 if(likely(cg->cpuacct_usage.updated && cg->cpuacct_usage.enabled == CONFIG_BOOLEAN_YES)) {
3673 char id[RRD_ID_LENGTH_MAX + 1];
3674 unsigned int i;
web/gui/dashboard_info.js
+10
@@ -3889,6 +3889,16 @@ netdataDashboard.context = {
3889 '<a href="https://en.wikipedia.org/wiki/CPU_modes#Mode_types" target="_blank">user and kernel</a> modes.'
3890 },
3891
3892 + 'cgroup.throttled': {
3893 + info: 'The percentage of runnable periods when tasks in a cgroup have been throttled. '+
3894 + 'The tasks have not been allowed to run because they have exhausted all of the available time as specified by their CPU quota.'
3895 + },
3896 +
3897 + 'cgroup.throttled_duration': {
3898 + info: 'The total time duration for which tasks in a cgroup have been throttled. '+
3899 + 'When an application has used its allotted CPU quota for a given period, it gets throttled until the next period.'
3900 + },
3901 +
3902 'cgroup.cpu_per_core': {
3903 info: 'Total CPU utilization per core within the system-wide CPU resources.'
3904 },