@cryptotaxi247 / netdata-1 / commits / 30f7324a6

Add cgroup cpu and memory limits and alarms (#5172)

* Add memory limit variables * Add memory usage alarms * Add CPU limit variables * Add cpu usage alarm * Fix quota calculation, minor cleanup * Update the documentation * Add charts with limits * Fix Codacy issues * Change units for the mem_usage_limit chart * Change the behaviour of the cpu_limit chart

Vladimir Kobal committed Feb 11, 2019 at 13:35 UTC 30f7324a6cdeaefbfee2960ffcc2a3b60e58c168
5 files changed +411 -9
collectors/cgroups.plugin/README.md
+4
@@ -89,6 +89,10 @@ The whole point for the additional pattern list, is to limit the number of times
89
90 The above pattern list is matched against the path of the cgroup. For matched cgroups, netdata calls the script [cgroup-name.sh](cgroup-name.sh.in) to get its name. This script queries `docker`, or applies heuristics to find give a name for the cgroup.
91
92 +### alarms
93 +
94 +CPU and memory limits are watched and used to rise alarms. Memory usage for every cgroup is checked against `ram` and `ram+swap` limits. CPU usage for every cgroup is checked against `cpuset.cpus` and `cpu.cfs_period_us` + `cpu.cfs_quota_us` pair assigned for the cgroup. Configuration for the alarms is available in `health.d/cgroups.conf` file.
95 +
96 ## Monitoring systemd services
97
98 netdata monitors **systemd services**. Example:
collectors/cgroups.plugin/sys_fs_cgroup.c
+304 -6
@@ -39,6 +39,7 @@ static int cgroup_recheck_zero_mem_failcnt_every_iterations = 10;
39 static int cgroup_recheck_zero_mem_detailed_every_iterations = 10;
40
41 static char *cgroup_cpuacct_base = NULL;
42 +static char *cgroup_cpuset_base = NULL;
43 static char *cgroup_blkio_base = NULL;
44 static char *cgroup_memory_base = NULL;
45 static char *cgroup_devices_base = NULL;
@@ -114,6 +115,16 @@ void read_cgroup_plugin_configuration() {
115 snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
116 cgroup_cpuacct_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/cpuacct", filename);
117
118 + mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "cpuset");
119 + if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "cpuset");
120 + if(!mi) {
121 + error("CGROUP: cannot find cpuset mountinfo. Assuming default: /sys/fs/cgroup/cpuset");
122 + s = "/sys/fs/cgroup/cpuset";
123 + }
124 + else s = mi->mount_point;
125 + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
126 + cgroup_cpuset_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/cpuset", filename);
127 +
128 mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "blkio");
129 if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "blkio");
130 if(!mi) {
@@ -391,12 +402,14 @@ struct cgroup {
402
403 // per cgroup charts
404 RRDSET *st_cpu;
405 + RRDSET *st_cpu_limit;
406 RRDSET *st_cpu_per_core;
407 RRDSET *st_mem;
408 RRDSET *st_writeback;
409 RRDSET *st_mem_activity;
410 RRDSET *st_pgfaults;
411 RRDSET *st_mem_usage;
412 + RRDSET *st_mem_usage_limit;
413 RRDSET *st_mem_failcnt;
414 RRDSET *st_io;
415 RRDSET *st_serviced_ops;
@@ -405,6 +418,27 @@ struct cgroup {
418 RRDSET *st_queued_ops;
419 RRDSET *st_merged_ops;
420
421 + // per cgroup chart variables
422 + char *filename_cpuset_cpus;
423 + unsigned long long cpuset_cpus;
424 +
425 + char *filename_cpu_cfs_period;
426 + unsigned long long cpu_cfs_period;
427 +
428 + char *filename_cpu_cfs_quota;
429 + unsigned long long cpu_cfs_quota;
430 +
431 + RRDSETVAR *chart_var_cpu_limit;
432 + calculated_number prev_cpu_usage;
433 +
434 + char *filename_memory_limit;
435 + unsigned long long memory_limit;
436 + RRDSETVAR *chart_var_memory_limit;
437 +
438 + char *filename_memoryswap_limit;
439 + unsigned long long memoryswap_limit;
440 + RRDSETVAR *chart_var_memoryswap_limit;
441 +
442 // services
443 RRDDIM *rd_cpu;
444 RRDDIM *rd_mem_usage;
@@ -1007,13 +1041,26 @@ static inline struct cgroup *cgroup_add(const char *id) {
1041 static inline void cgroup_free(struct cgroup *cg) {
1042 debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available");
1043
1010 - if(cg->st_cpu) rrdset_is_obsolete(cg->st_cpu);
1044 + if(cg->st_cpu) {
1045 + rrdset_wrlock(cg->st_cpu);
1046 + rrdsetvar_free(cg->chart_var_cpu_limit);
1047 + rrdset_unlock(cg->st_cpu);
1048 + rrdset_is_obsolete(cg->st_cpu);
1049 + rrdset_is_obsolete(cg->st_cpu_limit);
1050 + }
1051 if(cg->st_cpu_per_core) rrdset_is_obsolete(cg->st_cpu_per_core);
1052 if(cg->st_mem) rrdset_is_obsolete(cg->st_mem);
1053 if(cg->st_writeback) rrdset_is_obsolete(cg->st_writeback);
1054 if(cg->st_mem_activity) rrdset_is_obsolete(cg->st_mem_activity);
1055 if(cg->st_pgfaults) rrdset_is_obsolete(cg->st_pgfaults);
1016 - if(cg->st_mem_usage) rrdset_is_obsolete(cg->st_mem_usage);
1056 + if(cg->st_mem_usage) {
1057 + rrdset_wrlock(cg->st_mem_usage);
1058 + rrdsetvar_free(cg->chart_var_memory_limit);
1059 + rrdsetvar_free(cg->chart_var_memoryswap_limit);
1060 + rrdset_unlock(cg->st_mem_usage);
1061 + rrdset_is_obsolete(cg->st_mem_usage);
1062 + rrdset_is_obsolete(cg->st_mem_usage_limit);
1063 + }
1064 if(cg->st_mem_failcnt) rrdset_is_obsolete(cg->st_mem_failcnt);
1065 if(cg->st_io) rrdset_is_obsolete(cg->st_io);
1066 if(cg->st_serviced_ops) rrdset_is_obsolete(cg->st_serviced_ops);
@@ -1022,6 +1069,12 @@ static inline void cgroup_free(struct cgroup *cg) {
1069 if(cg->st_queued_ops) rrdset_is_obsolete(cg->st_queued_ops);
1070 if(cg->st_merged_ops) rrdset_is_obsolete(cg->st_merged_ops);
1071
1072 + freez(cg->filename_cpuset_cpus);
1073 + freez(cg->filename_cpu_cfs_period);
1074 + freez(cg->filename_cpu_cfs_quota);
1075 + freez(cg->filename_memory_limit);
1076 + freez(cg->filename_memoryswap_limit);
1077 +
1078 free_cgroup_network_interfaces(cg);
1079
1080 freez(cg->cpuacct_usage.cpu_percpu);
@@ -1272,6 +1325,12 @@ static inline void find_all_cgroups() {
1325 if(likely(stat(filename, &buf) != -1)) {
1326 cg->cpuacct_stat.filename = strdupz(filename);
1327 cg->cpuacct_stat.enabled = cgroup_enable_cpuacct_stat;
1328 + snprintfz(filename, FILENAME_MAX, "%s%s/cpuset.cpus", cgroup_cpuset_base, cg->id);
1329 + cg->filename_cpuset_cpus = strdupz(filename);
1330 + snprintfz(filename, FILENAME_MAX, "%s%s/cpu.cfs_period_us", cgroup_cpuacct_base, cg->id);
1331 + cg->filename_cpu_cfs_period = strdupz(filename);
1332 + snprintfz(filename, FILENAME_MAX, "%s%s/cpu.cfs_quota_us", cgroup_cpuacct_base, cg->id);
1333 + cg->filename_cpu_cfs_quota = strdupz(filename);
1334 debug(D_CGROUP, "cpuacct.stat filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_stat.filename);
1335 }
1336 else
@@ -1306,16 +1365,20 @@ static inline void find_all_cgroups() {
1365 cg->memory.filename_usage_in_bytes = strdupz(filename);
1366 cg->memory.enabled_usage_in_bytes = cgroup_enable_memory;
1367 debug(D_CGROUP, "memory.usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_usage_in_bytes);
1368 + snprintfz(filename, FILENAME_MAX, "%s%s/memory.limit_in_bytes", cgroup_memory_base, cg->id);
1369 + cg->filename_memory_limit = strdupz(filename);
1370 }
1371 else
1372 debug(D_CGROUP, "memory.usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1373 }
1374
1375 if(unlikely(cgroup_enable_swap && !cg->memory.filename_msw_usage_in_bytes)) {
1315 - snprintfz(filename, FILENAME_MAX, "%s%s/memory.msw_usage_in_bytes", cgroup_memory_base, cg->id);
1376 + snprintfz(filename, FILENAME_MAX, "%s%s/memory.memsw.usage_in_bytes", cgroup_memory_base, cg->id);
1377 if(likely(stat(filename, &buf) != -1)) {
1378 cg->memory.filename_msw_usage_in_bytes = strdupz(filename);
1379 cg->memory.enabled_msw_usage_in_bytes = cgroup_enable_swap;
1380 + snprintfz(filename, FILENAME_MAX, "%s%s/memory.memsw.limit_in_bytes", cgroup_memory_base, cg->id);
1381 + cg->filename_memoryswap_limit = strdupz(filename);
1382 debug(D_CGROUP, "memory.msw_usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_msw_usage_in_bytes);
1383 }
1384 else
@@ -1465,7 +1528,7 @@ void update_systemd_services_charts(
1528 , "cpu"
1529 , "services.cpu"
1530 , title
1468 - , "%"
1531 + , "percentage"
1532 , PLUGIN_CGROUPS_NAME
1533 , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
1534 , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD
@@ -2197,6 +2260,96 @@ static inline char *cgroup_chart_type(char *buffer, const char *id, size_t len)
2260 return buffer;
2261 }
2262
2263 +static inline unsigned long long cpuset_str2ull(char **s) {
2264 + unsigned long long n = 0;
2265 + char c;
2266 + for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) {
2267 + n *= 10;
2268 + n += c - '0';
2269 + }
2270 + return n;
2271 +}
2272 +
2273 +static inline void update_cpu_limits(char **filename, unsigned long long *value, struct cgroup *cg) {
2274 + if(*filename) {
2275 + int ret = -1;
2276 +
2277 + if(value == &cg->cpuset_cpus) {
2278 + static char *buf = NULL;
2279 + static size_t buf_size = 0;
2280 +
2281 + if(!buf) {
2282 + buf_size = 100U + 6 * get_system_cpus(); // taken from kernel/cgroup/cpuset.c
2283 + buf = mallocz(buf_size + 1);
2284 + }
2285 +
2286 + ret = read_file(*filename, buf, buf_size);
2287 +
2288 + if(!ret) {
2289 + char *s = buf;
2290 + unsigned long long ncpus = 0;
2291 +
2292 + // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use
2293 + while(*s) {
2294 + unsigned long long n = cpuset_str2ull(&s);
2295 + if(*s == ',') {
2296 + s++;
2297 + ncpus++;
2298 + continue;
2299 + }
2300 + if(*s == '-') {
2301 + s++;
2302 + unsigned long long m = cpuset_str2ull(&s);
2303 + ncpus += m - n + 1; // calculate the number of cpus in the region
2304 + }
2305 + s++;
2306 + }
2307 +
2308 + if(likely(ncpus)) *value = ncpus;
2309 + }
2310 + }
2311 + else if(value == &cg->cpu_cfs_period) {
2312 + ret = read_single_number_file(*filename, value);
2313 + }
2314 + else if(value == &cg->cpu_cfs_quota) {
2315 + ret = read_single_number_file(*filename, value);
2316 + }
2317 + else ret = -1;
2318 +
2319 + if(ret) {
2320 + error("Cannot refresh cgroup %s cpu limit by reading '%s'. Will not update its limit anymore.", cg->id, *filename);
2321 + freez(*filename);
2322 + *filename = NULL;
2323 + }
2324 + }
2325 +}
2326 +
2327 +static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, unsigned long long *value, const char *chart_var_name, struct cgroup *cg) {
2328 + if(*filename) {
2329 + if(unlikely(!*chart_var)) {
2330 + *chart_var = rrdsetvar_custom_chart_variable_create(cg->st_mem_usage, chart_var_name);
2331 + if(!*chart_var) {
2332 + error("Cannot create cgroup %s chart variable '%s'. Will not update its limit anymore.", cg->id, chart_var_name);
2333 + freez(*filename);
2334 + *filename = NULL;
2335 + }
2336 + }
2337 +
2338 + if(*filename && *chart_var) {
2339 + if(read_single_number_file(*filename, value)) {
2340 + error("Cannot refresh cgroup %s memory limit by reading '%s'. Will not update its limit anymore.", cg->id, *filename);
2341 + freez(*filename);
2342 + *filename = NULL;
2343 + }
2344 + else {
2345 + rrdsetvar_custom_chart_variable_set(*chart_var, (calculated_number)(*value / (1024 * 1024)));
2346 + return 1;
2347 + }
2348 + }
2349 + }
2350 + return 0;
2351 +}
2352 +
2353 void update_cgroup_charts(int update_every) {
2354 debug(D_CGROUP, "updating cgroups charts");
2355
@@ -2250,7 +2403,7 @@ void update_cgroup_charts(int update_every) {
2403 , "cpu"
2404 , "cgroup.cpu"
2405 , title
2253 - , "%"
2406 + , "percentage"
2407 , PLUGIN_CGROUPS_NAME
2408 , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
2409 , NETDATA_CHART_PRIO_CGROUPS_CONTAINERS
@@ -2267,6 +2420,82 @@ void update_cgroup_charts(int update_every) {
2420 rrddim_set(cg->st_cpu, "user", cg->cpuacct_stat.user);
2421 rrddim_set(cg->st_cpu, "system", cg->cpuacct_stat.system);
2422 rrdset_done(cg->st_cpu);
2423 +
2424 + if(likely(cg->filename_cpuset_cpus || cg->filename_cpu_cfs_period || cg->filename_cpu_cfs_quota)) {
2425 + update_cpu_limits(&cg->filename_cpuset_cpus, &cg->cpuset_cpus, cg);
2426 + update_cpu_limits(&cg->filename_cpu_cfs_period, &cg->cpu_cfs_period, cg);
2427 + update_cpu_limits(&cg->filename_cpu_cfs_quota, &cg->cpu_cfs_quota, cg);
2428 +
2429 + if(unlikely(!cg->chart_var_cpu_limit)) {
2430 + cg->chart_var_cpu_limit = rrdsetvar_custom_chart_variable_create(cg->st_cpu, "cpu_limit");
2431 + if(!cg->chart_var_cpu_limit) {
2432 + error("Cannot create cgroup %s chart variable 'cpu_limit'. Will not update its limit anymore.", cg->id);
2433 + freez(cg->filename_cpuset_cpus);
2434 + cg->filename_cpuset_cpus = NULL;
2435 + freez(cg->filename_cpu_cfs_period);
2436 + cg->filename_cpu_cfs_period = NULL;
2437 + freez(cg->filename_cpu_cfs_quota);
2438 + cg->filename_cpu_cfs_quota = NULL;
2439 + }
2440 + }
2441 + else {
2442 + calculated_number value = 0, quota = 0;
2443 +
2444 + if(likely(cg->filename_cpuset_cpus || (cg->filename_cpu_cfs_period && cg->filename_cpu_cfs_quota))) {
2445 + if(unlikely(cg->cpu_cfs_quota > 0))
2446 + quota = (calculated_number)cg->cpu_cfs_quota / (calculated_number)cg->cpu_cfs_period;
2447 +
2448 + if(unlikely(quota > 0 && quota < cg->cpuset_cpus))
2449 + value = quota * 100;
2450 + else
2451 + value = (calculated_number)cg->cpuset_cpus * 100;
2452 + }
2453 + if(likely(value)) {
2454 + rrdsetvar_custom_chart_variable_set(cg->chart_var_cpu_limit, value);
2455 +
2456 + if(unlikely(!cg->st_cpu_limit)) {
2457 + snprintfz(title, CHART_TITLE_MAX, "CPU Usage within the limits for cgroup %s", cg->chart_title);
2458 +
2459 + cg->st_cpu_limit = rrdset_create_localhost(
2460 + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2461 + , "cpu_limit"
2462 + , NULL
2463 + , "cpu"
2464 + , "cgroup.cpu_limit"
2465 + , title
2466 + , "percentage"
2467 + , PLUGIN_CGROUPS_NAME
2468 + , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
2469 + , NETDATA_CHART_PRIO_CGROUPS_CONTAINERS - 1
2470 + , update_every
2471 + , RRDSET_TYPE_LINE
2472 + );
2473 +
2474 + rrddim_add(cg->st_cpu_limit, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
2475 + }
2476 + else
2477 + rrdset_next(cg->st_cpu_limit);
2478 +
2479 + calculated_number cpu_usage = (cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100 / system_hz;
2480 + calculated_number cpu_used = 100 * (cpu_usage - cg->prev_cpu_usage) / (value * update_every);
2481 +
2482 + rrdset_isnot_obsolete(cg->st_cpu_limit);
2483 +
2484 + rrddim_set(cg->st_cpu_limit, "used", (cpu_used > 0)?cpu_used:0);
2485 +
2486 + cg->prev_cpu_usage = cpu_usage;
2487 +
2488 + rrdset_done(cg->st_cpu_limit);
2489 + }
2490 + else {
2491 + rrdsetvar_custom_chart_variable_set(cg->chart_var_cpu_limit, NAN);
2492 + if(unlikely(cg->st_cpu_limit)) {
2493 + rrdset_is_obsolete(cg->st_cpu_limit);
2494 + cg->st_cpu_limit = NULL;
2495 + }
2496 + }
2497 + }
2498 + }
2499 }
2500
2501 if(likely(cg->cpuacct_usage.updated && cg->cpuacct_usage.enabled == CONFIG_BOOLEAN_YES)) {
@@ -2283,7 +2512,7 @@ void update_cgroup_charts(int update_every) {
2512 , "cpu"
2513 , "cgroup.cpu_per_core"
2514 , title
2286 - , "%"
2515 + , "percentage"
2516 , PLUGIN_CGROUPS_NAME
2517 , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
2518 , NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 100
@@ -2464,6 +2693,75 @@ void update_cgroup_charts(int update_every) {
2693 rrddim_set(cg->st_mem_usage, "ram", cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.cache:0));
2694 rrddim_set(cg->st_mem_usage, "swap", (cg->memory.msw_usage_in_bytes > cg->memory.usage_in_bytes)?cg->memory.msw_usage_in_bytes - cg->memory.usage_in_bytes:0);
2695 rrdset_done(cg->st_mem_usage);
2696 +
2697 + if (likely(update_memory_limits(&cg->filename_memory_limit, &cg->chart_var_memory_limit, &cg->memory_limit, "memory_limit", cg))) {
2698 + static unsigned long long ram_total = 0;
2699 +
2700 + if(unlikely(!ram_total)) {
2701 + procfile *ff = NULL;
2702 +
2703 + char filename[FILENAME_MAX + 1];
2704 + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/meminfo");
2705 + ff = procfile_open(config_get("plugin:cgroups", "meminfo filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
2706 +
2707 + if(likely(ff))
2708 + ff = procfile_readall(ff);
2709 + if(likely(ff && procfile_lines(ff) && !strncmp(procfile_word(ff, 0), "MemTotal", 8)))
2710 + ram_total = str2ull(procfile_word(ff, 1)) * 1024;
2711 + else {
2712 + error("Cannot read file %s. Will not update cgroup %s RAM limit anymore.", filename, cg->id);
2713 + freez(cg->filename_memory_limit);
2714 + cg->filename_memory_limit = NULL;
2715 + }
2716 +
2717 + procfile_close(ff);
2718 + }
2719 +
2720 + if(likely(ram_total)) {
2721 + unsigned long long memory_limit = ram_total;
2722 +
2723 + if(unlikely(cg->memory_limit < ram_total))
2724 + memory_limit = cg->memory_limit;
2725 +
2726 + if(unlikely(!cg->st_mem_usage_limit)) {
2727 + snprintfz(title, CHART_TITLE_MAX, "Used RAM without Cache within the limits for cgroup %s", cg->chart_title);
2728 +
2729 + cg->st_mem_usage_limit = rrdset_create_localhost(
2730 + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2731 + , "mem_usage_limit"
2732 + , NULL
2733 + , "mem"
2734 + , "cgroup.mem_usage_limit"
2735 + , title
2736 + , "MiB"
2737 + , PLUGIN_CGROUPS_NAME
2738 + , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
2739 + , NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 199
2740 + , update_every
2741 + , RRDSET_TYPE_STACKED
2742 + );
2743 +
2744 + rrddim_add(cg->st_mem_usage_limit, "available", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2745 + rrddim_add(cg->st_mem_usage_limit, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2746 + }
2747 + else
2748 + rrdset_next(cg->st_mem_usage_limit);
2749 +
2750 + rrdset_isnot_obsolete(cg->st_mem_usage_limit);
2751 +
2752 + rrddim_set(cg->st_mem_usage_limit, "available", memory_limit - (cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.cache:0)));
2753 + rrddim_set(cg->st_mem_usage_limit, "used", cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.cache:0));
2754 + rrdset_done(cg->st_mem_usage_limit);
2755 + }
2756 + }
2757 + else {
2758 + if(unlikely(cg->st_mem_usage_limit)) {
2759 + rrdset_is_obsolete(cg->st_mem_usage_limit);
2760 + cg->st_mem_usage_limit = NULL;
2761 + }
2762 + }
2763 +
2764 + update_memory_limits(&cg->filename_memoryswap_limit, &cg->chart_var_memoryswap_limit, &cg->memoryswap_limit, "memory_and_swap_limit", cg);
2765 }
2766
2767 if(likely(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_BOOLEAN_YES)) {
health/Makefile.am
+1
@@ -31,6 +31,7 @@ dist_healthconfig_DATA = \
31 health.d/boinc.conf \
32 health.d/btrfs.conf \
33 health.d/ceph.conf \
34 + health.d/cgroups.conf \
35 health.d/cpu.conf \
36 health.d/couchdb.conf \
37 health.d/disks.conf \
health/health.d/cgroups.conf new
+41
@@ -0,0 +1,41 @@
1 +
2 +# you can disable an alarm notification by setting the 'to' line to: silent
3 +
4 +template: cgroup_10min_cpu_usage
5 + on: cgroup.cpu_limit
6 + os: linux
7 + hosts: *
8 + lookup: average -10m unaligned
9 + units: %
10 + every: 1m
11 + warn: $this > (($status >= $WARNING) ? (75) : (85))
12 + crit: $this > (($status == $CRITICAL) ? (85) : (95))
13 + delay: down 15m multiplier 1.5 max 1h
14 + info: cpu utilization for the last 10 minutes
15 + to: sysadmin
16 +
17 +template: cgroup_ram_in_use
18 + on: cgroup.mem_usage
19 + os: linux
20 + hosts: *
21 + calc: ($ram) * 100 / $memory_limit
22 + units: %
23 + every: 10s
24 + warn: $this > (($status >= $WARNING) ? (80) : (90))
25 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
26 + delay: down 15m multiplier 1.5 max 1h
27 + info: RAM used by cgroup
28 + to: sysadmin
29 +
30 +template: cgroup_ram_and_swap_in_use
31 + on: cgroup.mem_usage
32 + os: linux
33 + hosts: *
34 + calc: ($ram + $swap) * 100 / $memory_and_swap_limit
35 + units: %
36 + every: 10s
37 + warn: $this > (($status >= $WARNING) ? (80) : (90))
38 + crit: $this > (($status == $CRITICAL) ? (90) : (98))
39 + delay: down 15m multiplier 1.5 max 1h
40 + info: RAM and Swap used by cgroup
41 + to: sysadmin
web/gui/dashboard_info.js
+61 -3
@@ -618,6 +618,10 @@ netdataDashboard.submenu = {
618 // colors: the dimension colors of the chart (the default colors are appended)
619 // height: the ratio of the chart height relative to the default
620 //
621 +
622 +var cgroupCPULimitIsSet = 0;
623 +var cgroupMemLimitIsSet = 0;
624 +
625 netdataDashboard.context = {
626 'system.cpu': {
627 info: function (os) {
@@ -1416,11 +1420,15 @@ netdataDashboard.context = {
1420 // ------------------------------------------------------------------------
1421 // containers
1422
1419 - 'cgroup.cpu': {
1423 + 'cgroup.cpu_limit': {
1424 + valueRange: "[0, null]",
1425 mainheads: [
1426 function (os, id) {
1427 void(os);
1428 + cgroupCPULimitIsSet = 1;
1429 return '<div data-netdata="' + id + '"'
1430 + + ' data-dimensions="used"'
1431 + + ' data-gauge-max-value="100"'
1432 + ' data-chart-library="gauge"'
1433 + ' data-title="CPU"'
1434 + ' data-units="%"'
@@ -1435,14 +1443,41 @@ netdataDashboard.context = {
1443 ]
1444 },
1445
1438 - 'cgroup.mem_usage': {
1446 + 'cgroup.cpu': {
1447 + mainheads: [
1448 + function (os, id) {
1449 + void(os);
1450 + if (cgroupCPULimitIsSet === 0) {
1451 + return '<div data-netdata="' + id + '"'
1452 + + ' data-chart-library="gauge"'
1453 + + ' data-title="CPU"'
1454 + + ' data-units="%"'
1455 + + ' data-gauge-adjust="width"'
1456 + + ' data-width="12%"'
1457 + + ' data-before="0"'
1458 + + ' data-after="-CHART_DURATION"'
1459 + + ' data-points="CHART_DURATION"'
1460 + + ' data-colors="' + NETDATA.colors[4] + '"'
1461 + + ' role="application"></div>';
1462 + }
1463 + else
1464 + return '';
1465 + }
1466 + ]
1467 + },
1468 +
1469 + 'cgroup.mem_usage_limit': {
1470 mainheads: [
1471 function (os, id) {
1472 void(os);
1473 + cgroupMemLimitIsSet = 1;
1474 return '<div data-netdata="' + id + '"'
1475 + + ' data-dimensions="used"'
1476 + + ' data-append-options="percentage"'
1477 + + ' data-gauge-max-value="100"'
1478 + ' data-chart-library="gauge"'
1479 + ' data-title="Memory"'
1445 - + ' data-units="MB"'
1480 + + ' data-units="%"'
1481 + ' data-gauge-adjust="width"'
1482 + ' data-width="12%"'
1483 + ' data-before="0"'
@@ -1454,6 +1489,29 @@ netdataDashboard.context = {
1489 ]
1490 },
1491
1492 + 'cgroup.mem_usage': {
1493 + mainheads: [
1494 + function (os, id) {
1495 + void(os);
1496 + if (cgroupMemLimitIsSet === 0) {
1497 + return '<div data-netdata="' + id + '"'
1498 + + ' data-chart-library="gauge"'
1499 + + ' data-title="Memory"'
1500 + + ' data-units="MB"'
1501 + + ' data-gauge-adjust="width"'
1502 + + ' data-width="12%"'
1503 + + ' data-before="0"'
1504 + + ' data-after="-CHART_DURATION"'
1505 + + ' data-points="CHART_DURATION"'
1506 + + ' data-colors="' + NETDATA.colors[1] + '"'
1507 + + ' role="application"></div>';
1508 + }
1509 + else
1510 + return '';
1511 + }
1512 + ]
1513 + },
1514 +
1515 'cgroup.throttle_io': {
1516 mainheads: [
1517 function (os, id) {