7ba16b5ad · cryptotaxi247/netdata-1

@@ -584,6 +584,7 @@ struct cgroup_network_interface {

584 #define CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE 0x00000002

585 #define CGROUP_OPTIONS_IS_UNIFIED 0x00000004

586

587 +// *** WARNING *** The fields are not thread safe. Take care of safe usage.

588 struct cgroup {

589 uint32_t options;

590

@@ -691,9 +692,22 @@ struct cgroup {

692 RRDDIM *rd_io_merged_write;

693

694 struct cgroup *next;

695 + struct cgroup *discovered_next;

696

697 } *cgroup_root = NULL;

698

699 +uv_mutex_t cgroup_root_mutex;

700 +

701 +struct cgroup *discovered_cgroup_root = NULL;

702 +

703 +struct discovery_thread {

704 + uv_thread_t thread;

705 + uv_mutex_t mutex;

706 + uv_cond_t cond_var;

707 + int start_discovery;

708 + int exited;

709 +} discovery_thread;

710 +

711 // ----------------------------------------------------------------------------

712 // read values from /sys

713

@@ -1168,7 +1182,7 @@ static inline void read_all_cgroups(struct cgroup *root) {

1182 struct cgroup *cg;

1183

1184 for(cg = root; cg ; cg = cg->next)

1171 - if(cg->enabled && cg->available && !cg->pending_renames)

1185 + if(cg->enabled && !cg->pending_renames)

1186 cgroup_read(cg);

1187 }

1188

@@ -1380,13 +1394,13 @@ static inline struct cgroup *cgroup_add(const char *id) {

1394

1395 if(cgroup_use_unified_cgroups) cg->options |= CGROUP_OPTIONS_IS_UNIFIED;

1396

1383 - if(!cgroup_root)

1384 - cgroup_root = cg;

1397 + if(!discovered_cgroup_root)

1398 + discovered_cgroup_root = cg;

1399 else {

1400 // append it

1401 struct cgroup *e;

1388 - for(e = cgroup_root; e->next ;e = e->next) ;

1389 - e->next = cg;

1402 + for(e = discovered_cgroup_root; e->discovered_next ;e = e->discovered_next) ;

1403 + e->discovered_next = cg;

1404 }

1405

1406 cgroup_root_count++;

@@ -1452,24 +1466,23 @@ static inline struct cgroup *cgroup_add(const char *id) {

1466 // detect duplicate cgroups

1467 if(cg->enabled) {

1468 struct cgroup *t;

1455 - for (t = cgroup_root; t; t = t->next) {

1469 + for (t = discovered_cgroup_root; t; t = t->discovered_next) {

1470 if (t != cg && t->enabled && t->hash_chart == cg->hash_chart && !strcmp(t->chart_id, cg->chart_id)) {

1457 - if (!strncmp(t->chart_id, "/system.slice/", 14) && !strncmp(cg->chart_id, "/init.scope/system.slice/", 25)) {

1458 - error("CGROUP: chart id '%s' already exists with id '%s' and is enabled. Swapping them by enabling cgroup with id '%s' and disabling cgroup with id '%s'.",

1459 - cg->chart_id, t->id, cg->id, t->id);

1460 - debug(D_CGROUP, "Control group with chart id '%s' already exists with id '%s' and is enabled. Swapping them by enabling cgroup with id '%s' and disabling cgroup with id '%s'.",

1461 - cg->chart_id, t->id, cg->id, t->id);

1462 - t->enabled = 0;

1463 - t->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;

1464 - }

1465 - else {

1466 - error("CGROUP: chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.",

1467 - cg->chart_id, t->id, cg->id);

1468 - debug(D_CGROUP, "Control group with chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.",

1469 - cg->chart_id, t->id, cg->id);

1470 - cg->enabled = 0;

1471 - cg->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;

1472 - }

1471 + // TODO: use it after refactoring if system.slice might be scanned before init.scope/system.slice

1472 + //

1473 + // if (!strncmp(t->id, "/system.slice/", 14) && !strncmp(cg->id, "/init.scope/system.slice/", 25)) {

1474 + // error("CGROUP: chart id '%s' already exists with id '%s' and is enabled. Swapping them by enabling cgroup with id '%s' and disabling cgroup with id '%s'.",

1475 + // cg->chart_id, t->id, cg->id, t->id);

1476 + // t->enabled = 0;

1477 + // t->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;

1478 + // }

1479 + // else {}

1480 + //

1481 + // https://github.com/netdata/netdata/issues/797#issuecomment-241248884

1482 + error("CGROUP: chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.",

1483 + cg->chart_id, t->id, cg->id);

1484 + cg->enabled = 0;

1485 + cg->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;

1486

1487 break;

1488 }

@@ -1560,7 +1573,7 @@ static inline struct cgroup *cgroup_find(const char *id) {

1573 uint32_t hash = simple_hash(id);

1574

1575 struct cgroup *cg;

1563 - for(cg = cgroup_root; cg ; cg = cg->next) {

1576 + for(cg = discovered_cgroup_root; cg ; cg = cg->discovered_next) {

1577 if(hash == cg->hash && strcmp(id, cg->id) == 0)

1578 break;

1579 }

@@ -1686,103 +1699,16 @@ static inline void mark_all_cgroups_as_not_available() {

1699 struct cgroup *cg;

1700

1701 // mark all as not available

1689 - for(cg = cgroup_root; cg ; cg = cg->next) {

1702 + for(cg = discovered_cgroup_root; cg ; cg = cg->discovered_next) {

1703 cg->available = 0;

1704 }

1705 }

1706

1694 -static inline void cleanup_all_cgroups() {

1695 - struct cgroup *cg = cgroup_root, *last = NULL;

1696 -

1697 - for(; cg ;) {

1698 - if(!cg->available) {

1699 - // enable the first duplicate cgroup

1700 - {

1701 - struct cgroup *t;

1702 - for(t = cgroup_root; t ; t = t->next) {

1703 - if(t != cg && t->available && !t->enabled && t->options & CGROUP_OPTIONS_DISABLED_DUPLICATE && t->hash_chart == cg->hash_chart && !strcmp(t->chart_id, cg->chart_id)) {

1704 - debug(D_CGROUP, "Enabling duplicate of cgroup '%s' with id '%s', because the original with id '%s' stopped.", t->chart_id, t->id, cg->id);

1705 - t->enabled = 1;

1706 - t->options &= ~CGROUP_OPTIONS_DISABLED_DUPLICATE;

1707 - break;

1708 - }

1709 - }

1710 - }

1711 -

1712 - if(!last)

1713 - cgroup_root = cg->next;

1714 - else

1715 - last->next = cg->next;

1716 -

1717 - cgroup_free(cg);

1718 -

1719 - if(!last)

1720 - cg = cgroup_root;

1721 - else

1722 - cg = last->next;

1723 - }

1724 - else {

1725 - last = cg;

1726 - cg = cg->next;

1727 - }

1728 - }

1729 -}

1730 -

1731 -static inline void find_all_cgroups() {

1732 - debug(D_CGROUP, "searching for cgroups");

1733 -

1734 - mark_all_cgroups_as_not_available();

1735 - if(!cgroup_use_unified_cgroups) {

1736 - if(cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_usage) {

1737 - if(find_dir_in_subdirs(cgroup_cpuacct_base, NULL, found_subdir_in_dir) == -1) {

1738 - cgroup_enable_cpuacct_stat =

1739 - cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_NO;

1740 - error("CGROUP: disabled cpu statistics.");

1741 - }

1742 - }

1743 -

1744 - if(cgroup_enable_blkio_io || cgroup_enable_blkio_ops || cgroup_enable_blkio_throttle_io || cgroup_enable_blkio_throttle_ops || cgroup_enable_blkio_merged_ops || cgroup_enable_blkio_queued_ops) {

1745 - if(find_dir_in_subdirs(cgroup_blkio_base, NULL, found_subdir_in_dir) == -1) {

1746 - cgroup_enable_blkio_io =

1747 - cgroup_enable_blkio_ops =

1748 - cgroup_enable_blkio_throttle_io =

1749 - cgroup_enable_blkio_throttle_ops =

1750 - cgroup_enable_blkio_merged_ops =

1751 - cgroup_enable_blkio_queued_ops = CONFIG_BOOLEAN_NO;

1752 - error("CGROUP: disabled blkio statistics.");

1753 - }

1754 - }

1755 -

1756 - if(cgroup_enable_memory || cgroup_enable_detailed_memory || cgroup_enable_swap || cgroup_enable_memory_failcnt) {

1757 - if(find_dir_in_subdirs(cgroup_memory_base, NULL, found_subdir_in_dir) == -1) {

1758 - cgroup_enable_memory =

1759 - cgroup_enable_detailed_memory =

1760 - cgroup_enable_swap =

1761 - cgroup_enable_memory_failcnt = CONFIG_BOOLEAN_NO;

1762 - error("CGROUP: disabled memory statistics.");

1763 - }

1764 - }

1765 -

1766 - if(cgroup_search_in_devices) {

1767 - if(find_dir_in_subdirs(cgroup_devices_base, NULL, found_subdir_in_dir) == -1) {

1768 - cgroup_search_in_devices = 0;

1769 - error("CGROUP: disabled devices statistics.");

1770 - }

1771 - }

1772 - }

1773 - else {

1774 - if (find_dir_in_subdirs(cgroup_unified_base, NULL, found_subdir_in_dir) == -1) {

1775 - cgroup_unified_exist = CONFIG_BOOLEAN_NO;

1776 - error("CGROUP: disabled unified cgroups statistics.");

1777 - }

1778 - }

1779 -

1780 - // remove any non-existing cgroups

1781 - cleanup_all_cgroups();

1782 -

1707 +static inline void update_filenames()

1708 +{

1709 struct cgroup *cg;

1710 struct stat buf;

1785 - for(cg = cgroup_root; cg ; cg = cg->next) {

1711 + for(cg = discovered_cgroup_root; cg ; cg = cg->discovered_next) {

1712 // fprintf(stderr, " >>> CGROUP '%s' (%u - %s) with name '%s'\n", cg->id, cg->hash, cg->available?"available":"stopped", cg->name);

1713

1714 if(unlikely(cg->pending_renames))

@@ -2046,10 +1972,137 @@ static inline void find_all_cgroups() {

1972 }

1973 }

1974 }

1975 +}

1976 +

1977 +static inline void cleanup_all_cgroups() {

1978 + struct cgroup *cg = discovered_cgroup_root, *last = NULL;

1979 +

1980 + for(; cg ;) {

1981 + if(!cg->available) {

1982 + // enable the first duplicate cgroup

1983 + {

1984 + struct cgroup *t;

1985 + for(t = discovered_cgroup_root; t ; t = t->discovered_next) {

1986 + if(t != cg && t->available && !t->enabled && t->options & CGROUP_OPTIONS_DISABLED_DUPLICATE && t->hash_chart == cg->hash_chart && !strcmp(t->chart_id, cg->chart_id)) {

1987 + debug(D_CGROUP, "Enabling duplicate of cgroup '%s' with id '%s', because the original with id '%s' stopped.", t->chart_id, t->id, cg->id);

1988 + t->enabled = 1;

1989 + t->options &= ~CGROUP_OPTIONS_DISABLED_DUPLICATE;

1990 + break;

1991 + }

1992 + }

1993 + }

1994 +

1995 + if(!last)

1996 + discovered_cgroup_root = cg->discovered_next;

1997 + else

1998 + last->discovered_next = cg->discovered_next;

1999 +

2000 + cgroup_free(cg);

2001 +

2002 + if(!last)

2003 + cg = discovered_cgroup_root;

2004 + else

2005 + cg = last->discovered_next;

2006 + }

2007 + else {

2008 + last = cg;

2009 + cg = cg->discovered_next;

2010 + }

2011 + }

2012 +}

2013 +

2014 +static inline void copy_discovered_cgroups()

2015 +{

2016 + debug(D_CGROUP, "copy discovered cgroups to the main group list");

2017 +

2018 + struct cgroup *cg;

2019 +

2020 + for(cg = discovered_cgroup_root; cg ; cg = cg->discovered_next) {

2021 + cg->next = cg->discovered_next;

2022 + }

2023 +

2024 + cgroup_root = discovered_cgroup_root;

2025 +}

2026 +

2027 +static inline void find_all_cgroups() {

2028 + debug(D_CGROUP, "searching for cgroups");

2029 +

2030 + mark_all_cgroups_as_not_available();

2031 + if(!cgroup_use_unified_cgroups) {

2032 + if(cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_usage) {

2033 + if(find_dir_in_subdirs(cgroup_cpuacct_base, NULL, found_subdir_in_dir) == -1) {

2034 + cgroup_enable_cpuacct_stat =

2035 + cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_NO;

2036 + error("CGROUP: disabled cpu statistics.");

2037 + }

2038 + }

2039 +

2040 + if(cgroup_enable_blkio_io || cgroup_enable_blkio_ops || cgroup_enable_blkio_throttle_io || cgroup_enable_blkio_throttle_ops || cgroup_enable_blkio_merged_ops || cgroup_enable_blkio_queued_ops) {

2041 + if(find_dir_in_subdirs(cgroup_blkio_base, NULL, found_subdir_in_dir) == -1) {

2042 + cgroup_enable_blkio_io =

2043 + cgroup_enable_blkio_ops =

2044 + cgroup_enable_blkio_throttle_io =

2045 + cgroup_enable_blkio_throttle_ops =

2046 + cgroup_enable_blkio_merged_ops =

2047 + cgroup_enable_blkio_queued_ops = CONFIG_BOOLEAN_NO;

2048 + error("CGROUP: disabled blkio statistics.");

2049 + }

2050 + }

2051 +

2052 + if(cgroup_enable_memory || cgroup_enable_detailed_memory || cgroup_enable_swap || cgroup_enable_memory_failcnt) {

2053 + if(find_dir_in_subdirs(cgroup_memory_base, NULL, found_subdir_in_dir) == -1) {

2054 + cgroup_enable_memory =

2055 + cgroup_enable_detailed_memory =

2056 + cgroup_enable_swap =

2057 + cgroup_enable_memory_failcnt = CONFIG_BOOLEAN_NO;

2058 + error("CGROUP: disabled memory statistics.");

2059 + }

2060 + }

2061 +

2062 + if(cgroup_search_in_devices) {

2063 + if(find_dir_in_subdirs(cgroup_devices_base, NULL, found_subdir_in_dir) == -1) {

2064 + cgroup_search_in_devices = 0;

2065 + error("CGROUP: disabled devices statistics.");

2066 + }

2067 + }

2068 + }

2069 + else {

2070 + if (find_dir_in_subdirs(cgroup_unified_base, NULL, found_subdir_in_dir) == -1) {

2071 + cgroup_unified_exist = CONFIG_BOOLEAN_NO;

2072 + error("CGROUP: disabled unified cgroups statistics.");

2073 + }

2074 + }

2075 +

2076 + update_filenames();

2077 +

2078 + uv_mutex_lock(&cgroup_root_mutex);

2079 + cleanup_all_cgroups();

2080 + copy_discovered_cgroups();

2081 + uv_mutex_unlock(&cgroup_root_mutex);

2082

2083 debug(D_CGROUP, "done searching for cgroups");

2084 }

2085

2086 +void cgroup_discovery_worker(void *ptr)

2087 +{

2088 + UNUSED(ptr);

2089 +

2090 + while (!netdata_exit) {

2091 + uv_mutex_lock(&discovery_thread.mutex);

2092 + while (!discovery_thread.start_discovery)

2093 + uv_cond_wait(&discovery_thread.cond_var, &discovery_thread.mutex);

2094 + discovery_thread.start_discovery = 0;

2095 + uv_mutex_unlock(&discovery_thread.mutex);

2096 +

2097 + if (unlikely(netdata_exit))

2098 + break;

2099 +

2100 + find_all_cgroups();

2101 + }

2102 +

2103 + discovery_thread.exited = 1;

2104 +}

2105 +

2106 // ----------------------------------------------------------------------------

2107 // generate charts

2108

@@ -2631,7 +2684,7 @@ void update_systemd_services_charts(

2684 // update the values

2685 struct cgroup *cg;

2686 for(cg = cgroup_root; cg ; cg = cg->next) {

2634 - if(unlikely(!cg->available || !cg->enabled || cg->pending_renames || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))

2687 + if(unlikely(!cg->enabled || cg->pending_renames || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))

2688 continue;

2689

2690 if(likely(do_cpu && cg->cpuacct_stat.updated)) {

@@ -3022,7 +3075,7 @@ void update_cgroup_charts(int update_every) {

3075

3076 struct cgroup *cg;

3077 for(cg = cgroup_root; cg ; cg = cg->next) {

3025 - if(unlikely(!cg->available || !cg->enabled || cg->pending_renames))

3078 + if(unlikely(!cg->enabled || cg->pending_renames))

3079 continue;

3080

3081 if(likely(cgroup_enable_systemd_services && cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)) {

@@ -3880,6 +3933,21 @@ static void cgroup_main_cleanup(void *ptr) {

3933

3934 info("cleaning up...");

3935

3936 + usec_t max = 2 * USEC_PER_SEC, step = 50000;

3937 +

3938 + if (!discovery_thread.exited) {

3939 + info("stopping discovery thread worker");

3940 + uv_mutex_unlock(&discovery_thread.mutex);

3941 + discovery_thread.start_discovery = 1;

3942 + uv_cond_signal(&discovery_thread.cond_var);

3943 + }

3944 +

3945 + while (!discovery_thread.exited && max > 0) {

3946 + max -= step;

3947 + info("waiting for discovery thread to finish...");

3948 + sleep_usec(step);

3949 + }

3950 +

3951 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

3952 }

3953

@@ -3895,6 +3963,31 @@ void *cgroups_main(void *ptr) {

3963

3964 RRDSET *stcpu_thread = NULL;

3965

3966 + if (uv_mutex_init(&cgroup_root_mutex)) {

3967 + error("CGROUP: cannot initialize mutex for the main cgroup list");

3968 + goto exit;

3969 + }

3970 +

3971 + // dispatch a discovery worker thread

3972 + discovery_thread.start_discovery = 0;

3973 + discovery_thread.exited = 0;

3974 +

3975 + if (uv_mutex_init(&discovery_thread.mutex)) {

3976 + error("CGROUP: cannot initialize mutex for discovery thread");

3977 + goto exit;

3978 + }

3979 + if (uv_cond_init(&discovery_thread.cond_var)) {

3980 + error("CGROUP: cannot initialize conditional variable for discovery thread");

3981 + goto exit;

3982 + }

3983 +

3984 + int error = uv_thread_create(&discovery_thread.thread, cgroup_discovery_worker, NULL);

3985 + if (error) {

3986 + error("CGROUP: cannot create tread worker. uv_thread_create(): %s", uv_strerror(error));

3987 + goto exit;

3988 + }

3989 + uv_thread_set_name_np(discovery_thread.thread, "PLUGIN[cgroups]");

3990 +

3991 heartbeat_t hb;

3992 heartbeat_init(&hb);

3993 usec_t step = cgroup_update_every * USEC_PER_SEC;

@@ -3904,19 +3997,18 @@ void *cgroups_main(void *ptr) {

3997 usec_t hb_dt = heartbeat_next(&hb, step);

3998 if(unlikely(netdata_exit)) break;

3999

3907 - // BEGIN -- the job to be done

3908 -

4000 find_dt += hb_dt;

4001 if(unlikely(find_dt >= find_every || cgroups_check)) {

3911 - find_all_cgroups();

4002 + uv_cond_signal(&discovery_thread.cond_var);

4003 + discovery_thread.start_discovery = 1;

4004 find_dt = 0;

4005 cgroups_check = 0;

4006 }

4007

4008 + uv_mutex_lock(&cgroup_root_mutex);

4009 read_all_cgroups(cgroup_root);

4010 update_cgroup_charts(cgroup_update_every);

3918 -

3919 - // END -- the job is done

4011 + uv_mutex_unlock(&cgroup_root_mutex);

4012

4013 // --------------------------------------------------------------------

4014

@@ -3952,6 +4044,7 @@ void *cgroups_main(void *ptr) {

4044 }

4045 }

4046

4047 +exit:

4048 netdata_thread_cleanup_pop(1);

4049 return NULL;

4050 }

Dispatch cgroup discovery into another thread (#10399)