Delay renaming for docker containers (#5557)
Fixes #5512 * Delay cgroup renaming * Restrict renaming delay to problematic containers only * Use pending_renames instead of needs_renaming * Use an exit code instead of a parameter
Vladimir Kobal committed
Mar 9, 2019 at 18:12 UTC
2e05371992130803b546b7b6d37a98fb089fc5a3
2 files changed
+45
-9
collectors/cgroups.plugin/cgroup-name.sh.in
+8
@@ -73,6 +73,7 @@ function docker_get_name() {
73
fi
74
if [ -z "${NAME}" ]; then
75
warning "cannot find the name of docker container '${id}'"
76
+ NAME_NOT_FOUND=1
77
NAME="${id:0:12}"
78
else
79
info "docker container '${id}' is named '${NAME}'"
@@ -95,6 +96,7 @@ function docker_validate_id() {
96
97
DOCKER_HOST="${DOCKER_HOST:=/var/run/docker.sock}"
98
CGROUP="${1}"
99
+NAME_NOT_FOUND=0
100
NAME=
101
102
# -----------------------------------------------------------------------------
@@ -174,3 +176,9 @@ fi
176
177
info "cgroup '${CGROUP}' is called '${NAME}'"
178
echo "${NAME}"
179
+
180
+if [ "${NAME_NOT_FOUND}" -eq 1 ]; then
181
+ exit 2
182
+else
183
+ exit 0
184
+fi
collectors/cgroups.plugin/sys_fs_cgroup.c
+37
-9
@@ -376,6 +376,8 @@ struct cgroup {
376
char available; // found in the filesystem
377
char enabled; // enabled in the config
378
379
+ char pending_renames;
380
+
381
char *id;
382
uint32_t hash;
383
@@ -776,7 +778,7 @@ static inline void read_all_cgroups(struct cgroup *root) {
778
struct cgroup *cg;
779
780
for(cg = root; cg ; cg = cg->next)
779
- if(cg->enabled && cg->available)
781
+ if(cg->enabled && cg->available && !cg->pending_renames)
782
cgroup_read(cg);
783
}
784
@@ -886,7 +888,7 @@ static inline void cgroup_get_chart_name(struct cgroup *cg) {
888
pid_t cgroup_pid;
889
char command[CGROUP_CHARTID_LINE_MAX + 1];
890
889
- snprintfz(command, CGROUP_CHARTID_LINE_MAX, "exec %s '%s' '%s'", cgroups_rename_script, cg->chart_id, cg->id);
891
+ snprintfz(command, CGROUP_CHARTID_LINE_MAX, "exec %s '%s'", cgroups_rename_script, cg->id);
892
893
debug(D_CGROUP, "executing command \"%s\" for cgroup '%s'", command, cg->id);
894
FILE *fp = mypopen(command, &cgroup_pid);
@@ -895,7 +897,7 @@ static inline void cgroup_get_chart_name(struct cgroup *cg) {
897
char buffer[CGROUP_CHARTID_LINE_MAX + 1];
898
char *s = fgets(buffer, CGROUP_CHARTID_LINE_MAX, fp);
899
// debug(D_CGROUP, "closing command for cgroup '%s'", cg->id);
898
- mypclose(fp, cgroup_pid);
900
+ int name_error = mypclose(fp, cgroup_pid);
901
// debug(D_CGROUP, "closed command for cgroup '%s'", cg->id);
902
903
if(s && *s && *s != '\n') {
@@ -903,6 +905,9 @@ static inline void cgroup_get_chart_name(struct cgroup *cg) {
905
906
s = trim(s);
907
if (s) {
908
+ if(likely(!name_error))
909
+ cg->pending_renames = 0;
910
+
911
freez(cg->chart_title);
912
cg->chart_title = cgroup_title_strdupz(s);
913
@@ -950,6 +955,7 @@ static inline struct cgroup *cgroup_add(const char *id) {
955
// fix the chart_id and title by calling the external script
956
if(simple_pattern_matches(enabled_cgroup_renames, cg->id)) {
957
958
+ cg->pending_renames = 2;
959
cgroup_get_chart_name(cg);
960
961
debug(D_CGROUP, "cgroup '%s' renamed to '%s' (title: '%s')", cg->id, cg->chart_id, cg->chart_title);
@@ -1031,7 +1037,7 @@ static inline struct cgroup *cgroup_add(const char *id) {
1037
}
1038
}
1039
1034
- if(cg->enabled && !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))
1040
+ if(cg->enabled && !cg->pending_renames && !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))
1041
read_cgroup_network_interfaces(cg);
1042
1043
debug(D_CGROUP, "ADDED CGROUP: '%s' with chart id '%s' and title '%s' as %s (default was %s)", cg->id, cg->chart_id, cg->chart_title, (cg->enabled)?"enabled":"disabled", (def)?"enabled":"disabled");
@@ -1050,7 +1056,7 @@ static inline void cgroup_free(struct cgroup *cg) {
1056
if(cg->st_mem_activity) rrdset_is_obsolete(cg->st_mem_activity);
1057
if(cg->st_pgfaults) rrdset_is_obsolete(cg->st_pgfaults);
1058
if(cg->st_mem_usage) rrdset_is_obsolete(cg->st_mem_usage);
1053
- if (cg->st_mem_usage_limit) rrdset_is_obsolete(cg->st_mem_usage_limit);
1059
+ if(cg->st_mem_usage_limit) rrdset_is_obsolete(cg->st_mem_usage_limit);
1060
if(cg->st_mem_failcnt) rrdset_is_obsolete(cg->st_mem_failcnt);
1061
if(cg->st_io) rrdset_is_obsolete(cg->st_io);
1062
if(cg->st_serviced_ops) rrdset_is_obsolete(cg->st_serviced_ops);
@@ -1138,7 +1144,26 @@ static inline void found_subdir_in_dir(const char *dir) {
1144
cg = cgroup_add(dir);
1145
}
1146
1141
- if(cg) cg->available = 1;
1147
+ if(cg) {
1148
+ // delay renaming of the cgroup and looking for network interfaces to deal with the docker lag when starting the container
1149
+ if(unlikely(cg->pending_renames == 1)) {
1150
+ // fix the chart_id and title by calling the external script
1151
+ if(simple_pattern_matches(enabled_cgroup_renames, cg->id)) {
1152
+
1153
+ cgroup_get_chart_name(cg);
1154
+ cg->pending_renames = 0;
1155
+
1156
+ if(cg->enabled && !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))
1157
+ read_cgroup_network_interfaces(cg);
1158
+
1159
+ debug(D_CGROUP, "cgroup '%s' renamed to '%s' (title: '%s')", cg->id, cg->chart_id, cg->chart_title);
1160
+ }
1161
+ else
1162
+ debug(D_CGROUP, "cgroup '%s' will not be renamed - it matches the list of disabled cgroup renames (will be shown as '%s')", cg->id, cg->chart_id);
1163
+ }
1164
+
1165
+ cg->available = 1;
1166
+ }
1167
}
1168
1169
static inline int find_dir_in_subdirs(const char *base, const char *this, void (*callback)(const char *)) {
@@ -1302,7 +1327,10 @@ static inline void find_all_cgroups() {
1327
for(cg = cgroup_root; cg ; cg = cg->next) {
1328
// fprintf(stderr, " >>> CGROUP '%s' (%u - %s) with name '%s'\n", cg->id, cg->hash, cg->available?"available":"stopped", cg->name);
1329
1305
- if(unlikely(!cg->available))
1330
+ if(unlikely(cg->pending_renames))
1331
+ cg->pending_renames--;
1332
+
1333
+ if(unlikely(!cg->available || cg->pending_renames))
1334
continue;
1335
1336
debug(D_CGROUP, "checking paths for cgroup '%s'", cg->id);
@@ -2037,7 +2065,7 @@ void update_systemd_services_charts(
2065
// update the values
2066
struct cgroup *cg;
2067
for(cg = cgroup_root; cg ; cg = cg->next) {
2040
- if(unlikely(!cg->available || !cg->enabled || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))
2068
+ if(unlikely(!cg->available || !cg->enabled || cg->pending_renames || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))
2069
continue;
2070
2071
if(likely(do_cpu && cg->cpuacct_stat.updated)) {
@@ -2360,7 +2388,7 @@ void update_cgroup_charts(int update_every) {
2388
2389
struct cgroup *cg;
2390
for(cg = cgroup_root; cg ; cg = cg->next) {
2363
- if(unlikely(!cg->available || !cg->enabled))
2391
+ if(unlikely(!cg->available || !cg->enabled || cg->pending_renames))
2392
continue;
2393
2394
if(likely(cgroup_enable_systemd_services && cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)) {