collectors/cgroups: fix resolving container names in k8s (#10072)
Ilya Mashchenko committed
Oct 20, 2020 at 11:59 UTC
9798a042db0a74a7cb272491b99d0f3f1faeefd4
1 file changed
+171
-44
collectors/cgroups.plugin/cgroup-name.sh.in
+171
-44
@@ -76,51 +76,178 @@ function docker_like_get_name_api() {
76
return 0
77
}
78
79
-function k8s_get_name() {
80
- if [[ "${1}" =~ ^kube.*_pod.*$ ]]; then
81
- local id="${1##*_pod}"
82
- else
83
- # Take the last part of the delimited path identifier (expecting either _ or / as a delimiter).
84
- local id="${1##*_}"
85
- if [ "${id}" == "${1}" ]; then
86
- id="${1##*/}"
87
- fi
88
- fi
89
- if command -v jq >/dev/null 2>&1; then
90
- if [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ]; then
91
- KUBE_TOKEN="$(</var/run/secrets/kubernetes.io/serviceaccount/token)"
92
- NAME="$(
93
- curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/pods" |
94
- jq -r '.items[] | "k8s_\(.metadata.namespace)_\(.metadata.name)_\(.metadata.uid)_" + (.status.containerStatuses[]? | "\(.name) \(.containerID)")' |
95
- grep "$id" |
96
- cut -d' ' -f1
97
- )"
98
- elif ps -C kubelet >/dev/null 2>&1 && command -v kubectl >/dev/null 2>&1; then
99
- if [[ -z ${KUBE_CONFIG+x} ]]; then
100
- KUBE_CONFIG="/etc/kubernetes/admin.conf"
101
- fi
102
- if kubectl --kubeconfig=$KUBE_CONFIG get pod --all-namespaces >/dev/null 2>&1; then
103
- #shellcheck disable=SC2086
104
- NAME="$(kubectl --kubeconfig=$KUBE_CONFIG get pod --all-namespaces --output='json' |
105
- jq -r '.items[] | "k8s_\(.metadata.namespace)_\(.metadata.name)_\(.metadata.uid)_" + (.status.containerStatuses[]? | "\(.name) \(.containerID)")' |
106
- grep "$id" |
107
- cut -d' ' -f1
108
- )"
109
- else
110
- warning "kubectl cannot get pod list, check for configuration file in $KUBE_CONFIG, or set this path to env \$KUBE_CONFIG"
111
- fi
112
- fi
113
- else
114
- warning "jq command not available, k8s_get_name() cannot execute. Please install jq should you wish for k8s to be fully functional"
115
- fi
79
+# get_lbl_val returns the value for the label with the given name.
80
+# Returns "null" string if the label doesn't exist.
81
+# Expected labels format: 'name="value",...'.
82
+function get_lbl_val() {
83
+ local labels want_name
84
+ labels="${1}"
85
+ want_name="${2}"
86
+
87
+ IFS=, read -ra labels <<< "$labels"
88
+
89
+ local lname lval
90
+ for l in "${labels[@]}"; do
91
+ IFS="=" read -r lname lval <<< "$l"
92
+ if [ "$want_name" = "$lname" ] && [ -n "$lval" ]; then
93
+ echo "${lval:1:-1}" # trim "
94
+ return 0
95
+ fi
96
+ done
97
+
98
+ echo "null"
99
+ return 1
100
+}
101
117
- if [ -z "${NAME}" ]; then
118
- warning "cannot find the name of k8s pod with containerID '${id}'. Setting name to ${id} and disabling it"
119
- NAME="${id}"
120
- NAME_NOT_FOUND=3
121
- else
122
- info "k8s containerID '${id}' has chart name (namespace_podname_poduid_containername) '${NAME}'"
123
- fi
102
+# k8s_get_kubepod_name resolves */kubepods/* cgroup name.
103
+# pod level cgroup name format: 'pod_<namespace>_<pod_name>'
104
+# container level cgroup name format: 'cntr_<namespace>_<pod_name>_<container_name>'
105
+function k8s_get_kubepod_name() {
106
+ # GKE /sys/fs/cgroup/*/ tree:
107
+ # |-- kubepods
108
+ # | |-- burstable
109
+ # | | |-- pod98cee708-023b-11eb-933d-42010a800193
110
+ # | | | |-- 922161c98e6ea450bf665226cdc64ca2aa3e889934c2cff0aec4325f8f78ac03
111
+ # | | `-- a5d223eec35e00f5a1c6fa3e3a5faac6148cdc1f03a2e762e873b7efede012d7
112
+ # | `-- pode314bbac-d577-11ea-a171-42010a80013b
113
+ # | |-- 7d505356b04507de7b710016d540b2759483ed5f9136bb01a80872b08f771930
114
+ # | `-- 88ab4683b99cfa7cc8c5f503adf7987dd93a3faa7c4ce0d17d419962b3220d50
115
+ #
116
+ # Minikube (v1.8.2) /sys/fs/cgroup/*/ tree:
117
+ # |-- kubepods.slice
118
+ # | |-- kubepods-besteffort.slice
119
+ # | | |-- kubepods-besteffort-pod10fb5647_c724_400c_b9cc_0e6eae3110e7.slice
120
+ # | | | |-- docker-36e5eb5056dfdf6dbb75c0c44a1ecf23217fe2c50d606209d8130fcbb19fb5a7.scope
121
+ # | | | `-- docker-87e18c2323621cf0f635c53c798b926e33e9665c348c60d489eef31ee1bd38d7.scope
122
+ #
123
+ # NOTE: cgroups plugin uses '_' to join dir names, so it is <parent>_<child>_<child>_...
124
+
125
+ local funcname="${FUNCNAME[0]}"
126
+ local id="${1}"
127
+
128
+ if [[ ! $id =~ ^kubepods ]]; then
129
+ warning "${funcname}: '${id}' is not kubepod cgroup."
130
+ return 1
131
+ fi
132
+
133
+ local clean_id="$id"
134
+ clean_id=${clean_id//.slice/}
135
+ clean_id=${clean_id//.scope/}
136
+
137
+ local name pod_uid cntr_id
138
+ if [[ $clean_id == "kubepods" ]]; then
139
+ name="$clean_id"
140
+ elif [[ $clean_id =~ .+(besteffort|burstable|guaranteed)$ ]]; then
141
+ # kubepods_<QOS_CLASS>
142
+ # kubepods_kubepods-<QOS_CLASS>
143
+ name=${clean_id//-/_}
144
+ name=${name/#kubepods_kubepods/kubepods}
145
+ elif [[ $clean_id =~ .+pod[a-f0-9_-]+_docker-([a-f0-9]+)$ ]]; then
146
+ # ...pod<POD_UID>_docker-<CONTAINER_ID> (POD_UID w/ "_")
147
+ cntr_id=${BASH_REMATCH[1]}
148
+ elif [[ $clean_id =~ .+pod[a-f0-9-]+_([a-f0-9]+)$ ]]; then
149
+ # ...pod<POD_UID>_<CONTAINER_ID>
150
+ cntr_id=${BASH_REMATCH[1]}
151
+ elif [[ $clean_id =~ .+pod([a-f0-9_-]+)$ ]]; then
152
+ # ...pod<POD_UID> (POD_UID w/ and w/o "_")
153
+ pod_uid=${BASH_REMATCH[1]}
154
+ pod_uid=${pod_uid//_/-}
155
+ fi
156
+
157
+ if [ -n "$name" ]; then
158
+ echo "$name"
159
+ return 0
160
+ fi
161
+
162
+ if [ -z "$pod_uid" ] && [ -z "$cntr_id" ]; then
163
+ warning "${funcname}: can't extract pod_uid or container_id from the cgroup '$id'."
164
+ return 1
165
+ fi
166
+
167
+ [ -n "$pod_uid" ] && info "${funcname}: cgroup '$id' is a pod(uid:$pod_uid)"
168
+ [ -n "$cntr_id" ] && info "${funcname}: cgroup '$id' is a container(id:$cntr_id)"
169
+
170
+ if ! command -v jq > /dev/null 2>&1; then
171
+ warning "${funcname}: 'jq' command not available."
172
+ return 1
173
+ fi
174
+
175
+ local pods
176
+ if [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ]; then
177
+ local token header url
178
+ token="$(< /var/run/secrets/kubernetes.io/serviceaccount/token)"
179
+ header="Authorization: Bearer $token"
180
+ url="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/pods"
181
+ if ! pods=$(curl -sSk -H "$header" "$url" 2>&1); then
182
+ warning "${funcname}: error on curl '${url}': ${pods}."
183
+ return 1
184
+ fi
185
+ elif ps -C kubelet > /dev/null 2>&1 && command -v kubectl > /dev/null 2>&1; then
186
+ [[ -z ${KUBE_CONFIG+x} ]] && KUBE_CONFIG="/etc/kubernetes/admin.conf"
187
+ if ! pods=$(kubectl --kubeconfig="$KUBE_CONFIG" get pods --all-namespaces -o json 2>&1); then
188
+ warning "${funcname}: error on 'kubectl': ${pods}."
189
+ return 1
190
+ fi
191
+ else
192
+ warning "${funcname}: not inside the k8s cluster and 'kubectl' command not available."
193
+ return 1
194
+ fi
195
+
196
+ local jq_filter
197
+ # 'namespace="<NAMESPACE>",pod_name="<NAME>",pod_uid="<UID>",container_name="<NAME>",container_ID="<ID>"'
198
+ # wrong field value is 'null'
199
+ jq_filter+='.items[] | '
200
+ jq_filter+='"namespace=\"\(.metadata.namespace)\",pod_name=\"\(.metadata.name)\",pod_uid=\"\(.metadata.uid)\"" + '
201
+ jq_filter+='(.status.containerStatuses[]? | '
202
+ jq_filter+='",container_name=\"\(.name)\",container_id=\"\(.containerID)\""'
203
+ jq_filter+=') | '
204
+ jq_filter+='sub("docker://";"")'
205
+
206
+ local containers
207
+ if ! containers=$(jq -r "${jq_filter}" <<< "$pods" 2>&1); then
208
+ warning "${funcname}: error on 'jq' parse: ${containers}."
209
+ return 1
210
+ fi
211
+
212
+ # available labels:
213
+ # namespace, pod_name, pod_uid, container_name, container_id
214
+ local labels
215
+ if [ -n "$cntr_id" ]; then
216
+ if labels=$(grep "$cntr_id" <<< "$containers" 2> /dev/null); then
217
+ name="cntr_$(get_lbl_val "$labels" namespace)_$(get_lbl_val "$labels" pod_name)_$(get_lbl_val "$labels" container_name)"
218
+ fi
219
+ elif [ -n "$pod_uid" ]; then
220
+ if labels=$(grep "$pod_uid" -m 1 <<< "$containers" 2> /dev/null); then
221
+ labels="${labels%%,cont*}"
222
+ name="pod_$(get_lbl_val "$labels" namespace)_$(get_lbl_val "$labels" pod_name)"
223
+ fi
224
+ fi
225
+
226
+ # jq filter nonexistent field and nonexistent label value is 'null'
227
+ if [[ $name =~ _null(_|$) ]]; then
228
+ warning "${funcname}: invalid name: $name (cgroup '$id')"
229
+ name=""
230
+ fi
231
+
232
+ echo "$name"
233
+ [ -n "$name" ]
234
+ return
235
+}
236
+
237
+function k8s_get_name() {
238
+ local funcname="${FUNCNAME[0]}"
239
+ local id="${1}"
240
+
241
+ NAME=$(k8s_get_kubepod_name "$id")
242
+
243
+ if [ -z "${NAME}" ]; then
244
+ warning "${funcname}: cannot find the name of cgroup with id '${id}'. Setting name to ${id} and disabling it."
245
+ NAME="${id}"
246
+ NAME_NOT_FOUND=3
247
+ else
248
+ NAME="k8s_${NAME}"
249
+ info "${funcname}: cgroup '${id}' has chart name '${NAME}'"
250
+ fi
251
}
252
253
function docker_get_name() {