When running from within a k8s pod, use the k8s API to get the cgroup name (#5576)
* When running from within a k8s pod, use the k8s API to get the pod name * Check the last part of an underscore-delimited or slash-delimited container id * Add a docker image builder that adds a single image to a user-specified registry, for use in k8s * When running in k8s, disable cgroups that the pod API does not return * Use longer name for k8s containers * Add reference to build-test.sh to packaging/docker/README.md * Anonymous statistics should not break when /proc/1/sched is not available
Chris Akritidis committed
Mar 13, 2019 at 12:47 UTC
b7998ec82d579bd0550e23f880decdcee61264f5
5 files changed
+144
-22
collectors/cgroups.plugin/cgroup-name.sh.in
+30
-13
@@ -64,6 +64,28 @@ function docker_get_name_api() {
64
return 0
65
}
66
67
+function k8s_get_name() {
68
+ # Take the last part of the delimited path identifier (expecting either _ or / as a delimiter).
69
+ local id="${1##*_}"
70
+ if [ "${id}" == "${1}" ]; then
71
+ id="${1##*/}"
72
+ fi
73
+ KUBE_TOKEN="$(</var/run/secrets/kubernetes.io/serviceaccount/token)"
74
+ NAME="$(
75
+curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/pods" |
76
+jq -r '.items[] | "k8s_\(.metadata.namespace)_\(.metadata.name)_\(.metadata.uid)_\(.status.containerStatuses[0].name) \(.status.containerStatuses[0].containerID)"' |
77
+grep "$id" |
78
+cut -d' ' -f1
79
+)"
80
+ if [ -z "${NAME}" ]; then
81
+ warning "cannot find the name of k8s pod with containerID '${id}'. Setting name to ${id} and disabling it"
82
+ NAME="${id}"
83
+ NAME_NOT_FOUND=3
84
+ else
85
+ info "k8s containerID '${id}' has chart name (namespace_podname_poduid_containername) '${NAME}'"
86
+ fi
87
+}
88
+
89
function docker_get_name() {
90
local id="${1}"
91
if hash docker 2>/dev/null; then
@@ -73,7 +95,7 @@ function docker_get_name() {
95
fi
96
if [ -z "${NAME}" ]; then
97
warning "cannot find the name of docker container '${id}'"
76
- NAME_NOT_FOUND=1
98
+ NAME_NOT_FOUND=2
99
NAME="${id:0:12}"
100
else
101
info "docker container '${id}' is named '${NAME}'"
@@ -89,6 +111,7 @@ function docker_validate_id() {
111
fi
112
}
113
114
+
115
# -----------------------------------------------------------------------------
116
117
[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
@@ -118,25 +141,22 @@ for CONFIG in "${NETDATA_USER_CONFIG_DIR}/cgroups-names.conf" "${NETDATA_STOCK_C
141
fi
142
done
143
144
+if [ -z "${NAME}" ] && [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ] && [[ ${CGROUP} =~ ^.*kubepods.* ]]; then
145
+ k8s_get_name "${CGROUP}"
146
+fi
147
+
148
if [ -z "${NAME}" ]; then
149
if [[ ${CGROUP} =~ ^.*docker[-_/\.][a-fA-F0-9]+[-_\.]?.*$ ]]; then
150
# docker containers
151
#shellcheck disable=SC1117
152
DOCKERID="$(echo "${CGROUP}" | sed "s|^.*docker[-_/]\([a-fA-F0-9]\+\)[-_\.]\?.*$|\1|")"
153
docker_validate_id "${DOCKERID}"
127
-
154
elif [[ ${CGROUP} =~ ^.*ecs[-_/\.][a-fA-F0-9]+[-_\.]?.*$ ]]; then
155
# ECS
156
#shellcheck disable=SC1117
157
DOCKERID="$(echo "${CGROUP}" | sed "s|^.*ecs[-_/].*[-_/]\([a-fA-F0-9]\+\)[-_\.]\?.*$|\1|")"
158
docker_validate_id "${DOCKERID}"
159
134
- elif [[ ${CGROUP} =~ ^.*kubepods[_/].*[_/]pod[a-fA-F0-9-]+[_/][a-fA-F0-9]+$ ]]; then
135
- # kubernetes
136
- #shellcheck disable=SC1117
137
- DOCKERID="$(echo "${CGROUP}" | sed "s|^.*kubepods[_/].*[_/]pod[a-fA-F0-9-]\+[_/]\([a-fA-F0-9]\+\)$|\1|")"
138
- docker_validate_id "${DOCKERID}"
139
-
160
elif [[ ${CGROUP} =~ machine.slice[_/].*\.service ]]; then
161
# systemd-nspawn
162
NAME="$(echo "${CGROUP}" | sed 's/.*machine.slice[_\/]\(.*\)\.service/\1/g')"
@@ -177,8 +197,5 @@ fi
197
info "cgroup '${CGROUP}' is called '${NAME}'"
198
echo "${NAME}"
199
180
-if [ "${NAME_NOT_FOUND}" -eq 1 ]; then
181
- exit 2
182
-else
183
- exit 0
184
-fi
200
+exit ${NAME_NOT_FOUND}
201
+
collectors/cgroups.plugin/sys_fs_cgroup.c
+7
-3
@@ -893,7 +893,7 @@ static inline void cgroup_get_chart_name(struct cgroup *cg) {
893
894
snprintfz(command, CGROUP_CHARTID_LINE_MAX, "exec %s '%s'", cgroups_rename_script, cg->chart_id);
895
896
- debug(D_CGROUP, "executing command \"%s\" for cgroup '%s'", command, cg->id);
896
+ debug(D_CGROUP, "executing command \"%s\" for cgroup '%s'", command, cg->chart_id);
897
FILE *fp = mypopen(command, &cgroup_pid);
898
if(fp) {
899
// debug(D_CGROUP, "reading from command '%s' for cgroup '%s'", command, cg->id);
@@ -904,12 +904,16 @@ static inline void cgroup_get_chart_name(struct cgroup *cg) {
904
// debug(D_CGROUP, "closed command for cgroup '%s'", cg->id);
905
906
if(s && *s && *s != '\n') {
907
- debug(D_CGROUP, "cgroup '%s' should be renamed to '%s'", cg->id, s);
907
+ debug(D_CGROUP, "cgroup '%s' should be renamed to '%s'", cg->chart_id, s);
908
909
s = trim(s);
910
if (s) {
911
- if(likely(!name_error))
911
+ if(likely(name_error==0))
912
cg->pending_renames = 0;
913
+ else if (unlikely(name_error==3)) {
914
+ debug(D_CGROUP, "cgroup '%s' disabled based due to rename command output", cg->chart_id);
915
+ cg->enabled = 0;
916
+ }
917
918
if(likely(cg->pending_renames < 2)) {
919
freez(cg->chart_title);
daemon/anonymous-statistics.sh.in
+7
-6
@@ -85,13 +85,14 @@ fi
85
# -------------------------------------------------------------------------------------------------
86
# detect containers with heuristics
87
88
-if [ "${CONTAINER}" = "unknown" ] ; then
89
- IFS='(, ' read -r process _ </proc/1/sched
90
- if [ "${process}" = "netdata" ]; then
91
- CONTAINER="container"
92
- CONT_DETECTION="process"
88
+if [ "${CONTAINER}" = "unknown" ]; then
89
+ if [ -f /proc/1/sched ] ; then
90
+ IFS='(, ' read -r process _ </proc/1/sched
91
+ if [ "${process}" = "netdata" ]; then
92
+ CONTAINER="container"
93
+ CONT_DETECTION="process"
94
+ fi
95
fi
94
-
96
# ubuntu and debian supply /bin/running-in-container
97
# https://www.apt-browse.org/browse/ubuntu/trusty/main/i386/upstart/1.12.1-0ubuntu4/file/bin/running-in-container
98
if /bin/running-in-container >/dev/null 2>&1; then
packaging/docker/README.md
+32
@@ -124,3 +124,35 @@ services:
124
You can restrict access by following [official caddy guide](https://caddyserver.com/docs/basicauth) and adding lines to Caddyfile.
125
126
[]()
127
+
128
+### Publish a test image to your own repository
129
+
130
+The script `packaging/docker/build-test.sh` can be used to create an image and upload it to a repository of your choosing.
131
+
132
+```
133
+Usage: packaging/docker/build-test.sh -r <REPOSITORY> -v <VERSION> -u <DOCKER_USERNAME> -p <DOCKER_PASSWORD> [-s]
134
+ -s skip build, just push the image
135
+Builds an amd64 image and pushes it to the docker hub repository REPOSITORY
136
+```
137
+
138
+This is especially useful when testing a Pull Request for Kubernetes, since you can set `image` to an immutable repository and tag, set the `imagePullPolicy` to `Always` and just keep uploading new images.
139
+
140
+Example:
141
+
142
+We get a local copy of the Helm chart at https://github.com/netdata/helmchart. We modify `values.yaml` to have the following:
143
+
144
+```
145
+image:
146
+ repository: cakrit/netdata-prs
147
+ tag: PR5576
148
+ pullPolicy: Always
149
+```
150
+
151
+We check out PR5576 and run the following:
152
+```
153
+./packaging/docker/build-test.sh -r cakrit/netdata-prs -v PR5576 -u cakrit -p 'XXX'
154
+```
155
+
156
+Then we can run `helm install [path to our helmchart clone]`.
157
+
158
+If we make changes to the code, we execute the same `build-test.sh` command, followed by `helm upgrade [name] [path to our helmchart clone]`
\ No newline at end of file
packaging/docker/build-test.sh
new
+68
@@ -0,0 +1,68 @@
1
+#!/bin/bash
2
+# SPDX-License-Identifier: GPL-3.0-or-later
3
+# Author : Chris Akritidis (cakrit)
4
+# Cross-arch docker build helper script
5
+
6
+printhelp() {
7
+ echo "Usage: packaging/docker/build-test.sh -r <REPOSITORY> -v <VERSION> -u <DOCKER_USERNAME> -p <DOCKER_PASSWORD> [-s]
8
+ -s skip build, just push the image
9
+Builds an amd64 image and pushes it to the docker hub repository REPOSITORY"
10
+}
11
+
12
+set -e
13
+
14
+if [ ! -f .gitignore ]; then
15
+ echo "Run as ./packaging/docker/$(basename "$0") from top level directory of git repository"
16
+ exit 1
17
+fi
18
+
19
+DOBUILD=1
20
+while getopts :r:v:u:p:s option
21
+do
22
+ case "$option" in
23
+ r)
24
+ REPOSITORY=$OPTARG
25
+ ;;
26
+ v)
27
+ VERSION=$OPTARG
28
+ ;;
29
+ u)
30
+ DOCKER_USERNAME=$OPTARG
31
+ ;;
32
+ p)
33
+ DOCKER_PASSWORD=$OPTARG
34
+ ;;
35
+ s)
36
+ DOBUILD=0
37
+ ;;
38
+ *)
39
+ printhelp
40
+ exit 1
41
+ ;;
42
+ esac
43
+done
44
+
45
+if [ -n "${REPOSITORY}" ] && [ -n "${VERSION}" ] && [ -n "${DOCKER_USERNAME}" ] && [ -n "${DOCKER_PASSWORD}" ] ; then
46
+ if [ $DOBUILD -eq 1 ] ; then
47
+ echo "Building ${VERSION} of ${REPOSITORY} container"
48
+ docker run --rm --privileged multiarch/qemu-user-static:register --reset
49
+
50
+ # Build images using multi-arch Dockerfile.
51
+ eval docker build --build-arg ARCH="amd64" --tag "${REPOSITORY}:${VERSION}" --file packaging/docker/Dockerfile ./
52
+
53
+ # Create temporary docker CLI config with experimental features enabled (manifests v2 need it)
54
+ mkdir -p /tmp/docker
55
+ #echo '{"experimental":"enabled"}' > /tmp/docker/config.json
56
+ fi
57
+
58
+ # Login to docker hub to allow futher operations
59
+ echo "Logging into docker"
60
+ echo "$DOCKER_PASSWORD" | docker --config /tmp/docker login -u "$DOCKER_USERNAME" --password-stdin
61
+
62
+ echo "Pushing ${REPOSITORY}:${VERSION}"
63
+ docker --config /tmp/docker push "${REPOSITORY}:${VERSION}"
64
+else
65
+ echo "Missing parameter. REPOSITORY=${REPOSITORY} VERSION=${VERSION} DOCKER_USERNAME=${DOCKER_USERNAME} DOCKER_PASSWORD=${DOCKER_PASSWORD}"
66
+ printhelp
67
+ exit 1
68
+fi