@cryptotaxi247 / netdata-1 / commits / 6e17cb0fd

improvement(go.d/k8sstate): collect pod status reason (#18887)

Ilya Mashchenko committed Oct 28, 2024 at 18:59 UTC 6e17cb0fd453ff687fc7f8c999f8042072e93a40
7 files changed +159 -85
src/go/plugin/go.d/modules/k8s_state/charts.go
+38 -27
@@ -43,6 +43,7 @@ const (
43 prioPodMemLimitsUsed
44 prioPodCondition
45 prioPodPhase
46 + prioPodStatusReason
47 prioPodAge
48 prioPodContainersCount
49 prioPodContainersState
@@ -106,6 +107,7 @@ var podChartsTmpl = module.Charts{
107 podMemLimitsUsedChartTmpl.Copy(),
108 podConditionChartTmpl.Copy(),
109 podPhaseChartTmpl.Copy(),
110 + podStatusReasonChartTmpl.Copy(),
111 podAgeChartTmpl.Copy(),
112 podContainersCountChartTmpl.Copy(),
113 podContainersStateChartTmpl.Copy(),
@@ -247,15 +249,24 @@ var (
249 },
250 }
251 // condition
250 - nodeConditionsChartTmpl = module.Chart{
251 - IDSep: true,
252 - ID: "node_%s.condition_status",
253 - Title: "Condition status",
254 - Units: "status",
255 - Fam: "node condition",
256 - Ctx: "k8s_state.node_condition",
257 - Priority: prioNodeConditions,
258 - }
252 + nodeConditionsChartTmpl = func() module.Chart {
253 + chart := module.Chart{
254 + IDSep: true,
255 + ID: "node_%s.condition_status",
256 + Title: "Condition status",
257 + Units: "status",
258 + Fam: "node condition",
259 + Ctx: "k8s_state.node_condition",
260 + Priority: prioNodeConditions,
261 + }
262 + for _, v := range nodeConditionStatuses {
263 + chart.Dims = append(chart.Dims, &module.Dim{
264 + ID: "node_%s_cond_" + v,
265 + Name: v,
266 + })
267 + }
268 + return chart
269 + }()
270 nodeSchedulabilityChartTmpl = module.Chart{
271 IDSep: true,
272 ID: "node_%s.schedulability",
@@ -426,24 +437,6 @@ func (ks *KubeState) removeNodeCharts(ns *nodeState) {
437 }
438 }
439
429 -func (ks *KubeState) addNodeConditionToCharts(ns *nodeState, cond string) {
430 - id := fmt.Sprintf(nodeConditionsChartTmpl.ID, replaceDots(ns.id()))
431 - c := ks.Charts().Get(id)
432 - if c == nil {
433 - ks.Warningf("chart '%s' does not exist", id)
434 - return
435 - }
436 - dim := &module.Dim{
437 - ID: fmt.Sprintf("node_%s_cond_%s", ns.id(), strings.ToLower(cond)),
438 - Name: cond,
439 - }
440 - if err := c.AddDim(dim); err != nil {
441 - ks.Warning(err)
442 - return
443 - }
444 - c.MarkNotCreated()
445 -}
446 -
440 var (
441 podCPURequestsUsedChartTmpl = module.Chart{
442 IDSep: true,
@@ -523,6 +516,24 @@ var (
516 {ID: "pod_%s_phase_pending", Name: "pending"},
517 },
518 }
519 + podStatusReasonChartTmpl = func() module.Chart {
520 + chart := module.Chart{
521 + IDSep: true,
522 + ID: "pod_%s.status_reason",
523 + Title: "Status reason",
524 + Units: "status",
525 + Fam: "pod status",
526 + Ctx: "k8s_state.pod_status_reason",
527 + Priority: prioPodStatusReason,
528 + }
529 + for _, v := range podStatusReasons {
530 + chart.Dims = append(chart.Dims, &module.Dim{
531 + ID: "pod_%s_status_reason_" + v,
532 + Name: v,
533 + })
534 + }
535 + return chart
536 + }()
537 podAgeChartTmpl = module.Chart{
538 IDSep: true,
539 ID: "pod_%s.age",
src/go/plugin/go.d/modules/k8s_state/collect.go
+48 -14
@@ -6,7 +6,6 @@ import (
6 "errors"
7 "fmt"
8 "slices"
9 - "strings"
9 "time"
10
11 "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
@@ -17,28 +16,47 @@ import (
16 const precision = 1000
17
18 var (
19 + podStatusReasons = []string{
20 + "Evicted",
21 + "NodeAffinity",
22 + "NodeLost",
23 + "Shutdown",
24 + "UnexpectedAdmissionError",
25 + "Other",
26 + }
27 +
28 containerWaitingStateReasons = []string{
21 - "PodInitializing",
29 "ContainerCreating",
30 "CrashLoopBackOff",
31 "CreateContainerConfigError",
32 + "CreateContainerError",
33 "ErrImagePull",
34 "ImagePullBackOff",
27 - "CreateContainerError",
35 "InvalidImageName",
36 + "PodInitializing",
37 "Other",
38 }
39 containerTerminatedStateReasons = []string{
32 - "OOMKilled",
40 "Completed",
34 - "Error",
41 "ContainerCannotRun",
42 "DeadlineExceeded",
43 + "Error",
44 "Evicted",
45 + "OOMKilled",
46 "Other",
47 }
48 )
49
50 +var (
51 + nodeConditionStatuses = []string{
52 + "Ready",
53 + "DiskPressure",
54 + "MemoryPressure",
55 + "NetworkUnavailable",
56 + "PIDPressure",
57 + }
58 +)
59 +
60 func (ks *KubeState) collect() (map[string]int64, error) {
61 if ks.discoverer == nil {
62 return nil, errors.New("nil discoverer")
@@ -56,6 +74,7 @@ func (ks *KubeState) collect() (map[string]int64, error) {
74
75 ks.kubeClusterID = ks.getKubeClusterID()
76 ks.kubeClusterName = ks.getKubeClusterName()
77 +
78 if chart := ks.Charts().Get(discoveryStatusChart.ID); chart != nil {
79 chart.Labels = []module.Label{
80 {Key: labelKeyClusterID, Value: ks.kubeClusterID, Source: module.LabelSourceK8s},
@@ -92,7 +111,7 @@ func (ks *KubeState) collectKubeState(mx map[string]int64) {
111 func (ks *KubeState) collectPodsState(mx map[string]int64) {
112 now := time.Now()
113 for _, ps := range ks.state.pods {
95 - // Skip cronjobs (each of them is a unique container because name contains hash)
114 + // Skip cronjobs (each of them is a unique container because the name contains hash)
115 // to avoid overwhelming Netdata with high cardinality metrics.
116 // Related issue https://github.com/netdata/netdata/issues/16412
117 if ps.controllerKind == "Job" {
@@ -104,6 +123,7 @@ func (ks *KubeState) collectPodsState(mx map[string]int64) {
123 ks.removePodCharts(ps)
124 continue
125 }
126 +
127 if ps.new {
128 ps.new = false
129 ks.addPodCharts(ps)
@@ -130,12 +150,14 @@ func (ks *KubeState) collectPodsState(mx map[string]int64) {
150 ns.stats.podsPhaseRunning += boolToInt(ps.phase == corev1.PodRunning)
151 ns.stats.podsPhaseSucceeded += boolToInt(ps.phase == corev1.PodSucceeded)
152 ns.stats.podsPhaseFailed += boolToInt(ps.phase == corev1.PodFailed)
153 +
154 for _, cs := range ps.initContainers {
155 ns.stats.initContainers++
156 ns.stats.initContStateRunning += boolToInt(cs.stateRunning)
157 ns.stats.initContStateWaiting += boolToInt(cs.stateWaiting)
158 ns.stats.initContStateTerminated += boolToInt(cs.stateTerminated)
159 }
160 +
161 for _, cs := range ps.containers {
162 ns.stats.containers++
163 ns.stats.contStateRunning += boolToInt(cs.stateRunning)
@@ -155,6 +177,17 @@ func (ks *KubeState) collectPodsState(mx map[string]int64) {
177 mx[px+"phase_succeeded"] = boolToInt(ps.phase == corev1.PodSucceeded)
178 mx[px+"phase_pending"] = boolToInt(ps.phase == corev1.PodPending)
179 mx[px+"age"] = int64(now.Sub(ps.creationTime).Seconds())
180 +
181 + for _, v := range podStatusReasons {
182 + mx[px+"status_reason_"+v] = 0
183 + }
184 + if v := ps.statusReason; v != "" {
185 + if !slices.Contains(podStatusReasons, v) {
186 + v = "Other"
187 + }
188 + mx[px+"status_reason_"+v] = 1
189 + }
190 +
191 mx[px+"cpu_requests_used"] = ps.reqCPU
192 mx[px+"cpu_limits_used"] = ps.limitCPU
193 mx[px+"mem_requests_used"] = ps.reqMem
@@ -166,6 +199,7 @@ func (ks *KubeState) collectPodsState(mx map[string]int64) {
199 mx[px+"init_containers_state_running"] = 0
200 mx[px+"init_containers_state_waiting"] = 0
201 mx[px+"init_containers_state_terminated"] = 0
202 +
203 for _, cs := range ps.initContainers {
204 mx[px+"init_containers_state_running"] += boolToInt(cs.stateRunning)
205 mx[px+"init_containers_state_waiting"] += boolToInt(cs.stateWaiting)
@@ -174,6 +208,7 @@ func (ks *KubeState) collectPodsState(mx map[string]int64) {
208 mx[px+"containers_state_running"] = 0
209 mx[px+"containers_state_waiting"] = 0
210 mx[px+"containers_state_terminated"] = 0
211 +
212 for _, cs := range ps.containers {
213 if cs.new {
214 cs.new = false
@@ -194,7 +229,7 @@ func (ks *KubeState) collectPodsState(mx map[string]int64) {
229 mx[ppx+"state_waiting_reason_"+v] = 0
230 }
231 if v := cs.waitingReason; v != "" {
197 - if !slices.Contains(containerWaitingStateReasons, cs.waitingReason) {
232 + if !slices.Contains(containerWaitingStateReasons, v) {
233 v = "Other"
234 }
235 mx[ppx+"state_waiting_reason_"+v] = 1
@@ -204,7 +239,7 @@ func (ks *KubeState) collectPodsState(mx map[string]int64) {
239 mx[ppx+"state_terminated_reason_"+v] = 0
240 }
241 if v := cs.terminatedReason; v != "" {
207 - if !slices.Contains(containerTerminatedStateReasons, cs.terminatedReason) {
242 + if !slices.Contains(containerTerminatedStateReasons, v) {
243 v = "Other"
244 }
245 mx[ppx+"state_terminated_reason_"+v] = 1
@@ -228,12 +263,11 @@ func (ks *KubeState) collectNodesState(mx map[string]int64) {
263
264 px := fmt.Sprintf("node_%s_", ns.id())
265
231 - for typ, cond := range ns.conditions {
232 - if cond.new {
233 - cond.new = false
234 - ks.addNodeConditionToCharts(ns, typ)
235 - }
236 - mx[px+"cond_"+strings.ToLower(typ)] = condStatusToInt(cond.status)
266 + for _, v := range nodeConditionStatuses {
267 + mx[px+"cond_"+v] = 0
268 + }
269 + for _, v := range ns.conditions {
270 + mx[px+"cond_"+string(v.Type)] = condStatusToInt(v.Status)
271 }
272
273 mx[px+"age"] = int64(now.Sub(ns.creationTime).Seconds())
src/go/plugin/go.d/modules/k8s_state/kube_state_test.go
+46 -20
@@ -213,11 +213,11 @@ func TestKubeState_Collect(t *testing.T) {
213 "node_node01_alloc_pods_allocated": 0,
214 "node_node01_alloc_pods_available": 110,
215 "node_node01_alloc_pods_util": 0,
216 - "node_node01_cond_diskpressure": 0,
217 - "node_node01_cond_memorypressure": 0,
218 - "node_node01_cond_networkunavailable": 0,
219 - "node_node01_cond_pidpressure": 0,
220 - "node_node01_cond_ready": 1,
216 + "node_node01_cond_DiskPressure": 0,
217 + "node_node01_cond_MemoryPressure": 0,
218 + "node_node01_cond_NetworkUnavailable": 0,
219 + "node_node01_cond_PIDPressure": 0,
220 + "node_node01_cond_Ready": 1,
221 "node_node01_schedulability_schedulable": 1,
222 "node_node01_schedulability_unschedulable": 0,
223 "node_node01_containers": 0,
@@ -240,6 +240,7 @@ func TestKubeState_Collect(t *testing.T) {
240 "node_node01_pods_readiness_ready": 0,
241 "node_node01_pods_readiness_unready": 0,
242 }
243 +
244 copyAge(expected, mx)
245
246 assert.Equal(t, expected, mx)
@@ -331,6 +332,12 @@ func TestKubeState_Collect(t *testing.T) {
332 "pod_default_pod01_phase_pending": 0,
333 "pod_default_pod01_phase_running": 1,
334 "pod_default_pod01_phase_succeeded": 0,
335 + "pod_default_pod01_status_reason_Evicted": 0,
336 + "pod_default_pod01_status_reason_NodeAffinity": 0,
337 + "pod_default_pod01_status_reason_NodeLost": 0,
338 + "pod_default_pod01_status_reason_Other": 0,
339 + "pod_default_pod01_status_reason_Shutdown": 0,
340 + "pod_default_pod01_status_reason_UnexpectedAdmissionError": 0,
341 }
342
343 copyAge(expected, mx)
@@ -375,11 +382,11 @@ func TestKubeState_Collect(t *testing.T) {
382 "node_node01_alloc_pods_allocated": 1,
383 "node_node01_alloc_pods_available": 109,
384 "node_node01_alloc_pods_util": 909,
378 - "node_node01_cond_diskpressure": 0,
379 - "node_node01_cond_memorypressure": 0,
380 - "node_node01_cond_networkunavailable": 0,
381 - "node_node01_cond_pidpressure": 0,
382 - "node_node01_cond_ready": 1,
385 + "node_node01_cond_DiskPressure": 0,
386 + "node_node01_cond_MemoryPressure": 0,
387 + "node_node01_cond_NetworkUnavailable": 0,
388 + "node_node01_cond_PIDPressure": 0,
389 + "node_node01_cond_Ready": 1,
390 "node_node01_containers": 2,
391 "node_node01_containers_state_running": 2,
392 "node_node01_containers_state_terminated": 0,
@@ -464,6 +471,12 @@ func TestKubeState_Collect(t *testing.T) {
471 "pod_default_pod01_phase_pending": 0,
472 "pod_default_pod01_phase_running": 1,
473 "pod_default_pod01_phase_succeeded": 0,
474 + "pod_default_pod01_status_reason_Evicted": 0,
475 + "pod_default_pod01_status_reason_NodeAffinity": 0,
476 + "pod_default_pod01_status_reason_NodeLost": 0,
477 + "pod_default_pod01_status_reason_Other": 0,
478 + "pod_default_pod01_status_reason_Shutdown": 0,
479 + "pod_default_pod01_status_reason_UnexpectedAdmissionError": 0,
480 }
481
482 copyAge(expected, mx)
@@ -513,11 +526,11 @@ func TestKubeState_Collect(t *testing.T) {
526 "node_node01_alloc_pods_allocated": 0,
527 "node_node01_alloc_pods_available": 110,
528 "node_node01_alloc_pods_util": 0,
516 - "node_node01_cond_diskpressure": 0,
517 - "node_node01_cond_memorypressure": 0,
518 - "node_node01_cond_networkunavailable": 0,
519 - "node_node01_cond_pidpressure": 0,
520 - "node_node01_cond_ready": 1,
529 + "node_node01_cond_DiskPressure": 0,
530 + "node_node01_cond_MemoryPressure": 0,
531 + "node_node01_cond_NetworkUnavailable": 0,
532 + "node_node01_cond_PIDPressure": 0,
533 + "node_node01_cond_Ready": 1,
534 "node_node01_schedulability_schedulable": 1,
535 "node_node01_schedulability_unschedulable": 0,
536 "node_node01_containers": 0,
@@ -632,11 +645,11 @@ func TestKubeState_Collect(t *testing.T) {
645 "node_node01_alloc_pods_allocated": 2,
646 "node_node01_alloc_pods_available": 108,
647 "node_node01_alloc_pods_util": 1818,
635 - "node_node01_cond_diskpressure": 0,
636 - "node_node01_cond_memorypressure": 0,
637 - "node_node01_cond_networkunavailable": 0,
638 - "node_node01_cond_pidpressure": 0,
639 - "node_node01_cond_ready": 1,
648 + "node_node01_cond_DiskPressure": 0,
649 + "node_node01_cond_MemoryPressure": 0,
650 + "node_node01_cond_NetworkUnavailable": 0,
651 + "node_node01_cond_PIDPressure": 0,
652 + "node_node01_cond_Ready": 1,
653 "node_node01_containers": 4,
654 "node_node01_containers_state_running": 4,
655 "node_node01_containers_state_terminated": 0,
@@ -721,6 +734,12 @@ func TestKubeState_Collect(t *testing.T) {
734 "pod_default_pod01_phase_pending": 0,
735 "pod_default_pod01_phase_running": 1,
736 "pod_default_pod01_phase_succeeded": 0,
737 + "pod_default_pod01_status_reason_Evicted": 0,
738 + "pod_default_pod01_status_reason_NodeAffinity": 0,
739 + "pod_default_pod01_status_reason_NodeLost": 0,
740 + "pod_default_pod01_status_reason_Other": 0,
741 + "pod_default_pod01_status_reason_Shutdown": 0,
742 + "pod_default_pod01_status_reason_UnexpectedAdmissionError": 0,
743 "pod_default_pod02_age": 4,
744 "pod_default_pod02_cond_containersready": 1,
745 "pod_default_pod02_cond_podinitialized": 1,
@@ -784,7 +803,14 @@ func TestKubeState_Collect(t *testing.T) {
803 "pod_default_pod02_phase_pending": 0,
804 "pod_default_pod02_phase_running": 1,
805 "pod_default_pod02_phase_succeeded": 0,
806 + "pod_default_pod02_status_reason_Evicted": 0,
807 + "pod_default_pod02_status_reason_NodeAffinity": 0,
808 + "pod_default_pod02_status_reason_NodeLost": 0,
809 + "pod_default_pod02_status_reason_Other": 0,
810 + "pod_default_pod02_status_reason_Shutdown": 0,
811 + "pod_default_pod02_status_reason_UnexpectedAdmissionError": 0,
812 }
813 +
814 copyAge(expected, mx)
815
816 assert.Equal(t, expected, mx)
src/go/plugin/go.d/modules/k8s_state/metadata.yaml
+20 -5
@@ -141,7 +141,11 @@ modules:
141 unit: status
142 chart_type: line
143 dimensions:
144 - - name: a dimension per condition
144 + - name: Ready
145 + - name: DiskPressure
146 + - name: MemoryPressure
147 + - name: NetworkUnavailable
148 + - name: PIDPressure
149 - name: k8s_state.node_schedulability
150 description: Schedulability
151 unit: state
@@ -271,6 +275,17 @@ modules:
275 - name: failed
276 - name: succeeded
277 - name: pending
278 + - name: k8s_state.pod_status_reason
279 + description: Status reason
280 + unit: status
281 + chart_type: line
282 + dimensions:
283 + - name: Evicted
284 + - name: NodeAffinity
285 + - name: NodeLost
286 + - name: Shutdown
287 + - name: UnexpectedAdmissionError
288 + - name: Other
289 - name: k8s_state.pod_age
290 description: Age
291 unit: seconds
@@ -347,24 +362,24 @@ modules:
362 unit: state
363 chart_type: line
364 dimensions:
350 - - name: PodInitializing
365 - name: ContainerCreating
366 - name: CrashLoopBackOff
367 - name: CreateContainerConfigError
368 + - name: CreateContainerError
369 - name: ErrImagePull
370 - name: ImagePullBackOff
356 - - name: CreateContainerError
371 - name: InvalidImageName
372 + - name: PodInitializing
373 - name: Other
374 - name: k8s_state.pod_container_terminated_state_reason
375 description: Container terminated state reason
376 unit: state
377 chart_type: line
378 dimensions:
364 - - name: OOMKilled
379 - name: Completed
366 - - name: Error
380 - name: ContainerCannotRun
381 - name: DeadlineExceeded
382 + - name: Error
383 - name: Evicted
384 + - name: OOMKilled
385 - name: Other
src/go/plugin/go.d/modules/k8s_state/state.go
+5 -11
@@ -19,9 +19,8 @@ func newKubeState() *kubeState {
19
20 func newNodeState() *nodeState {
21 return &nodeState{
22 - new: true,
23 - labels: make(map[string]string),
24 - conditions: make(map[string]*nodeStateCondition),
22 + new: true,
23 + labels: make(map[string]string),
24 }
25 }
26
@@ -58,16 +57,10 @@ type (
57 allocatableCPU int64
58 allocatableMem int64
59 allocatablePods int64
61 - conditions map[string]*nodeStateCondition
60 + conditions []corev1.NodeCondition
61
62 stats nodeStateStats
63 }
65 - nodeStateCondition struct {
66 - new bool
67 - // https://kubernetes.io/docs/concepts/architecture/nodes/#condition
68 - //typ corev1.NodeConditionType
69 - status corev1.ConditionStatus
70 - }
64 nodeStateStats struct {
65 reqCPU int64
66 limitCPU int64
@@ -127,7 +120,8 @@ type (
120 condPodInitialized corev1.ConditionStatus
121 condPodReady corev1.ConditionStatus
122 // https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
130 - phase corev1.PodPhase
123 + phase corev1.PodPhase
124 + statusReason string
125
126 initContainers map[string]*containerState
127 containers map[string]*containerState
src/go/plugin/go.d/modules/k8s_state/update_node_state.go
+1 -8
@@ -36,12 +36,5 @@ func (ks *KubeState) updateNodeState(r resource) {
36 }
37
38 ns.unSchedulable = node.Spec.Unschedulable
39 -
40 - for _, c := range node.Status.Conditions {
41 - if v, ok := ns.conditions[string(c.Type)]; !ok {
42 - ns.conditions[string(c.Type)] = &nodeStateCondition{new: true, status: c.Status}
43 - } else {
44 - v.status = c.Status
45 - }
46 - }
39 + ns.conditions = node.Status.Conditions
40 }
src/go/plugin/go.d/modules/k8s_state/update_pod_state.go
+1
@@ -78,6 +78,7 @@ func (ks *KubeState) updatePodState(r resource) {
78 }
79
80 ps.phase = pod.Status.Phase
81 + ps.statusReason = pod.Status.Reason
82
83 for _, cntr := range pod.Status.ContainerStatuses {
84 cs, ok := ps.containers[cntr.Name]