@cryptotaxi247 / netdata-1 / commits / 955b78ba5

improvement(go.d/k8s_state): add more CronJob metrics (#19796)

Ilya Mashchenko committed Mar 7, 2025 at 23:21 UTC 955b78ba523dc386218d77a14a10c387b0abc3ef
7 files changed +219 -51
src/go/plugin/go.d/collector/k8s_state/charts.go
+73 -1
@@ -63,6 +63,11 @@ const (
63
64 const (
65 prioCronJobJobsCountByStatus = 50700 + iota
66 + prioCronJobJobsFailedByReason
67 + prioCronJobLastExecutionStatus
68 + prioCronJobLastCompletionDuration
69 + prioCronJobLastCompletedTimeAgo
70 + prioCronJobLastScheduleTimeAgo
71 prioCronJobAge
72 )
73
@@ -143,6 +148,11 @@ var deploymentChartsTmpl = module.Charts{
148
149 var cronJobChartsTmpl = module.Charts{
150 cronJobJobsCountByStatusChartTmpl.Copy(),
151 + cronJobJobsFailedByReasonChartTmpl.Copy(),
152 + cronJobLastExecutionStatusChartTmpl.Copy(),
153 + cronJobLastCompletionDurationChartTmpl.Copy(),
154 + cronJobLastCompletedTimeAgoChartTmpl.Copy(),
155 + cronJobLastScheduleTimeAgoChartTmpl.Copy(),
156 cronJobAgeChartTmpl.Copy(),
157 }
158
@@ -849,7 +859,6 @@ var (
859 Fam: "cronjob jobs",
860 Ctx: "k8s_state.cronjob_jobs_count_by_status",
861 Priority: prioCronJobJobsCountByStatus,
852 - Type: module.Stacked,
862 Dims: module.Dims{
863 {ID: "cronjob_%s_complete_jobs", Name: "completed"},
864 {ID: "cronjob_%s_failed_jobs", Name: "failed"},
@@ -857,6 +866,69 @@ var (
866 {ID: "cronjob_%s_suspended_jobs", Name: "suspended"},
867 },
868 }
869 + cronJobJobsFailedByReasonChartTmpl = module.Chart{
870 + IDSep: true,
871 + ID: "cronjob_%s.jobs_failed_by_reason",
872 + Title: "CronJob Jobs Failed by Reason",
873 + Units: "jobs",
874 + Fam: "cronjob jobs",
875 + Ctx: "k8s_state.cronjob_jobs_failed_by_reason",
876 + Priority: prioCronJobJobsFailedByReason,
877 + Dims: module.Dims{
878 + {ID: "cronjob_%s_failed_jobs_reason_pod_failure_policy", Name: "pod_failure_policy"},
879 + {ID: "cronjob_%s_failed_jobs_reason_backoff_limit_exceeded", Name: "backoff_limit_exceeded"},
880 + {ID: "cronjob_%s_failed_jobs_reason_deadline_exceeded", Name: "deadline_exceeded"},
881 + },
882 + }
883 + cronJobLastExecutionStatusChartTmpl = module.Chart{
884 + IDSep: true,
885 + ID: "cronjob_%s.last_execution_status",
886 + Title: "CronJob Last Execution Status",
887 + Units: "status",
888 + Fam: "cronjob execution",
889 + Ctx: "k8s_state.cronjob_last_execution_status",
890 + Priority: prioCronJobLastExecutionStatus,
891 + Dims: module.Dims{
892 + {ID: "cronjob_%s_last_execution_status_succeeded", Name: "completed"},
893 + {ID: "cronjob_%s_last_execution_status_failed", Name: "failed"},
894 + },
895 + }
896 + cronJobLastCompletionDurationChartTmpl = module.Chart{
897 + IDSep: true,
898 + ID: "cronjob_%s.last_completion_duration",
899 + Title: "CronJob Last Completion Duration",
900 + Units: "seconds",
901 + Fam: "cronjob execution",
902 + Ctx: "k8s_state.cronjob_last_completion_duration",
903 + Priority: prioCronJobLastCompletionDuration,
904 + Dims: module.Dims{
905 + {ID: "cronjob_%s_last_completion_duration", Name: "last_completion"},
906 + },
907 + }
908 + cronJobLastCompletedTimeAgoChartTmpl = module.Chart{
909 + IDSep: true,
910 + ID: "cronjob_%s.last_completed_time_ago",
911 + Title: "CronJob Last Completed Time Ago",
912 + Units: "seconds",
913 + Fam: "cronjob execution",
914 + Ctx: "k8s_state.cronjob_last_completed_time_ago",
915 + Priority: prioCronJobLastCompletedTimeAgo,
916 + Dims: module.Dims{
917 + {ID: "cronjob_%s_last_successful_seconds_ago", Name: "last_completed_ago"},
918 + },
919 + }
920 + cronJobLastScheduleTimeAgoChartTmpl = module.Chart{
921 + IDSep: true,
922 + ID: "cronjob_%s.last_schedule_time_ago",
923 + Title: "CronJob Last Schedule Time Ago",
924 + Units: "seconds",
925 + Fam: "cronjob execution",
926 + Ctx: "k8s_state.cronjob_last_schedule_time_ago",
927 + Priority: prioCronJobLastScheduleTimeAgo,
928 + Dims: module.Dims{
929 + {ID: "cronjob_%s_last_schedule_seconds_ago", Name: "last_schedule_ago"},
930 + },
931 + }
932 cronJobAgeChartTmpl = module.Chart{
933 IDSep: true,
934 ID: "cronjob_%s.age",
src/go/plugin/go.d/collector/k8s_state/collect.go
+52 -14
@@ -375,31 +375,69 @@ func (c *Collector) collectCronJobState(mx map[string]int64) {
375 px := fmt.Sprintf("cronjob_%s_", st.id())
376
377 mx[px+"age"] = int64(now.Sub(st.creationTime).Seconds())
378 + if st.lastScheduleTime != nil {
379 + mx[px+"last_schedule_seconds_ago"] = int64(now.Sub(*st.lastScheduleTime).Seconds())
380 + }
381 + if st.lastSuccessfulTime != nil {
382 + mx[px+"last_successful_seconds_ago"] = int64(now.Sub(*st.lastSuccessfulTime).Seconds())
383 + }
384
385 mx[px+"running_jobs"] = 0
386 mx[px+"failed_jobs"] = 0
387 mx[px+"complete_jobs"] = 0
388 + mx[px+"complete_jobs"] = 0
389 mx[px+"suspended_jobs"] = 0
390
391 + mx[px+"failed_jobs_reason_pod_failure_policy"] = 0
392 + mx[px+"failed_jobs_reason_backoff_limit_exceeded"] = 0
393 + mx[px+"failed_jobs_reason_deadline_exceeded"] = 0
394 +
395 + mx[px+"last_execution_status_succeeded"] = 0
396 + mx[px+"last_execution_status_failed"] = 0
397 +
398 + var lastExecutedEndTime time.Time
399 + var lastCompleteTime time.Time
400 +
401 for _, job := range c.state.jobs {
385 - switch {
386 - case job.controller.kind != "CronJob", job.controller.uid != st.uid, job.startTime == nil:
402 + if job.controller.kind != "CronJob" || job.controller.uid != st.uid || job.startTime == nil {
403 continue
388 - case job.active > 0:
404 + }
405 + if job.active > 0 {
406 mx[px+"running_jobs"]++
390 - case len(job.conditions) > 0:
391 - for _, cond := range job.conditions {
392 - if cond.Status != corev1.ConditionTrue {
393 - continue
407 + continue
408 + }
409 +
410 + for _, cond := range job.conditions {
411 + if cond.Status != corev1.ConditionTrue {
412 + continue
413 + }
414 +
415 + switch cond.Type {
416 + case batchv1.JobComplete:
417 + mx[px+"complete_jobs"]++
418 + if job.completionTime != nil {
419 + if job.completionTime.After(lastExecutedEndTime) {
420 + lastExecutedEndTime = *job.completionTime
421 + mx[px+"last_execution_status_succeeded"] = 1
422 + mx[px+"last_execution_status_failed"] = 0
423 + }
424 + if job.completionTime.After(lastCompleteTime) {
425 + lastCompleteTime = *job.completionTime
426 + mx[px+"last_completion_duration"] = int64(job.completionTime.Sub(*job.startTime).Seconds())
427 + }
428 }
395 - switch cond.Type {
396 - case batchv1.JobFailed:
397 - mx[px+"failed_jobs"]++
398 - case batchv1.JobComplete:
399 - mx[px+"complete_jobs"]++
400 - case batchv1.JobSuspended:
401 - mx[px+"suspended_jobs"]++
429 + case batchv1.JobFailed:
430 + mx[px+"failed_jobs"]++
431 + mx[px+"failed_jobs_reason_pod_failure_policy"] += metrix.Bool(cond.Reason == batchv1.JobReasonPodFailurePolicy)
432 + mx[px+"failed_jobs_reason_backoff_limit_exceeded"] += metrix.Bool(cond.Reason == batchv1.JobReasonBackoffLimitExceeded)
433 + mx[px+"failed_jobs_reason_deadline_exceeded"] += metrix.Bool(cond.Reason == batchv1.JobReasonDeadlineExceeded)
434 + if cond.LastTransitionTime.Time.After(lastExecutedEndTime) {
435 + lastExecutedEndTime = cond.LastTransitionTime.Time
436 + mx[px+"last_execution_status_succeeded"] = 0
437 + mx[px+"last_execution_status_failed"] = 1
438 }
439 + case batchv1.JobSuspended:
440 + mx[px+"suspended_jobs"]++
441 }
442 }
443 }
src/go/plugin/go.d/collector/k8s_state/collector_test.go
+39 -16
@@ -533,16 +533,24 @@ func TestCollector_Collect(t *testing.T) {
533 step1 := func(t *testing.T, collr *Collector) {
534 mx := collr.Collect(context.Background())
535 expected := map[string]int64{
536 - "cronjob_default_cronjob01_age": 10,
537 - "cronjob_default_cronjob01_complete_jobs": 1,
538 - "cronjob_default_cronjob01_failed_jobs": 1,
539 - "cronjob_default_cronjob01_running_jobs": 1,
540 - "cronjob_default_cronjob01_suspended_jobs": 1,
541 - "discovery_node_discoverer_state": 1,
542 - "discovery_pod_discoverer_state": 1,
536 + "cronjob_default_cronjob01_age": 10,
537 + "cronjob_default_cronjob01_complete_jobs": 1,
538 + "cronjob_default_cronjob01_failed_jobs": 1,
539 + "cronjob_default_cronjob01_failed_jobs_reason_backoff_limit_exceeded": 0,
540 + "cronjob_default_cronjob01_failed_jobs_reason_deadline_exceeded": 1,
541 + "cronjob_default_cronjob01_failed_jobs_reason_pod_failure_policy": 0,
542 + "cronjob_default_cronjob01_last_completion_duration": 60,
543 + "cronjob_default_cronjob01_last_execution_status_failed": 0,
544 + "cronjob_default_cronjob01_last_execution_status_succeeded": 1,
545 + "cronjob_default_cronjob01_last_schedule_seconds_ago": 130,
546 + "cronjob_default_cronjob01_last_successful_seconds_ago": 70,
547 + "cronjob_default_cronjob01_running_jobs": 1,
548 + "cronjob_default_cronjob01_suspended_jobs": 1,
549 + "discovery_node_discoverer_state": 1,
550 + "discovery_pod_discoverer_state": 1,
551 }
552
545 - copyAge(expected, mx)
553 + copyIfSuffix(expected, mx, "age", "ago")
554
555 assert.Equal(t, expected, mx)
556 assert.Equal(t,
@@ -1068,6 +1076,10 @@ func prepareCronJob(name string) *batchv1.CronJob {
1076 UID: types.UID(name),
1077 CreationTimestamp: metav1.Time{Time: time.Now()},
1078 },
1079 + Status: batchv1.CronJobStatus{
1080 + LastScheduleTime: &metav1.Time{Time: time.Now().Add(-2 * time.Minute)},
1081 + LastSuccessfulTime: &metav1.Time{Time: time.Now().Add(-1 * time.Minute)},
1082 + },
1083 }
1084 }
1085
@@ -1084,7 +1096,8 @@ func prepareCronJobRunningJob(name string, cj *batchv1.CronJob) *batchv1.Job {
1096
1097 func prepareCronJobCompleteJob(name string, cj *batchv1.CronJob) *batchv1.Job {
1098 job := prepareCronJobJob(name, cj)
1087 - job.Status.StartTime = &metav1.Time{Time: time.Now()}
1099 + job.Status.StartTime = &metav1.Time{Time: time.Now().Add(-1 * time.Minute)}
1100 + job.Status.CompletionTime = &metav1.Time{Time: time.Now()}
1101 job.Status.Conditions = []batchv1.JobCondition{
1102 {Type: batchv1.JobComplete, Status: corev1.ConditionTrue},
1103 }
@@ -1095,7 +1108,12 @@ func prepareCronJobFailedJob(name string, cj *batchv1.CronJob) *batchv1.Job {
1108 job := prepareCronJobJob(name, cj)
1109 job.Status.StartTime = &metav1.Time{Time: time.Now()}
1110 job.Status.Conditions = []batchv1.JobCondition{
1098 - {Type: batchv1.JobFailed, Status: corev1.ConditionTrue},
1111 + {
1112 + Type: batchv1.JobFailed,
1113 + Status: corev1.ConditionTrue,
1114 + LastTransitionTime: metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
1115 + Reason: batchv1.JobReasonDeadlineExceeded,
1116 + },
1117 }
1118 return job
1119 }
@@ -1161,17 +1179,22 @@ func mustQuantity(s string) apiresource.Quantity {
1179 return q
1180 }
1181
1164 -func copyAge(dst, src map[string]int64) {
1182 +func copyIfSuffix(dst, src map[string]int64, suffixes ...string) {
1183 for k, v := range src {
1166 - if !strings.HasSuffix(k, "_age") {
1167 - continue
1168 - }
1169 - if _, ok := dst[k]; ok {
1170 - dst[k] = v
1184 + for _, suffix := range suffixes {
1185 + if strings.HasSuffix(k, suffix) {
1186 + if _, ok := dst[k]; ok {
1187 + dst[k] = v
1188 + }
1189 + }
1190 }
1191 }
1192 }
1193
1194 +func copyAge(dst, src map[string]int64) {
1195 + copyIfSuffix(dst, src, "_age")
1196 +}
1197 +
1198 func isLabelValueSet(c *module.Chart, name string) bool {
1199 for _, l := range c.Labels {
1200 if l.Key == name {
src/go/plugin/go.d/collector/k8s_state/metadata.yaml
+34 -1
@@ -262,12 +262,45 @@ modules:
262 - name: k8s_state.cronjob_jobs_count_by_status
263 description: CronJob Jobs Count by Status
264 unit: 'jobs'
265 - chart_type: stacked
265 + chart_type: line
266 dimensions:
267 - name: completed
268 - name: failed
269 - name: running
270 - name: suspended
271 + - name: k8s_state.cronjob_jobs_failed_by_reason
272 + description: CronJob Jobs Failed by Reason
273 + unit: 'jobs'
274 + chart_type: line
275 + dimensions:
276 + - name: pod_failure_policy
277 + - name: backoff_limit_exceeded
278 + - name: deadline_exceeded
279 + - name: k8s_state.cronjob_last_execution_status
280 + description: CronJob Last Execution Status
281 + unit: 'status'
282 + chart_type: line
283 + dimensions:
284 + - name: completed
285 + - name: failed
286 + - name: k8s_state.cronjob_last_completion_duration
287 + description: CronJob Last Completion Duration
288 + unit: 'seconds'
289 + chart_type: line
290 + dimensions:
291 + - name: last_completion
292 + - name: k8s_state.cronjob_last_completed_time_ago
293 + description: CronJob Last Completed Time Ago
294 + unit: 'seconds'
295 + chart_type: line
296 + dimensions:
297 + - name: last_completed_ago
298 + - name: k8s_state.cronjob_last_schedule_time_ago
299 + description: CronJob Last Schedule Time Ago
300 + unit: 'seconds'
301 + chart_type: line
302 + dimensions:
303 + - name: last_schedule_ago
304 - name: k8s_state.cronjob_age
305 description: CronJob Age
306 unit: 'seconds'
src/go/plugin/go.d/collector/k8s_state/state.go
+6 -6
@@ -202,6 +202,9 @@ type cronJobState struct {
202 name string
203 namespace string
204 creationTime time.Time
205 +
206 + lastScheduleTime *time.Time
207 + lastSuccessfulTime *time.Time
208 }
209
210 func (cs cronJobState) id() string { return cs.namespace + "_" + cs.name }
@@ -223,10 +226,7 @@ type jobState struct {
226
227 conditions []batchv1.JobCondition
228
226 - startTime *time.Time
227 - active int32
228 - uncountedTerminatedPods struct {
229 - succeeded int
230 - failed int
231 - }
229 + startTime *time.Time
230 + completionTime *time.Time
231 + active int32
232 }
src/go/plugin/go.d/collector/k8s_state/update_cronjob_state.go
+12 -2
@@ -16,9 +16,9 @@ func (c *Collector) updateCronJobState(r resource) {
16 return
17 }
18
19 - _, ok := c.state.cronJobs[r.source()]
19 + st, ok := c.state.cronJobs[r.source()]
20 if !ok {
21 - st := newCronJobState()
21 + st = newCronJobState()
22 c.state.cronJobs[r.source()] = st
23
24 st.uid = string(cj.UID)
@@ -26,4 +26,14 @@ func (c *Collector) updateCronJobState(r resource) {
26 st.namespace = cj.Namespace
27 st.creationTime = cj.CreationTimestamp.Time
28 }
29 +
30 + st.lastScheduleTime = nil
31 + st.lastSuccessfulTime = nil
32 +
33 + if cj.Status.LastScheduleTime != nil {
34 + st.lastScheduleTime = ptr(cj.Status.LastScheduleTime.Time)
35 + }
36 + if cj.Status.LastSuccessfulTime != nil {
37 + st.lastSuccessfulTime = ptr(cj.Status.LastSuccessfulTime.Time)
38 + }
39 }
src/go/plugin/go.d/collector/k8s_state/update_job_state.go
+3 -11
@@ -50,17 +50,9 @@ func (c *Collector) updateJobState(r resource) {
50 if job.Status.StartTime != nil {
51 st.startTime = ptr(job.Status.StartTime.Time)
52 }
53 -
53 + if job.Status.CompletionTime != nil {
54 + st.completionTime = ptr(job.Status.CompletionTime.Time)
55 + }
56 st.active = job.Status.Active
55 -
57 st.conditions = job.Status.Conditions
57 -
58 - st.uncountedTerminatedPods.succeeded = 0
59 - st.uncountedTerminatedPods.failed = 0
60 -
61 - if v := job.Status.UncountedTerminatedPods; v != nil {
62 - st.uncountedTerminatedPods.succeeded = len(v.Succeeded)
63 - st.uncountedTerminatedPods.failed = len(v.Failed)
64 - }
65 -
58 }