| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package k8s_state |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "regexp" |
| 8 | "strings" |
| 9 | |
| 10 | "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" |
| 11 | ) |
| 12 | |
| 13 | // NETDATA_CHART_PRIO_CGROUPS_CONTAINERS 40000 |
| 14 | const prioDiscoveryDiscovererState = 50999 |
| 15 | |
| 16 | const ( |
| 17 | prioNodeAllocatableCPURequestsUtil = 50100 + iota |
| 18 | prioNodeAllocatableCPURequestsUsed |
| 19 | prioNodeAllocatableCPULimitsUtil |
| 20 | prioNodeAllocatableCPULimitsUsed |
| 21 | prioNodeAllocatableMemRequestsUtil |
| 22 | prioNodeAllocatableMemRequestsUsed |
| 23 | prioNodeAllocatableMemLimitsUtil |
| 24 | prioNodeAllocatableMemLimitsUsed |
| 25 | prioNodeAllocatablePodsUtil |
| 26 | prioNodeAllocatablePodsUsage |
| 27 | prioNodeConditions |
| 28 | prioNodeSchedulability |
| 29 | prioNodePodsReadiness |
| 30 | prioNodePodsReadinessState |
| 31 | prioNodePodsCondition |
| 32 | prioNodePodsPhase |
| 33 | prioNodeContainersCount |
| 34 | prioNodeContainersState |
| 35 | prioNodeInitContainersState |
| 36 | prioNodeAge |
| 37 | ) |
| 38 | |
| 39 | const ( |
| 40 | prioPodCPURequestsUsed = 50300 + iota |
| 41 | prioPodCPULimitsUsed |
| 42 | prioPodMemRequestsUsed |
| 43 | prioPodMemLimitsUsed |
| 44 | prioPodCondition |
| 45 | prioPodPhase |
| 46 | prioPodStatusReason |
| 47 | prioPodAge |
| 48 | prioPodContainersCount |
| 49 | prioPodContainersState |
| 50 | prioPodInitContainersState |
| 51 | prioPodContainerReadinessState |
| 52 | prioPodContainerRestarts |
| 53 | prioPodContainerState |
| 54 | prioPodContainerWaitingStateReason |
| 55 | prioPodContainerTerminatedStateReason |
| 56 | ) |
| 57 | |
| 58 | const ( |
| 59 | prioDeploymentConditions = 50500 + iota |
| 60 | prioDeploymentReplicas |
| 61 | prioDeploymentAge |
| 62 | ) |
| 63 | |
| 64 | const ( |
| 65 | prioCronJobJobsCountByStatus = 50700 + iota |
| 66 | prioCronJobJobsFailedByReason |
| 67 | prioCronJobLastExecutionStatus |
| 68 | prioCronJobLastCompletionDuration |
| 69 | prioCronJobLastCompletedTimeAgo |
| 70 | prioCronJobLastScheduleTimeAgo |
| 71 | prioCronJobSuspendStatus |
| 72 | prioCronJobAge |
| 73 | ) |
| 74 | |
| 75 | const ( |
| 76 | labelKeyPrefix = "k8s_" |
| 77 | //labelKeyLabelPrefix = labelKeyPrefix + "label_" |
| 78 | //labelKeyAnnotationPrefix = labelKeyPrefix + "annotation_" |
| 79 | labelKeyClusterID = labelKeyPrefix + "cluster_id" |
| 80 | labelKeyClusterName = labelKeyPrefix + "cluster_name" |
| 81 | labelKeyNamespace = labelKeyPrefix + "namespace" |
| 82 | labelKeyKind = labelKeyPrefix + "kind" |
| 83 | labelKeyPodName = labelKeyPrefix + "pod_name" |
| 84 | labelKeyNodeName = labelKeyPrefix + "node_name" |
| 85 | labelKeyPodUID = labelKeyPrefix + "pod_uid" |
| 86 | labelKeyControllerKind = labelKeyPrefix + "controller_kind" |
| 87 | labelKeyControllerName = labelKeyPrefix + "controller_name" |
| 88 | labelKeyContainerName = labelKeyPrefix + "container_name" |
| 89 | labelKeyContainerID = labelKeyPrefix + "container_id" |
| 90 | labelKeyQoSClass = labelKeyPrefix + "qos_class" |
| 91 | labelKeyDeploymentName = labelKeyPrefix + "deployment_name" |
| 92 | labelKeyCronJobName = labelKeyPrefix + "cronjob_name" |
| 93 | ) |
| 94 | |
| 95 | var baseCharts = collectorapi.Charts{ |
| 96 | discoveryStatusChart.Copy(), |
| 97 | } |
| 98 | |
| 99 | var nodeChartsTmpl = collectorapi.Charts{ |
| 100 | nodeAllocatableCPURequestsUtilChartTmpl.Copy(), |
| 101 | nodeAllocatableCPURequestsUsedChartTmpl.Copy(), |
| 102 | nodeAllocatableCPULimitsUtilChartTmpl.Copy(), |
| 103 | nodeAllocatableCPULimitsUsedChartTmpl.Copy(), |
| 104 | nodeAllocatableMemRequestsUtilChartTmpl.Copy(), |
| 105 | nodeAllocatableMemRequestsUsedChartTmpl.Copy(), |
| 106 | nodeAllocatableMemLimitsUtilChartTmpl.Copy(), |
| 107 | nodeAllocatableMemLimitsUsedChartTmpl.Copy(), |
| 108 | nodeAllocatablePodsUtilizationChartTmpl.Copy(), |
| 109 | nodeAllocatablePodsUsageChartTmpl.Copy(), |
| 110 | nodeConditionsChartTmpl.Copy(), |
| 111 | nodeSchedulabilityChartTmpl.Copy(), |
| 112 | nodePodsReadinessChartTmpl.Copy(), |
| 113 | nodePodsReadinessStateChartTmpl.Copy(), |
| 114 | nodePodsConditionChartTmpl.Copy(), |
| 115 | nodePodsPhaseChartTmpl.Copy(), |
| 116 | nodeContainersChartTmpl.Copy(), |
| 117 | nodeContainersStateChartTmpl.Copy(), |
| 118 | nodeInitContainersStateChartTmpl.Copy(), |
| 119 | nodeAgeChartTmpl.Copy(), |
| 120 | } |
| 121 | |
| 122 | var podChartsTmpl = collectorapi.Charts{ |
| 123 | podCPURequestsUsedChartTmpl.Copy(), |
| 124 | podCPULimitsUsedChartTmpl.Copy(), |
| 125 | podMemRequestsUsedChartTmpl.Copy(), |
| 126 | podMemLimitsUsedChartTmpl.Copy(), |
| 127 | podConditionChartTmpl.Copy(), |
| 128 | podPhaseChartTmpl.Copy(), |
| 129 | podStatusReasonChartTmpl.Copy(), |
| 130 | podAgeChartTmpl.Copy(), |
| 131 | podContainersCountChartTmpl.Copy(), |
| 132 | podContainersStateChartTmpl.Copy(), |
| 133 | podInitContainersStateChartTmpl.Copy(), |
| 134 | } |
| 135 | |
| 136 | var containerChartsTmpl = collectorapi.Charts{ |
| 137 | containerReadinessStateChartTmpl.Copy(), |
| 138 | containerRestartsChartTmpl.Copy(), |
| 139 | containersStateChartTmpl.Copy(), |
| 140 | containersStateWaitingChartTmpl.Copy(), |
| 141 | containersStateTerminatedChartTmpl.Copy(), |
| 142 | } |
| 143 | |
| 144 | var deploymentChartsTmpl = collectorapi.Charts{ |
| 145 | deploymentConditionStatusChartTmpl.Copy(), |
| 146 | deploymentReplicasChartTmpl.Copy(), |
| 147 | deploymentAgeChartTmpl.Copy(), |
| 148 | } |
| 149 | |
| 150 | var cronJobChartsTmpl = collectorapi.Charts{ |
| 151 | cronJobJobsCountByStatusChartTmpl.Copy(), |
| 152 | cronJobJobsFailedByReasonChartTmpl.Copy(), |
| 153 | cronJobLastExecutionStatusChartTmpl.Copy(), |
| 154 | cronJobLastCompletionDurationChartTmpl.Copy(), |
| 155 | cronJobLastCompletedTimeAgoChartTmpl.Copy(), |
| 156 | cronJobLastScheduleTimeAgoChartTmpl.Copy(), |
| 157 | cronJobSuspendStatusChartTmpl.Copy(), |
| 158 | cronJobAgeChartTmpl.Copy(), |
| 159 | } |
| 160 | |
| 161 | var ( |
| 162 | // CPU resource |
| 163 | nodeAllocatableCPURequestsUtilChartTmpl = collectorapi.Chart{ |
| 164 | IDSep: true, |
| 165 | ID: "node_%s.allocatable_cpu_requests_utilization", |
| 166 | Title: "CPU requests utilization", |
| 167 | Units: "%", |
| 168 | Fam: "node cpu resource", |
| 169 | Ctx: "k8s_state.node_allocatable_cpu_requests_utilization", |
| 170 | Priority: prioNodeAllocatableCPURequestsUtil, |
| 171 | Dims: collectorapi.Dims{ |
| 172 | {ID: "node_%s_alloc_cpu_requests_util", Name: "requests", Div: precision}, |
| 173 | }, |
| 174 | } |
| 175 | nodeAllocatableCPURequestsUsedChartTmpl = collectorapi.Chart{ |
| 176 | IDSep: true, |
| 177 | ID: "node_%s.allocatable_cpu_requests_used", |
| 178 | Title: "CPU requests used", |
| 179 | Units: "millicpu", |
| 180 | Fam: "node cpu resource", |
| 181 | Ctx: "k8s_state.node_allocatable_cpu_requests_used", |
| 182 | Priority: prioNodeAllocatableCPURequestsUsed, |
| 183 | Dims: collectorapi.Dims{ |
| 184 | {ID: "node_%s_alloc_cpu_requests_used", Name: "requests"}, |
| 185 | }, |
| 186 | } |
| 187 | nodeAllocatableCPULimitsUtilChartTmpl = collectorapi.Chart{ |
| 188 | IDSep: true, |
| 189 | ID: "node_%s.allocatable_cpu_limits_utilization", |
| 190 | Title: "CPU limits utilization", |
| 191 | Units: "%", |
| 192 | Fam: "node cpu resource", |
| 193 | Ctx: "k8s_state.node_allocatable_cpu_limits_utilization", |
| 194 | Priority: prioNodeAllocatableCPULimitsUtil, |
| 195 | Dims: collectorapi.Dims{ |
| 196 | {ID: "node_%s_alloc_cpu_limits_util", Name: "limits", Div: precision}, |
| 197 | }, |
| 198 | } |
| 199 | nodeAllocatableCPULimitsUsedChartTmpl = collectorapi.Chart{ |
| 200 | IDSep: true, |
| 201 | ID: "node_%s.allocatable_cpu_limits_used", |
| 202 | Title: "CPU limits used", |
| 203 | Units: "millicpu", |
| 204 | Fam: "node cpu resource", |
| 205 | Ctx: "k8s_state.node_allocatable_cpu_limits_used", |
| 206 | Priority: prioNodeAllocatableCPULimitsUsed, |
| 207 | Dims: collectorapi.Dims{ |
| 208 | {ID: "node_%s_alloc_cpu_limits_used", Name: "limits"}, |
| 209 | }, |
| 210 | } |
| 211 | // memory resource |
| 212 | nodeAllocatableMemRequestsUtilChartTmpl = collectorapi.Chart{ |
| 213 | IDSep: true, |
| 214 | ID: "node_%s.allocatable_mem_requests_utilization", |
| 215 | Title: "Memory requests utilization", |
| 216 | Units: "%", |
| 217 | Fam: "node mem resource", |
| 218 | Ctx: "k8s_state.node_allocatable_mem_requests_utilization", |
| 219 | Priority: prioNodeAllocatableMemRequestsUtil, |
| 220 | Dims: collectorapi.Dims{ |
| 221 | {ID: "node_%s_alloc_mem_requests_util", Name: "requests", Div: precision}, |
| 222 | }, |
| 223 | } |
| 224 | nodeAllocatableMemRequestsUsedChartTmpl = collectorapi.Chart{ |
| 225 | IDSep: true, |
| 226 | ID: "node_%s.allocatable_mem_requests_used", |
| 227 | Title: "Memory requests used", |
| 228 | Units: "bytes", |
| 229 | Fam: "node mem resource", |
| 230 | Ctx: "k8s_state.node_allocatable_mem_requests_used", |
| 231 | Priority: prioNodeAllocatableMemRequestsUsed, |
| 232 | Dims: collectorapi.Dims{ |
| 233 | {ID: "node_%s_alloc_mem_requests_used", Name: "requests"}, |
| 234 | }, |
| 235 | } |
| 236 | nodeAllocatableMemLimitsUtilChartTmpl = collectorapi.Chart{ |
| 237 | IDSep: true, |
| 238 | ID: "node_%s.allocatable_mem_limits_utilization", |
| 239 | Title: "Memory limits utilization", |
| 240 | Units: "%", |
| 241 | Fam: "node mem resource", |
| 242 | Ctx: "k8s_state.node_allocatable_mem_limits_utilization", |
| 243 | Priority: prioNodeAllocatableMemLimitsUtil, |
| 244 | Dims: collectorapi.Dims{ |
| 245 | {ID: "node_%s_alloc_mem_limits_util", Name: "limits", Div: precision}, |
| 246 | }, |
| 247 | } |
| 248 | nodeAllocatableMemLimitsUsedChartTmpl = collectorapi.Chart{ |
| 249 | IDSep: true, |
| 250 | ID: "node_%s.allocatable_mem_limits_used", |
| 251 | Title: "Memory limits used", |
| 252 | Units: "bytes", |
| 253 | Fam: "node mem resource", |
| 254 | Ctx: "k8s_state.node_allocatable_mem_limits_used", |
| 255 | Priority: prioNodeAllocatableMemLimitsUsed, |
| 256 | Dims: collectorapi.Dims{ |
| 257 | {ID: "node_%s_alloc_mem_limits_used", Name: "limits"}, |
| 258 | }, |
| 259 | } |
| 260 | // pods resource |
| 261 | nodeAllocatablePodsUtilizationChartTmpl = collectorapi.Chart{ |
| 262 | IDSep: true, |
| 263 | ID: "node_%s.allocatable_pods_utilization", |
| 264 | Title: "Pods resource utilization", |
| 265 | Units: "%", |
| 266 | Fam: "node pods resource", |
| 267 | Ctx: "k8s_state.node_allocatable_pods_utilization", |
| 268 | Priority: prioNodeAllocatablePodsUtil, |
| 269 | Dims: collectorapi.Dims{ |
| 270 | {ID: "node_%s_alloc_pods_util", Name: "allocated", Div: precision}, |
| 271 | }, |
| 272 | } |
| 273 | nodeAllocatablePodsUsageChartTmpl = collectorapi.Chart{ |
| 274 | IDSep: true, |
| 275 | ID: "node_%s.allocated_pods_usage", |
| 276 | Title: "Pods resource usage", |
| 277 | Units: "pods", |
| 278 | Fam: "node pods resource", |
| 279 | Ctx: "k8s_state.node_allocatable_pods_usage", |
| 280 | Type: collectorapi.Stacked, |
| 281 | Priority: prioNodeAllocatablePodsUsage, |
| 282 | Dims: collectorapi.Dims{ |
| 283 | {ID: "node_%s_alloc_pods_available", Name: "available"}, |
| 284 | {ID: "node_%s_alloc_pods_allocated", Name: "allocated"}, |
| 285 | }, |
| 286 | } |
| 287 | // condition |
| 288 | nodeConditionsChartTmpl = func() collectorapi.Chart { |
| 289 | chart := collectorapi.Chart{ |
| 290 | IDSep: true, |
| 291 | ID: "node_%s.condition_status", |
| 292 | Title: "Condition status", |
| 293 | Units: "status", |
| 294 | Fam: "node condition", |
| 295 | Ctx: "k8s_state.node_condition", |
| 296 | Priority: prioNodeConditions, |
| 297 | } |
| 298 | for _, v := range nodeConditionStatuses { |
| 299 | chart.Dims = append(chart.Dims, &collectorapi.Dim{ |
| 300 | ID: "node_%s_cond_" + v, |
| 301 | Name: v, |
| 302 | }) |
| 303 | } |
| 304 | return chart |
| 305 | }() |
| 306 | nodeSchedulabilityChartTmpl = collectorapi.Chart{ |
| 307 | IDSep: true, |
| 308 | ID: "node_%s.schedulability", |
| 309 | Title: "Schedulability", |
| 310 | Units: "state", |
| 311 | Fam: "node schedulability", |
| 312 | Ctx: "k8s_state.node_schedulability", |
| 313 | Priority: prioNodeSchedulability, |
| 314 | Dims: collectorapi.Dims{ |
| 315 | {ID: "node_%s_schedulability_schedulable", Name: "schedulable"}, |
| 316 | {ID: "node_%s_schedulability_unschedulable", Name: "unschedulable"}, |
| 317 | }, |
| 318 | } |
| 319 | // pods readiness |
| 320 | nodePodsReadinessChartTmpl = collectorapi.Chart{ |
| 321 | IDSep: true, |
| 322 | ID: "node_%s.pods_readiness", |
| 323 | Title: "Pods readiness", |
| 324 | Units: "%", |
| 325 | Fam: "node pods readiness", |
| 326 | Ctx: "k8s_state.node_pods_readiness", |
| 327 | Priority: prioNodePodsReadiness, |
| 328 | Dims: collectorapi.Dims{ |
| 329 | {ID: "node_%s_pods_readiness", Name: "ready", Div: precision}, |
| 330 | }, |
| 331 | } |
| 332 | nodePodsReadinessStateChartTmpl = collectorapi.Chart{ |
| 333 | IDSep: true, |
| 334 | ID: "node_%s.pods_readiness_state", |
| 335 | Title: "Pods readiness state", |
| 336 | Units: "pods", |
| 337 | Fam: "node pods readiness", |
| 338 | Ctx: "k8s_state.node_pods_readiness_state", |
| 339 | Type: collectorapi.Stacked, |
| 340 | Priority: prioNodePodsReadinessState, |
| 341 | Dims: collectorapi.Dims{ |
| 342 | {ID: "node_%s_pods_readiness_ready", Name: "ready"}, |
| 343 | {ID: "node_%s_pods_readiness_unready", Name: "unready"}, |
| 344 | }, |
| 345 | } |
| 346 | // pods condition |
| 347 | nodePodsConditionChartTmpl = collectorapi.Chart{ |
| 348 | IDSep: true, |
| 349 | ID: "node_%s.pods_condition", |
| 350 | Title: "Pods condition", |
| 351 | Units: "pods", |
| 352 | Fam: "node pods condition", |
| 353 | Ctx: "k8s_state.node_pods_condition", |
| 354 | Priority: prioNodePodsCondition, |
| 355 | Dims: collectorapi.Dims{ |
| 356 | {ID: "node_%s_pods_cond_podready", Name: "pod_ready"}, |
| 357 | {ID: "node_%s_pods_cond_podscheduled", Name: "pod_scheduled"}, |
| 358 | {ID: "node_%s_pods_cond_podinitialized", Name: "pod_initialized"}, |
| 359 | {ID: "node_%s_pods_cond_containersready", Name: "containers_ready"}, |
| 360 | }, |
| 361 | } |
| 362 | // pods phase |
| 363 | nodePodsPhaseChartTmpl = collectorapi.Chart{ |
| 364 | IDSep: true, |
| 365 | ID: "node_%s.pods_phase", |
| 366 | Title: "Pods phase", |
| 367 | Units: "pods", |
| 368 | Fam: "node pods phase", |
| 369 | Ctx: "k8s_state.node_pods_phase", |
| 370 | Type: collectorapi.Stacked, |
| 371 | Priority: prioNodePodsPhase, |
| 372 | Dims: collectorapi.Dims{ |
| 373 | {ID: "node_%s_pods_phase_running", Name: "running"}, |
| 374 | {ID: "node_%s_pods_phase_failed", Name: "failed"}, |
| 375 | {ID: "node_%s_pods_phase_succeeded", Name: "succeeded"}, |
| 376 | {ID: "node_%s_pods_phase_pending", Name: "pending"}, |
| 377 | }, |
| 378 | } |
| 379 | // containers |
| 380 | nodeContainersChartTmpl = collectorapi.Chart{ |
| 381 | IDSep: true, |
| 382 | ID: "node_%s.containers", |
| 383 | Title: "Containers", |
| 384 | Units: "containers", |
| 385 | Fam: "node containers", |
| 386 | Ctx: "k8s_state.node_containers", |
| 387 | Priority: prioNodeContainersCount, |
| 388 | Dims: collectorapi.Dims{ |
| 389 | {ID: "node_%s_containers", Name: "containers"}, |
| 390 | {ID: "node_%s_init_containers", Name: "init_containers"}, |
| 391 | }, |
| 392 | } |
| 393 | nodeContainersStateChartTmpl = collectorapi.Chart{ |
| 394 | IDSep: true, |
| 395 | ID: "node_%s.containers_state", |
| 396 | Title: "Containers state", |
| 397 | Units: "containers", |
| 398 | Fam: "node containers", |
| 399 | Ctx: "k8s_state.node_containers_state", |
| 400 | Type: collectorapi.Stacked, |
| 401 | Priority: prioNodeContainersState, |
| 402 | Dims: collectorapi.Dims{ |
| 403 | {ID: "node_%s_containers_state_running", Name: "running"}, |
| 404 | {ID: "node_%s_containers_state_waiting", Name: "waiting"}, |
| 405 | {ID: "node_%s_containers_state_terminated", Name: "terminated"}, |
| 406 | }, |
| 407 | } |
| 408 | nodeInitContainersStateChartTmpl = collectorapi.Chart{ |
| 409 | IDSep: true, |
| 410 | ID: "node_%s.init_containers_state", |
| 411 | Title: "Init containers state", |
| 412 | Units: "containers", |
| 413 | Fam: "node containers", |
| 414 | Ctx: "k8s_state.node_init_containers_state", |
| 415 | Type: collectorapi.Stacked, |
| 416 | Priority: prioNodeInitContainersState, |
| 417 | Dims: collectorapi.Dims{ |
| 418 | {ID: "node_%s_init_containers_state_running", Name: "running"}, |
| 419 | {ID: "node_%s_init_containers_state_waiting", Name: "waiting"}, |
| 420 | {ID: "node_%s_init_containers_state_terminated", Name: "terminated"}, |
| 421 | }, |
| 422 | } |
| 423 | // age |
| 424 | nodeAgeChartTmpl = collectorapi.Chart{ |
| 425 | IDSep: true, |
| 426 | ID: "node_%s.age", |
| 427 | Title: "Age", |
| 428 | Units: "seconds", |
| 429 | Fam: "node age", |
| 430 | Ctx: "k8s_state.node_age", |
| 431 | Priority: prioNodeAge, |
| 432 | Dims: collectorapi.Dims{ |
| 433 | {ID: "node_%s_age", Name: "age"}, |
| 434 | }, |
| 435 | } |
| 436 | ) |
| 437 | |
| 438 | func (c *Collector) newNodeCharts(ns *nodeState) *collectorapi.Charts { |
| 439 | charts := nodeChartsTmpl.Copy() |
| 440 | for _, chart := range *charts { |
| 441 | chart.ID = fmt.Sprintf(chart.ID, replaceDots(ns.id())) |
| 442 | chart.Labels = c.newNodeChartLabels(ns) |
| 443 | for _, d := range chart.Dims { |
| 444 | d.ID = fmt.Sprintf(d.ID, ns.id()) |
| 445 | } |
| 446 | } |
| 447 | return charts |
| 448 | } |
| 449 | |
| 450 | func (c *Collector) newNodeChartLabels(ns *nodeState) []collectorapi.Label { |
| 451 | labels := []collectorapi.Label{ |
| 452 | {Key: labelKeyNodeName, Value: ns.name, Source: collectorapi.LabelSourceK8s}, |
| 453 | {Key: labelKeyClusterID, Value: c.kubeClusterID, Source: collectorapi.LabelSourceK8s}, |
| 454 | {Key: labelKeyClusterName, Value: c.kubeClusterName, Source: collectorapi.LabelSourceK8s}, |
| 455 | } |
| 456 | return labels |
| 457 | } |
| 458 | |
| 459 | func (c *Collector) addNodeCharts(ns *nodeState) { |
| 460 | cs := c.newNodeCharts(ns) |
| 461 | if err := c.Charts().Add(*cs...); err != nil { |
| 462 | c.Warning(err) |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | func (c *Collector) removeNodeCharts(ns *nodeState) { |
| 467 | prefix := fmt.Sprintf("node_%s", replaceDots(ns.id())) |
| 468 | c.removeCharts(prefix) |
| 469 | } |
| 470 | |
| 471 | var ( |
| 472 | podCPURequestsUsedChartTmpl = collectorapi.Chart{ |
| 473 | IDSep: true, |
| 474 | ID: "pod_%s.cpu_requests_used", |
| 475 | Title: "CPU requests used", |
| 476 | Units: "millicpu", |
| 477 | Fam: "pod allocated cpu", |
| 478 | Ctx: "k8s_state.pod_cpu_requests_used", |
| 479 | Priority: prioPodCPURequestsUsed, |
| 480 | Dims: collectorapi.Dims{ |
| 481 | {ID: "pod_%s_cpu_requests_used", Name: "requests"}, |
| 482 | }, |
| 483 | } |
| 484 | podCPULimitsUsedChartTmpl = collectorapi.Chart{ |
| 485 | IDSep: true, |
| 486 | ID: "pod_%s.cpu_limits_used", |
| 487 | Title: "CPU limits used", |
| 488 | Units: "millicpu", |
| 489 | Fam: "pod allocated cpu", |
| 490 | Ctx: "k8s_state.pod_cpu_limits_used", |
| 491 | Priority: prioPodCPULimitsUsed, |
| 492 | Dims: collectorapi.Dims{ |
| 493 | {ID: "pod_%s_cpu_limits_used", Name: "limits"}, |
| 494 | }, |
| 495 | } |
| 496 | podMemRequestsUsedChartTmpl = collectorapi.Chart{ |
| 497 | IDSep: true, |
| 498 | ID: "pod_%s.mem_requests_used", |
| 499 | Title: "Memory requests used", |
| 500 | Units: "bytes", |
| 501 | Fam: "pod allocated mem", |
| 502 | Ctx: "k8s_state.pod_mem_requests_used", |
| 503 | Priority: prioPodMemRequestsUsed, |
| 504 | Dims: collectorapi.Dims{ |
| 505 | {ID: "pod_%s_mem_requests_used", Name: "requests"}, |
| 506 | }, |
| 507 | } |
| 508 | podMemLimitsUsedChartTmpl = collectorapi.Chart{ |
| 509 | IDSep: true, |
| 510 | ID: "pod_%s.mem_limits_used", |
| 511 | Title: "Memory limits used", |
| 512 | Units: "bytes", |
| 513 | Fam: "pod allocated mem", |
| 514 | Ctx: "k8s_state.pod_mem_limits_used", |
| 515 | Priority: prioPodMemLimitsUsed, |
| 516 | Dims: collectorapi.Dims{ |
| 517 | {ID: "pod_%s_mem_limits_used", Name: "limits"}, |
| 518 | }, |
| 519 | } |
| 520 | podConditionChartTmpl = collectorapi.Chart{ |
| 521 | IDSep: true, |
| 522 | ID: "pod_%s.condition", |
| 523 | Title: "Condition", |
| 524 | Units: "state", |
| 525 | Fam: "pod condition", |
| 526 | Ctx: "k8s_state.pod_condition", |
| 527 | Priority: prioPodCondition, |
| 528 | Dims: collectorapi.Dims{ |
| 529 | {ID: "pod_%s_cond_podready", Name: "pod_ready"}, |
| 530 | {ID: "pod_%s_cond_podscheduled", Name: "pod_scheduled"}, |
| 531 | {ID: "pod_%s_cond_podinitialized", Name: "pod_initialized"}, |
| 532 | {ID: "pod_%s_cond_containersready", Name: "containers_ready"}, |
| 533 | }, |
| 534 | } |
| 535 | podPhaseChartTmpl = collectorapi.Chart{ |
| 536 | IDSep: true, |
| 537 | ID: "pod_%s.phase", |
| 538 | Title: "Phase", |
| 539 | Units: "state", |
| 540 | Fam: "pod phase", |
| 541 | Ctx: "k8s_state.pod_phase", |
| 542 | Priority: prioPodPhase, |
| 543 | Dims: collectorapi.Dims{ |
| 544 | {ID: "pod_%s_phase_running", Name: "running"}, |
| 545 | {ID: "pod_%s_phase_failed", Name: "failed"}, |
| 546 | {ID: "pod_%s_phase_succeeded", Name: "succeeded"}, |
| 547 | {ID: "pod_%s_phase_pending", Name: "pending"}, |
| 548 | }, |
| 549 | } |
| 550 | podStatusReasonChartTmpl = func() collectorapi.Chart { |
| 551 | chart := collectorapi.Chart{ |
| 552 | IDSep: true, |
| 553 | ID: "pod_%s.status_reason", |
| 554 | Title: "Status reason", |
| 555 | Units: "status", |
| 556 | Fam: "pod status", |
| 557 | Ctx: "k8s_state.pod_status_reason", |
| 558 | Priority: prioPodStatusReason, |
| 559 | } |
| 560 | for _, v := range podStatusReasons { |
| 561 | chart.Dims = append(chart.Dims, &collectorapi.Dim{ |
| 562 | ID: "pod_%s_status_reason_" + v, |
| 563 | Name: v, |
| 564 | }) |
| 565 | } |
| 566 | return chart |
| 567 | }() |
| 568 | podAgeChartTmpl = collectorapi.Chart{ |
| 569 | IDSep: true, |
| 570 | ID: "pod_%s.age", |
| 571 | Title: "Age", |
| 572 | Units: "seconds", |
| 573 | Fam: "pod age", |
| 574 | Ctx: "k8s_state.pod_age", |
| 575 | Priority: prioPodAge, |
| 576 | Dims: collectorapi.Dims{ |
| 577 | {ID: "pod_%s_age", Name: "age"}, |
| 578 | }, |
| 579 | } |
| 580 | podContainersCountChartTmpl = collectorapi.Chart{ |
| 581 | IDSep: true, |
| 582 | ID: "pod_%s.containers_count", |
| 583 | Title: "Containers", |
| 584 | Units: "containers", |
| 585 | Fam: "pod containers", |
| 586 | Ctx: "k8s_state.pod_containers", |
| 587 | Priority: prioPodContainersCount, |
| 588 | Dims: collectorapi.Dims{ |
| 589 | {ID: "pod_%s_containers", Name: "containers"}, |
| 590 | {ID: "pod_%s_init_containers", Name: "init_containers"}, |
| 591 | }, |
| 592 | } |
| 593 | podContainersStateChartTmpl = collectorapi.Chart{ |
| 594 | IDSep: true, |
| 595 | ID: "pod_%s.containers_state", |
| 596 | Title: "Containers state", |
| 597 | Units: "containers", |
| 598 | Fam: "pod containers", |
| 599 | Ctx: "k8s_state.pod_containers_state", |
| 600 | Type: collectorapi.Stacked, |
| 601 | Priority: prioPodContainersState, |
| 602 | Dims: collectorapi.Dims{ |
| 603 | {ID: "pod_%s_containers_state_running", Name: "running"}, |
| 604 | {ID: "pod_%s_containers_state_waiting", Name: "waiting"}, |
| 605 | {ID: "pod_%s_containers_state_terminated", Name: "terminated"}, |
| 606 | }, |
| 607 | } |
| 608 | podInitContainersStateChartTmpl = collectorapi.Chart{ |
| 609 | IDSep: true, |
| 610 | ID: "pod_%s.init_containers_state", |
| 611 | Title: "Init containers state", |
| 612 | Units: "containers", |
| 613 | Fam: "pod containers", |
| 614 | Ctx: "k8s_state.pod_init_containers_state", |
| 615 | Type: collectorapi.Stacked, |
| 616 | Priority: prioPodInitContainersState, |
| 617 | Dims: collectorapi.Dims{ |
| 618 | {ID: "pod_%s_init_containers_state_running", Name: "running"}, |
| 619 | {ID: "pod_%s_init_containers_state_waiting", Name: "waiting"}, |
| 620 | {ID: "pod_%s_init_containers_state_terminated", Name: "terminated"}, |
| 621 | }, |
| 622 | } |
| 623 | ) |
| 624 | |
| 625 | func (c *Collector) newPodCharts(ps *podState) *collectorapi.Charts { |
| 626 | charts := podChartsTmpl.Copy() |
| 627 | for _, chart := range *charts { |
| 628 | chart.ID = fmt.Sprintf(chart.ID, replaceDots(ps.id())) |
| 629 | chart.Labels = c.newPodChartLabels(ps) |
| 630 | for _, d := range chart.Dims { |
| 631 | d.ID = fmt.Sprintf(d.ID, ps.id()) |
| 632 | } |
| 633 | } |
| 634 | return charts |
| 635 | } |
| 636 | |
| 637 | func (c *Collector) newPodChartLabels(ps *podState) []collectorapi.Label { |
| 638 | labels := []collectorapi.Label{ |
| 639 | {Key: labelKeyNamespace, Value: ps.namespace, Source: collectorapi.LabelSourceK8s}, |
| 640 | {Key: labelKeyPodName, Value: ps.name, Source: collectorapi.LabelSourceK8s}, |
| 641 | {Key: labelKeyNodeName, Value: ps.nodeName, Source: collectorapi.LabelSourceK8s}, |
| 642 | {Key: labelKeyQoSClass, Value: ps.qosClass, Source: collectorapi.LabelSourceK8s}, |
| 643 | {Key: labelKeyControllerKind, Value: ps.controllerKind, Source: collectorapi.LabelSourceK8s}, |
| 644 | {Key: labelKeyControllerName, Value: ps.controllerName, Source: collectorapi.LabelSourceK8s}, |
| 645 | {Key: labelKeyClusterID, Value: c.kubeClusterID, Source: collectorapi.LabelSourceK8s}, |
| 646 | {Key: labelKeyClusterName, Value: c.kubeClusterName, Source: collectorapi.LabelSourceK8s}, |
| 647 | } |
| 648 | return labels |
| 649 | } |
| 650 | |
| 651 | func (c *Collector) addPodCharts(ps *podState) { |
| 652 | charts := c.newPodCharts(ps) |
| 653 | if err := c.Charts().Add(*charts...); err != nil { |
| 654 | c.Warning(err) |
| 655 | } |
| 656 | } |
| 657 | |
| 658 | func (c *Collector) updatePodChartsNodeLabel(ps *podState) { |
| 659 | prefix := fmt.Sprintf("pod_%s", replaceDots(ps.id())) |
| 660 | for _, c := range *c.Charts() { |
| 661 | if strings.HasPrefix(c.ID, prefix) { |
| 662 | updateNodeLabel(c, ps.nodeName) |
| 663 | c.MarkNotCreated() |
| 664 | } |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | func updateNodeLabel(c *collectorapi.Chart, nodeName string) { |
| 669 | for i, l := range c.Labels { |
| 670 | if l.Key == labelKeyNodeName { |
| 671 | c.Labels[i].Value = nodeName |
| 672 | break |
| 673 | } |
| 674 | } |
| 675 | } |
| 676 | |
| 677 | func (c *Collector) removePodCharts(ps *podState) { |
| 678 | prefix := fmt.Sprintf("pod_%s", replaceDots(ps.id())) |
| 679 | c.removeCharts(prefix) |
| 680 | } |
| 681 | |
| 682 | var ( |
| 683 | containerReadinessStateChartTmpl = collectorapi.Chart{ |
| 684 | IDSep: true, |
| 685 | ID: "pod_%s_container_%s.readiness_state", |
| 686 | Title: "Readiness state", |
| 687 | Units: "state", |
| 688 | Fam: "container readiness", |
| 689 | Ctx: "k8s_state.pod_container_readiness_state", |
| 690 | Priority: prioPodContainerReadinessState, |
| 691 | Dims: collectorapi.Dims{ |
| 692 | {ID: "pod_%s_container_%s_readiness", Name: "ready"}, |
| 693 | }, |
| 694 | } |
| 695 | containerRestartsChartTmpl = collectorapi.Chart{ |
| 696 | IDSep: true, |
| 697 | ID: "pod_%s_container_%s.restarts", |
| 698 | Title: "Restarts", |
| 699 | Units: "restarts", |
| 700 | Fam: "container restarts", |
| 701 | Ctx: "k8s_state.pod_container_restarts", |
| 702 | Priority: prioPodContainerRestarts, |
| 703 | Dims: collectorapi.Dims{ |
| 704 | {ID: "pod_%s_container_%s_restarts", Name: "restarts"}, |
| 705 | }, |
| 706 | } |
| 707 | containersStateChartTmpl = collectorapi.Chart{ |
| 708 | IDSep: true, |
| 709 | ID: "pod_%s_container_%s.state", |
| 710 | Title: "Container state", |
| 711 | Units: "state", |
| 712 | Fam: "container state", |
| 713 | Ctx: "k8s_state.pod_container_state", |
| 714 | Priority: prioPodContainerState, |
| 715 | Dims: collectorapi.Dims{ |
| 716 | {ID: "pod_%s_container_%s_state_running", Name: "running"}, |
| 717 | {ID: "pod_%s_container_%s_state_waiting", Name: "waiting"}, |
| 718 | {ID: "pod_%s_container_%s_state_terminated", Name: "terminated"}, |
| 719 | }, |
| 720 | } |
| 721 | containersStateWaitingChartTmpl = func() collectorapi.Chart { |
| 722 | chart := collectorapi.Chart{ |
| 723 | IDSep: true, |
| 724 | ID: "pod_%s_container_%s.state_waiting_reason", |
| 725 | Title: "Container waiting state reason", |
| 726 | Units: "state", |
| 727 | Fam: "container waiting reason", |
| 728 | Ctx: "k8s_state.pod_container_waiting_state_reason", |
| 729 | Priority: prioPodContainerWaitingStateReason, |
| 730 | } |
| 731 | for _, v := range containerWaitingStateReasons { |
| 732 | chart.Dims = append(chart.Dims, &collectorapi.Dim{ |
| 733 | ID: "pod_%s_container_%s_state_waiting_reason_" + v, |
| 734 | Name: v, |
| 735 | }) |
| 736 | } |
| 737 | return chart |
| 738 | }() |
| 739 | containersStateTerminatedChartTmpl = func() collectorapi.Chart { |
| 740 | chart := collectorapi.Chart{ |
| 741 | IDSep: true, |
| 742 | ID: "pod_%s_container_%s.state_terminated_reason", |
| 743 | Title: "Container terminated state reason", |
| 744 | Units: "state", |
| 745 | Fam: "container terminated reason", |
| 746 | Ctx: "k8s_state.pod_container_terminated_state_reason", |
| 747 | Priority: prioPodContainerTerminatedStateReason, |
| 748 | } |
| 749 | for _, v := range containerTerminatedStateReasons { |
| 750 | chart.Dims = append(chart.Dims, &collectorapi.Dim{ |
| 751 | ID: "pod_%s_container_%s_state_terminated_reason_" + v, |
| 752 | Name: v, |
| 753 | }) |
| 754 | } |
| 755 | return chart |
| 756 | }() |
| 757 | ) |
| 758 | |
| 759 | func (c *Collector) newContainerCharts(ps *podState, cs *containerState) *collectorapi.Charts { |
| 760 | charts := containerChartsTmpl.Copy() |
| 761 | for _, chart := range *charts { |
| 762 | chart.ID = fmt.Sprintf(chart.ID, replaceDots(ps.id()), cs.name) |
| 763 | chart.Labels = c.newContainerChartLabels(ps, cs) |
| 764 | for _, d := range chart.Dims { |
| 765 | d.ID = fmt.Sprintf(d.ID, ps.id(), cs.name) |
| 766 | } |
| 767 | } |
| 768 | return charts |
| 769 | } |
| 770 | |
| 771 | func (c *Collector) newContainerChartLabels(ps *podState, cs *containerState) []collectorapi.Label { |
| 772 | labels := c.newPodChartLabels(ps) |
| 773 | labels = append( |
| 774 | labels, collectorapi.Label{Key: labelKeyContainerName, Value: cs.name, Source: collectorapi.LabelSourceK8s}, |
| 775 | ) |
| 776 | return labels |
| 777 | } |
| 778 | |
| 779 | func (c *Collector) addContainerCharts(ps *podState, cs *containerState) { |
| 780 | charts := c.newContainerCharts(ps, cs) |
| 781 | if err := c.Charts().Add(*charts...); err != nil { |
| 782 | c.Warning(err) |
| 783 | } |
| 784 | } |
| 785 | |
| 786 | var ( |
| 787 | deploymentConditionStatusChartTmpl = collectorapi.Chart{ |
| 788 | IDSep: true, |
| 789 | ID: "deployment_%s.conditions", |
| 790 | Title: "Deployment Conditions", |
| 791 | Units: "status", |
| 792 | Fam: "deployment conditions", |
| 793 | Ctx: "k8s_state.deployment_conditions", |
| 794 | Priority: prioDeploymentConditions, |
| 795 | Dims: collectorapi.Dims{ |
| 796 | {ID: "deploy_%s_condition_available", Name: "available"}, |
| 797 | {ID: "deploy_%s_condition_replica_failure", Name: "replica_failure"}, |
| 798 | {ID: "deploy_%s_condition_progressing", Name: "progressing"}, |
| 799 | }, |
| 800 | } |
| 801 | deploymentReplicasChartTmpl = collectorapi.Chart{ |
| 802 | IDSep: true, |
| 803 | ID: "deployment_%s.replicas", |
| 804 | Title: "Deployment Replicas", |
| 805 | Units: "replicas", |
| 806 | Fam: "deployment replicas", |
| 807 | Ctx: "k8s_state.deployment_replicas", |
| 808 | Priority: prioDeploymentReplicas, |
| 809 | Dims: collectorapi.Dims{ |
| 810 | {ID: "deploy_%s_desired_replicas", Name: "desired"}, |
| 811 | {ID: "deploy_%s_current_replicas", Name: "current"}, |
| 812 | {ID: "deploy_%s_ready_replicas", Name: "ready"}, |
| 813 | }, |
| 814 | } |
| 815 | deploymentAgeChartTmpl = collectorapi.Chart{ |
| 816 | IDSep: true, |
| 817 | ID: "deployment_%s.age", |
| 818 | Title: "Deployment Age", |
| 819 | Units: "seconds", |
| 820 | Fam: "deployment age", |
| 821 | Ctx: "k8s_state.deployment_age", |
| 822 | Priority: prioDeploymentAge, |
| 823 | Dims: collectorapi.Dims{ |
| 824 | {ID: "deploy_%s_age", Name: "age"}, |
| 825 | }, |
| 826 | } |
| 827 | ) |
| 828 | |
| 829 | func (c *Collector) addDeploymentCharts(rs *deploymentState) { |
| 830 | charts := deploymentChartsTmpl.Copy() |
| 831 | |
| 832 | for _, chart := range *charts { |
| 833 | chart.ID = fmt.Sprintf(chart.ID, replaceDots(rs.id())) |
| 834 | chart.Labels = []collectorapi.Label{ |
| 835 | {Key: labelKeyClusterID, Value: c.kubeClusterID, Source: collectorapi.LabelSourceK8s}, |
| 836 | {Key: labelKeyClusterName, Value: c.kubeClusterName, Source: collectorapi.LabelSourceK8s}, |
| 837 | {Key: labelKeyDeploymentName, Value: rs.name, Source: collectorapi.LabelSourceK8s}, |
| 838 | {Key: labelKeyNamespace, Value: rs.namespace, Source: collectorapi.LabelSourceK8s}, |
| 839 | } |
| 840 | for _, d := range chart.Dims { |
| 841 | d.ID = fmt.Sprintf(d.ID, rs.id()) |
| 842 | } |
| 843 | } |
| 844 | |
| 845 | if err := c.Charts().Add(*charts...); err != nil { |
| 846 | c.Warning(err) |
| 847 | } |
| 848 | } |
| 849 | |
| 850 | func (c *Collector) removeDeploymentCharts(st *deploymentState) { |
| 851 | prefix := fmt.Sprintf("deployment_%s", replaceDots(st.id())) |
| 852 | c.removeCharts(prefix) |
| 853 | } |
| 854 | |
| 855 | var ( |
| 856 | cronJobJobsCountByStatusChartTmpl = collectorapi.Chart{ |
| 857 | IDSep: true, |
| 858 | ID: "cronjob_%s.jobs_count_by_status", |
| 859 | Title: "CronJob Jobs Count by Status", |
| 860 | Units: "jobs", |
| 861 | Fam: "cronjob jobs", |
| 862 | Ctx: "k8s_state.cronjob_jobs_count_by_status", |
| 863 | Priority: prioCronJobJobsCountByStatus, |
| 864 | Dims: collectorapi.Dims{ |
| 865 | {ID: "cronjob_%s_complete_jobs", Name: "completed"}, |
| 866 | {ID: "cronjob_%s_failed_jobs", Name: "failed"}, |
| 867 | {ID: "cronjob_%s_running_jobs", Name: "running"}, |
| 868 | {ID: "cronjob_%s_suspended_jobs", Name: "suspended"}, |
| 869 | }, |
| 870 | } |
| 871 | cronJobJobsFailedByReasonChartTmpl = collectorapi.Chart{ |
| 872 | IDSep: true, |
| 873 | ID: "cronjob_%s.jobs_failed_by_reason", |
| 874 | Title: "CronJob Jobs Failed by Reason", |
| 875 | Units: "jobs", |
| 876 | Fam: "cronjob jobs", |
| 877 | Ctx: "k8s_state.cronjob_jobs_failed_by_reason", |
| 878 | Priority: prioCronJobJobsFailedByReason, |
| 879 | Dims: collectorapi.Dims{ |
| 880 | {ID: "cronjob_%s_failed_jobs_reason_pod_failure_policy", Name: "pod_failure_policy"}, |
| 881 | {ID: "cronjob_%s_failed_jobs_reason_backoff_limit_exceeded", Name: "backoff_limit_exceeded"}, |
| 882 | {ID: "cronjob_%s_failed_jobs_reason_deadline_exceeded", Name: "deadline_exceeded"}, |
| 883 | }, |
| 884 | } |
| 885 | cronJobLastExecutionStatusChartTmpl = collectorapi.Chart{ |
| 886 | IDSep: true, |
| 887 | ID: "cronjob_%s.last_execution_status", |
| 888 | Title: "CronJob Last Execution Status", |
| 889 | Units: "status", |
| 890 | Fam: "cronjob execution", |
| 891 | Ctx: "k8s_state.cronjob_last_execution_status", |
| 892 | Priority: prioCronJobLastExecutionStatus, |
| 893 | Dims: collectorapi.Dims{ |
| 894 | {ID: "cronjob_%s_last_execution_status_succeeded", Name: "completed"}, |
| 895 | {ID: "cronjob_%s_last_execution_status_failed", Name: "failed"}, |
| 896 | }, |
| 897 | } |
| 898 | cronJobLastCompletionDurationChartTmpl = collectorapi.Chart{ |
| 899 | IDSep: true, |
| 900 | ID: "cronjob_%s.last_completion_duration", |
| 901 | Title: "CronJob Last Completion Duration", |
| 902 | Units: "seconds", |
| 903 | Fam: "cronjob execution", |
| 904 | Ctx: "k8s_state.cronjob_last_completion_duration", |
| 905 | Priority: prioCronJobLastCompletionDuration, |
| 906 | Dims: collectorapi.Dims{ |
| 907 | {ID: "cronjob_%s_last_completion_duration", Name: "last_completion"}, |
| 908 | }, |
| 909 | } |
| 910 | cronJobLastCompletedTimeAgoChartTmpl = collectorapi.Chart{ |
| 911 | IDSep: true, |
| 912 | ID: "cronjob_%s.last_completed_time_ago", |
| 913 | Title: "CronJob Last Completed Time Ago", |
| 914 | Units: "seconds", |
| 915 | Fam: "cronjob execution", |
| 916 | Ctx: "k8s_state.cronjob_last_completed_time_ago", |
| 917 | Priority: prioCronJobLastCompletedTimeAgo, |
| 918 | Dims: collectorapi.Dims{ |
| 919 | {ID: "cronjob_%s_last_successful_seconds_ago", Name: "last_completed_ago"}, |
| 920 | }, |
| 921 | } |
| 922 | cronJobLastScheduleTimeAgoChartTmpl = collectorapi.Chart{ |
| 923 | IDSep: true, |
| 924 | ID: "cronjob_%s.last_schedule_time_ago", |
| 925 | Title: "CronJob Last Schedule Time Ago", |
| 926 | Units: "seconds", |
| 927 | Fam: "cronjob execution", |
| 928 | Ctx: "k8s_state.cronjob_last_schedule_time_ago", |
| 929 | Priority: prioCronJobLastScheduleTimeAgo, |
| 930 | Dims: collectorapi.Dims{ |
| 931 | {ID: "cronjob_%s_last_schedule_seconds_ago", Name: "last_schedule_ago"}, |
| 932 | }, |
| 933 | } |
| 934 | cronJobSuspendStatusChartTmpl = collectorapi.Chart{ |
| 935 | IDSep: true, |
| 936 | ID: "cronjob_%s.suspend_status", |
| 937 | Title: "CronJob Suspend Status", |
| 938 | Units: "status", |
| 939 | Fam: "cronjob status", |
| 940 | Ctx: "k8s_state.cronjob_suspend_status", |
| 941 | Priority: prioCronJobSuspendStatus, |
| 942 | Dims: collectorapi.Dims{ |
| 943 | {ID: "cronjob_%s_suspend_status_enabled", Name: "enabled"}, |
| 944 | {ID: "cronjob_%s_suspend_status_suspended", Name: "suspended"}, |
| 945 | }, |
| 946 | } |
| 947 | cronJobAgeChartTmpl = collectorapi.Chart{ |
| 948 | IDSep: true, |
| 949 | ID: "cronjob_%s.age", |
| 950 | Title: "CronJob Age", |
| 951 | Units: "seconds", |
| 952 | Fam: "cronjob age", |
| 953 | Ctx: "k8s_state.cronjob_age", |
| 954 | Priority: prioCronJobAge, |
| 955 | Dims: collectorapi.Dims{ |
| 956 | {ID: "cronjob_%s_age", Name: "age"}, |
| 957 | }, |
| 958 | } |
| 959 | ) |
| 960 | |
| 961 | func (c *Collector) addCronJobCharts(st *cronJobState) { |
| 962 | charts := cronJobChartsTmpl.Copy() |
| 963 | |
| 964 | for _, chart := range *charts { |
| 965 | chart.ID = fmt.Sprintf(chart.ID, replaceDots(st.id())) |
| 966 | chart.Labels = []collectorapi.Label{ |
| 967 | {Key: labelKeyClusterID, Value: c.kubeClusterID, Source: collectorapi.LabelSourceK8s}, |
| 968 | {Key: labelKeyClusterName, Value: c.kubeClusterName, Source: collectorapi.LabelSourceK8s}, |
| 969 | {Key: labelKeyCronJobName, Value: st.name, Source: collectorapi.LabelSourceK8s}, |
| 970 | {Key: labelKeyNamespace, Value: st.namespace, Source: collectorapi.LabelSourceK8s}, |
| 971 | } |
| 972 | for _, d := range chart.Dims { |
| 973 | d.ID = fmt.Sprintf(d.ID, st.id()) |
| 974 | } |
| 975 | } |
| 976 | |
| 977 | if err := c.Charts().Add(*charts...); err != nil { |
| 978 | c.Warning(err) |
| 979 | } |
| 980 | } |
| 981 | |
| 982 | func (c *Collector) removeCronJobCharts(st *cronJobState) { |
| 983 | prefix := fmt.Sprintf("cronjob_%s", replaceDots(st.id())) |
| 984 | c.removeCharts(prefix) |
| 985 | } |
| 986 | |
| 987 | func (c *Collector) removeCharts(prefix string) { |
| 988 | for _, c := range *c.Charts() { |
| 989 | if strings.HasPrefix(c.ID, prefix) { |
| 990 | c.MarkRemove() |
| 991 | c.MarkNotCreated() |
| 992 | } |
| 993 | } |
| 994 | } |
| 995 | |
| 996 | var discoveryStatusChart = collectorapi.Chart{ |
| 997 | ID: "discovery_discoverers_state", |
| 998 | Title: "Running discoverers state", |
| 999 | Units: "state", |
| 1000 | Fam: "discovery", |
| 1001 | Ctx: "k8s_state.discovery_discoverers_state", |
| 1002 | Priority: prioDiscoveryDiscovererState, |
| 1003 | Opts: collectorapi.Opts{Hidden: true}, |
| 1004 | Dims: collectorapi.Dims{ |
| 1005 | {ID: "discovery_node_discoverer_state", Name: "node"}, |
| 1006 | {ID: "discovery_pod_discoverer_state", Name: "pod"}, |
| 1007 | }, |
| 1008 | } |
| 1009 | |
| 1010 | var reDots = regexp.MustCompile(`\.`) |
| 1011 | |
| 1012 | func replaceDots(v string) string { |
| 1013 | return reDots.ReplaceAllString(v, "-") |
| 1014 | } |