| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package vsphere |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "strings" |
| 8 | "time" |
| 9 | "unicode" |
| 10 | |
| 11 | "github.com/netdata/netdata/go/plugins/pkg/metrix" |
| 12 | rs "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/vsphere/resources" |
| 13 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/oldmetrix" |
| 14 | |
| 15 | "github.com/vmware/govmomi/performance" |
| 16 | "github.com/vmware/govmomi/vim25/mo" |
| 17 | "github.com/vmware/govmomi/vim25/types" |
| 18 | ) |
| 19 | |
| 20 | // ManagedEntityStatus |
| 21 | var overallStatuses = []string{"green", "red", "yellow", "gray"} |
| 22 | |
| 23 | var vmPowerStates = []struct { |
| 24 | value string |
| 25 | key string |
| 26 | }{ |
| 27 | {value: string(types.VirtualMachinePowerStatePoweredOn), key: "poweredOn"}, |
| 28 | {value: string(types.VirtualMachinePowerStatePoweredOff), key: "poweredOff"}, |
| 29 | {value: string(types.VirtualMachinePowerStateSuspended), key: "suspended"}, |
| 30 | } |
| 31 | |
| 32 | var vmConnectionStates = []struct { |
| 33 | value string |
| 34 | key string |
| 35 | }{ |
| 36 | {value: string(types.VirtualMachineConnectionStateConnected), key: "connected"}, |
| 37 | {value: string(types.VirtualMachineConnectionStateDisconnected), key: "disconnected"}, |
| 38 | {value: string(types.VirtualMachineConnectionStateOrphaned), key: "orphaned"}, |
| 39 | {value: string(types.VirtualMachineConnectionStateInaccessible), key: "inaccessible"}, |
| 40 | {value: string(types.VirtualMachineConnectionStateInvalid), key: "invalid"}, |
| 41 | } |
| 42 | |
| 43 | var vmToolsRunningStatuses = []struct { |
| 44 | value string |
| 45 | key string |
| 46 | }{ |
| 47 | {value: string(types.VirtualMachineToolsRunningStatusGuestToolsRunning), key: "running"}, |
| 48 | {value: string(types.VirtualMachineToolsRunningStatusGuestToolsNotRunning), key: "notRunning"}, |
| 49 | {value: string(types.VirtualMachineToolsRunningStatusGuestToolsExecutingScripts), key: "executingScripts"}, |
| 50 | } |
| 51 | |
| 52 | var vmToolsVersionStatuses = []struct { |
| 53 | value string |
| 54 | key string |
| 55 | }{ |
| 56 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsCurrent), key: "current"}, |
| 57 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsNeedUpgrade), key: "needUpgrade"}, |
| 58 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsNotInstalled), key: "notInstalled"}, |
| 59 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsUnmanaged), key: "unmanaged"}, |
| 60 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsTooOld), key: "tooOld"}, |
| 61 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsSupportedOld), key: "supportedOld"}, |
| 62 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsSupportedNew), key: "supportedNew"}, |
| 63 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsTooNew), key: "tooNew"}, |
| 64 | {value: string(types.VirtualMachineToolsVersionStatusGuestToolsBlacklisted), key: "blacklisted"}, |
| 65 | } |
| 66 | |
| 67 | var hostPowerStates = []struct { |
| 68 | value string |
| 69 | key string |
| 70 | }{ |
| 71 | {value: string(types.HostSystemPowerStatePoweredOn), key: "poweredOn"}, |
| 72 | {value: string(types.HostSystemPowerStatePoweredOff), key: "poweredOff"}, |
| 73 | {value: string(types.HostSystemPowerStateStandBy), key: "standBy"}, |
| 74 | {value: string(types.HostSystemPowerStateUnknown), key: "unknown"}, |
| 75 | } |
| 76 | |
| 77 | var hostConnectionStates = []struct { |
| 78 | value string |
| 79 | key string |
| 80 | }{ |
| 81 | {value: string(types.HostSystemConnectionStateConnected), key: "connected"}, |
| 82 | {value: string(types.HostSystemConnectionStateNotResponding), key: "notResponding"}, |
| 83 | {value: string(types.HostSystemConnectionStateDisconnected), key: "disconnected"}, |
| 84 | } |
| 85 | |
| 86 | var datastoreMaintenanceModes = []struct { |
| 87 | value string |
| 88 | key string |
| 89 | }{ |
| 90 | {value: string(types.DatastoreSummaryMaintenanceModeStateNormal), key: "normal"}, |
| 91 | {value: string(types.DatastoreSummaryMaintenanceModeStateEnteringMaintenance), key: "enteringMaintenance"}, |
| 92 | {value: string(types.DatastoreSummaryMaintenanceModeStateInMaintenance), key: "inMaintenance"}, |
| 93 | } |
| 94 | |
| 95 | var clusterDRSModes = []struct { |
| 96 | value string |
| 97 | key string |
| 98 | }{ |
| 99 | {value: string(types.DrsBehaviorManual), key: "manual"}, |
| 100 | {value: string(types.DrsBehaviorPartiallyAutomated), key: "partiallyAutomated"}, |
| 101 | {value: string(types.DrsBehaviorFullyAutomated), key: "fullyAutomated"}, |
| 102 | } |
| 103 | |
| 104 | var clusterHAServiceStates = []struct { |
| 105 | value string |
| 106 | key string |
| 107 | }{ |
| 108 | {value: string(types.ClusterDasConfigInfoServiceStateEnabled), key: "enabled"}, |
| 109 | {value: string(types.ClusterDasConfigInfoServiceStateDisabled), key: "disabled"}, |
| 110 | } |
| 111 | |
| 112 | var clusterHAVMMonitoringStates = []struct { |
| 113 | value string |
| 114 | key string |
| 115 | }{ |
| 116 | {value: string(types.ClusterDasConfigInfoVmMonitoringStateVmMonitoringDisabled), key: "vmMonitoringDisabled"}, |
| 117 | {value: string(types.ClusterDasConfigInfoVmMonitoringStateVmMonitoringOnly), key: "vmMonitoringOnly"}, |
| 118 | {value: string(types.ClusterDasConfigInfoVmMonitoringStateVmAndAppMonitoring), key: "vmAndAppMonitoring"}, |
| 119 | } |
| 120 | |
| 121 | var hostPerfMetricByCounter = map[string]string{ |
| 122 | "cpu.usage.average": "host_cpu_utilization_used", |
| 123 | "mem.usage.average": "host_mem_utilization_used", |
| 124 | "mem.granted.average": "host_mem_usage_granted", |
| 125 | "mem.consumed.average": "host_mem_usage_consumed", |
| 126 | "mem.active.average": "host_mem_usage_active", |
| 127 | "mem.shared.average": "host_mem_usage_shared", |
| 128 | "mem.sharedcommon.average": "host_mem_usage_sharedcommon", |
| 129 | "mem.swapinRate.average": "host_mem_swap_io_in", |
| 130 | "mem.swapoutRate.average": "host_mem_swap_io_out", |
| 131 | "disk.read.average": "host_disk_io_read", |
| 132 | "disk.write.average": "host_disk_io_write", |
| 133 | "disk.maxTotalLatency.latest": "host_disk_max_latency_latency", |
| 134 | "net.bytesRx.average": "host_net_traffic_received", |
| 135 | "net.bytesTx.average": "host_net_traffic_sent", |
| 136 | "net.packetsRx.summation": "host_net_packets_received", |
| 137 | "net.packetsTx.summation": "host_net_packets_sent", |
| 138 | "net.droppedRx.summation": "host_net_drops_received", |
| 139 | "net.droppedTx.summation": "host_net_drops_sent", |
| 140 | "net.errorsRx.summation": "host_net_errors_received", |
| 141 | "net.errorsTx.summation": "host_net_errors_sent", |
| 142 | "sys.uptime.latest": "host_system_uptime_uptime", |
| 143 | } |
| 144 | |
| 145 | var vmPerfMetricByCounter = map[string]string{ |
| 146 | "cpu.usage.average": "vm_cpu_utilization_used", |
| 147 | "mem.usage.average": "vm_mem_utilization_used", |
| 148 | "mem.granted.average": "vm_mem_usage_granted", |
| 149 | "mem.consumed.average": "vm_mem_usage_consumed", |
| 150 | "mem.active.average": "vm_mem_usage_active", |
| 151 | "mem.shared.average": "vm_mem_usage_shared", |
| 152 | "mem.swapped.average": "vm_mem_swap_usage_swapped", |
| 153 | "mem.swapinRate.average": "vm_mem_swap_io_in", |
| 154 | "mem.swapoutRate.average": "vm_mem_swap_io_out", |
| 155 | "disk.read.average": "vm_disk_io_read", |
| 156 | "disk.write.average": "vm_disk_io_write", |
| 157 | "disk.maxTotalLatency.latest": "vm_disk_max_latency_latency", |
| 158 | "net.bytesRx.average": "vm_net_traffic_received", |
| 159 | "net.bytesTx.average": "vm_net_traffic_sent", |
| 160 | "net.packetsRx.summation": "vm_net_packets_received", |
| 161 | "net.packetsTx.summation": "vm_net_packets_sent", |
| 162 | "net.droppedRx.summation": "vm_net_drops_received", |
| 163 | "net.droppedTx.summation": "vm_net_drops_sent", |
| 164 | "sys.uptime.latest": "vm_system_uptime_uptime", |
| 165 | } |
| 166 | |
| 167 | var datastorePerfMetricByCounter = map[string]string{ |
| 168 | "datastore.read.average": "datastore_disk_io_read", |
| 169 | "datastore.write.average": "datastore_disk_io_write", |
| 170 | "datastore.numberReadAveraged.average": "datastore_disk_iops_reads", |
| 171 | "datastore.numberWriteAveraged.average": "datastore_disk_iops_writes", |
| 172 | "datastore.totalReadLatency.average": "datastore_disk_latency_read", |
| 173 | "datastore.totalWriteLatency.average": "datastore_disk_latency_write", |
| 174 | } |
| 175 | |
| 176 | var clusterPerfMetricByCounter = map[string]string{ |
| 177 | "cpu.usage.average": "cluster_cpu_utilization_used", |
| 178 | "cpu.usagemhz.average": "cluster_cpu_usage_used", |
| 179 | "cpu.totalmhz.average": "cluster_cpu_usage_total", |
| 180 | "mem.usage.average": "cluster_mem_utilization_used", |
| 181 | "mem.consumed.average": "cluster_mem_usage_consumed", |
| 182 | "mem.active.average": "cluster_mem_usage_active", |
| 183 | "mem.granted.average": "cluster_mem_usage_granted", |
| 184 | "mem.shared.average": "cluster_mem_usage_shared", |
| 185 | "mem.overhead.average": "cluster_mem_usage_overhead", |
| 186 | "mem.swapused.average": "cluster_mem_usage_swap_used", |
| 187 | "clusterServices.effectivecpu.average": "cluster_services_effective_cpu_effective_cpu", |
| 188 | "clusterServices.effectivemem.average": "cluster_services_effective_mem_effective_mem", |
| 189 | "clusterServices.cpufairness.latest": "cluster_services_fairness_cpu", |
| 190 | "clusterServices.memfairness.latest": "cluster_services_fairness_memory", |
| 191 | "clusterServices.failover.latest": "cluster_services_failover_failures_tolerable", |
| 192 | "vmop.numVMotion.latest": "cluster_vm_migrations_vmotion", |
| 193 | "vmop.numSVMotion.latest": "cluster_vm_migrations_svmotion", |
| 194 | "vmop.numXVMotion.latest": "cluster_vm_migrations_xvmotion", |
| 195 | "vmop.numPoweron.latest": "cluster_vm_lifecycle_poweron", |
| 196 | "vmop.numPoweroff.latest": "cluster_vm_lifecycle_poweroff", |
| 197 | "vmop.numCreate.latest": "cluster_vm_lifecycle_create", |
| 198 | "vmop.numDestroy.latest": "cluster_vm_lifecycle_destroy", |
| 199 | "vmop.numClone.latest": "cluster_vm_lifecycle_clone", |
| 200 | "vmop.numDeploy.latest": "cluster_vm_lifecycle_deploy", |
| 201 | "vmop.numReconfigure.latest": "cluster_vm_management_reconfigure", |
| 202 | "vmop.numReset.latest": "cluster_vm_management_reset", |
| 203 | "vmop.numSuspend.latest": "cluster_vm_management_suspend", |
| 204 | "vmop.numRegister.latest": "cluster_vm_management_register", |
| 205 | "vmop.numUnregister.latest": "cluster_vm_management_unregister", |
| 206 | "vmop.numRebootGuest.latest": "cluster_vm_guest_ops_reboot", |
| 207 | "vmop.numShutdownGuest.latest": "cluster_vm_guest_ops_shutdown", |
| 208 | "vmop.numStandbyGuest.latest": "cluster_vm_guest_ops_standby", |
| 209 | "vmop.numChangeDS.latest": "cluster_vm_cold_migrations_change_ds", |
| 210 | "vmop.numChangeHost.latest": "cluster_vm_cold_migrations_change_host", |
| 211 | "vmop.numChangeHostDS.latest": "cluster_vm_cold_migrations_change_host_ds", |
| 212 | } |
| 213 | |
| 214 | const ( |
| 215 | recurringLogEvery = time.Hour |
| 216 | |
| 217 | logKeyHostNoPerfSamples = "vsphere:host-no-perf-samples" |
| 218 | logKeyVMNoPerfSamples = "vsphere:vm-no-perf-samples" |
| 219 | logKeyDatastorePropertyRefreshError = "vsphere:datastore-property-refresh-error" |
| 220 | logKeyClusterPropertyRefreshError = "vsphere:cluster-property-refresh-error" |
| 221 | logKeyResourcePoolRefreshError = "vsphere:resource-pool-property-refresh-error" |
| 222 | logKeyDiscoveryError = "vsphere:periodic-discovery-error" |
| 223 | ) |
| 224 | |
| 225 | func (c *Collector) collectLocked() error { |
| 226 | c.Debug("starting collection process") |
| 227 | t := time.Now() |
| 228 | c.hostPowerPerfSamples = nil |
| 229 | c.vmPowerPerfSamples = nil |
| 230 | c.vsanMetrics = nil |
| 231 | |
| 232 | c.collectInventory() |
| 233 | |
| 234 | err := c.collectHosts() |
| 235 | if err != nil { |
| 236 | return fmt.Errorf("collect host metrics from vSphere resources: %w", err) |
| 237 | } |
| 238 | |
| 239 | err = c.collectVMs() |
| 240 | if err != nil { |
| 241 | return fmt.Errorf("collect VM metrics from vSphere resources: %w", err) |
| 242 | } |
| 243 | |
| 244 | c.collectDatastores() |
| 245 | c.collectClusters() |
| 246 | c.collectResourcePools() |
| 247 | c.collectVSAN() |
| 248 | c.writeDatastoreClusterMetrics() |
| 249 | c.writePowerMetrics() |
| 250 | c.writeVSANMetrics() |
| 251 | |
| 252 | c.Debugf("metrics collected, process took %s", time.Since(t)) |
| 253 | |
| 254 | return nil |
| 255 | } |
| 256 | |
| 257 | func (c *Collector) collectVSAN() { |
| 258 | if !c.CollectVSAN || c.resources == nil { |
| 259 | return |
| 260 | } |
| 261 | clusters, hosts, vms := c.vsanResources() |
| 262 | c.vsanMetrics = c.ScrapeVSAN(clusters, hosts, vms) |
| 263 | } |
| 264 | |
| 265 | func (c *Collector) collectInventory() { |
| 266 | if c.resources == nil { |
| 267 | return |
| 268 | } |
| 269 | |
| 270 | labels := c.inventoryLabelSet() |
| 271 | c.observeGauge("inventory_objects_datacenters", int64(len(c.resources.DataCenters)), labels) |
| 272 | c.observeGauge("inventory_objects_folders", int64(len(c.resources.Folders)), labels) |
| 273 | c.observeGauge("inventory_objects_clusters", int64(len(c.resources.Clusters)), labels) |
| 274 | c.observeGauge("inventory_objects_hosts", int64(len(c.resources.Hosts)), labels) |
| 275 | c.observeGauge("inventory_objects_vms", int64(len(c.resources.VMs)), labels) |
| 276 | c.observeGauge("inventory_objects_datastores", int64(len(c.resources.Datastores)), labels) |
| 277 | c.observeGauge("inventory_objects_resource_pools", int64(len(c.resources.ResourcePools)), labels) |
| 278 | } |
| 279 | |
| 280 | func (c *Collector) collectHosts() error { |
| 281 | if len(c.resources.Hosts) == 0 { |
| 282 | return nil |
| 283 | } |
| 284 | c.collectHostsPropertyMetrics() |
| 285 | |
| 286 | poweredOnHosts := numPoweredOnHosts(c.resources.Hosts) |
| 287 | if poweredOnHosts == 0 { |
| 288 | return nil |
| 289 | } |
| 290 | |
| 291 | // NOTE: returns unsorted if at least one types.PerfMetricId Instance is not "" |
| 292 | metrics := c.ScrapeHosts(c.resources.Hosts) |
| 293 | if len(metrics) == 0 { |
| 294 | c.Limit(logKeyHostNoPerfSamples, 1, recurringLogEvery). |
| 295 | Warningf("collect host performance metrics: vSphere returned no samples for %d powered-on host(s) out of %d discovered host(s)", poweredOnHosts, len(c.resources.Hosts)) |
| 296 | return nil |
| 297 | } |
| 298 | |
| 299 | c.collectHostsMetrics(metrics) |
| 300 | |
| 301 | return nil |
| 302 | } |
| 303 | |
| 304 | func (c *Collector) collectHostsPropertyMetrics() { |
| 305 | for _, host := range c.resources.Hosts { |
| 306 | c.writeHostPropertyMetrics(host) |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | func (c *Collector) collectHostsMetrics(metrics []performance.EntityMetric) { |
| 311 | for _, metric := range metrics { |
| 312 | if host := c.resources.Hosts.Get(metric.Entity.Value); host != nil { |
| 313 | c.writeHostPerfMetrics(host, metric.Value) |
| 314 | c.collectHostPowerMetrics(host, metric.Value) |
| 315 | } |
| 316 | } |
| 317 | } |
| 318 | |
| 319 | func numPoweredOnHosts(hosts rs.Hosts) (num int) { |
| 320 | for _, host := range hosts { |
| 321 | if host.IsPoweredOn() { |
| 322 | num++ |
| 323 | } |
| 324 | } |
| 325 | return num |
| 326 | } |
| 327 | |
| 328 | func (c *Collector) writeHostPerfMetrics(host *rs.Host, metrics []performance.MetricSeries) { |
| 329 | labels := c.hostLabelSet(host) |
| 330 | for _, metric := range metrics { |
| 331 | if _, ok := hostPowerMetricByCounter[metric.Name]; ok { |
| 332 | continue |
| 333 | } |
| 334 | if metric.Instance != "" { |
| 335 | continue |
| 336 | } |
| 337 | if len(metric.Value) == 0 || metric.Value[0] == -1 { |
| 338 | continue |
| 339 | } |
| 340 | name := hostPerfMetricByCounter[metric.Name] |
| 341 | if name == "" { |
| 342 | continue |
| 343 | } |
| 344 | c.observeGauge(name, metric.Value[0], labels) |
| 345 | } |
| 346 | } |
| 347 | |
| 348 | func (c *Collector) writeHostPropertyMetrics(host *rs.Host) { |
| 349 | labels := c.hostLabelSet(host) |
| 350 | for _, v := range overallStatuses { |
| 351 | c.observeGauge("host_overall_status_"+v, oldmetrix.Bool(host.OverallStatus == v), labels) |
| 352 | } |
| 353 | for _, v := range hostPowerStates { |
| 354 | c.observeGauge("host_power_state_"+snakeStatus(v.key), oldmetrix.Bool(host.PowerState == v.value), labels) |
| 355 | } |
| 356 | for _, v := range hostConnectionStates { |
| 357 | c.observeGauge("host_connection_state_"+snakeStatus(v.key), oldmetrix.Bool(host.ConnectionState == v.value), labels) |
| 358 | } |
| 359 | c.observeGauge("host_maintenance_status_in_maintenance", oldmetrix.Bool(host.InMaintenanceMode), labels) |
| 360 | c.observeGauge("host_maintenance_status_normal", oldmetrix.Bool(!host.InMaintenanceMode), labels) |
| 361 | } |
| 362 | |
| 363 | func (c *Collector) collectVMs() error { |
| 364 | if len(c.resources.VMs) == 0 { |
| 365 | return nil |
| 366 | } |
| 367 | c.collectVMsPropertyMetrics() |
| 368 | |
| 369 | poweredOnVMs := numPoweredOnVMs(c.resources.VMs) |
| 370 | if poweredOnVMs == 0 { |
| 371 | return nil |
| 372 | } |
| 373 | |
| 374 | // NOTE: returns unsorted if at least one types.PerfMetricId Instance is not "" |
| 375 | ems := c.ScrapeVMs(c.resources.VMs) |
| 376 | if len(ems) == 0 { |
| 377 | c.Limit(logKeyVMNoPerfSamples, 1, recurringLogEvery). |
| 378 | Warningf("collect VM performance metrics: vSphere returned no samples for %d powered-on VM(s) out of %d discovered VM(s)", poweredOnVMs, len(c.resources.VMs)) |
| 379 | return nil |
| 380 | } |
| 381 | |
| 382 | c.collectVMsMetrics(ems) |
| 383 | |
| 384 | return nil |
| 385 | } |
| 386 | |
| 387 | func (c *Collector) collectVMsPropertyMetrics() { |
| 388 | for _, vm := range c.resources.VMs { |
| 389 | c.writeVMPropertyMetrics(vm) |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | func (c *Collector) collectVMsMetrics(metrics []performance.EntityMetric) { |
| 394 | for _, metric := range metrics { |
| 395 | if vm := c.resources.VMs.Get(metric.Entity.Value); vm != nil { |
| 396 | c.writeVMPerfMetrics(vm, metric.Value) |
| 397 | c.collectVMPowerMetrics(vm, metric.Value) |
| 398 | } |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | func numPoweredOnVMs(vms rs.VMs) (num int) { |
| 403 | for _, vm := range vms { |
| 404 | if vm.IsPoweredOn() { |
| 405 | num++ |
| 406 | } |
| 407 | } |
| 408 | return num |
| 409 | } |
| 410 | |
| 411 | func (c *Collector) writeVMPerfMetrics(vm *rs.VM, metrics []performance.MetricSeries) { |
| 412 | labels := c.vmLabelSet(vm) |
| 413 | for _, metric := range metrics { |
| 414 | if _, ok := vmPowerMetricByCounter[metric.Name]; ok { |
| 415 | continue |
| 416 | } |
| 417 | if metric.Instance != "" { |
| 418 | continue |
| 419 | } |
| 420 | if len(metric.Value) == 0 || metric.Value[0] == -1 { |
| 421 | continue |
| 422 | } |
| 423 | name := vmPerfMetricByCounter[metric.Name] |
| 424 | if name == "" { |
| 425 | continue |
| 426 | } |
| 427 | c.observeGauge(name, metric.Value[0], labels) |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | func (c *Collector) writeVMPropertyMetrics(vm *rs.VM) { |
| 432 | labels := c.vmLabelSet(vm) |
| 433 | for _, v := range overallStatuses { |
| 434 | c.observeGauge("vm_overall_status_"+v, oldmetrix.Bool(vm.OverallStatus == v), labels) |
| 435 | } |
| 436 | for _, v := range vmPowerStates { |
| 437 | c.observeGauge("vm_power_state_"+snakeStatus(v.key), oldmetrix.Bool(vm.PowerState == v.value), labels) |
| 438 | } |
| 439 | for _, v := range vmConnectionStates { |
| 440 | c.observeGauge("vm_connection_state_"+snakeStatus(v.key), oldmetrix.Bool(vm.ConnectionState == v.value), labels) |
| 441 | } |
| 442 | toolsRunningStatusKnown := false |
| 443 | for _, v := range vmToolsRunningStatuses { |
| 444 | ok := vm.ToolsRunningStatus == v.value |
| 445 | c.observeGauge("vm_tools_running_status_"+snakeStatus(v.key), oldmetrix.Bool(ok), labels) |
| 446 | toolsRunningStatusKnown = toolsRunningStatusKnown || ok |
| 447 | } |
| 448 | c.observeGauge("vm_tools_running_status_unknown", oldmetrix.Bool(!toolsRunningStatusKnown), labels) |
| 449 | |
| 450 | toolsVersionStatusKnown := false |
| 451 | for _, v := range vmToolsVersionStatuses { |
| 452 | ok := vm.ToolsVersionStatus == v.value |
| 453 | c.observeGauge("vm_tools_version_status_"+snakeStatus(v.key), oldmetrix.Bool(ok), labels) |
| 454 | toolsVersionStatusKnown = toolsVersionStatusKnown || ok |
| 455 | } |
| 456 | c.observeGauge("vm_tools_version_status_unknown", oldmetrix.Bool(!toolsVersionStatusKnown), labels) |
| 457 | |
| 458 | c.observeGauge("vm_consolidation_needed_needed", oldmetrix.Bool(vm.ConsolidationNeeded), labels) |
| 459 | c.observeGauge("vm_consolidation_needed_not_needed", oldmetrix.Bool(!vm.ConsolidationNeeded), labels) |
| 460 | c.observeGauge("vm_config_cpu_vcpus", vm.ConfigCPU, labels) |
| 461 | c.observeGauge("vm_config_memory_memory", vm.ConfigMemory, labels) |
| 462 | c.observeGauge("vm_config_devices_disks", vm.ConfigDisks, labels) |
| 463 | c.observeGauge("vm_config_devices_nics", vm.ConfigNICs, labels) |
| 464 | c.observeGauge("vm_storage_usage_committed", vm.StorageCommitted, labels) |
| 465 | c.observeGauge("vm_storage_usage_uncommitted", vm.StorageUncommitted, labels) |
| 466 | c.observeGauge("vm_storage_usage_unshared", vm.StorageUnshared, labels) |
| 467 | c.observeGauge("vm_snapshot_count_count", vm.SnapshotCount, labels) |
| 468 | c.observeGauge("vm_snapshot_max_chain_depth_depth", vm.SnapshotMaxChainDepth, labels) |
| 469 | c.observeGauge("vm_snapshot_max_age_age", snapshotMaxAgeSeconds(vm.SnapshotOldestCreateTime), labels) |
| 470 | } |
| 471 | |
| 472 | func snapshotMaxAgeSeconds(oldest time.Time) int64 { |
| 473 | if oldest.IsZero() { |
| 474 | return 0 |
| 475 | } |
| 476 | age := time.Since(oldest).Seconds() |
| 477 | if age < 0 { |
| 478 | return 0 |
| 479 | } |
| 480 | return int64(age) |
| 481 | } |
| 482 | |
| 483 | func snakeStatus(s string) string { |
| 484 | switch s { |
| 485 | // Chart selectors use the VMware UI spelling, not the enum's camel-case suffix. |
| 486 | case "standBy": |
| 487 | return "standby" |
| 488 | // The chart dimension is "disabled"; the enum name includes the monitored subsystem. |
| 489 | case "vmMonitoringDisabled": |
| 490 | return "disabled" |
| 491 | } |
| 492 | |
| 493 | var b strings.Builder |
| 494 | for i, r := range s { |
| 495 | if unicode.IsUpper(r) { |
| 496 | if i > 0 { |
| 497 | b.WriteByte('_') |
| 498 | } |
| 499 | r = unicode.ToLower(r) |
| 500 | } |
| 501 | b.WriteRune(r) |
| 502 | } |
| 503 | return b.String() |
| 504 | } |
| 505 | |
| 506 | func (c *Collector) collectDatastores() { |
| 507 | if len(c.resources.Datastores) == 0 { |
| 508 | return |
| 509 | } |
| 510 | |
| 511 | refreshed := c.refreshDatastoreProperties() |
| 512 | |
| 513 | metrics := c.ScrapeDatastores(c.resources.Datastores) |
| 514 | // Datastore perf counters may return empty for vSAN or when no historical data is available yet. |
| 515 | // This is not an error — we still collect capacity and status from properties. |
| 516 | c.collectDatastoresMetrics(metrics, refreshed) |
| 517 | } |
| 518 | |
| 519 | func (c *Collector) refreshDatastoreProperties() map[string]bool { |
| 520 | refreshed := make(map[string]bool) |
| 521 | |
| 522 | if c.dsPropertyCollector == nil { |
| 523 | return refreshed |
| 524 | } |
| 525 | |
| 526 | refs := make([]types.ManagedObjectReference, 0, len(c.resources.Datastores)) |
| 527 | for _, ds := range c.resources.Datastores { |
| 528 | refs = append(refs, ds.Ref) |
| 529 | } |
| 530 | |
| 531 | pathSet := []string{"summary", "overallStatus"} |
| 532 | dsList, err := c.dsPropertyCollector.DatastoresByRef(refs, pathSet...) |
| 533 | if err != nil { |
| 534 | c.Limit(logKeyDatastorePropertyRefreshError, 1, recurringLogEvery). |
| 535 | Warningf("collect vSphere datastore properties refresh: refs=%d pathSet=%v: %v", len(refs), pathSet, err) |
| 536 | return refreshed |
| 537 | } |
| 538 | |
| 539 | for _, raw := range dsList { |
| 540 | ds := c.resources.Datastores.Get(raw.Reference().Value) |
| 541 | if ds == nil { |
| 542 | continue |
| 543 | } |
| 544 | refreshed[ds.ID] = true |
| 545 | ds.Type = raw.Summary.Type |
| 546 | ds.Capacity = raw.Summary.Capacity |
| 547 | ds.FreeSpace = raw.Summary.FreeSpace |
| 548 | ds.Uncommitted = raw.Summary.Uncommitted |
| 549 | ds.Accessible = raw.Summary.Accessible |
| 550 | ds.MaintenanceMode = raw.Summary.MaintenanceMode |
| 551 | ds.MultipleHostAccess = raw.Summary.MultipleHostAccess |
| 552 | ds.OverallStatus = string(raw.OverallStatus) |
| 553 | } |
| 554 | |
| 555 | return refreshed |
| 556 | } |
| 557 | |
| 558 | func (c *Collector) collectDatastoresMetrics(metrics []performance.EntityMetric, refreshed map[string]bool) { |
| 559 | // Property metrics reflect cached resource fields, so emit them only after |
| 560 | // the current refresh succeeds. Perf samples are fetched separately. |
| 561 | for _, ds := range c.resources.Datastores { |
| 562 | if refreshed[ds.ID] { |
| 563 | c.writeDatastoreMetrics(ds) |
| 564 | } |
| 565 | } |
| 566 | |
| 567 | for _, metric := range metrics { |
| 568 | if ds := c.resources.Datastores.Get(metric.Entity.Value); ds != nil { |
| 569 | c.writeDatastorePerfMetrics(ds, metric.Value) |
| 570 | } |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | func (c *Collector) writeDatastoreMetrics(ds *rs.Datastore) { |
| 575 | // VMware docs: Capacity and FreeSpace are guaranteed valid only when Accessible is true. |
| 576 | var capacity, freeSpace, used, uncommitted int64 |
| 577 | if ds.Accessible { |
| 578 | capacity = ds.Capacity |
| 579 | freeSpace = ds.FreeSpace |
| 580 | used = max(capacity-freeSpace, 0) |
| 581 | uncommitted = ds.Uncommitted |
| 582 | } |
| 583 | |
| 584 | labels := c.datastoreLabelSet(ds) |
| 585 | c.observeGauge("datastore_space_usage_capacity", capacity, labels) |
| 586 | c.observeGauge("datastore_space_usage_free", freeSpace, labels) |
| 587 | c.observeGauge("datastore_space_usage_used", used, labels) |
| 588 | c.observeGauge("datastore_space_usage_uncommitted", uncommitted, labels) |
| 589 | |
| 590 | if capacity > 0 { |
| 591 | // use float64 to avoid int64 overflow on datastores larger than 922 TB |
| 592 | c.observeGauge("datastore_space_utilization_used", int64(float64(used)/float64(capacity)*scaledPercent), labels) |
| 593 | } else { |
| 594 | c.observeGauge("datastore_space_utilization_used", 0, labels) |
| 595 | } |
| 596 | |
| 597 | for _, v := range overallStatuses { |
| 598 | c.observeGauge("datastore_overall_status_"+v, oldmetrix.Bool(ds.OverallStatus == v), labels) |
| 599 | } |
| 600 | |
| 601 | c.observeGauge("datastore_accessibility_status_accessible", oldmetrix.Bool(ds.Accessible), labels) |
| 602 | c.observeGauge("datastore_accessibility_status_inaccessible", oldmetrix.Bool(!ds.Accessible), labels) |
| 603 | |
| 604 | maintenanceModeKnown := false |
| 605 | for _, mode := range datastoreMaintenanceModes { |
| 606 | ok := ds.MaintenanceMode == mode.value |
| 607 | maintenanceModeKnown = maintenanceModeKnown || ok |
| 608 | c.observeGauge("datastore_maintenance_status_"+snakeStatus(mode.key), oldmetrix.Bool(ok), labels) |
| 609 | } |
| 610 | c.observeGauge("datastore_maintenance_status_unknown", oldmetrix.Bool(!maintenanceModeKnown), labels) |
| 611 | |
| 612 | c.observeGauge("datastore_multiple_host_access_enabled", oldmetrix.Bool(ds.MultipleHostAccess != nil && *ds.MultipleHostAccess), labels) |
| 613 | c.observeGauge("datastore_multiple_host_access_disabled", oldmetrix.Bool(ds.MultipleHostAccess != nil && !*ds.MultipleHostAccess), labels) |
| 614 | c.observeGauge("datastore_multiple_host_access_unknown", oldmetrix.Bool(ds.MultipleHostAccess == nil), labels) |
| 615 | } |
| 616 | |
| 617 | func (c *Collector) writeDatastorePerfMetrics(ds *rs.Datastore, metrics []performance.MetricSeries) { |
| 618 | labels := c.datastoreLabelSet(ds) |
| 619 | for _, metric := range metrics { |
| 620 | if len(metric.Value) == 0 || metric.Value[0] == -1 { |
| 621 | continue |
| 622 | } |
| 623 | name := datastorePerfMetricByCounter[metric.Name] |
| 624 | if name == "" { |
| 625 | continue |
| 626 | } |
| 627 | c.observeGauge(name, metric.Value[0], labels) |
| 628 | } |
| 629 | } |
| 630 | |
| 631 | func (c *Collector) collectClusters() { |
| 632 | if len(c.resources.Clusters) == 0 { |
| 633 | return |
| 634 | } |
| 635 | |
| 636 | refreshed := c.refreshClusterProperties() |
| 637 | |
| 638 | metrics := c.ScrapeClusters(c.resources.Clusters) |
| 639 | c.collectClustersMetrics(metrics, refreshed) |
| 640 | } |
| 641 | |
| 642 | func (c *Collector) refreshClusterProperties() map[string]bool { |
| 643 | refreshed := make(map[string]bool) |
| 644 | |
| 645 | if c.clusterPropertyCollector == nil { |
| 646 | return refreshed |
| 647 | } |
| 648 | |
| 649 | refs := make([]types.ManagedObjectReference, 0, len(c.resources.Clusters)) |
| 650 | for _, cl := range c.resources.Clusters { |
| 651 | refs = append(refs, cl.Ref) |
| 652 | } |
| 653 | |
| 654 | pathSet := []string{"name", "summary", "configurationEx", "overallStatus"} |
| 655 | clusters, err := c.clusterPropertyCollector.ClustersByRef(refs, pathSet...) |
| 656 | if err != nil { |
| 657 | c.Limit(logKeyClusterPropertyRefreshError, 1, recurringLogEvery). |
| 658 | Warningf("collect vSphere cluster properties refresh: refs=%d pathSet=%v: %v", len(refs), pathSet, err) |
| 659 | return refreshed |
| 660 | } |
| 661 | |
| 662 | for _, raw := range clusters { |
| 663 | cl := c.resources.Clusters.Get(raw.Reference().Value) |
| 664 | if cl == nil { |
| 665 | continue |
| 666 | } |
| 667 | refreshed[cl.ID] = true |
| 668 | updateClusterFromProperties(cl, raw) |
| 669 | } |
| 670 | |
| 671 | return refreshed |
| 672 | } |
| 673 | |
| 674 | func updateClusterFromProperties(cl *rs.Cluster, raw mo.ClusterComputeResource) { |
| 675 | cl.OverallStatus = string(raw.OverallStatus) |
| 676 | |
| 677 | var s *types.ComputeResourceSummary |
| 678 | if raw.Summary != nil { |
| 679 | s = raw.Summary.GetComputeResourceSummary() |
| 680 | } |
| 681 | if s != nil { |
| 682 | cl.TotalCpu = s.TotalCpu |
| 683 | cl.TotalMemory = s.TotalMemory |
| 684 | cl.NumCpuCores = s.NumCpuCores |
| 685 | cl.NumCpuThreads = s.NumCpuThreads |
| 686 | cl.EffectiveCpu = s.EffectiveCpu |
| 687 | cl.EffectiveMemory = s.EffectiveMemory |
| 688 | cl.NumHosts = s.NumHosts |
| 689 | cl.NumEffectiveHosts = s.NumEffectiveHosts |
| 690 | } |
| 691 | |
| 692 | // Reset conditional fields to avoid stale values |
| 693 | cl.NumVmotions = 0 |
| 694 | cl.CurrentBalance = 0 |
| 695 | cl.TargetBalance = 0 |
| 696 | cl.DrsScore = 0 |
| 697 | cl.UsageCpuDemandMhz = 0 |
| 698 | cl.UsageMemDemandMB = 0 |
| 699 | cl.UsageCpuEntitledMhz = 0 |
| 700 | cl.UsageMemEntitledMB = 0 |
| 701 | cl.UsageCpuReservationMhz = 0 |
| 702 | cl.UsageMemReservationMB = 0 |
| 703 | cl.UsageTotalVmCount = 0 |
| 704 | cl.UsagePoweredOffVmCount = 0 |
| 705 | cl.DrsEnabled = false |
| 706 | cl.DrsMode = "" |
| 707 | cl.DrsVmotionRate = 0 |
| 708 | cl.HaEnabled = false |
| 709 | cl.HaAdmCtrlEnabled = false |
| 710 | cl.HaHostMonitoring = "" |
| 711 | cl.HaVMMonitoring = "" |
| 712 | cl.HaVMComponentProtection = "" |
| 713 | |
| 714 | // Cluster-specific summary fields |
| 715 | if cs, ok := raw.Summary.(*types.ClusterComputeResourceSummary); ok { |
| 716 | cl.NumVmotions = cs.NumVmotions |
| 717 | cl.CurrentBalance = cs.CurrentBalance |
| 718 | cl.TargetBalance = cs.TargetBalance |
| 719 | cl.DrsScore = cs.DrsScore |
| 720 | if cs.UsageSummary != nil { |
| 721 | cl.UsageCpuDemandMhz = cs.UsageSummary.CpuDemandMhz |
| 722 | cl.UsageMemDemandMB = cs.UsageSummary.MemDemandMB |
| 723 | cl.UsageCpuEntitledMhz = cs.UsageSummary.CpuEntitledMhz |
| 724 | cl.UsageMemEntitledMB = cs.UsageSummary.MemEntitledMB |
| 725 | cl.UsageCpuReservationMhz = cs.UsageSummary.CpuReservationMhz |
| 726 | cl.UsageMemReservationMB = cs.UsageSummary.MemReservationMB |
| 727 | cl.UsageTotalVmCount = cs.UsageSummary.TotalVmCount |
| 728 | cl.UsagePoweredOffVmCount = cs.UsageSummary.PoweredOffVmCount |
| 729 | } |
| 730 | } |
| 731 | |
| 732 | // DRS and HA config from configurationEx |
| 733 | if cfg, ok := raw.ConfigurationEx.(*types.ClusterConfigInfoEx); ok { |
| 734 | rs.SetClusterVSANInfo(cl, cfg) |
| 735 | if cfg.DrsConfig.Enabled != nil { |
| 736 | cl.DrsEnabled = *cfg.DrsConfig.Enabled |
| 737 | } |
| 738 | cl.DrsMode = string(cfg.DrsConfig.DefaultVmBehavior) |
| 739 | cl.DrsVmotionRate = cfg.DrsConfig.VmotionRate |
| 740 | if cfg.DasConfig.Enabled != nil { |
| 741 | cl.HaEnabled = *cfg.DasConfig.Enabled |
| 742 | } |
| 743 | if cfg.DasConfig.AdmissionControlEnabled != nil { |
| 744 | cl.HaAdmCtrlEnabled = *cfg.DasConfig.AdmissionControlEnabled |
| 745 | } |
| 746 | cl.HaHostMonitoring = cfg.DasConfig.HostMonitoring |
| 747 | cl.HaVMMonitoring = cfg.DasConfig.VmMonitoring |
| 748 | cl.HaVMComponentProtection = cfg.DasConfig.VmComponentProtecting |
| 749 | } |
| 750 | } |
| 751 | |
| 752 | func (c *Collector) collectClustersMetrics(metrics []performance.EntityMetric, refreshed map[string]bool) { |
| 753 | // Property metrics reflect cached resource fields, so emit them only after |
| 754 | // the current refresh succeeds. Perf samples are fetched separately. |
| 755 | for _, cl := range c.resources.Clusters { |
| 756 | if refreshed[cl.ID] { |
| 757 | c.writeClusterPropertyMetrics(cl) |
| 758 | } |
| 759 | } |
| 760 | |
| 761 | for _, metric := range metrics { |
| 762 | if cl := c.resources.Clusters.Get(metric.Entity.Value); cl != nil { |
| 763 | c.writeClusterPerfMetrics(cl, metric.Value) |
| 764 | } |
| 765 | } |
| 766 | } |
| 767 | |
| 768 | func (c *Collector) writeClusterPropertyMetrics(cl *rs.Cluster) { |
| 769 | labels := c.clusterLabelSet(cl) |
| 770 | c.observeGauge("cluster_hosts_total", int64(cl.NumHosts), labels) |
| 771 | c.observeGauge("cluster_hosts_effective", int64(cl.NumEffectiveHosts), labels) |
| 772 | c.observeGauge("cluster_cpu_capacity_total", int64(cl.TotalCpu), labels) |
| 773 | c.observeGauge("cluster_cpu_capacity_effective", int64(cl.EffectiveCpu), labels) |
| 774 | c.observeGauge("cluster_mem_capacity_total", cl.TotalMemory, labels) |
| 775 | // EffectiveMemory is MB from API, convert to bytes for consistency with TotalMemory |
| 776 | c.observeGauge("cluster_mem_capacity_effective", cl.EffectiveMemory*1024*1024, labels) |
| 777 | c.observeGauge("cluster_cpu_topology_cores", int64(cl.NumCpuCores), labels) |
| 778 | c.observeGauge("cluster_cpu_topology_threads", int64(cl.NumCpuThreads), labels) |
| 779 | c.observeGauge("cluster_vmotions_vmotions", int64(cl.NumVmotions), labels) |
| 780 | c.observeGauge("cluster_drs_score_score", int64(cl.DrsScore), labels) |
| 781 | c.observeGauge("cluster_drs_balance_current", int64(cl.CurrentBalance), labels) |
| 782 | c.observeGauge("cluster_drs_balance_target", int64(cl.TargetBalance), labels) |
| 783 | |
| 784 | c.observeGauge("cluster_drs_config_enabled", oldmetrix.Bool(cl.DrsEnabled), labels) |
| 785 | drsModeKnown := false |
| 786 | for _, v := range clusterDRSModes { |
| 787 | ok := cl.DrsMode == v.value |
| 788 | c.observeGauge("cluster_drs_mode_"+snakeStatus(v.key), oldmetrix.Bool(ok), labels) |
| 789 | drsModeKnown = drsModeKnown || ok |
| 790 | } |
| 791 | c.observeGauge("cluster_drs_mode_unknown", oldmetrix.Bool(!drsModeKnown), labels) |
| 792 | c.observeGauge("cluster_drs_vmotion_rate_rate", int64(cl.DrsVmotionRate), labels) |
| 793 | |
| 794 | c.observeGauge("cluster_ha_config_enabled", oldmetrix.Bool(cl.HaEnabled), labels) |
| 795 | c.observeGauge("cluster_ha_config_admission_control", oldmetrix.Bool(cl.HaAdmCtrlEnabled), labels) |
| 796 | c.writeClusterHAServiceState("cluster_ha_host_monitoring", cl.HaHostMonitoring, labels) |
| 797 | c.writeClusterHAVMMonitoringState(cl.HaVMMonitoring, labels) |
| 798 | c.writeClusterHAServiceState("cluster_ha_vm_component_protection", cl.HaVMComponentProtection, labels) |
| 799 | |
| 800 | c.observeGauge("cluster_usage_cpu_demand", int64(cl.UsageCpuDemandMhz), labels) |
| 801 | c.observeGauge("cluster_usage_mem_demand", int64(cl.UsageMemDemandMB), labels) |
| 802 | c.observeGauge("cluster_usage_cpu_entitled", int64(cl.UsageCpuEntitledMhz), labels) |
| 803 | c.observeGauge("cluster_usage_mem_entitled", int64(cl.UsageMemEntitledMB), labels) |
| 804 | c.observeGauge("cluster_usage_cpu_reserved", int64(cl.UsageCpuReservationMhz), labels) |
| 805 | c.observeGauge("cluster_usage_mem_reserved", int64(cl.UsageMemReservationMB), labels) |
| 806 | c.observeGauge("cluster_vm_count_total", int64(cl.UsageTotalVmCount), labels) |
| 807 | c.observeGauge("cluster_vm_count_powered_off", int64(cl.UsagePoweredOffVmCount), labels) |
| 808 | |
| 809 | for _, v := range overallStatuses { |
| 810 | c.observeGauge("cluster_overall_status_"+v, oldmetrix.Bool(cl.OverallStatus == v), labels) |
| 811 | } |
| 812 | } |
| 813 | |
| 814 | func (c *Collector) writeClusterHAServiceState(prefix, state string, labels metrix.LabelSet) { |
| 815 | known := false |
| 816 | for _, v := range clusterHAServiceStates { |
| 817 | ok := state == v.value |
| 818 | c.observeGauge(prefix+"_"+snakeStatus(v.key), oldmetrix.Bool(ok), labels) |
| 819 | known = known || ok |
| 820 | } |
| 821 | c.observeGauge(prefix+"_unknown", oldmetrix.Bool(!known), labels) |
| 822 | } |
| 823 | |
| 824 | func (c *Collector) writeClusterHAVMMonitoringState(state string, labels metrix.LabelSet) { |
| 825 | known := false |
| 826 | for _, v := range clusterHAVMMonitoringStates { |
| 827 | ok := state == v.value |
| 828 | c.observeGauge("cluster_ha_vm_monitoring_"+snakeStatus(v.key), oldmetrix.Bool(ok), labels) |
| 829 | known = known || ok |
| 830 | } |
| 831 | c.observeGauge("cluster_ha_vm_monitoring_unknown", oldmetrix.Bool(!known), labels) |
| 832 | } |
| 833 | |
| 834 | func (c *Collector) writeClusterPerfMetrics(cl *rs.Cluster, metrics []performance.MetricSeries) { |
| 835 | labels := c.clusterLabelSet(cl) |
| 836 | for _, metric := range metrics { |
| 837 | if len(metric.Value) == 0 || metric.Value[0] == -1 { |
| 838 | continue |
| 839 | } |
| 840 | name := clusterPerfMetricByCounter[metric.Name] |
| 841 | if name == "" { |
| 842 | continue |
| 843 | } |
| 844 | c.observeGauge(name, metric.Value[0], labels) |
| 845 | } |
| 846 | } |
| 847 | |
| 848 | func (c *Collector) collectResourcePools() { |
| 849 | if len(c.resources.ResourcePools) == 0 { |
| 850 | return |
| 851 | } |
| 852 | |
| 853 | refreshed := c.refreshResourcePoolProperties() |
| 854 | |
| 855 | c.collectResourcePoolsMetrics(refreshed) |
| 856 | } |
| 857 | |
| 858 | func (c *Collector) refreshResourcePoolProperties() map[string]bool { |
| 859 | refreshed := make(map[string]bool) |
| 860 | |
| 861 | if c.rpPropertyCollector == nil { |
| 862 | return refreshed |
| 863 | } |
| 864 | |
| 865 | refs := make([]types.ManagedObjectReference, 0, len(c.resources.ResourcePools)) |
| 866 | for _, rp := range c.resources.ResourcePools { |
| 867 | refs = append(refs, rp.Ref) |
| 868 | } |
| 869 | |
| 870 | pathSet := []string{"name", "summary", "config", "runtime", "overallStatus"} |
| 871 | pools, err := c.rpPropertyCollector.ResourcePoolsByRef(refs, pathSet...) |
| 872 | if err != nil { |
| 873 | c.Limit(logKeyResourcePoolRefreshError, 1, recurringLogEvery). |
| 874 | Warningf("collect vSphere resource pool properties refresh: refs=%d pathSet=%v: %v", len(refs), pathSet, err) |
| 875 | return refreshed |
| 876 | } |
| 877 | |
| 878 | for _, raw := range pools { |
| 879 | rp := c.resources.ResourcePools.Get(raw.Reference().Value) |
| 880 | if rp == nil { |
| 881 | continue |
| 882 | } |
| 883 | refreshed[rp.ID] = true |
| 884 | updateResourcePoolFromProperties(rp, raw) |
| 885 | } |
| 886 | |
| 887 | return refreshed |
| 888 | } |
| 889 | |
| 890 | func updateResourcePoolFromProperties(rp *rs.ResourcePool, raw mo.ResourcePool) { |
| 891 | rp.OverallStatus = string(raw.OverallStatus) |
| 892 | |
| 893 | // Reset conditional fields to avoid stale values |
| 894 | rp.OverallCpuUsage = 0 |
| 895 | rp.OverallCpuDemand = 0 |
| 896 | rp.GuestMemoryUsage = 0 |
| 897 | rp.HostMemoryUsage = 0 |
| 898 | rp.DistributedCpuEntitlement = 0 |
| 899 | rp.DistributedMemoryEntitlement = 0 |
| 900 | rp.PrivateMemory = 0 |
| 901 | rp.SharedMemory = 0 |
| 902 | rp.SwappedMemory = 0 |
| 903 | rp.BalloonedMemory = 0 |
| 904 | rp.OverheadMemory = 0 |
| 905 | rp.ConsumedOverheadMemory = 0 |
| 906 | rp.CompressedMemory = 0 |
| 907 | |
| 908 | var qs *types.ResourcePoolQuickStats |
| 909 | if raw.Summary != nil { |
| 910 | if s := raw.Summary.GetResourcePoolSummary(); s != nil { |
| 911 | qs = s.QuickStats |
| 912 | } |
| 913 | } |
| 914 | if qs != nil { |
| 915 | rp.OverallCpuUsage = qs.OverallCpuUsage |
| 916 | rp.OverallCpuDemand = qs.OverallCpuDemand |
| 917 | rp.GuestMemoryUsage = qs.GuestMemoryUsage |
| 918 | rp.HostMemoryUsage = qs.HostMemoryUsage |
| 919 | rp.DistributedCpuEntitlement = qs.DistributedCpuEntitlement |
| 920 | rp.DistributedMemoryEntitlement = qs.DistributedMemoryEntitlement |
| 921 | rp.PrivateMemory = qs.PrivateMemory |
| 922 | rp.SharedMemory = qs.SharedMemory |
| 923 | rp.SwappedMemory = qs.SwappedMemory |
| 924 | rp.BalloonedMemory = qs.BalloonedMemory |
| 925 | rp.OverheadMemory = qs.OverheadMemory |
| 926 | rp.ConsumedOverheadMemory = qs.ConsumedOverheadMemory |
| 927 | rp.CompressedMemory = qs.CompressedMemory |
| 928 | } |
| 929 | |
| 930 | // Runtime and Config are value structs in mo.ResourcePool; missing properties |
| 931 | // decode as zero values rather than nil pointers. |
| 932 | rp.CpuReservationUsed = raw.Runtime.Cpu.ReservationUsed |
| 933 | rp.CpuMaxUsage = raw.Runtime.Cpu.MaxUsage |
| 934 | rp.CpuUnreservedForVm = raw.Runtime.Cpu.UnreservedForVm |
| 935 | rp.MemReservationUsed = raw.Runtime.Memory.ReservationUsed |
| 936 | rp.MemMaxUsage = raw.Runtime.Memory.MaxUsage |
| 937 | rp.MemUnreservedForVm = raw.Runtime.Memory.UnreservedForVm |
| 938 | |
| 939 | // Config — reservation, limit |
| 940 | rp.CpuReservation = 0 |
| 941 | rp.CpuLimit = -1 |
| 942 | rp.MemReservation = 0 |
| 943 | rp.MemLimit = -1 |
| 944 | if raw.Config.CpuAllocation.Reservation != nil { |
| 945 | rp.CpuReservation = *raw.Config.CpuAllocation.Reservation |
| 946 | } |
| 947 | if raw.Config.CpuAllocation.Limit != nil { |
| 948 | rp.CpuLimit = *raw.Config.CpuAllocation.Limit |
| 949 | } |
| 950 | if raw.Config.MemoryAllocation.Reservation != nil { |
| 951 | rp.MemReservation = *raw.Config.MemoryAllocation.Reservation |
| 952 | } |
| 953 | if raw.Config.MemoryAllocation.Limit != nil { |
| 954 | rp.MemLimit = *raw.Config.MemoryAllocation.Limit |
| 955 | } |
| 956 | |
| 957 | // CpuSharesLevel / MemSharesLevel are strings (low/normal/high/custom) — not exported as metrics |
| 958 | } |
| 959 | |
| 960 | func (c *Collector) collectResourcePoolsMetrics(refreshed map[string]bool) { |
| 961 | // Property metrics reflect cached resource fields, so emit them only after |
| 962 | // the current refresh succeeds. |
| 963 | for _, rp := range c.resources.ResourcePools { |
| 964 | if refreshed[rp.ID] { |
| 965 | c.writeResourcePoolMetrics(rp) |
| 966 | } |
| 967 | } |
| 968 | } |
| 969 | |
| 970 | func (c *Collector) writeResourcePoolMetrics(rp *rs.ResourcePool) { |
| 971 | labels := c.resourcePoolLabelSet(rp) |
| 972 | c.observeGauge("resource_pool_cpu_usage_usage", rp.OverallCpuUsage, labels) |
| 973 | c.observeGauge("resource_pool_cpu_usage_demand", rp.OverallCpuDemand, labels) |
| 974 | c.observeGauge("resource_pool_cpu_entitlement_distributed", rp.DistributedCpuEntitlement, labels) |
| 975 | c.observeGauge("resource_pool_mem_usage_guest", rp.GuestMemoryUsage, labels) |
| 976 | c.observeGauge("resource_pool_mem_usage_host", rp.HostMemoryUsage, labels) |
| 977 | c.observeGauge("resource_pool_mem_entitlement_distributed", rp.DistributedMemoryEntitlement, labels) |
| 978 | |
| 979 | c.observeGauge("resource_pool_mem_breakdown_private", rp.PrivateMemory, labels) |
| 980 | c.observeGauge("resource_pool_mem_breakdown_shared", rp.SharedMemory, labels) |
| 981 | c.observeGauge("resource_pool_mem_breakdown_swapped", rp.SwappedMemory, labels) |
| 982 | c.observeGauge("resource_pool_mem_breakdown_ballooned", rp.BalloonedMemory, labels) |
| 983 | c.observeGauge("resource_pool_mem_breakdown_overhead", rp.OverheadMemory, labels) |
| 984 | c.observeGauge("resource_pool_mem_breakdown_consumed_overhead", rp.ConsumedOverheadMemory, labels) |
| 985 | // vSphere reports CompressedMemory in KiB; the chart keeps V1's MB display scale. |
| 986 | c.observeGauge("resource_pool_mem_breakdown_compressed", rp.CompressedMemory, labels) |
| 987 | |
| 988 | c.observeGauge("resource_pool_cpu_allocation_reservation_used", rp.CpuReservationUsed, labels) |
| 989 | c.observeGauge("resource_pool_cpu_allocation_max_usage", rp.CpuMaxUsage, labels) |
| 990 | c.observeGauge("resource_pool_cpu_allocation_unreserved_for_vm", rp.CpuUnreservedForVm, labels) |
| 991 | c.observeGauge("resource_pool_mem_allocation_reservation_used", rp.MemReservationUsed, labels) |
| 992 | c.observeGauge("resource_pool_mem_allocation_max_usage", rp.MemMaxUsage, labels) |
| 993 | c.observeGauge("resource_pool_mem_allocation_unreserved_for_vm", rp.MemUnreservedForVm, labels) |
| 994 | |
| 995 | c.observeGauge("resource_pool_cpu_config_reservation", rp.CpuReservation, labels) |
| 996 | c.observeGauge("resource_pool_cpu_config_limit", rp.CpuLimit, labels) |
| 997 | c.observeGauge("resource_pool_mem_config_reservation", rp.MemReservation, labels) |
| 998 | c.observeGauge("resource_pool_mem_config_limit", rp.MemLimit, labels) |
| 999 | |
| 1000 | for _, v := range overallStatuses { |
| 1001 | c.observeGauge("resource_pool_overall_status_"+v, oldmetrix.Bool(rp.OverallStatus == v), labels) |
| 1002 | } |
| 1003 | } |