| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package scrape |
| 4 | |
| 5 | import ( |
| 6 | "errors" |
| 7 | "reflect" |
| 8 | "sort" |
| 9 | "strconv" |
| 10 | "strings" |
| 11 | "time" |
| 12 | |
| 13 | rs "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/vsphere/resources" |
| 14 | |
| 15 | "github.com/vmware/govmomi/vim25/soap" |
| 16 | "github.com/vmware/govmomi/vim25/types" |
| 17 | vsantypes "github.com/vmware/govmomi/vsan/types" |
| 18 | ) |
| 19 | |
| 20 | const ( |
| 21 | defaultVSANPerfInterval = 300 |
| 22 | maxVSANWarningKeys = 64 |
| 23 | ) |
| 24 | |
| 25 | const ( |
| 26 | vsanQueryClusterPrefix = "cluster-domclient:" |
| 27 | vsanQueryHostPrefix = "host-domclient:" |
| 28 | vsanQueryVMPrefix = "virtual-machine:" |
| 29 | ) |
| 30 | |
| 31 | type VSANMetrics struct { |
| 32 | Clusters map[string]VSANEntityMetrics |
| 33 | Hosts map[string]VSANEntityMetrics |
| 34 | VMs map[string]VSANEntityMetrics |
| 35 | Space map[string]VSANSpaceUsage |
| 36 | Health map[string]string |
| 37 | } |
| 38 | |
| 39 | type VSANEntityMetrics map[string]float64 |
| 40 | |
| 41 | type VSANSpaceUsage struct { |
| 42 | Total int64 |
| 43 | Free int64 |
| 44 | } |
| 45 | |
| 46 | type vsanMetricSpec struct { |
| 47 | name string |
| 48 | rate bool |
| 49 | } |
| 50 | |
| 51 | var ( |
| 52 | vsanClusterMetricSpecs = map[string]vsanMetricSpec{ |
| 53 | "iopsRead": {name: "read_operations"}, |
| 54 | "iopsWrite": {name: "write_operations"}, |
| 55 | "throughputRead": {name: "read_throughput", rate: true}, |
| 56 | "throughputWrite": {name: "write_throughput", rate: true}, |
| 57 | "latencyAvgRead": {name: "read_latency"}, |
| 58 | "latencyAvgWrite": {name: "write_latency"}, |
| 59 | "congestion": {name: "congestions", rate: true}, |
| 60 | } |
| 61 | vsanHostMetricSpecs = map[string]vsanMetricSpec{ |
| 62 | "iopsRead": {name: "read_operations"}, |
| 63 | "iopsWrite": {name: "write_operations"}, |
| 64 | "throughputRead": {name: "read_throughput", rate: true}, |
| 65 | "throughputWrite": {name: "write_throughput", rate: true}, |
| 66 | "latencyAvgRead": {name: "read_latency"}, |
| 67 | "latencyAvgWrite": {name: "write_latency"}, |
| 68 | "congestion": {name: "congestions", rate: true}, |
| 69 | "clientCacheHitRate": {name: "cache_hit_rate"}, |
| 70 | } |
| 71 | vsanVMMetricSpecs = map[string]vsanMetricSpec{ |
| 72 | "iopsRead": {name: "read_operations"}, |
| 73 | "iopsWrite": {name: "write_operations"}, |
| 74 | "throughputRead": {name: "read_throughput", rate: true}, |
| 75 | "throughputWrite": {name: "write_throughput", rate: true}, |
| 76 | "latencyRead": {name: "read_latency"}, |
| 77 | "latencyWrite": {name: "write_latency"}, |
| 78 | } |
| 79 | ) |
| 80 | |
| 81 | func (s *Scraper) ScrapeVSAN(clusters rs.Clusters, hosts rs.Hosts, vms rs.VMs) *VSANMetrics { |
| 82 | out := &VSANMetrics{ |
| 83 | Clusters: make(map[string]VSANEntityMetrics), |
| 84 | Hosts: make(map[string]VSANEntityMetrics), |
| 85 | VMs: make(map[string]VSANEntityMetrics), |
| 86 | Space: make(map[string]VSANSpaceUsage), |
| 87 | Health: make(map[string]string), |
| 88 | } |
| 89 | |
| 90 | vsanClusters := sortedVSANClusters(clusters) |
| 91 | if len(vsanClusters) == 0 { |
| 92 | return out |
| 93 | } |
| 94 | |
| 95 | clusterByUUID := clusterIDByVSANUUID(clusters) |
| 96 | hostByUUID := hostIDByVSANNodeUUID(hosts) |
| 97 | vmByUUID := vmIDByInstanceUUID(vms) |
| 98 | |
| 99 | for _, cluster := range vsanClusters { |
| 100 | s.scrapeVSANClusterSummary(out, cluster) |
| 101 | s.scrapeVSANPerf(out.Clusters, cluster.Ref, vsanClusterQueryIDs(cluster), vsanClusterMetricSpecs, clusterByUUID) |
| 102 | s.scrapeVSANPerf(out.Hosts, cluster.Ref, vsanHostQueryIDs(cluster, hosts), vsanHostMetricSpecs, hostByUUID) |
| 103 | s.scrapeVSANPerf(out.VMs, cluster.Ref, vsanVMQueryIDs(cluster, vms), vsanVMMetricSpecs, vmByUUID) |
| 104 | } |
| 105 | |
| 106 | return out |
| 107 | } |
| 108 | |
| 109 | func (s *Scraper) scrapeVSANClusterSummary(out *VSANMetrics, cluster *rs.Cluster) { |
| 110 | space, err := s.VSANSpaceUsage(cluster.Ref) |
| 111 | if err != nil { |
| 112 | s.warnVSANOnce("space:"+vsanFaultName(err), "failed to query vSAN space usage for cluster %s: %v", cluster.ID, err) |
| 113 | } else if space != nil { |
| 114 | out.Space[cluster.ID] = VSANSpaceUsage{ |
| 115 | Total: space.TotalCapacityB, |
| 116 | Free: space.FreeCapacityB, |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | health, err := s.VSANHealth(cluster.Ref) |
| 121 | if err != nil { |
| 122 | s.warnVSANOnce("health:"+vsanFaultName(err), "failed to query vSAN health for cluster %s: %v", cluster.ID, err) |
| 123 | return |
| 124 | } |
| 125 | out.Health[cluster.ID] = health |
| 126 | } |
| 127 | |
| 128 | func (s *Scraper) scrapeVSANPerf(dst map[string]VSANEntityMetrics, cluster types.ManagedObjectReference, queries []string, specs map[string]vsanMetricSpec, idByUUID map[string]string) { |
| 129 | if len(queries) == 0 { |
| 130 | return |
| 131 | } |
| 132 | now := time.Now() |
| 133 | start := now.Add(-defaultVSANPerfInterval * time.Second) |
| 134 | qspecs := make([]vsantypes.VsanPerfQuerySpec, 0, len(queries)) |
| 135 | labels := sortedVSANMetricLabels(specs) |
| 136 | for _, query := range queries { |
| 137 | qspecs = append(qspecs, vsantypes.VsanPerfQuerySpec{ |
| 138 | EntityRefId: query, |
| 139 | StartTime: &start, |
| 140 | EndTime: &now, |
| 141 | Labels: labels, |
| 142 | }) |
| 143 | } |
| 144 | raw, err := s.VSANPerfMetrics(cluster, qspecs) |
| 145 | if err != nil { |
| 146 | s.warnVSANOnce("perf:"+queries[0]+":"+vsanFaultName(err), "failed to query %d vSAN performance entity refs for cluster %s: %v", len(queries), cluster.Value, err) |
| 147 | return |
| 148 | } |
| 149 | |
| 150 | values, err := parseVSANEntityMetrics(raw, specs) |
| 151 | if err != nil { |
| 152 | s.warnVSANOnce("parse:"+queries[0], "failed to parse vSAN performance metrics for cluster %s: %v", cluster.Value, err) |
| 153 | return |
| 154 | } |
| 155 | |
| 156 | for uuid, metrics := range values { |
| 157 | id := idByUUID[uuid] |
| 158 | if id == "" || len(metrics) == 0 { |
| 159 | continue |
| 160 | } |
| 161 | dst[id] = metrics |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | func parseVSANEntityMetrics(raw []vsantypes.VsanPerfEntityMetricCSV, specs map[string]vsanMetricSpec) (map[string]VSANEntityMetrics, error) { |
| 166 | out := make(map[string]VSANEntityMetrics) |
| 167 | for _, entity := range raw { |
| 168 | uuid, ok := vsanEntityUUID(entity.EntityRefId) |
| 169 | if !ok { |
| 170 | continue |
| 171 | } |
| 172 | values := make(VSANEntityMetrics) |
| 173 | sampleIndex := latestVSANSampleIndex(entity) |
| 174 | for _, series := range entity.Value { |
| 175 | spec, ok := specs[series.MetricId.Label] |
| 176 | if !ok { |
| 177 | continue |
| 178 | } |
| 179 | value, ok := latestVSANValue(series.Values, sampleIndex) |
| 180 | if !ok { |
| 181 | continue |
| 182 | } |
| 183 | if spec.rate { |
| 184 | interval := series.MetricId.MetricsCollectInterval |
| 185 | if interval == 0 { |
| 186 | interval = defaultVSANPerfInterval |
| 187 | } |
| 188 | value /= float64(interval) |
| 189 | } |
| 190 | values[spec.name] = value |
| 191 | } |
| 192 | if len(values) > 0 { |
| 193 | out[uuid] = values |
| 194 | } |
| 195 | } |
| 196 | return out, nil |
| 197 | } |
| 198 | |
| 199 | func latestVSANSampleIndex(entity vsantypes.VsanPerfEntityMetricCSV) int { |
| 200 | sampleCount := len(csvParts(entity.SampleInfo)) |
| 201 | if sampleCount == 0 { |
| 202 | return -1 |
| 203 | } |
| 204 | latest := -1 |
| 205 | for _, series := range entity.Value { |
| 206 | parts := csvParts(series.Values) |
| 207 | limit := len(parts) - 1 |
| 208 | if sampleCount > 0 && limit >= sampleCount { |
| 209 | limit = sampleCount - 1 |
| 210 | } |
| 211 | for i := limit; i >= 0; i-- { |
| 212 | if strings.TrimSpace(parts[i]) == "" { |
| 213 | continue |
| 214 | } |
| 215 | if i > latest { |
| 216 | latest = i |
| 217 | } |
| 218 | break |
| 219 | } |
| 220 | } |
| 221 | return latest |
| 222 | } |
| 223 | |
| 224 | func latestVSANValue(csv string, sampleIndex int) (float64, bool) { |
| 225 | parts := csvParts(csv) |
| 226 | if sampleIndex >= 0 { |
| 227 | if sampleIndex >= len(parts) { |
| 228 | return 0, false |
| 229 | } |
| 230 | part := strings.TrimSpace(parts[sampleIndex]) |
| 231 | if part == "" { |
| 232 | return 0, false |
| 233 | } |
| 234 | v, err := strconv.ParseFloat(part, 64) |
| 235 | return v, err == nil |
| 236 | } |
| 237 | |
| 238 | return latestNonEmptyVSANValue(parts) |
| 239 | } |
| 240 | |
| 241 | func latestNonEmptyVSANValue(parts []string) (float64, bool) { |
| 242 | if len(parts) == 0 { |
| 243 | return 0, false |
| 244 | } |
| 245 | for i := len(parts) - 1; i >= 0; i-- { |
| 246 | part := strings.TrimSpace(parts[i]) |
| 247 | if part == "" { |
| 248 | continue |
| 249 | } |
| 250 | v, err := strconv.ParseFloat(part, 64) |
| 251 | return v, err == nil |
| 252 | } |
| 253 | return 0, false |
| 254 | } |
| 255 | |
| 256 | func csvParts(csv string) []string { |
| 257 | if csv == "" { |
| 258 | return nil |
| 259 | } |
| 260 | return strings.Split(csv, ",") |
| 261 | } |
| 262 | |
| 263 | func vsanEntityUUID(refID string) (string, bool) { |
| 264 | _, uuid, ok := strings.Cut(refID, ":") |
| 265 | return uuid, ok && uuid != "" |
| 266 | } |
| 267 | |
| 268 | func sortedVSANMetricLabels(specs map[string]vsanMetricSpec) []string { |
| 269 | labels := make([]string, 0, len(specs)) |
| 270 | for label := range specs { |
| 271 | labels = append(labels, label) |
| 272 | } |
| 273 | sort.Strings(labels) |
| 274 | return labels |
| 275 | } |
| 276 | |
| 277 | func sortedVSANClusters(clusters rs.Clusters) []*rs.Cluster { |
| 278 | out := make([]*rs.Cluster, 0, len(clusters)) |
| 279 | for _, cluster := range clusters { |
| 280 | if cluster.VSANEnabled { |
| 281 | out = append(out, cluster) |
| 282 | } |
| 283 | } |
| 284 | sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) |
| 285 | return out |
| 286 | } |
| 287 | |
| 288 | func vsanClusterQueryIDs(cluster *rs.Cluster) []string { |
| 289 | if cluster == nil || cluster.VSANUUID == "" { |
| 290 | return nil |
| 291 | } |
| 292 | return []string{vsanQueryClusterPrefix + cluster.VSANUUID} |
| 293 | } |
| 294 | |
| 295 | func vsanHostQueryIDs(cluster *rs.Cluster, hosts rs.Hosts) []string { |
| 296 | if cluster == nil { |
| 297 | return nil |
| 298 | } |
| 299 | var out []string |
| 300 | for _, host := range hosts { |
| 301 | if host.Hier.Cluster.ID == cluster.ID && host.VSANNodeUUID != "" { |
| 302 | out = append(out, vsanQueryHostPrefix+host.VSANNodeUUID) |
| 303 | } |
| 304 | } |
| 305 | sort.Strings(out) |
| 306 | return out |
| 307 | } |
| 308 | |
| 309 | func vsanVMQueryIDs(cluster *rs.Cluster, vms rs.VMs) []string { |
| 310 | if cluster == nil { |
| 311 | return nil |
| 312 | } |
| 313 | var out []string |
| 314 | for _, vm := range vms { |
| 315 | if vm.Hier.Cluster.ID == cluster.ID && vm.InstanceUUID != "" { |
| 316 | out = append(out, vsanQueryVMPrefix+vm.InstanceUUID) |
| 317 | } |
| 318 | } |
| 319 | sort.Strings(out) |
| 320 | return out |
| 321 | } |
| 322 | |
| 323 | func clusterIDByVSANUUID(clusters rs.Clusters) map[string]string { |
| 324 | out := make(map[string]string, len(clusters)) |
| 325 | for _, cluster := range clusters { |
| 326 | if cluster.VSANUUID != "" { |
| 327 | out[cluster.VSANUUID] = cluster.ID |
| 328 | } |
| 329 | } |
| 330 | return out |
| 331 | } |
| 332 | |
| 333 | func hostIDByVSANNodeUUID(hosts rs.Hosts) map[string]string { |
| 334 | out := make(map[string]string, len(hosts)) |
| 335 | for _, host := range hosts { |
| 336 | if host.VSANNodeUUID != "" { |
| 337 | out[host.VSANNodeUUID] = host.ID |
| 338 | } |
| 339 | } |
| 340 | return out |
| 341 | } |
| 342 | |
| 343 | func vmIDByInstanceUUID(vms rs.VMs) map[string]string { |
| 344 | out := make(map[string]string, len(vms)) |
| 345 | for _, vm := range vms { |
| 346 | if vm.InstanceUUID != "" { |
| 347 | out[vm.InstanceUUID] = vm.ID |
| 348 | } |
| 349 | } |
| 350 | return out |
| 351 | } |
| 352 | |
| 353 | func (s *Scraper) warnVSANOnce(key, format string, args ...any) { |
| 354 | s.vsanWarningsLock.Lock() |
| 355 | defer s.vsanWarningsLock.Unlock() |
| 356 | |
| 357 | if s.vsanWarnings == nil { |
| 358 | s.vsanWarnings = make(map[string]bool) |
| 359 | } |
| 360 | if s.vsanWarnings[key] { |
| 361 | return |
| 362 | } |
| 363 | if len(s.vsanWarnings) >= maxVSANWarningKeys { |
| 364 | return |
| 365 | } |
| 366 | s.vsanWarnings[key] = true |
| 367 | s.Warningf(format, args...) |
| 368 | } |
| 369 | |
| 370 | func vsanFaultName(err error) string { |
| 371 | for e := err; e != nil; e = errors.Unwrap(e) { |
| 372 | if !soap.IsSoapFault(e) { |
| 373 | continue |
| 374 | } |
| 375 | fault := soap.ToSoapFault(e) |
| 376 | if fault.Detail.Fault != nil { |
| 377 | t := reflect.TypeOf(fault.Detail.Fault) |
| 378 | if t.Kind() == reflect.Pointer { |
| 379 | t = t.Elem() |
| 380 | } |
| 381 | return t.Name() |
| 382 | } |
| 383 | if fault.String != "" { |
| 384 | return fault.String |
| 385 | } |
| 386 | } |
| 387 | return "unknown" |
| 388 | } |