| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | package vsphere |
| 3 | |
| 4 | import ( |
| 5 | "bytes" |
| 6 | "context" |
| 7 | "crypto/tls" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/stretchr/testify/assert" |
| 15 | "github.com/stretchr/testify/require" |
| 16 | "github.com/vmware/govmomi/performance" |
| 17 | "github.com/vmware/govmomi/simulator" |
| 18 | mo25 "github.com/vmware/govmomi/vim25/mo" |
| 19 | "github.com/vmware/govmomi/vim25/types" |
| 20 | |
| 21 | "github.com/netdata/netdata/go/plugins/logger" |
| 22 | "github.com/netdata/netdata/go/plugins/pkg/confopt" |
| 23 | "github.com/netdata/netdata/go/plugins/pkg/metrix" |
| 24 | metrixselector "github.com/netdata/netdata/go/plugins/pkg/metrix/selector" |
| 25 | "github.com/netdata/netdata/go/plugins/plugin/framework/chartengine" |
| 26 | "github.com/netdata/netdata/go/plugins/plugin/framework/charttpl" |
| 27 | "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/vsphere/match" |
| 28 | rs "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/vsphere/resources" |
| 29 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/collecttest" |
| 30 | ) |
| 31 | |
| 32 | var ( |
| 33 | dataConfigJSON, _ = os.ReadFile("testdata/config.json") |
| 34 | dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") |
| 35 | ) |
| 36 | |
| 37 | func Test_testDataIsValid(t *testing.T) { |
| 38 | for name, data := range map[string][]byte{ |
| 39 | "dataConfigJSON": dataConfigJSON, |
| 40 | "dataConfigYAML": dataConfigYAML, |
| 41 | } { |
| 42 | require.NotNil(t, data, name) |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | func TestCollector_ConfigurationSerialize(t *testing.T) { |
| 47 | collecttest.TestConfigurationSerialize(t, &Collector{}, dataConfigJSON, dataConfigYAML) |
| 48 | } |
| 49 | |
| 50 | func TestCollector_Init(t *testing.T) { |
| 51 | tests := map[string]struct { |
| 52 | setup func(*Collector) |
| 53 | wantErr string |
| 54 | check func(*testing.T, *Collector) |
| 55 | }{ |
| 56 | "success": { |
| 57 | check: func(t *testing.T, collr *Collector) { |
| 58 | assert.NotNil(t, collr.discoverer) |
| 59 | assert.NotNil(t, collr.scraper) |
| 60 | assert.NotNil(t, collr.resources) |
| 61 | assert.NotNil(t, collr.discoveryTask) |
| 62 | assert.True(t, collr.discoveryTask.isRunning()) |
| 63 | }, |
| 64 | }, |
| 65 | "URL not set": { |
| 66 | setup: func(c *Collector) { c.URL = "" }, |
| 67 | wantErr: "url", |
| 68 | }, |
| 69 | "username not set": { |
| 70 | setup: func(c *Collector) { c.Username = "" }, |
| 71 | wantErr: "username", |
| 72 | }, |
| 73 | "password not set": { |
| 74 | setup: func(c *Collector) { c.Password = "" }, |
| 75 | wantErr: "password", |
| 76 | }, |
| 77 | "discovery interval not positive": { |
| 78 | setup: func(c *Collector) { c.DiscoveryInterval = 0 }, |
| 79 | wantErr: "discovery_interval must be greater than zero", |
| 80 | }, |
| 81 | "wrong TLS CA": { |
| 82 | setup: func(c *Collector) { c.ClientConfig.TLSConfig.TLSCA = "testdata/tls" }, |
| 83 | wantErr: "testdata/tls", |
| 84 | }, |
| 85 | "connection refused": { |
| 86 | setup: func(c *Collector) { c.URL = "http://127.0.0.1:32001" }, |
| 87 | wantErr: "connect", |
| 88 | }, |
| 89 | "invalid host include format": { |
| 90 | setup: func(c *Collector) { c.HostsInclude = match.HostIncludes{"invalid"} }, |
| 91 | wantErr: "host_include", |
| 92 | }, |
| 93 | "invalid VM include format": { |
| 94 | setup: func(c *Collector) { c.VMsInclude = match.VMIncludes{"invalid"} }, |
| 95 | wantErr: "vm_include", |
| 96 | }, |
| 97 | "invalid datastore include format": { |
| 98 | setup: func(c *Collector) { c.DatastoresInclude = match.DatastoreIncludes{"invalid"} }, |
| 99 | wantErr: "datastore_include", |
| 100 | }, |
| 101 | "invalid cluster include format": { |
| 102 | setup: func(c *Collector) { c.ClustersInclude = match.ClusterIncludes{"invalid"} }, |
| 103 | wantErr: "cluster_include", |
| 104 | }, |
| 105 | } |
| 106 | |
| 107 | for name, tc := range tests { |
| 108 | t.Run(name, func(t *testing.T) { |
| 109 | collr, _, teardown := prepareVSphereSim(t) |
| 110 | defer teardown() |
| 111 | if tc.setup != nil { |
| 112 | tc.setup(collr) |
| 113 | } |
| 114 | |
| 115 | err := collr.Init(context.Background()) |
| 116 | if tc.wantErr != "" { |
| 117 | require.ErrorContains(t, err, tc.wantErr) |
| 118 | return |
| 119 | } |
| 120 | require.NoError(t, err) |
| 121 | if tc.check != nil { |
| 122 | tc.check(t, collr) |
| 123 | } |
| 124 | }) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | func TestCollector_InitReentrantResetsRuntimeState(t *testing.T) { |
| 129 | collr, _, teardown := prepareVSphereSim(t) |
| 130 | defer teardown() |
| 131 | |
| 132 | require.NoError(t, collr.Init(context.Background())) |
| 133 | collr.scraper = mockScraper{collr.scraper} |
| 134 | firstRun := collectScalarSeriesForTest(t, collr) |
| 135 | require.NotEmpty(t, firstRun) |
| 136 | |
| 137 | var keepHostName, keepHostID string |
| 138 | for _, host := range collr.resources.Hosts { |
| 139 | if keepHostID == "" { |
| 140 | keepHostName = host.Name |
| 141 | keepHostID = host.ID |
| 142 | } |
| 143 | } |
| 144 | require.NotEmpty(t, keepHostName) |
| 145 | |
| 146 | collr.HostsInclude = match.HostIncludes{"/*/*/" + keepHostName} |
| 147 | require.NoError(t, collr.Init(context.Background())) |
| 148 | require.Len(t, collr.resources.Hosts, 1) |
| 149 | require.NotNil(t, collr.resources.Hosts.Get(keepHostID)) |
| 150 | |
| 151 | collr.scraper = mockScraper{collr.scraper} |
| 152 | secondRun := collectScalarSeriesForTest(t, collr) |
| 153 | require.NotEmpty(t, secondRun) |
| 154 | require.True(t, scalarSeriesHasLabel(secondRun, "id", keepHostID)) |
| 155 | } |
| 156 | |
| 157 | func TestCollector_validateConfig_IgnoresDisabledOptionalSelectors(t *testing.T) { |
| 158 | collr := New() |
| 159 | collr.URL = "https://vcenter.local" |
| 160 | collr.Username = "user" |
| 161 | collr.Password = "[REDACTED_SECRET]" |
| 162 | collr.DatastoreClustersInclude = match.DatastoreClusterIncludes{"!*"} |
| 163 | collr.VSANClustersInclude = match.VSANClusterIncludes{"!*"} |
| 164 | collr.VSANHostsInclude = match.VSANHostIncludes{"!*"} |
| 165 | collr.VSANVMsInclude = match.VSANVMIncludes{"!*"} |
| 166 | |
| 167 | require.NoError(t, collr.validateConfig()) |
| 168 | } |
| 169 | |
| 170 | func TestCollector_validateConfig_ValidatesEnabledOptionalSelectors(t *testing.T) { |
| 171 | tests := map[string]struct { |
| 172 | setup func(*Collector) |
| 173 | want string |
| 174 | }{ |
| 175 | "datastore clusters": { |
| 176 | setup: func(c *Collector) { |
| 177 | c.CollectDatastoreClusters = true |
| 178 | c.DatastoreClustersInclude = match.DatastoreClusterIncludes{"!*"} |
| 179 | }, |
| 180 | want: "datastore_cluster_include must include at least one positive pattern", |
| 181 | }, |
| 182 | "vSAN clusters": { |
| 183 | setup: func(c *Collector) { |
| 184 | c.CollectVSAN = true |
| 185 | c.VSANClustersInclude = match.VSANClusterIncludes{"!*"} |
| 186 | }, |
| 187 | want: "vsan_cluster_include must include at least one positive pattern", |
| 188 | }, |
| 189 | "vSAN hosts": { |
| 190 | setup: func(c *Collector) { |
| 191 | c.CollectVSAN = true |
| 192 | c.VSANHostsInclude = match.VSANHostIncludes{"!*"} |
| 193 | }, |
| 194 | want: "vsan_host_include must include at least one positive pattern", |
| 195 | }, |
| 196 | "vSAN VMs": { |
| 197 | setup: func(c *Collector) { |
| 198 | c.CollectVSAN = true |
| 199 | c.VSANVMsInclude = match.VSANVMIncludes{"!*"} |
| 200 | }, |
| 201 | want: "vsan_vm_include must include at least one positive pattern", |
| 202 | }, |
| 203 | } |
| 204 | |
| 205 | for name, tc := range tests { |
| 206 | t.Run(name, func(t *testing.T) { |
| 207 | collr := New() |
| 208 | collr.URL = "https://vcenter.local" |
| 209 | collr.Username = "user" |
| 210 | collr.Password = "[REDACTED_SECRET]" |
| 211 | tc.setup(collr) |
| 212 | |
| 213 | require.ErrorContains(t, collr.validateConfig(), tc.want) |
| 214 | }) |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | func TestCollector_Check(t *testing.T) { |
| 219 | assert.NoError(t, New().Check(context.Background())) |
| 220 | } |
| 221 | |
| 222 | func TestCollector_Cleanup(t *testing.T) { |
| 223 | tests := map[string]struct { |
| 224 | prepare func(t *testing.T) (*Collector, func()) |
| 225 | check func(t *testing.T, collr *Collector) |
| 226 | }{ |
| 227 | "initialized": { |
| 228 | prepare: func(t *testing.T) (*Collector, func()) { |
| 229 | collr, _, teardown := prepareVSphereSim(t) |
| 230 | require.NoError(t, collr.Init(context.Background())) |
| 231 | return collr, teardown |
| 232 | }, |
| 233 | check: func(t *testing.T, collr *Collector) { |
| 234 | assert.True(t, collr.discoveryTask.isStopped()) |
| 235 | assert.False(t, collr.discoveryTask.isRunning()) |
| 236 | assert.Nil(t, collr.vsClient) |
| 237 | }, |
| 238 | }, |
| 239 | "not initialized": { |
| 240 | prepare: func(t *testing.T) (*Collector, func()) { |
| 241 | return New(), func() {} |
| 242 | }, |
| 243 | check: func(t *testing.T, collr *Collector) { |
| 244 | assert.Nil(t, collr.vsClient) |
| 245 | }, |
| 246 | }, |
| 247 | } |
| 248 | |
| 249 | for name, tc := range tests { |
| 250 | t.Run(name, func(t *testing.T) { |
| 251 | collr, cleanup := tc.prepare(t) |
| 252 | defer cleanup() |
| 253 | |
| 254 | assert.NotPanics(t, func() { collr.Cleanup(context.Background()) }) |
| 255 | tc.check(t, collr) |
| 256 | }) |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | func TestCollector_Collect(t *testing.T) { |
| 261 | collr, _, teardown := prepareVSphereSim(t) |
| 262 | defer teardown() |
| 263 | |
| 264 | require.NoError(t, collr.Init(context.Background())) |
| 265 | collr.scraper = mockScraper{collr.scraper} |
| 266 | |
| 267 | series := collectScalarSeriesForTest(t, collr) |
| 268 | require.NotEmpty(t, series) |
| 269 | |
| 270 | requireScalarSeriesValue(t, series, "inventory_objects_datacenters", "inventory", 1) |
| 271 | requireScalarSeriesValue(t, series, "inventory_objects_hosts", "inventory", 4) |
| 272 | requireScalarSeriesValue(t, series, "inventory_objects_vms", "inventory", 4) |
| 273 | requireScalarSeriesValue(t, series, "inventory_objects_datastores", "inventory", 1) |
| 274 | requireScalarSeriesValue(t, series, "inventory_objects_resource_pools", "inventory", 1) |
| 275 | |
| 276 | requireScalarSeriesValue(t, series, "host_cpu_utilization_used", "host-21", 100) |
| 277 | requireScalarSeriesValue(t, series, "host_disk_max_latency_latency", "host-21", 100) |
| 278 | requireScalarSeriesValue(t, series, "host_net_errors_received", "host-21", 100) |
| 279 | requireScalarSeriesValue(t, series, "host_overall_status_gray", "host-21", 1) |
| 280 | requireScalarSeriesValue(t, series, "host_power_state_powered_on", "host-21", 1) |
| 281 | requireScalarSeriesValue(t, series, "host_connection_state_connected", "host-21", 1) |
| 282 | requireScalarSeriesValue(t, series, "host_maintenance_status_normal", "host-21", 1) |
| 283 | requireScalarSeriesValue(t, series, "host_system_uptime_uptime", "host-21", 100) |
| 284 | |
| 285 | requireScalarSeriesValue(t, series, "vm_cpu_utilization_used", "vm-62", 200) |
| 286 | requireScalarSeriesValue(t, series, "vm_disk_max_latency_latency", "vm-62", 200) |
| 287 | requireScalarSeriesValue(t, series, "vm_net_drops_received", "vm-62", 200) |
| 288 | requireScalarSeriesValue(t, series, "vm_overall_status_green", "vm-62", 1) |
| 289 | requireScalarSeriesValue(t, series, "vm_power_state_powered_on", "vm-62", 1) |
| 290 | requireScalarSeriesValue(t, series, "vm_tools_running_status_unknown", "vm-62", 1) |
| 291 | requireScalarSeriesValue(t, series, "vm_tools_version_status_unknown", "vm-62", 1) |
| 292 | requireScalarSeriesValue(t, series, "vm_config_cpu_vcpus", "vm-62", 1) |
| 293 | requireScalarSeriesValue(t, series, "vm_config_devices_disks", "vm-62", 1) |
| 294 | requireScalarSeriesValue(t, series, "vm_storage_usage_uncommitted", "vm-62", 10737418240) |
| 295 | requireScalarSeriesValue(t, series, "vm_snapshot_count_count", "vm-62", 0) |
| 296 | |
| 297 | requireScalarSeriesValue(t, series, "datastore_space_usage_capacity", "datastore-59", 4398046511104) |
| 298 | requireScalarSeriesValue(t, series, "datastore_space_usage_used", "datastore-59", 42949672960) |
| 299 | requireScalarSeriesValue(t, series, "datastore_space_utilization_used", "datastore-59", 97) |
| 300 | requireScalarSeriesValue(t, series, "datastore_overall_status_green", "datastore-59", 1) |
| 301 | requireScalarSeriesValue(t, series, "datastore_accessibility_status_accessible", "datastore-59", 1) |
| 302 | requireScalarSeriesValue(t, series, "datastore_maintenance_status_normal", "datastore-59", 1) |
| 303 | requireScalarSeriesValue(t, series, "datastore_multiple_host_access_unknown", "datastore-59", 1) |
| 304 | requireScalarSeriesValue(t, series, "datastore_disk_iops_reads", "datastore-59", 300) |
| 305 | |
| 306 | requireScalarSeriesValue(t, series, "cluster_hosts_total", "domain-c28", 3) |
| 307 | requireScalarSeriesValue(t, series, "cluster_cpu_capacity_total", "domain-c28", 6882) |
| 308 | requireScalarSeriesValue(t, series, "cluster_drs_config_enabled", "domain-c28", 1) |
| 309 | requireScalarSeriesValue(t, series, "cluster_drs_mode_unknown", "domain-c28", 1) |
| 310 | requireScalarSeriesValue(t, series, "cluster_ha_host_monitoring_unknown", "domain-c28", 1) |
| 311 | requireScalarSeriesValue(t, series, "cluster_overall_status_green", "domain-c28", 1) |
| 312 | requireScalarSeriesValue(t, series, "cluster_cpu_utilization_used", "domain-c28", 400) |
| 313 | requireScalarSeriesValue(t, series, "cluster_services_fairness_cpu", "domain-c28", 400) |
| 314 | requireScalarSeriesValue(t, series, "cluster_vm_migrations_vmotion", "domain-c28", 400) |
| 315 | |
| 316 | requireScalarSeriesValue(t, series, "resource_pool_cpu_usage_usage", "resgroup-27", 0) |
| 317 | requireScalarSeriesValue(t, series, "resource_pool_cpu_allocation_max_usage", "resgroup-27", 4121) |
| 318 | requireScalarSeriesValue(t, series, "resource_pool_mem_allocation_max_usage", "resgroup-27", 1007681536) |
| 319 | requireScalarSeriesValue(t, series, "resource_pool_cpu_config_reservation", "resgroup-27", 4121) |
| 320 | requireScalarSeriesValue(t, series, "resource_pool_mem_config_limit", "resgroup-27", 961) |
| 321 | requireScalarSeriesValue(t, series, "resource_pool_overall_status_green", "resgroup-27", 1) |
| 322 | |
| 323 | collecttest.AssertChartCoverage(t, collr, collecttest.ChartCoverageExpectation{}) |
| 324 | } |
| 325 | |
| 326 | func TestCollector_V2CompatibilitySurface(t *testing.T) { |
| 327 | collr, _, teardown := prepareVSphereSim(t) |
| 328 | defer teardown() |
| 329 | |
| 330 | require.NoError(t, collr.Init(context.Background())) |
| 331 | collr.scraper = mockScraper{collr.scraper} |
| 332 | |
| 333 | require.NotEmpty(t, collectScalarSeriesForTest(t, collr)) |
| 334 | |
| 335 | plan := buildV2PlanForTest(t, collr) |
| 336 | createdCharts, createdDims := v2CreatedChartsAndDims(plan) |
| 337 | require.NotEmpty(t, createdCharts) |
| 338 | |
| 339 | for chartID, chart := range createdCharts { |
| 340 | require.NotEmpty(t, chart.Labels["id"], "chart %s must have the V2 instance id label", chartID) |
| 341 | require.NotEmpty(t, createdDims[chartID], "chart %s must have dimensions", chartID) |
| 342 | } |
| 343 | collecttest.AssertChartCoverage(t, collr, collecttest.ChartCoverageExpectation{}) |
| 344 | } |
| 345 | |
| 346 | func TestSnapshotMaxAgeSeconds(t *testing.T) { |
| 347 | now := time.Now() |
| 348 | tests := map[string]struct { |
| 349 | created time.Time |
| 350 | want int64 |
| 351 | delta float64 |
| 352 | }{ |
| 353 | "zero time": { |
| 354 | created: time.Time{}, |
| 355 | }, |
| 356 | "future time": { |
| 357 | created: now.Add(time.Hour), |
| 358 | }, |
| 359 | "past time": { |
| 360 | created: now.Add(-2 * time.Hour), |
| 361 | want: 7200, |
| 362 | delta: 1, |
| 363 | }, |
| 364 | } |
| 365 | |
| 366 | for name, tc := range tests { |
| 367 | t.Run(name, func(t *testing.T) { |
| 368 | if tc.delta > 0 { |
| 369 | assert.InDelta(t, tc.want, snapshotMaxAgeSeconds(tc.created), tc.delta) |
| 370 | return |
| 371 | } |
| 372 | assert.EqualValues(t, tc.want, snapshotMaxAgeSeconds(tc.created)) |
| 373 | }) |
| 374 | } |
| 375 | } |
| 376 | |
| 377 | func TestCollectInventory(t *testing.T) { |
| 378 | collr := New() |
| 379 | collr.resources = &rs.Resources{ |
| 380 | DataCenters: rs.DataCenters{"dc-1": &rs.Datacenter{}}, |
| 381 | Folders: rs.Folders{"folder-1": &rs.Folder{}, "folder-2": &rs.Folder{}}, |
| 382 | Clusters: rs.Clusters{"domain-c1": &rs.Cluster{}}, |
| 383 | Hosts: rs.Hosts{"host-1": &rs.Host{}, "host-2": &rs.Host{}}, |
| 384 | VMs: rs.VMs{"vm-1": &rs.VM{}, "vm-2": &rs.VM{}, "vm-3": &rs.VM{}}, |
| 385 | Datastores: rs.Datastores{"datastore-1": &rs.Datastore{}}, |
| 386 | ResourcePools: rs.ResourcePools{"resgroup-1": &rs.ResourcePool{}}, |
| 387 | } |
| 388 | |
| 389 | series := runMetricWriteForTest(t, collr, collr.collectInventory) |
| 390 | |
| 391 | requireScalarSeriesValue(t, series, "inventory_objects_datacenters", "inventory", 1) |
| 392 | requireScalarSeriesValue(t, series, "inventory_objects_folders", "inventory", 2) |
| 393 | requireScalarSeriesValue(t, series, "inventory_objects_clusters", "inventory", 1) |
| 394 | requireScalarSeriesValue(t, series, "inventory_objects_hosts", "inventory", 2) |
| 395 | requireScalarSeriesValue(t, series, "inventory_objects_vms", "inventory", 3) |
| 396 | requireScalarSeriesValue(t, series, "inventory_objects_datastores", "inventory", 1) |
| 397 | requireScalarSeriesValue(t, series, "inventory_objects_resource_pools", "inventory", 1) |
| 398 | } |
| 399 | |
| 400 | func TestCollector_Collect_NonPoweredResourcePropertyOnly(t *testing.T) { |
| 401 | tests := map[string]struct { |
| 402 | setup func(*Collector) |
| 403 | collect func(*Collector) error |
| 404 | want map[string]int64 |
| 405 | missing []string |
| 406 | }{ |
| 407 | "host": { |
| 408 | setup: func(c *Collector) { |
| 409 | c.resources = &rs.Resources{ |
| 410 | Hosts: rs.Hosts{ |
| 411 | "host-1": &rs.Host{ |
| 412 | ID: "host-1", |
| 413 | PowerState: string(types.HostSystemPowerStatePoweredOff), |
| 414 | OverallStatus: "gray", |
| 415 | }, |
| 416 | }, |
| 417 | } |
| 418 | }, |
| 419 | collect: (*Collector).collectHosts, |
| 420 | want: map[string]int64{ |
| 421 | "host_overall_status_gray": 1, |
| 422 | "host_power_state_powered_off": 1, |
| 423 | "host_power_state_powered_on": 0, |
| 424 | "host_connection_state_connected": 0, |
| 425 | }, |
| 426 | missing: []string{"host_cpu_utilization_used"}, |
| 427 | }, |
| 428 | "VM": { |
| 429 | setup: func(c *Collector) { |
| 430 | c.resources = &rs.Resources{ |
| 431 | VMs: rs.VMs{ |
| 432 | "vm-1": &rs.VM{ |
| 433 | ID: "vm-1", |
| 434 | PowerState: string(types.VirtualMachinePowerStateSuspended), |
| 435 | OverallStatus: "yellow", |
| 436 | SnapshotCount: 2, |
| 437 | SnapshotMaxChainDepth: 1, |
| 438 | }, |
| 439 | }, |
| 440 | } |
| 441 | }, |
| 442 | collect: (*Collector).collectVMs, |
| 443 | want: map[string]int64{ |
| 444 | "vm_overall_status_yellow": 1, |
| 445 | "vm_power_state_suspended": 1, |
| 446 | "vm_power_state_powered_on": 0, |
| 447 | "vm_snapshot_count_count": 2, |
| 448 | "vm_snapshot_max_chain_depth_depth": 1, |
| 449 | }, |
| 450 | missing: []string{"vm_cpu_utilization_used"}, |
| 451 | }, |
| 452 | } |
| 453 | |
| 454 | for name, tc := range tests { |
| 455 | t.Run(name, func(t *testing.T) { |
| 456 | collr := New() |
| 457 | tc.setup(collr) |
| 458 | |
| 459 | series := runMetricCollectForTest(t, collr, func() error { return tc.collect(collr) }) |
| 460 | for metric, want := range tc.want { |
| 461 | requireScalarSeriesValue(t, series, metric, firstResourceID(collr), want) |
| 462 | } |
| 463 | for _, metric := range tc.missing { |
| 464 | requireNoScalarSeries(t, series, metric, firstResourceID(collr)) |
| 465 | } |
| 466 | }) |
| 467 | } |
| 468 | } |
| 469 | |
| 470 | func TestCollector_Collect_NoPerfDataKeepsPropertyMetrics(t *testing.T) { |
| 471 | tests := map[string]struct { |
| 472 | setup func(*Collector) |
| 473 | check func(*testing.T, *Collector, map[string]metrix.SampleValue) |
| 474 | }{ |
| 475 | "hosts": { |
| 476 | setup: func(c *Collector) { c.scraper = mockScraperNoHostPerf{mockScraper{c.scraper}} }, |
| 477 | check: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue) { |
| 478 | for _, host := range collr.resources.Hosts { |
| 479 | requireScalarSeries(t, series, "host_overall_status_green", host.ID) |
| 480 | requireScalarSeries(t, series, "host_power_state_powered_on", host.ID) |
| 481 | requireNoScalarSeries(t, series, "host_cpu_utilization_used", host.ID) |
| 482 | } |
| 483 | for _, vm := range collr.resources.VMs { |
| 484 | requireScalarSeries(t, series, "vm_cpu_utilization_used", vm.ID) |
| 485 | } |
| 486 | }, |
| 487 | }, |
| 488 | "VMs": { |
| 489 | setup: func(c *Collector) { c.scraper = mockScraperNoVMPerf{mockScraper{c.scraper}} }, |
| 490 | check: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue) { |
| 491 | for _, host := range collr.resources.Hosts { |
| 492 | requireScalarSeries(t, series, "host_cpu_utilization_used", host.ID) |
| 493 | } |
| 494 | for _, vm := range collr.resources.VMs { |
| 495 | requireScalarSeries(t, series, "vm_overall_status_green", vm.ID) |
| 496 | requireScalarSeries(t, series, "vm_power_state_powered_on", vm.ID) |
| 497 | requireNoScalarSeries(t, series, "vm_cpu_utilization_used", vm.ID) |
| 498 | } |
| 499 | }, |
| 500 | }, |
| 501 | } |
| 502 | |
| 503 | for name, tc := range tests { |
| 504 | t.Run(name, func(t *testing.T) { |
| 505 | collr, _, teardown := prepareVSphereSim(t) |
| 506 | defer teardown() |
| 507 | |
| 508 | require.NoError(t, collr.Init(context.Background())) |
| 509 | tc.setup(collr) |
| 510 | |
| 511 | series := collectScalarSeriesForTest(t, collr) |
| 512 | require.NotNil(t, series) |
| 513 | tc.check(t, collr, series) |
| 514 | }) |
| 515 | } |
| 516 | } |
| 517 | |
| 518 | func TestCollector_Collect_PropertyRefreshFailureSkipsStalePropertyMetrics(t *testing.T) { |
| 519 | tests := map[string]struct { |
| 520 | setup func(*Collector) |
| 521 | check func(*testing.T, *Collector, map[string]metrix.SampleValue) |
| 522 | }{ |
| 523 | "datastores": { |
| 524 | setup: func(c *Collector) { c.dsPropertyCollector = nil }, |
| 525 | check: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue) { |
| 526 | for _, ds := range collr.resources.Datastores { |
| 527 | requireNoScalarSeries(t, series, "datastore_space_usage_capacity", ds.ID) |
| 528 | requireNoScalarSeries(t, series, "datastore_overall_status_green", ds.ID) |
| 529 | requireScalarSeries(t, series, "datastore_disk_io_read", ds.ID) |
| 530 | } |
| 531 | }, |
| 532 | }, |
| 533 | "clusters": { |
| 534 | setup: func(c *Collector) { c.clusterPropertyCollector = nil }, |
| 535 | check: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue) { |
| 536 | for _, cl := range collr.resources.Clusters { |
| 537 | requireNoScalarSeries(t, series, "cluster_hosts_total", cl.ID) |
| 538 | requireNoScalarSeries(t, series, "cluster_overall_status_green", cl.ID) |
| 539 | requireScalarSeries(t, series, "cluster_cpu_utilization_used", cl.ID) |
| 540 | } |
| 541 | }, |
| 542 | }, |
| 543 | "resource pools": { |
| 544 | setup: func(c *Collector) { c.rpPropertyCollector = nil }, |
| 545 | check: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue) { |
| 546 | for _, rp := range collr.resources.ResourcePools { |
| 547 | requireNoScalarSeries(t, series, "resource_pool_cpu_usage_usage", rp.ID) |
| 548 | requireNoScalarSeries(t, series, "resource_pool_overall_status_green", rp.ID) |
| 549 | } |
| 550 | }, |
| 551 | }, |
| 552 | } |
| 553 | |
| 554 | for name, tc := range tests { |
| 555 | t.Run(name, func(t *testing.T) { |
| 556 | collr, _, teardown := prepareVSphereSim(t) |
| 557 | defer teardown() |
| 558 | |
| 559 | require.NoError(t, collr.Init(context.Background())) |
| 560 | collr.scraper = mockScraper{collr.scraper} |
| 561 | tc.setup(collr) |
| 562 | |
| 563 | series := collectScalarSeriesForTest(t, collr) |
| 564 | require.NotNil(t, series) |
| 565 | tc.check(t, collr, series) |
| 566 | }) |
| 567 | } |
| 568 | } |
| 569 | |
| 570 | func TestCollector_Collect_NoPerfDataWarningIsRateLimited(t *testing.T) { |
| 571 | tests := map[string]struct { |
| 572 | setup func(*Collector) |
| 573 | collect func(*Collector) error |
| 574 | wantLog string |
| 575 | }{ |
| 576 | "hosts": { |
| 577 | setup: func(c *Collector) { |
| 578 | c.scraper = mockScraperNoHostPerf{} |
| 579 | c.resources = &rs.Resources{ |
| 580 | Hosts: rs.Hosts{ |
| 581 | "host-1": &rs.Host{ |
| 582 | ID: "host-1", |
| 583 | PowerState: string(types.HostSystemPowerStatePoweredOn), |
| 584 | }, |
| 585 | }, |
| 586 | } |
| 587 | }, |
| 588 | collect: (*Collector).collectHosts, |
| 589 | wantLog: "collect host performance metrics", |
| 590 | }, |
| 591 | "VMs": { |
| 592 | setup: func(c *Collector) { |
| 593 | c.scraper = mockScraperNoVMPerf{} |
| 594 | c.resources = &rs.Resources{ |
| 595 | VMs: rs.VMs{ |
| 596 | "vm-1": &rs.VM{ |
| 597 | ID: "vm-1", |
| 598 | PowerState: string(types.VirtualMachinePowerStatePoweredOn), |
| 599 | }, |
| 600 | }, |
| 601 | } |
| 602 | }, |
| 603 | collect: (*Collector).collectVMs, |
| 604 | wantLog: "collect VM performance metrics", |
| 605 | }, |
| 606 | } |
| 607 | |
| 608 | for name, tc := range tests { |
| 609 | t.Run(name, func(t *testing.T) { |
| 610 | var buf bytes.Buffer |
| 611 | collr := New() |
| 612 | collr.Logger = logger.NewWithWriter(&buf) |
| 613 | tc.setup(collr) |
| 614 | |
| 615 | runMetricCollectForTest(t, collr, func() error { return tc.collect(collr) }) |
| 616 | runMetricCollectForTest(t, collr, func() error { return tc.collect(collr) }) |
| 617 | |
| 618 | assert.Equal(t, 1, strings.Count(buf.String(), tc.wantLog)) |
| 619 | }) |
| 620 | } |
| 621 | } |
| 622 | |
| 623 | func TestWriteHostPropertyMetrics_RuntimeStatus(t *testing.T) { |
| 624 | host := &rs.Host{ |
| 625 | ID: "host-1", |
| 626 | ConnectionState: string(types.HostSystemConnectionStateNotResponding), |
| 627 | PowerState: string(types.HostSystemPowerStatePoweredOn), |
| 628 | InMaintenanceMode: true, |
| 629 | OverallStatus: "yellow", |
| 630 | } |
| 631 | collr := New() |
| 632 | |
| 633 | series := runMetricWriteForTest(t, collr, func() { collr.writeHostPropertyMetrics(host) }) |
| 634 | |
| 635 | requireScalarSeriesValue(t, series, "host_connection_state_not_responding", host.ID, 1) |
| 636 | requireScalarSeriesValue(t, series, "host_connection_state_connected", host.ID, 0) |
| 637 | requireScalarSeriesValue(t, series, "host_maintenance_status_in_maintenance", host.ID, 1) |
| 638 | requireScalarSeriesValue(t, series, "host_maintenance_status_normal", host.ID, 0) |
| 639 | } |
| 640 | |
| 641 | func TestWriteVMPropertyMetrics_StatusConfigAndStorage(t *testing.T) { |
| 642 | vm := &rs.VM{ |
| 643 | ID: "vm-1", |
| 644 | ConnectionState: string(types.VirtualMachineConnectionStateInaccessible), |
| 645 | PowerState: string(types.VirtualMachinePowerStatePoweredOn), |
| 646 | ToolsRunningStatus: string(types.VirtualMachineToolsRunningStatusGuestToolsRunning), |
| 647 | ToolsVersionStatus: string(types.VirtualMachineToolsVersionStatusGuestToolsTooOld), |
| 648 | ConsolidationNeeded: true, |
| 649 | ConfigCPU: 4, |
| 650 | ConfigMemory: 8192, |
| 651 | ConfigDisks: 2, |
| 652 | ConfigNICs: 3, |
| 653 | StorageCommitted: 100, |
| 654 | StorageUncommitted: 200, |
| 655 | StorageUnshared: 300, |
| 656 | OverallStatus: "green", |
| 657 | } |
| 658 | collr := New() |
| 659 | |
| 660 | series := runMetricWriteForTest(t, collr, func() { collr.writeVMPropertyMetrics(vm) }) |
| 661 | |
| 662 | requireScalarSeriesValue(t, series, "vm_connection_state_inaccessible", vm.ID, 1) |
| 663 | requireScalarSeriesValue(t, series, "vm_connection_state_connected", vm.ID, 0) |
| 664 | requireScalarSeriesValue(t, series, "vm_tools_running_status_running", vm.ID, 1) |
| 665 | requireScalarSeriesValue(t, series, "vm_tools_running_status_unknown", vm.ID, 0) |
| 666 | requireScalarSeriesValue(t, series, "vm_tools_version_status_too_old", vm.ID, 1) |
| 667 | requireScalarSeriesValue(t, series, "vm_tools_version_status_unknown", vm.ID, 0) |
| 668 | requireScalarSeriesValue(t, series, "vm_consolidation_needed_needed", vm.ID, 1) |
| 669 | requireScalarSeriesValue(t, series, "vm_consolidation_needed_not_needed", vm.ID, 0) |
| 670 | requireScalarSeriesValue(t, series, "vm_config_cpu_vcpus", vm.ID, 4) |
| 671 | requireScalarSeriesValue(t, series, "vm_config_memory_memory", vm.ID, 8192) |
| 672 | requireScalarSeriesValue(t, series, "vm_config_devices_disks", vm.ID, 2) |
| 673 | requireScalarSeriesValue(t, series, "vm_config_devices_nics", vm.ID, 3) |
| 674 | requireScalarSeriesValue(t, series, "vm_storage_usage_committed", vm.ID, 100) |
| 675 | requireScalarSeriesValue(t, series, "vm_storage_usage_uncommitted", vm.ID, 200) |
| 676 | requireScalarSeriesValue(t, series, "vm_storage_usage_unshared", vm.ID, 300) |
| 677 | } |
| 678 | |
| 679 | func TestWriteClusterPropertyMetrics_ConfigStates(t *testing.T) { |
| 680 | cluster := &rs.Cluster{ |
| 681 | ID: "domain-c1", |
| 682 | DrsEnabled: true, |
| 683 | DrsMode: string(types.DrsBehaviorFullyAutomated), |
| 684 | DrsVmotionRate: 3, |
| 685 | HaEnabled: true, |
| 686 | HaAdmCtrlEnabled: true, |
| 687 | HaHostMonitoring: string(types.ClusterDasConfigInfoServiceStateEnabled), |
| 688 | HaVMMonitoring: string(types.ClusterDasConfigInfoVmMonitoringStateVmAndAppMonitoring), |
| 689 | HaVMComponentProtection: string(types.ClusterDasConfigInfoServiceStateDisabled), |
| 690 | OverallStatus: "green", |
| 691 | } |
| 692 | collr := New() |
| 693 | |
| 694 | series := runMetricWriteForTest(t, collr, func() { collr.writeClusterPropertyMetrics(cluster) }) |
| 695 | |
| 696 | requireScalarSeriesValue(t, series, "cluster_drs_config_enabled", cluster.ID, 1) |
| 697 | requireScalarSeriesValue(t, series, "cluster_drs_mode_fully_automated", cluster.ID, 1) |
| 698 | requireScalarSeriesValue(t, series, "cluster_drs_mode_unknown", cluster.ID, 0) |
| 699 | requireScalarSeriesValue(t, series, "cluster_drs_vmotion_rate_rate", cluster.ID, 3) |
| 700 | requireScalarSeriesValue(t, series, "cluster_ha_config_enabled", cluster.ID, 1) |
| 701 | requireScalarSeriesValue(t, series, "cluster_ha_config_admission_control", cluster.ID, 1) |
| 702 | requireScalarSeriesValue(t, series, "cluster_ha_host_monitoring_enabled", cluster.ID, 1) |
| 703 | requireScalarSeriesValue(t, series, "cluster_ha_host_monitoring_unknown", cluster.ID, 0) |
| 704 | requireScalarSeriesValue(t, series, "cluster_ha_vm_monitoring_vm_and_app_monitoring", cluster.ID, 1) |
| 705 | requireScalarSeriesValue(t, series, "cluster_ha_vm_monitoring_unknown", cluster.ID, 0) |
| 706 | requireScalarSeriesValue(t, series, "cluster_ha_vm_component_protection_disabled", cluster.ID, 1) |
| 707 | requireScalarSeriesValue(t, series, "cluster_ha_vm_component_protection_unknown", cluster.ID, 0) |
| 708 | } |
| 709 | |
| 710 | func TestUpdateResourcePoolFromProperties_ZeroValueOptionalProperties(t *testing.T) { |
| 711 | rp := &rs.ResourcePool{ |
| 712 | ID: "resgroup-1", |
| 713 | OverallCpuUsage: 1, |
| 714 | OverallCpuDemand: 2, |
| 715 | GuestMemoryUsage: 3, |
| 716 | HostMemoryUsage: 4, |
| 717 | DistributedCpuEntitlement: 5, |
| 718 | DistributedMemoryEntitlement: 6, |
| 719 | PrivateMemory: 7, |
| 720 | SharedMemory: 8, |
| 721 | SwappedMemory: 9, |
| 722 | BalloonedMemory: 10, |
| 723 | OverheadMemory: 11, |
| 724 | ConsumedOverheadMemory: 12, |
| 725 | CompressedMemory: 13, |
| 726 | CpuReservationUsed: 14, |
| 727 | CpuMaxUsage: 15, |
| 728 | CpuUnreservedForVm: 16, |
| 729 | MemReservationUsed: 17, |
| 730 | MemMaxUsage: 18, |
| 731 | MemUnreservedForVm: 19, |
| 732 | CpuReservation: 20, |
| 733 | CpuLimit: 21, |
| 734 | MemReservation: 22, |
| 735 | MemLimit: 23, |
| 736 | } |
| 737 | |
| 738 | require.NotPanics(t, func() { |
| 739 | updateResourcePoolFromProperties(rp, mo25.ResourcePool{}) |
| 740 | }) |
| 741 | |
| 742 | assert.Zero(t, rp.OverallCpuUsage) |
| 743 | assert.Zero(t, rp.GuestMemoryUsage) |
| 744 | assert.Zero(t, rp.CpuReservationUsed) |
| 745 | assert.Zero(t, rp.MemReservationUsed) |
| 746 | assert.Zero(t, rp.CpuReservation) |
| 747 | assert.EqualValues(t, -1, rp.CpuLimit) |
| 748 | assert.Zero(t, rp.MemReservation) |
| 749 | assert.EqualValues(t, -1, rp.MemLimit) |
| 750 | } |
| 751 | |
| 752 | func collectScalarSeriesForTest(t *testing.T, collr *Collector) map[string]metrix.SampleValue { |
| 753 | t.Helper() |
| 754 | |
| 755 | mx, err := collecttest.CollectScalarSeries(collr, metrix.ReadRaw()) |
| 756 | require.NoError(t, err) |
| 757 | if len(mx) == 0 { |
| 758 | return nil |
| 759 | } |
| 760 | return mx |
| 761 | } |
| 762 | |
| 763 | func buildV2PlanForTest(t *testing.T, collr *Collector) chartengine.Plan { |
| 764 | t.Helper() |
| 765 | |
| 766 | engine, err := chartengine.New() |
| 767 | require.NoError(t, err) |
| 768 | require.NoError(t, engine.LoadYAML([]byte(collr.ChartTemplateYAML()), 1)) |
| 769 | |
| 770 | reader := collr.MetricStore().Read(metrix.ReadRaw(), metrix.ReadFlatten()) |
| 771 | attempt, err := engine.PreparePlan(reader) |
| 772 | require.NoError(t, err) |
| 773 | defer attempt.Abort() |
| 774 | |
| 775 | plan := attempt.Plan() |
| 776 | require.NoError(t, attempt.Commit()) |
| 777 | return plan |
| 778 | } |
| 779 | |
| 780 | func requireChartSelectorsMatchSeries(t *testing.T, collr *Collector, contextPrefixes ...string) { |
| 781 | t.Helper() |
| 782 | |
| 783 | spec, err := charttpl.DecodeYAML([]byte(collr.ChartTemplateYAML())) |
| 784 | require.NoError(t, err) |
| 785 | |
| 786 | reader := collr.MetricStore().Read(metrix.ReadRaw(), metrix.ReadFlatten()) |
| 787 | require.NotZero(t, requireChartGroupSelectorsMatchSeries(t, reader, spec.Groups, contextParts(spec.ContextNamespace), contextPrefixes)) |
| 788 | } |
| 789 | |
| 790 | func requireChartGroupSelectorsMatchSeries(t *testing.T, reader metrix.Reader, groups []charttpl.Group, parent, contextPrefixes []string) int { |
| 791 | t.Helper() |
| 792 | |
| 793 | matchedContexts := 0 |
| 794 | for _, group := range groups { |
| 795 | parts := append(append([]string(nil), parent...), contextParts(group.ContextNamespace)...) |
| 796 | for _, chart := range group.Charts { |
| 797 | contextName := strings.Join(append(append([]string(nil), parts...), strings.TrimSpace(chart.Context)), ".") |
| 798 | if !matchesAnyContextPrefix(contextName, contextPrefixes) { |
| 799 | continue |
| 800 | } |
| 801 | matchedContexts++ |
| 802 | for _, dim := range chart.Dimensions { |
| 803 | requireSelectorMatchesSeries(t, reader, contextName, dim.Selector) |
| 804 | } |
| 805 | } |
| 806 | matchedContexts += requireChartGroupSelectorsMatchSeries(t, reader, group.Groups, parts, contextPrefixes) |
| 807 | } |
| 808 | return matchedContexts |
| 809 | } |
| 810 | |
| 811 | func matchesAnyContextPrefix(contextName string, prefixes []string) bool { |
| 812 | for _, prefix := range prefixes { |
| 813 | if contextName == prefix || strings.HasPrefix(contextName, prefix) { |
| 814 | return true |
| 815 | } |
| 816 | } |
| 817 | return false |
| 818 | } |
| 819 | |
| 820 | func requireSelectorMatchesSeries(t *testing.T, reader metrix.Reader, contextName, selector string) { |
| 821 | t.Helper() |
| 822 | |
| 823 | sel, err := metrixselector.Parse(selector) |
| 824 | require.NoErrorf(t, err, "chart context %s selector %q", contextName, selector) |
| 825 | |
| 826 | matched := false |
| 827 | reader.ForEachSeriesIdentity(func(_ metrix.SeriesIdentity, _ metrix.SeriesMeta, metricName string, labels metrix.LabelView, _ metrix.SampleValue) { |
| 828 | if sel.Matches(metricName, labels) { |
| 829 | matched = true |
| 830 | } |
| 831 | }) |
| 832 | require.Truef(t, matched, "chart context %s selector %q matches no series", contextName, selector) |
| 833 | } |
| 834 | |
| 835 | func contextParts(value string) []string { |
| 836 | value = strings.TrimSpace(value) |
| 837 | if value == "" { |
| 838 | return nil |
| 839 | } |
| 840 | return []string{value} |
| 841 | } |
| 842 | |
| 843 | func v2CreatedChartsAndDims(plan chartengine.Plan) (map[string]chartengine.CreateChartAction, map[string]map[string]chartengine.CreateDimensionAction) { |
| 844 | charts := make(map[string]chartengine.CreateChartAction) |
| 845 | dims := make(map[string]map[string]chartengine.CreateDimensionAction) |
| 846 | for _, action := range plan.Actions { |
| 847 | switch v := action.(type) { |
| 848 | case chartengine.CreateChartAction: |
| 849 | charts[v.ChartID] = v |
| 850 | case chartengine.CreateDimensionAction: |
| 851 | if _, ok := dims[v.ChartID]; !ok { |
| 852 | dims[v.ChartID] = make(map[string]chartengine.CreateDimensionAction) |
| 853 | } |
| 854 | dims[v.ChartID][v.Name] = v |
| 855 | } |
| 856 | } |
| 857 | return charts, dims |
| 858 | } |
| 859 | |
| 860 | func scalarSeriesHasLabel(series map[string]metrix.SampleValue, key, value string) bool { |
| 861 | needle := fmt.Sprintf(`%s="%s"`, key, value) |
| 862 | for name := range series { |
| 863 | if strings.Contains(name, needle) { |
| 864 | return true |
| 865 | } |
| 866 | } |
| 867 | return false |
| 868 | } |
| 869 | |
| 870 | func runMetricWriteForTest(t *testing.T, collr *Collector, write func()) map[string]metrix.SampleValue { |
| 871 | t.Helper() |
| 872 | |
| 873 | cycle := mustCycleController(t, collr.MetricStore()) |
| 874 | cycle.BeginCycle() |
| 875 | write() |
| 876 | require.NoError(t, cycle.CommitCycleSuccess()) |
| 877 | return scalarSeriesFromReaderForTest(collr.MetricStore().Read(metrix.ReadRaw())) |
| 878 | } |
| 879 | |
| 880 | func runMetricCollectForTest(t *testing.T, collr *Collector, collect func() error) map[string]metrix.SampleValue { |
| 881 | t.Helper() |
| 882 | |
| 883 | cycle := mustCycleController(t, collr.MetricStore()) |
| 884 | cycle.BeginCycle() |
| 885 | if err := collect(); err != nil { |
| 886 | cycle.AbortCycle() |
| 887 | require.NoError(t, err) |
| 888 | } |
| 889 | require.NoError(t, cycle.CommitCycleSuccess()) |
| 890 | return scalarSeriesFromReaderForTest(collr.MetricStore().Read(metrix.ReadRaw())) |
| 891 | } |
| 892 | |
| 893 | func scalarSeriesFromReaderForTest(reader metrix.Reader) map[string]metrix.SampleValue { |
| 894 | out := make(map[string]metrix.SampleValue) |
| 895 | reader.ForEachSeries(func(name string, labels metrix.LabelView, value metrix.SampleValue) { |
| 896 | out[scalarSeriesKeyForTest(name, labels)] = value |
| 897 | }) |
| 898 | return out |
| 899 | } |
| 900 | |
| 901 | func scalarSeriesKeyForTest(name string, labels metrix.LabelView) string { |
| 902 | if labels == nil || labels.Len() == 0 { |
| 903 | return name |
| 904 | } |
| 905 | |
| 906 | var b strings.Builder |
| 907 | b.WriteString(name) |
| 908 | b.WriteByte('{') |
| 909 | first := true |
| 910 | labels.Range(func(key, value string) bool { |
| 911 | if !first { |
| 912 | b.WriteByte(',') |
| 913 | } |
| 914 | first = false |
| 915 | b.WriteString(key) |
| 916 | b.WriteString(`="`) |
| 917 | b.WriteString(value) |
| 918 | b.WriteByte('"') |
| 919 | return true |
| 920 | }) |
| 921 | b.WriteByte('}') |
| 922 | return b.String() |
| 923 | } |
| 924 | |
| 925 | func requireScalarSeries(t *testing.T, series map[string]metrix.SampleValue, metric, id string) { |
| 926 | t.Helper() |
| 927 | |
| 928 | _, ok := findScalarSeries(series, metric, id) |
| 929 | require.Truef(t, ok, "expected metric %s with id=%s in %v", metric, id, scalarSeriesKeys(series)) |
| 930 | } |
| 931 | |
| 932 | func requireNoScalarSeries(t *testing.T, series map[string]metrix.SampleValue, metric, id string) { |
| 933 | t.Helper() |
| 934 | |
| 935 | _, ok := findScalarSeries(series, metric, id) |
| 936 | require.Falsef(t, ok, "unexpected metric %s with id=%s", metric, id) |
| 937 | } |
| 938 | |
| 939 | func requireScalarSeriesValue(t *testing.T, series map[string]metrix.SampleValue, metric, id string, want int64) { |
| 940 | t.Helper() |
| 941 | |
| 942 | got, ok := findScalarSeries(series, metric, id) |
| 943 | require.Truef(t, ok, "expected metric %s with id=%s in %v", metric, id, scalarSeriesKeys(series)) |
| 944 | require.EqualValues(t, want, got) |
| 945 | } |
| 946 | |
| 947 | func findScalarSeries(series map[string]metrix.SampleValue, metric, id string) (metrix.SampleValue, bool) { |
| 948 | prefix := metric + "{" |
| 949 | idLabel := fmt.Sprintf(`id="%s"`, id) |
| 950 | for key, value := range series { |
| 951 | if (key == metric || strings.HasPrefix(key, prefix)) && strings.Contains(key, idLabel) { |
| 952 | return value, true |
| 953 | } |
| 954 | } |
| 955 | return 0, false |
| 956 | } |
| 957 | |
| 958 | func scalarSeriesKeys(series map[string]metrix.SampleValue) []string { |
| 959 | keys := make([]string, 0, len(series)) |
| 960 | for key := range series { |
| 961 | keys = append(keys, key) |
| 962 | } |
| 963 | return keys |
| 964 | } |
| 965 | |
| 966 | func firstResourceID(collr *Collector) string { |
| 967 | for id := range collr.resources.Hosts { |
| 968 | return id |
| 969 | } |
| 970 | for id := range collr.resources.VMs { |
| 971 | return id |
| 972 | } |
| 973 | return "" |
| 974 | } |
| 975 | |
| 976 | func mustCycleController(t *testing.T, store metrix.CollectorStore) metrix.CycleController { |
| 977 | t.Helper() |
| 978 | |
| 979 | managed, ok := metrix.AsCycleManagedStore(store) |
| 980 | require.True(t, ok) |
| 981 | return managed.CycleController() |
| 982 | } |
| 983 | |
| 984 | func TestCollector_Collect_Run(t *testing.T) { |
| 985 | collr, _, teardown := prepareVSphereSim(t) |
| 986 | defer teardown() |
| 987 | |
| 988 | collr.DiscoveryInterval = confopt.Duration(time.Second * 2) |
| 989 | require.NoError(t, collr.Init(context.Background())) |
| 990 | require.NoError(t, collr.Check(context.Background())) |
| 991 | |
| 992 | runs := 20 |
| 993 | for i := range runs { |
| 994 | assert.True(t, len(collectScalarSeriesForTest(t, collr)) > 0) |
| 995 | if i < 6 { |
| 996 | time.Sleep(time.Second) |
| 997 | } |
| 998 | } |
| 999 | collecttest.AssertChartCoverage(t, collr, collecttest.ChartCoverageExpectation{}) |
| 1000 | } |
| 1001 | |
| 1002 | func prepareVSphereSim(t *testing.T) (collr *Collector, model *simulator.Model, teardown func()) { |
| 1003 | model, srv := createSim(t) |
| 1004 | collr = New() |
| 1005 | teardown = func() { model.Remove(); srv.Close(); collr.Cleanup(context.Background()) } |
| 1006 | |
| 1007 | collr.Username = "administrator" |
| 1008 | collr.Password = "password" |
| 1009 | collr.URL = srv.URL.String() |
| 1010 | collr.TLSConfig.InsecureSkipVerify = true |
| 1011 | |
| 1012 | return collr, model, teardown |
| 1013 | } |
| 1014 | |
| 1015 | func createSim(t *testing.T) (*simulator.Model, *simulator.Server) { |
| 1016 | model := simulator.VPX() |
| 1017 | err := model.Create() |
| 1018 | require.NoError(t, err) |
| 1019 | model.Service.TLS = new(tls.Config) |
| 1020 | return model, model.Service.NewServer() |
| 1021 | } |
| 1022 | |
| 1023 | type mockScraper struct { |
| 1024 | scraper |
| 1025 | } |
| 1026 | |
| 1027 | func (s mockScraper) ScrapeHosts(hosts rs.Hosts) []performance.EntityMetric { |
| 1028 | ms := s.scraper.ScrapeHosts(hosts) |
| 1029 | return populateMetrics(ms, 100) |
| 1030 | } |
| 1031 | func (s mockScraper) ScrapeVMs(vms rs.VMs) []performance.EntityMetric { |
| 1032 | ms := s.scraper.ScrapeVMs(vms) |
| 1033 | return populateMetrics(ms, 200) |
| 1034 | } |
| 1035 | func (s mockScraper) ScrapeDatastores(datastores rs.Datastores) []performance.EntityMetric { |
| 1036 | ms := s.scraper.ScrapeDatastores(datastores) |
| 1037 | return populateMetrics(ms, 300) |
| 1038 | } |
| 1039 | func (s mockScraper) ScrapeClusters(clusters rs.Clusters) []performance.EntityMetric { |
| 1040 | ms := s.scraper.ScrapeClusters(clusters) |
| 1041 | return populateMetrics(ms, 400) |
| 1042 | } |
| 1043 | |
| 1044 | func populateMetrics(ms []performance.EntityMetric, value int64) []performance.EntityMetric { |
| 1045 | for i := range ms { |
| 1046 | for ii := range ms[i].Value { |
| 1047 | v := &ms[i].Value[ii].Value |
| 1048 | if *v == nil { |
| 1049 | *v = append(*v, value) |
| 1050 | } else { |
| 1051 | (*v)[0] = value |
| 1052 | } |
| 1053 | } |
| 1054 | } |
| 1055 | return ms |
| 1056 | } |
| 1057 | |
| 1058 | type mockHostMatcher struct{ name string } |
| 1059 | type mockVMMatcher struct{ name string } |
| 1060 | |
| 1061 | func (m mockHostMatcher) Match(host *rs.Host) bool { return m.name == host.ID } |
| 1062 | func (m mockVMMatcher) Match(vm *rs.VM) bool { return m.name == vm.ID } |
| 1063 | |
| 1064 | type mockScraperNoHostPerf struct { |
| 1065 | mockScraper |
| 1066 | } |
| 1067 | |
| 1068 | func (s mockScraperNoHostPerf) ScrapeHosts(rs.Hosts) []performance.EntityMetric { |
| 1069 | return nil |
| 1070 | } |
| 1071 | |
| 1072 | type mockScraperNoVMPerf struct { |
| 1073 | mockScraper |
| 1074 | } |
| 1075 | |
| 1076 | func (s mockScraperNoVMPerf) ScrapeVMs(rs.VMs) []performance.EntityMetric { |
| 1077 | return nil |
| 1078 | } |
| 1079 | |
| 1080 | // mockScraperNoDSPerf wraps a scraper but returns no perf data for datastores (simulates vSAN). |
| 1081 | type mockScraperNoDSPerf struct { |
| 1082 | scraper |
| 1083 | } |
| 1084 | |
| 1085 | func (s mockScraperNoDSPerf) ScrapeHosts(hosts rs.Hosts) []performance.EntityMetric { |
| 1086 | return populateMetrics(s.scraper.ScrapeHosts(hosts), 100) |
| 1087 | } |
| 1088 | func (s mockScraperNoDSPerf) ScrapeVMs(vms rs.VMs) []performance.EntityMetric { |
| 1089 | return populateMetrics(s.scraper.ScrapeVMs(vms), 200) |
| 1090 | } |
| 1091 | func (s mockScraperNoDSPerf) ScrapeDatastores(_ rs.Datastores) []performance.EntityMetric { |
| 1092 | return nil |
| 1093 | } |
| 1094 | func (s mockScraperNoDSPerf) ScrapeClusters(clusters rs.Clusters) []performance.EntityMetric { |
| 1095 | return populateMetrics(s.scraper.ScrapeClusters(clusters), 400) |
| 1096 | } |
| 1097 | |
| 1098 | // mockScraperNoClusterPerf wraps a scraper but returns no perf data for clusters. |
| 1099 | type mockScraperNoClusterPerf struct { |
| 1100 | scraper |
| 1101 | } |
| 1102 | |
| 1103 | func (s mockScraperNoClusterPerf) ScrapeHosts(hosts rs.Hosts) []performance.EntityMetric { |
| 1104 | return populateMetrics(s.scraper.ScrapeHosts(hosts), 100) |
| 1105 | } |
| 1106 | func (s mockScraperNoClusterPerf) ScrapeVMs(vms rs.VMs) []performance.EntityMetric { |
| 1107 | return populateMetrics(s.scraper.ScrapeVMs(vms), 200) |
| 1108 | } |
| 1109 | func (s mockScraperNoClusterPerf) ScrapeDatastores(datastores rs.Datastores) []performance.EntityMetric { |
| 1110 | return populateMetrics(s.scraper.ScrapeDatastores(datastores), 300) |
| 1111 | } |
| 1112 | func (s mockScraperNoClusterPerf) ScrapeClusters(_ rs.Clusters) []performance.EntityMetric { |
| 1113 | return nil |
| 1114 | } |
| 1115 | |
| 1116 | func TestCollector_Collect_NoPerfData(t *testing.T) { |
| 1117 | tests := map[string]struct { |
| 1118 | setNoPerfScraper func(*Collector) |
| 1119 | restoreScraper func(*Collector) |
| 1120 | checkNoPerf func(*testing.T, *Collector, map[string]metrix.SampleValue, simulator.Model) |
| 1121 | checkWithPerf func(*testing.T, *Collector, map[string]metrix.SampleValue, simulator.Model) |
| 1122 | }{ |
| 1123 | "datastores": { |
| 1124 | setNoPerfScraper: func(c *Collector) { c.scraper = mockScraperNoDSPerf{c.scraper} }, |
| 1125 | restoreScraper: func(c *Collector) { c.scraper = mockScraper{c.scraper.(mockScraperNoDSPerf).scraper} }, |
| 1126 | checkNoPerf: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue, count simulator.Model) { |
| 1127 | for _, ds := range collr.resources.Datastores { |
| 1128 | requireScalarSeries(t, series, "datastore_space_usage_capacity", ds.ID) |
| 1129 | requireScalarSeries(t, series, "datastore_space_usage_free", ds.ID) |
| 1130 | requireScalarSeries(t, series, "datastore_space_usage_used", ds.ID) |
| 1131 | requireScalarSeries(t, series, "datastore_space_utilization_used", ds.ID) |
| 1132 | requireScalarSeries(t, series, "datastore_space_usage_uncommitted", ds.ID) |
| 1133 | requireScalarSeries(t, series, "datastore_overall_status_green", ds.ID) |
| 1134 | requireScalarSeries(t, series, "datastore_accessibility_status_accessible", ds.ID) |
| 1135 | requireScalarSeries(t, series, "datastore_maintenance_status_normal", ds.ID) |
| 1136 | requireScalarSeries(t, series, "datastore_multiple_host_access_unknown", ds.ID) |
| 1137 | requireNoScalarSeries(t, series, "datastore_disk_io_read", ds.ID) |
| 1138 | requireNoScalarSeries(t, series, "datastore_disk_iops_reads", ds.ID) |
| 1139 | } |
| 1140 | |
| 1141 | collecttest.AssertChartCoverage(t, collr, collecttest.ChartCoverageExpectation{}) |
| 1142 | }, |
| 1143 | checkWithPerf: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue, count simulator.Model) { |
| 1144 | for _, ds := range collr.resources.Datastores { |
| 1145 | requireScalarSeries(t, series, "datastore_disk_io_read", ds.ID) |
| 1146 | } |
| 1147 | |
| 1148 | collecttest.AssertChartCoverage(t, collr, collecttest.ChartCoverageExpectation{}) |
| 1149 | }, |
| 1150 | }, |
| 1151 | "clusters": { |
| 1152 | setNoPerfScraper: func(c *Collector) { c.scraper = mockScraperNoClusterPerf{c.scraper} }, |
| 1153 | restoreScraper: func(c *Collector) { c.scraper = mockScraper{c.scraper.(mockScraperNoClusterPerf).scraper} }, |
| 1154 | checkNoPerf: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue, count simulator.Model) { |
| 1155 | for _, cl := range collr.resources.Clusters { |
| 1156 | requireScalarSeries(t, series, "cluster_hosts_total", cl.ID) |
| 1157 | requireScalarSeries(t, series, "cluster_cpu_capacity_total", cl.ID) |
| 1158 | requireScalarSeries(t, series, "cluster_overall_status_green", cl.ID) |
| 1159 | requireNoScalarSeries(t, series, "cluster_cpu_utilization_used", cl.ID) |
| 1160 | requireNoScalarSeries(t, series, "cluster_services_fairness_cpu", cl.ID) |
| 1161 | } |
| 1162 | |
| 1163 | collecttest.AssertChartCoverage(t, collr, collecttest.ChartCoverageExpectation{}) |
| 1164 | }, |
| 1165 | checkWithPerf: func(t *testing.T, collr *Collector, series map[string]metrix.SampleValue, count simulator.Model) { |
| 1166 | for _, cl := range collr.resources.Clusters { |
| 1167 | requireScalarSeries(t, series, "cluster_cpu_utilization_used", cl.ID) |
| 1168 | } |
| 1169 | |
| 1170 | collecttest.AssertChartCoverage(t, collr, collecttest.ChartCoverageExpectation{}) |
| 1171 | }, |
| 1172 | }, |
| 1173 | } |
| 1174 | |
| 1175 | for name, tc := range tests { |
| 1176 | t.Run(name, func(t *testing.T) { |
| 1177 | collr, model, teardown := prepareVSphereSim(t) |
| 1178 | defer teardown() |
| 1179 | |
| 1180 | require.NoError(t, collr.Init(context.Background())) |
| 1181 | tc.setNoPerfScraper(collr) |
| 1182 | |
| 1183 | mx := collectScalarSeriesForTest(t, collr) |
| 1184 | require.NotNil(t, mx) |
| 1185 | count := model.Count() |
| 1186 | tc.checkNoPerf(t, collr, mx, count) |
| 1187 | |
| 1188 | tc.restoreScraper(collr) |
| 1189 | mx = collectScalarSeriesForTest(t, collr) |
| 1190 | require.NotNil(t, mx) |
| 1191 | tc.checkWithPerf(t, collr, mx, count) |
| 1192 | }) |
| 1193 | } |
| 1194 | } |