| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package dcgm |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "net/http" |
| 8 | "net/http/httptest" |
| 9 | "os" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | |
| 13 | "github.com/stretchr/testify/assert" |
| 14 | "github.com/stretchr/testify/require" |
| 15 | |
| 16 | "github.com/netdata/netdata/go/plugins/pkg/web" |
| 17 | "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" |
| 18 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/collecttest" |
| 19 | ) |
| 20 | |
| 21 | var ( |
| 22 | dataConfigJSON, _ = os.ReadFile("testdata/config.json") |
| 23 | dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") |
| 24 | |
| 25 | dataMetricsValid, _ = os.ReadFile("testdata/metrics_valid.prom") |
| 26 | dataMetricsNonDCGM, _ = os.ReadFile("testdata/metrics_non_dcgm.prom") |
| 27 | dataAllFieldsList, _ = os.ReadFile("testdata/all_fields_nonlabel.txt") |
| 28 | ) |
| 29 | |
| 30 | func Test_testDataIsValid(t *testing.T) { |
| 31 | for name, data := range map[string][]byte{ |
| 32 | "dataConfigJSON": dataConfigJSON, |
| 33 | "dataConfigYAML": dataConfigYAML, |
| 34 | "dataMetricsValid": dataMetricsValid, |
| 35 | "dataMetricsNonDCGM": dataMetricsNonDCGM, |
| 36 | "dataAllFieldsList": dataAllFieldsList, |
| 37 | } { |
| 38 | require.NotNil(t, data, name) |
| 39 | } |
| 40 | } |
| 41 | |
| 42 | func TestCollector_ConfigurationSerialize(t *testing.T) { |
| 43 | collecttest.TestConfigurationSerialize(t, &Collector{}, dataConfigJSON, dataConfigYAML) |
| 44 | } |
| 45 | |
| 46 | func TestCollector_Init(t *testing.T) { |
| 47 | tests := map[string]struct { |
| 48 | config Config |
| 49 | wantFail bool |
| 50 | }{ |
| 51 | "valid URL": { |
| 52 | config: Config{ |
| 53 | HTTPConfig: web.HTTPConfig{ |
| 54 | RequestConfig: web.RequestConfig{URL: "http://127.0.0.1:9400/metrics"}, |
| 55 | }, |
| 56 | }, |
| 57 | wantFail: false, |
| 58 | }, |
| 59 | "empty URL": { |
| 60 | config: Config{}, |
| 61 | wantFail: true, |
| 62 | }, |
| 63 | } |
| 64 | |
| 65 | for name, test := range tests { |
| 66 | t.Run(name, func(t *testing.T) { |
| 67 | collr := New() |
| 68 | collr.Config = test.config |
| 69 | |
| 70 | if test.wantFail { |
| 71 | assert.Error(t, collr.Init(context.Background())) |
| 72 | } else { |
| 73 | assert.NoError(t, collr.Init(context.Background())) |
| 74 | } |
| 75 | }) |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | func TestCollector_Check(t *testing.T) { |
| 80 | tests := map[string]struct { |
| 81 | metrics []byte |
| 82 | prepare func(*Collector) |
| 83 | wantFail bool |
| 84 | }{ |
| 85 | "success valid dcgm metrics": { |
| 86 | metrics: dataMetricsValid, |
| 87 | wantFail: false, |
| 88 | }, |
| 89 | "fail if endpoint has no dcgm metric prefix": { |
| 90 | metrics: dataMetricsNonDCGM, |
| 91 | wantFail: true, |
| 92 | }, |
| 93 | "success when global limit counts only dcgm series": { |
| 94 | metrics: []byte(` |
| 95 | # HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %). |
| 96 | # TYPE DCGM_FI_DEV_GPU_UTIL gauge |
| 97 | DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa"} 80 |
| 98 | # HELP go_memstats_alloc_bytes Number of bytes allocated in heap. |
| 99 | # TYPE go_memstats_alloc_bytes gauge |
| 100 | go_memstats_alloc_bytes 12 |
| 101 | `), |
| 102 | prepare: func(c *Collector) { c.MaxTS = 1 }, |
| 103 | wantFail: false, |
| 104 | }, |
| 105 | "fail when per-metric series limit is exceeded": { |
| 106 | metrics: []byte(` |
| 107 | # HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %). |
| 108 | # TYPE DCGM_FI_DEV_GPU_UTIL gauge |
| 109 | DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa"} 80 |
| 110 | DCGM_FI_DEV_GPU_UTIL{gpu="1",UUID="GPU-bbb"} 70 |
| 111 | `), |
| 112 | prepare: func(c *Collector) { c.MaxTSPerMetric = 1 }, |
| 113 | wantFail: true, |
| 114 | }, |
| 115 | } |
| 116 | |
| 117 | for name, test := range tests { |
| 118 | t.Run(name, func(t *testing.T) { |
| 119 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 120 | _, _ = w.Write(test.metrics) |
| 121 | })) |
| 122 | defer srv.Close() |
| 123 | |
| 124 | collr := New() |
| 125 | collr.URL = srv.URL |
| 126 | if test.prepare != nil { |
| 127 | test.prepare(collr) |
| 128 | } |
| 129 | |
| 130 | require.NoError(t, collr.Init(context.Background())) |
| 131 | |
| 132 | if test.wantFail { |
| 133 | assert.Error(t, collr.Check(context.Background())) |
| 134 | } else { |
| 135 | assert.NoError(t, collr.Check(context.Background())) |
| 136 | } |
| 137 | }) |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | func TestCollector_Collect(t *testing.T) { |
| 142 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 143 | _, _ = w.Write(dataMetricsValid) |
| 144 | })) |
| 145 | defer srv.Close() |
| 146 | |
| 147 | collr := New() |
| 148 | collr.URL = srv.URL |
| 149 | require.NoError(t, collr.Init(context.Background())) |
| 150 | |
| 151 | mx := collr.Collect(context.Background()) |
| 152 | require.NotNil(t, mx) |
| 153 | |
| 154 | gpuKey := "gpu=0|uuid=GPU-aaa" |
| 155 | migKey := "gpu=0|uuid=GPU-aaa|gpu_i_id=2|gpu_i_profile=1g.10gb" |
| 156 | linkKey := "gpu=0|gpu_uuid=GPU-aaa|nvlink=1" |
| 157 | |
| 158 | expect := map[string]int64{ |
| 159 | makeID(makeID("dcgm.gpu.compute.utilization", gpuKey), "gpu"): 80000, |
| 160 | makeID(makeID("dcgm.mig.compute.utilization", migKey), "gpu"): 60000, |
| 161 | makeID(makeID("dcgm.gpu.memory.usage", gpuKey), "used"): 1073741824000, |
| 162 | makeID(makeID("dcgm.gpu.reliability.xid", gpuKey), "xid"): 31000, |
| 163 | makeID(makeID("dcgm.gpu.reliability.row_remap_status", gpuKey), "row_remap_failure"): 1000, |
| 164 | makeID(makeID("dcgm.gpu.reliability.row_remap_events", gpuKey), "uncorrectable_remapped_rows"): 7000, |
| 165 | makeID(makeID("dcgm.gpu.throttle.violations", gpuKey), "power_violation"): 2000, |
| 166 | makeID(makeID("dcgm.gpu.throttle.violations", gpuKey), "thermal_violation"): 5000, |
| 167 | makeID(makeID("dcgm.gpu.interconnect.pcie.throughput", gpuKey), "pcie_tx"): 123456000, |
| 168 | makeID(makeID("dcgm.gpu.interconnect.total.throughput", gpuKey), "pcie"): 123456000, |
| 169 | makeID(makeID("dcgm.nvlink.interconnect.error_rate", linkKey), "nvlink_replay_error"): 4000, |
| 170 | } |
| 171 | |
| 172 | assert.Len(t, mx, len(expect)) |
| 173 | for dimID, want := range expect { |
| 174 | assert.Equal(t, want, mx[dimID], dimID) |
| 175 | } |
| 176 | |
| 177 | assert.Len(t, *collr.Charts(), 10) |
| 178 | |
| 179 | seenCtx := make(map[string]bool) |
| 180 | for _, ch := range *collr.Charts() { |
| 181 | seenCtx[ch.Ctx] = true |
| 182 | assert.NotContains(t, ch.Title, "(gpu:") |
| 183 | assert.NotContains(t, ch.Title, "(uuid:") |
| 184 | for _, lbl := range ch.Labels { |
| 185 | assert.NotEqual(t, "hostname", lbl.Key) |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | assert.True(t, seenCtx["dcgm.gpu.compute.utilization"]) |
| 190 | assert.True(t, seenCtx["dcgm.mig.compute.utilization"]) |
| 191 | assert.True(t, seenCtx["dcgm.gpu.memory.usage"]) |
| 192 | assert.True(t, seenCtx["dcgm.gpu.reliability.xid"]) |
| 193 | assert.True(t, seenCtx["dcgm.gpu.reliability.row_remap_status"]) |
| 194 | assert.True(t, seenCtx["dcgm.gpu.reliability.row_remap_events"]) |
| 195 | assert.True(t, seenCtx["dcgm.gpu.throttle.violations"]) |
| 196 | assert.True(t, seenCtx["dcgm.gpu.interconnect.pcie.throughput"]) |
| 197 | assert.True(t, seenCtx["dcgm.gpu.interconnect.total.throughput"]) |
| 198 | assert.True(t, seenCtx["dcgm.nvlink.interconnect.error_rate"]) |
| 199 | assert.False(t, seenCtx["dcgm.gpu.thermal.temperature"]) |
| 200 | } |
| 201 | |
| 202 | func TestCollector_Collect_NVLinkTotalOnlyInOverviewAndCleanDimNames(t *testing.T) { |
| 203 | metrics := []byte(` |
| 204 | # HELP DCGM_FI_PROF_NVLINK_RX_BYTES NVLink RX bytes. |
| 205 | # TYPE DCGM_FI_PROF_NVLINK_RX_BYTES gauge |
| 206 | DCGM_FI_PROF_NVLINK_RX_BYTES{gpu="0",UUID="GPU-aaa"} 10 |
| 207 | # HELP DCGM_FI_PROF_NVLINK_TX_BYTES NVLink TX bytes. |
| 208 | # TYPE DCGM_FI_PROF_NVLINK_TX_BYTES gauge |
| 209 | DCGM_FI_PROF_NVLINK_TX_BYTES{gpu="0",UUID="GPU-aaa"} 20 |
| 210 | # HELP DCGM_FI_PROF_NVLINK_RX_BYTES NVLink RX bytes. |
| 211 | # TYPE DCGM_FI_PROF_NVLINK_RX_BYTES gauge |
| 212 | DCGM_FI_PROF_NVLINK_RX_BYTES{gpu="0",gpu_uuid="GPU-aaa",nvlink="1"} 11 |
| 213 | # HELP DCGM_FI_PROF_NVLINK_TX_BYTES NVLink TX bytes. |
| 214 | # TYPE DCGM_FI_PROF_NVLINK_TX_BYTES gauge |
| 215 | DCGM_FI_PROF_NVLINK_TX_BYTES{gpu="0",gpu_uuid="GPU-aaa",nvlink="1"} 22 |
| 216 | # HELP DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL Total NVLink bandwidth. |
| 217 | # TYPE DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL counter |
| 218 | DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL{gpu="0",UUID="GPU-aaa"} 400 |
| 219 | `) |
| 220 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 221 | _, _ = w.Write(metrics) |
| 222 | })) |
| 223 | defer srv.Close() |
| 224 | |
| 225 | collr := New() |
| 226 | collr.URL = srv.URL |
| 227 | require.NoError(t, collr.Init(context.Background())) |
| 228 | |
| 229 | mx := collr.Collect(context.Background()) |
| 230 | require.NotNil(t, mx) |
| 231 | |
| 232 | gpuKey := "gpu=0|uuid=GPU-aaa" |
| 233 | linkKey := "gpu=0|gpu_uuid=GPU-aaa|nvlink=1" |
| 234 | nvlinkCtxID := "dcgm.gpu.interconnect.nvlink.throughput" |
| 235 | nvlinkEntityCtxID := "dcgm.nvlink.interconnect.throughput" |
| 236 | totalCtxID := "dcgm.gpu.interconnect.total.throughput" |
| 237 | |
| 238 | assert.Equal(t, int64(10000), mx[makeID(makeID(nvlinkCtxID, gpuKey), "nvlink_rx")]) |
| 239 | assert.Equal(t, int64(20000), mx[makeID(makeID(nvlinkCtxID, gpuKey), "nvlink_tx")]) |
| 240 | assert.NotContains(t, mx, makeID(makeID(nvlinkCtxID, gpuKey), "nvlink_bandwidth")) |
| 241 | assert.Equal(t, int64(11000), mx[makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_rx")]) |
| 242 | assert.Equal(t, int64(22000), mx[makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_tx")]) |
| 243 | assert.NotContains(t, mx, makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_rx_bytes")) |
| 244 | assert.NotContains(t, mx, makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_tx_bytes")) |
| 245 | // Explicit NVLink total metric should win over rx+tx aggregation for overview. |
| 246 | assert.Equal(t, int64(400000), mx[makeID(makeID(totalCtxID, gpuKey), "nvlink")]) |
| 247 | } |
| 248 | |
| 249 | func TestCollector_Collect_XIDErrorCodeCreatesCleanDimensions(t *testing.T) { |
| 250 | metrics := []byte(` |
| 251 | # HELP DCGM_FI_DEV_XID_ERRORS Value of the last XID error encountered. |
| 252 | # TYPE DCGM_FI_DEV_XID_ERRORS gauge |
| 253 | DCGM_FI_DEV_XID_ERRORS{gpu="0",UUID="GPU-aaa",err_code="31",err_msg="MMU fault",DCGM_FI_DEV_BRAND="GeForce"} 31 |
| 254 | `) |
| 255 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 256 | _, _ = w.Write(metrics) |
| 257 | })) |
| 258 | defer srv.Close() |
| 259 | |
| 260 | collr := New() |
| 261 | collr.URL = srv.URL |
| 262 | require.NoError(t, collr.Init(context.Background())) |
| 263 | |
| 264 | mx := collr.Collect(context.Background()) |
| 265 | require.NotNil(t, mx) |
| 266 | require.Len(t, mx, 1) |
| 267 | |
| 268 | chartID := makeID("dcgm.gpu.reliability.xid", "gpu=0|uuid=GPU-aaa") |
| 269 | dimID := makeID(chartID, "xid") |
| 270 | assert.Equal(t, int64(31000), mx[dimID], dimID) |
| 271 | |
| 272 | for dimID := range mx { |
| 273 | assert.False(t, strings.Contains(dimID, "err_code"), dimID) |
| 274 | assert.False(t, strings.Contains(dimID, "dcgm_fi_dev_brand"), dimID) |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | func TestCollector_Collect_ExposesDatasetLabelsExceptHostname(t *testing.T) { |
| 279 | metrics := []byte(` |
| 280 | # HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %). |
| 281 | # TYPE DCGM_FI_DEV_GPU_UTIL gauge |
| 282 | DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa",Hostname="host1",DCGM_FI_PROCESS_NAME="/usr/bin/nv-hostengine",DCGM_FI_DRIVER_VERSION="590.48.01"} 80 |
| 283 | `) |
| 284 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 285 | _, _ = w.Write(metrics) |
| 286 | })) |
| 287 | defer srv.Close() |
| 288 | |
| 289 | collr := New() |
| 290 | collr.URL = srv.URL |
| 291 | require.NoError(t, collr.Init(context.Background())) |
| 292 | |
| 293 | mx := collr.Collect(context.Background()) |
| 294 | require.NotNil(t, mx) |
| 295 | |
| 296 | charts := *collr.Charts() |
| 297 | require.NotEmpty(t, charts) |
| 298 | |
| 299 | var labels []collectorapi.Label |
| 300 | for _, ch := range charts { |
| 301 | if ch.Ctx == "dcgm.gpu.compute.utilization" { |
| 302 | labels = ch.Labels |
| 303 | break |
| 304 | } |
| 305 | } |
| 306 | require.NotNil(t, labels) |
| 307 | |
| 308 | assertChartHasLabel(t, labels, "dcgm_fi_process_name") |
| 309 | assertChartHasLabel(t, labels, "dcgm_fi_driver_version") |
| 310 | assertChartHasNoLabel(t, labels, "hostname") |
| 311 | } |
| 312 | |
| 313 | func TestCollector_Collect_RatioAndBar1Classification(t *testing.T) { |
| 314 | metrics := []byte(` |
| 315 | # HELP DCGM_FI_PROF_SM_ACTIVE Ratio of cycles an SM has at least 1 warp assigned. |
| 316 | # TYPE DCGM_FI_PROF_SM_ACTIVE gauge |
| 317 | DCGM_FI_PROF_SM_ACTIVE{gpu="0",UUID="GPU-aaa"} 0.25 |
| 318 | # HELP DCGM_FI_DEV_FB_USED_PERCENT Framebuffer memory used percent. |
| 319 | # TYPE DCGM_FI_DEV_FB_USED_PERCENT gauge |
| 320 | DCGM_FI_DEV_FB_USED_PERCENT{gpu="0",UUID="GPU-aaa"} 0.921353 |
| 321 | # HELP DCGM_FI_DEV_FB_USED Framebuffer memory used (in MiB). |
| 322 | # TYPE DCGM_FI_DEV_FB_USED gauge |
| 323 | DCGM_FI_DEV_FB_USED{gpu="0",UUID="GPU-aaa"} 1024 |
| 324 | # HELP DCGM_FI_DEV_FB_TOTAL Framebuffer memory total (in MiB). |
| 325 | # TYPE DCGM_FI_DEV_FB_TOTAL gauge |
| 326 | DCGM_FI_DEV_FB_TOTAL{gpu="0",UUID="GPU-aaa"} 32768 |
| 327 | # HELP DCGM_FI_DEV_BAR1_USED BAR1 used (in MiB). |
| 328 | # TYPE DCGM_FI_DEV_BAR1_USED gauge |
| 329 | DCGM_FI_DEV_BAR1_USED{gpu="0",UUID="GPU-aaa"} 162 |
| 330 | # HELP DCGM_FI_DEV_BAR1_TOTAL BAR1 total (in MiB). |
| 331 | # TYPE DCGM_FI_DEV_BAR1_TOTAL gauge |
| 332 | DCGM_FI_DEV_BAR1_TOTAL{gpu="0",UUID="GPU-aaa"} 256 |
| 333 | `) |
| 334 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 335 | _, _ = w.Write(metrics) |
| 336 | })) |
| 337 | defer srv.Close() |
| 338 | |
| 339 | collr := New() |
| 340 | collr.URL = srv.URL |
| 341 | require.NoError(t, collr.Init(context.Background())) |
| 342 | |
| 343 | mx := collr.Collect(context.Background()) |
| 344 | require.NotNil(t, mx) |
| 345 | |
| 346 | gpuKey := "gpu=0|uuid=GPU-aaa" |
| 347 | expect := map[string]int64{ |
| 348 | makeID(makeID("dcgm.gpu.compute.activity", gpuKey), "sm_active"): 25000, |
| 349 | makeID(makeID("dcgm.gpu.memory.utilization", gpuKey), "used_percent"): 92135, |
| 350 | makeID(makeID("dcgm.gpu.memory.usage", gpuKey), "used"): 1073741824000, |
| 351 | makeID(makeID("dcgm.gpu.memory.capacity", gpuKey), "total"): 34359738368000, |
| 352 | makeID(makeID("dcgm.gpu.memory.bar1_usage", gpuKey), "used"): 169869312000, |
| 353 | makeID(makeID("dcgm.gpu.memory.bar1_capacity", gpuKey), "total"): 268435456000, |
| 354 | } |
| 355 | |
| 356 | assert.Len(t, mx, len(expect)) |
| 357 | for dimID, want := range expect { |
| 358 | assert.Equal(t, want, mx[dimID], dimID) |
| 359 | } |
| 360 | |
| 361 | seenUnits := make(map[string]string) |
| 362 | for _, ch := range *collr.Charts() { |
| 363 | seenUnits[ch.Ctx] = ch.Units |
| 364 | } |
| 365 | assert.Equal(t, "percentage", seenUnits["dcgm.gpu.compute.activity"]) |
| 366 | assert.Equal(t, "percentage", seenUnits["dcgm.gpu.memory.utilization"]) |
| 367 | assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.usage"]) |
| 368 | assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.capacity"]) |
| 369 | assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.bar1_usage"]) |
| 370 | assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.bar1_capacity"]) |
| 371 | } |
| 372 | |
| 373 | func TestCollector_Collect_AvoidsOtherContextsForKnownMetrics(t *testing.T) { |
| 374 | metrics := []byte(` |
| 375 | # HELP DCGM_FI_DEV_PSTATE Performance state. |
| 376 | # TYPE DCGM_FI_DEV_PSTATE gauge |
| 377 | DCGM_FI_DEV_PSTATE{gpu="0",UUID="GPU-aaa"} 0 |
| 378 | # HELP DCGM_FI_DEV_VGPU_LICENSE_STATUS vGPU license status. |
| 379 | # TYPE DCGM_FI_DEV_VGPU_LICENSE_STATUS gauge |
| 380 | DCGM_FI_DEV_VGPU_LICENSE_STATUS{gpu="0",UUID="GPU-aaa"} 1 |
| 381 | # HELP DCGM_FI_DEV_VIRTUAL_MODE GPU virtualization mode. |
| 382 | # TYPE DCGM_FI_DEV_VIRTUAL_MODE gauge |
| 383 | DCGM_FI_DEV_VIRTUAL_MODE{gpu="0",UUID="GPU-aaa"} 0 |
| 384 | # HELP DCGM_FI_DEV_CLOCK_THROTTLE_REASONS Clock throttle reasons. |
| 385 | # TYPE DCGM_FI_DEV_CLOCK_THROTTLE_REASONS gauge |
| 386 | DCGM_FI_DEV_CLOCK_THROTTLE_REASONS{gpu="0",UUID="GPU-aaa"} 0 |
| 387 | # HELP DCGM_FI_DEV_FAN_SPEED Fan speed (in %). |
| 388 | # TYPE DCGM_FI_DEV_FAN_SPEED gauge |
| 389 | DCGM_FI_DEV_FAN_SPEED{gpu="0",UUID="GPU-aaa"} 30 |
| 390 | # HELP DCGM_FI_DEV_ENFORCED_POWER_LIMIT Enforced power limit (in W). |
| 391 | # TYPE DCGM_FI_DEV_ENFORCED_POWER_LIMIT gauge |
| 392 | DCGM_FI_DEV_ENFORCED_POWER_LIMIT{gpu="0",UUID="GPU-aaa"} 600 |
| 393 | # HELP DCGM_FI_DEV_PCIE_LINK_GEN PCIe current link generation. |
| 394 | # TYPE DCGM_FI_DEV_PCIE_LINK_GEN gauge |
| 395 | DCGM_FI_DEV_PCIE_LINK_GEN{gpu="0",UUID="GPU-aaa"} 5 |
| 396 | # HELP DCGM_FI_DEV_PCIE_LINK_WIDTH PCIe current link width. |
| 397 | # TYPE DCGM_FI_DEV_PCIE_LINK_WIDTH gauge |
| 398 | DCGM_FI_DEV_PCIE_LINK_WIDTH{gpu="0",UUID="GPU-aaa"} 16 |
| 399 | # HELP DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS Time throttled by SW power cap (in ns). |
| 400 | # TYPE DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS counter |
| 401 | DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS{gpu="0",UUID="GPU-aaa"} 1000000 |
| 402 | `) |
| 403 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 404 | _, _ = w.Write(metrics) |
| 405 | })) |
| 406 | defer srv.Close() |
| 407 | |
| 408 | collr := New() |
| 409 | collr.URL = srv.URL |
| 410 | require.NoError(t, collr.Init(context.Background())) |
| 411 | |
| 412 | mx := collr.Collect(context.Background()) |
| 413 | require.NotNil(t, mx) |
| 414 | |
| 415 | seenCtx := make(map[string]bool) |
| 416 | for _, ch := range *collr.Charts() { |
| 417 | seenCtx[ch.Ctx] = true |
| 418 | } |
| 419 | |
| 420 | assert.True(t, seenCtx["dcgm.gpu.state.performance"]) |
| 421 | assert.True(t, seenCtx["dcgm.gpu.state.virtualization"]) |
| 422 | assert.True(t, seenCtx["dcgm.gpu.virtualization.vgpu.license"]) |
| 423 | assert.True(t, seenCtx["dcgm.gpu.throttle.reasons"]) |
| 424 | assert.True(t, seenCtx["dcgm.gpu.thermal.fan_speed"]) |
| 425 | assert.True(t, seenCtx["dcgm.gpu.power.usage"]) |
| 426 | assert.True(t, seenCtx["dcgm.gpu.interconnect.pcie.link.generation"]) |
| 427 | assert.True(t, seenCtx["dcgm.gpu.interconnect.pcie.link.width"]) |
| 428 | assert.True(t, seenCtx["dcgm.gpu.throttle.violations"]) |
| 429 | assert.False(t, seenCtx["dcgm.gpu.other.gauge"]) |
| 430 | assert.False(t, seenCtx["dcgm.gpu.other.counter"]) |
| 431 | } |
| 432 | |
| 433 | func TestCollector_Collect_HidesThresholdDimensionsByDefault(t *testing.T) { |
| 434 | metrics := []byte(` |
| 435 | # HELP DCGM_FI_DEV_SM_CLOCK SM clock in MHz. |
| 436 | # TYPE DCGM_FI_DEV_SM_CLOCK gauge |
| 437 | DCGM_FI_DEV_SM_CLOCK{gpu="0",UUID="GPU-aaa"} 2100 |
| 438 | # HELP DCGM_FI_DEV_MAX_SM_CLOCK Max SM clock in MHz. |
| 439 | # TYPE DCGM_FI_DEV_MAX_SM_CLOCK gauge |
| 440 | DCGM_FI_DEV_MAX_SM_CLOCK{gpu="0",UUID="GPU-aaa"} 3000 |
| 441 | # HELP DCGM_FI_DEV_APP_SM_CLOCK App SM clock in MHz. |
| 442 | # TYPE DCGM_FI_DEV_APP_SM_CLOCK gauge |
| 443 | DCGM_FI_DEV_APP_SM_CLOCK{gpu="0",UUID="GPU-aaa"} 2800 |
| 444 | # HELP DCGM_FI_DEV_GPU_TEMP GPU temperature in C. |
| 445 | # TYPE DCGM_FI_DEV_GPU_TEMP gauge |
| 446 | DCGM_FI_DEV_GPU_TEMP{gpu="0",UUID="GPU-aaa"} 55 |
| 447 | # HELP DCGM_FI_DEV_GPU_TEMP_LIMIT GPU temperature limit in C. |
| 448 | # TYPE DCGM_FI_DEV_GPU_TEMP_LIMIT gauge |
| 449 | DCGM_FI_DEV_GPU_TEMP_LIMIT{gpu="0",UUID="GPU-aaa"} 90 |
| 450 | # HELP DCGM_FI_DEV_SHUTDOWN_TEMP Shutdown temperature in C. |
| 451 | # TYPE DCGM_FI_DEV_SHUTDOWN_TEMP gauge |
| 452 | DCGM_FI_DEV_SHUTDOWN_TEMP{gpu="0",UUID="GPU-aaa"} 95 |
| 453 | # HELP DCGM_FI_DEV_POWER_USAGE Power draw in W. |
| 454 | # TYPE DCGM_FI_DEV_POWER_USAGE gauge |
| 455 | DCGM_FI_DEV_POWER_USAGE{gpu="0",UUID="GPU-aaa"} 320 |
| 456 | # HELP DCGM_FI_DEV_POWER_USAGE_INSTANT Instant power draw in W. |
| 457 | # TYPE DCGM_FI_DEV_POWER_USAGE_INSTANT gauge |
| 458 | DCGM_FI_DEV_POWER_USAGE_INSTANT{gpu="0",UUID="GPU-aaa"} 330 |
| 459 | # HELP DCGM_FI_DEV_ENFORCED_POWER_LIMIT Enforced power limit in W. |
| 460 | # TYPE DCGM_FI_DEV_ENFORCED_POWER_LIMIT gauge |
| 461 | DCGM_FI_DEV_ENFORCED_POWER_LIMIT{gpu="0",UUID="GPU-aaa"} 600 |
| 462 | # HELP DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX Maximum power limit in W. |
| 463 | # TYPE DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX gauge |
| 464 | DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX{gpu="0",UUID="GPU-aaa"} 650 |
| 465 | `) |
| 466 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 467 | _, _ = w.Write(metrics) |
| 468 | })) |
| 469 | defer srv.Close() |
| 470 | |
| 471 | collr := New() |
| 472 | collr.URL = srv.URL |
| 473 | require.NoError(t, collr.Init(context.Background())) |
| 474 | |
| 475 | mx := collr.Collect(context.Background()) |
| 476 | require.NotNil(t, mx) |
| 477 | |
| 478 | findChartByCtx := func(ctx string) *collectorapi.Chart { |
| 479 | for _, ch := range *collr.Charts() { |
| 480 | if ch.Ctx == ctx { |
| 481 | return ch |
| 482 | } |
| 483 | } |
| 484 | return nil |
| 485 | } |
| 486 | dimHiddenByName := func(ch *collectorapi.Chart, name string) (bool, bool) { |
| 487 | for _, d := range ch.Dims { |
| 488 | if d.Name == name { |
| 489 | return d.Hidden, true |
| 490 | } |
| 491 | } |
| 492 | return false, false |
| 493 | } |
| 494 | |
| 495 | clock := findChartByCtx("dcgm.gpu.clock.frequency") |
| 496 | require.NotNil(t, clock) |
| 497 | hidden, ok := dimHiddenByName(clock, "sm") |
| 498 | require.True(t, ok) |
| 499 | assert.False(t, hidden) |
| 500 | hidden, ok = dimHiddenByName(clock, "max_sm_clock") |
| 501 | require.True(t, ok) |
| 502 | assert.True(t, hidden) |
| 503 | hidden, ok = dimHiddenByName(clock, "app_sm_clock") |
| 504 | require.True(t, ok) |
| 505 | assert.True(t, hidden) |
| 506 | |
| 507 | thermal := findChartByCtx("dcgm.gpu.thermal.temperature") |
| 508 | require.NotNil(t, thermal) |
| 509 | hidden, ok = dimHiddenByName(thermal, "gpu") |
| 510 | require.True(t, ok) |
| 511 | assert.False(t, hidden) |
| 512 | hidden, ok = dimHiddenByName(thermal, "gpu_temp_limit") |
| 513 | require.True(t, ok) |
| 514 | assert.True(t, hidden) |
| 515 | hidden, ok = dimHiddenByName(thermal, "shutdown_temp") |
| 516 | require.True(t, ok) |
| 517 | assert.True(t, hidden) |
| 518 | |
| 519 | power := findChartByCtx("dcgm.gpu.power.usage") |
| 520 | require.NotNil(t, power) |
| 521 | hidden, ok = dimHiddenByName(power, "draw") |
| 522 | require.True(t, ok) |
| 523 | assert.False(t, hidden) |
| 524 | hidden, ok = dimHiddenByName(power, "power_usage_instant") |
| 525 | require.True(t, ok) |
| 526 | assert.False(t, hidden) |
| 527 | hidden, ok = dimHiddenByName(power, "enforced_limit") |
| 528 | require.True(t, ok) |
| 529 | assert.True(t, hidden) |
| 530 | hidden, ok = dimHiddenByName(power, "power_mgmt_limit_max") |
| 531 | require.True(t, ok) |
| 532 | assert.True(t, hidden) |
| 533 | } |
| 534 | |
| 535 | func TestCollector_Collect_UsesNVSwitchEntityContextToken(t *testing.T) { |
| 536 | metrics := []byte(` |
| 537 | # HELP DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX NVSwitch RX throughput. |
| 538 | # TYPE DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX counter |
| 539 | DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX{nvswitch="0"} 42 |
| 540 | `) |
| 541 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 542 | _, _ = w.Write(metrics) |
| 543 | })) |
| 544 | defer srv.Close() |
| 545 | |
| 546 | collr := New() |
| 547 | collr.URL = srv.URL |
| 548 | require.NoError(t, collr.Init(context.Background())) |
| 549 | |
| 550 | mx := collr.Collect(context.Background()) |
| 551 | require.NotNil(t, mx) |
| 552 | |
| 553 | seenCtx := make(map[string]bool) |
| 554 | for _, ch := range *collr.Charts() { |
| 555 | seenCtx[ch.Ctx] = true |
| 556 | } |
| 557 | |
| 558 | assert.True(t, seenCtx["dcgm.nvswitch.interconnect.nvswitch.throughput"]) |
| 559 | assert.False(t, seenCtx["dcgm.switch.interconnect.nvswitch.throughput"]) |
| 560 | } |
| 561 | |
| 562 | func TestCollector_Collect_SkipsUnsupportedSummaryHistogramFamilies(t *testing.T) { |
| 563 | metrics := []byte(` |
| 564 | # HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %). |
| 565 | # TYPE DCGM_FI_DEV_GPU_UTIL gauge |
| 566 | DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa"} 77 |
| 567 | # HELP DCGM_FI_DEV_FAKE_SUMMARY synthetic summary for test. |
| 568 | # TYPE DCGM_FI_DEV_FAKE_SUMMARY summary |
| 569 | DCGM_FI_DEV_FAKE_SUMMARY{gpu="0",UUID="GPU-aaa",quantile="0.5"} 1 |
| 570 | DCGM_FI_DEV_FAKE_SUMMARY_sum{gpu="0",UUID="GPU-aaa"} 2 |
| 571 | DCGM_FI_DEV_FAKE_SUMMARY_count{gpu="0",UUID="GPU-aaa"} 3 |
| 572 | # HELP DCGM_FI_DEV_FAKE_HIST synthetic histogram for test. |
| 573 | # TYPE DCGM_FI_DEV_FAKE_HIST histogram |
| 574 | DCGM_FI_DEV_FAKE_HIST_bucket{gpu="0",UUID="GPU-aaa",le="1"} 4 |
| 575 | DCGM_FI_DEV_FAKE_HIST_sum{gpu="0",UUID="GPU-aaa"} 5 |
| 576 | DCGM_FI_DEV_FAKE_HIST_count{gpu="0",UUID="GPU-aaa"} 6 |
| 577 | `) |
| 578 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 579 | _, _ = w.Write(metrics) |
| 580 | })) |
| 581 | defer srv.Close() |
| 582 | |
| 583 | collr := New() |
| 584 | collr.URL = srv.URL |
| 585 | require.NoError(t, collr.Init(context.Background())) |
| 586 | |
| 587 | mx := collr.Collect(context.Background()) |
| 588 | require.NotNil(t, mx) |
| 589 | |
| 590 | gpuKey := "gpu=0|uuid=GPU-aaa" |
| 591 | utilDimID := makeID(makeID("dcgm.gpu.compute.utilization", gpuKey), "gpu") |
| 592 | assert.Equal(t, int64(77000), mx[utilDimID], utilDimID) |
| 593 | assert.Len(t, mx, 1) |
| 594 | |
| 595 | for _, ch := range *collr.Charts() { |
| 596 | assert.NotContains(t, ch.Ctx, "fake_summary") |
| 597 | assert.NotContains(t, ch.Ctx, "fake_hist") |
| 598 | } |
| 599 | } |
| 600 | |
| 601 | func TestClassifier_StrictNIDLSplitsForRareFamilies(t *testing.T) { |
| 602 | tests := []struct { |
| 603 | name string |
| 604 | typ sampleKind |
| 605 | group string |
| 606 | }{ |
| 607 | {name: "DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL", typ: sampleCounter, group: "interconnect.nvlink.throughput"}, |
| 608 | {name: "DCGM_FI_DEV_NVLINK_COUNT_TX_PACKETS", typ: sampleCounter, group: "interconnect.nvlink.traffic"}, |
| 609 | {name: "DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER", typ: sampleGauge, group: "interconnect.nvlink.ber"}, |
| 610 | {name: "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC0", typ: sampleCounter, group: "interconnect.nvswitch.latency"}, |
| 611 | {name: "DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_BUS", typ: sampleGauge, group: "interconnect.nvswitch.topology"}, |
| 612 | {name: "DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_STATUS", typ: sampleGauge, group: "interconnect.connectx.error_status"}, |
| 613 | {name: "DCGM_FI_DEV_CLOCKS_EVENT_REASONS", typ: sampleGauge, group: "throttle.reasons"}, |
| 614 | {name: "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST_NS", typ: sampleCounter, group: "throttle.violations"}, |
| 615 | {name: "DCGM_FI_DEV_VGPU_MEMORY_USAGE", typ: sampleGauge, group: "virtualization.vgpu.memory"}, |
| 616 | {name: "DCGM_FI_DEV_VGPU_FRAME_RATE_LIMIT", typ: sampleGauge, group: "virtualization.vgpu.frame_rate"}, |
| 617 | {name: "DCGM_FI_DEV_VGPU_TYPE_NAME", typ: sampleGauge, group: "virtualization.vgpu.type"}, |
| 618 | {name: "DCGM_FI_DEV_VGPU_VM_NAME", typ: sampleGauge, group: "virtualization.vgpu.vm"}, |
| 619 | {name: "DCGM_FI_DEV_VGPU_INSTANCE_IDS", typ: sampleGauge, group: "virtualization.vgpu.instance"}, |
| 620 | {name: "DCGM_FI_DEV_VGPU_LICENSE_STATUS", typ: sampleGauge, group: "virtualization.vgpu.license"}, |
| 621 | {name: "DCGM_FI_DEV_VGPU_UTILIZATIONS", typ: sampleGauge, group: "virtualization.vgpu.utilization"}, |
| 622 | {name: "DCGM_FI_DEV_VGPU_ENC_SESSIONS_INFO", typ: sampleGauge, group: "virtualization.vgpu.sessions"}, |
| 623 | {name: "DCGM_FI_DEV_FB_TOTAL", typ: sampleGauge, group: "memory.capacity"}, |
| 624 | {name: "DCGM_FI_DEV_BAR1_TOTAL", typ: sampleGauge, group: "memory.bar1_capacity"}, |
| 625 | } |
| 626 | |
| 627 | for _, tc := range tests { |
| 628 | got := classifyMetricGroup(entityGPU, tc.name, tc.typ) |
| 629 | assert.Equal(t, tc.group, got, tc.name) |
| 630 | } |
| 631 | } |
| 632 | |
| 633 | func TestClassifier_AllKnownFieldsAvoidOtherContexts(t *testing.T) { |
| 634 | lines := strings.Split(string(dataAllFieldsList), "\n") |
| 635 | var unmapped []string |
| 636 | for _, line := range lines { |
| 637 | name := strings.TrimSpace(line) |
| 638 | if name == "" || strings.HasPrefix(name, "#") { |
| 639 | continue |
| 640 | } |
| 641 | for _, kind := range []sampleKind{sampleGauge, sampleCounter} { |
| 642 | group := classifyMetricGroup(entityGPU, name, kind) |
| 643 | if group == "other.gauge" || group == "other.counter" { |
| 644 | unmapped = append(unmapped, name) |
| 645 | break |
| 646 | } |
| 647 | } |
| 648 | } |
| 649 | |
| 650 | assert.Empty(t, unmapped, "unmapped DCGM fields fell into other contexts") |
| 651 | } |
| 652 | |
| 653 | func TestClassifier_NIDLInterconnectAndVGPUSplits(t *testing.T) { |
| 654 | lines := strings.SplitSeq(string(dataAllFieldsList), "\n") |
| 655 | |
| 656 | for line := range lines { |
| 657 | name := strings.TrimSpace(line) |
| 658 | if name == "" || strings.HasPrefix(name, "#") { |
| 659 | continue |
| 660 | } |
| 661 | |
| 662 | for _, kind := range []sampleKind{sampleGauge, sampleCounter} { |
| 663 | group := classifyMetricGroup(entityGPU, name, kind) |
| 664 | |
| 665 | if group == "interconnect.throughput" { |
| 666 | assert.True(t, |
| 667 | containsAny(name, "C2C_"), |
| 668 | "generic throughput grouping should only contain C2C-style throughput fields: %s", name) |
| 669 | } |
| 670 | |
| 671 | if group == "interconnect.pcie.throughput" { |
| 672 | assert.True(t, |
| 673 | strings.Contains(name, "PCIE") && containsAny(name, "BYTES", "THROUGHPUT", "BANDWIDTH"), |
| 674 | "pcie throughput grouping got non-PCIe throughput field: %s", name) |
| 675 | } |
| 676 | |
| 677 | if group == "interconnect.nvlink.throughput" { |
| 678 | assert.True(t, |
| 679 | strings.Contains(name, "NVLINK") && |
| 680 | containsAny(name, "BYTES", "THROUGHPUT", "BANDWIDTH"), |
| 681 | "nvlink throughput grouping got non-NVLink throughput field: %s", name) |
| 682 | } |
| 683 | |
| 684 | if group == "interconnect.pcie.traffic" || group == "interconnect.nvlink.traffic" || group == "interconnect.traffic" { |
| 685 | assert.True(t, containsAny(name, "PACKETS", "CODES"), "traffic grouping got non-traffic field: %s", name) |
| 686 | } |
| 687 | |
| 688 | if group == "interconnect.pcie.ber" || group == "interconnect.nvlink.ber" || group == "interconnect.ber" { |
| 689 | assert.True(t, containsAny(name, "BER"), "BER grouping got non-BER field: %s", name) |
| 690 | } |
| 691 | |
| 692 | if group == "virtualization.vgpu.utilization" { |
| 693 | assert.True(t, |
| 694 | containsAny(name, "UTILIZATION"), |
| 695 | "vGPU utilization grouping got non-utilization field: %s", name) |
| 696 | } |
| 697 | |
| 698 | if group == "virtualization.vgpu.memory" { |
| 699 | assert.True(t, containsAny(name, "MEMORY_USAGE"), "vGPU memory grouping got non-memory field: %s", name) |
| 700 | } |
| 701 | } |
| 702 | } |
| 703 | } |
| 704 | |
| 705 | func TestCatalog_GPUInterconnectFamiliesOnlyThreeVariants(t *testing.T) { |
| 706 | got := make(map[string]struct{}) |
| 707 | for _, g := range groupCatalog { |
| 708 | if !strings.HasPrefix(g.Suffix, "interconnect.") { |
| 709 | continue |
| 710 | } |
| 711 | got["gpu "+g.Family] = struct{}{} |
| 712 | } |
| 713 | |
| 714 | want := map[string]struct{}{ |
| 715 | "gpu interconnect/overview": {}, |
| 716 | "gpu interconnect/pcie": {}, |
| 717 | "gpu interconnect/nvlink": {}, |
| 718 | } |
| 719 | |
| 720 | assert.Equal(t, want, got) |
| 721 | } |
| 722 | |
| 723 | func TestCollector_Cleanup(t *testing.T) { |
| 724 | assert.NotPanics(t, func() { New().Cleanup(context.Background()) }) |
| 725 | |
| 726 | collr := New() |
| 727 | collr.URL = "http://127.0.0.1:9400/metrics" |
| 728 | require.NoError(t, collr.Init(context.Background())) |
| 729 | assert.NotPanics(t, func() { collr.Cleanup(context.Background()) }) |
| 730 | } |
| 731 | |
| 732 | func assertChartHasLabel(t *testing.T, labels []collectorapi.Label, key string) { |
| 733 | t.Helper() |
| 734 | for _, lbl := range labels { |
| 735 | if lbl.Key == key { |
| 736 | return |
| 737 | } |
| 738 | } |
| 739 | assert.Failf(t, "missing label", "expected chart label %q", key) |
| 740 | } |
| 741 | |
| 742 | func assertChartHasNoLabel(t *testing.T, labels []collectorapi.Label, key string) { |
| 743 | t.Helper() |
| 744 | for _, lbl := range labels { |
| 745 | if lbl.Key == key { |
| 746 | assert.Failf(t, "unexpected label", "did not expect chart label %q", key) |
| 747 | return |
| 748 | } |
| 749 | } |
| 750 | } |