@cryptotaxi247 / netdata-1 / commits / 8b3ede3ef

go.d: add dcgm-exporter collector with full field classification (#21721)

Costa Tsaousis committed Feb 12, 2026 at 05:09 UTC 8b3ede3ef3617d38ffce5226d849554ef4c3ba9a
23 files changed +7734 -10
src/go/pkg/prometheus/parse.go
+33 -10
@@ -164,7 +164,11 @@ func (p *promTextParser) setMetricFamilyBySeries() {
164 p.isSum, p.isCount, p.isQuantile, p.isBucket = false, false, false, false
165 p.currQuantile, p.currBucket = 0, 0
166
167 - name := p.currSeries[0].Value
167 + name, ok := metricNameValue(p.currSeries)
168 + if !ok {
169 + p.currMF = nil
170 + return
171 + }
172
173 if p.currMF != nil && p.currMF.name == name {
174 if p.currMF.typ == model.MetricTypeSummary {
@@ -180,7 +184,7 @@ func (p *promTextParser) setMetricFamilyBySeries() {
184 n := strings.TrimSuffix(name, sumSuffix)
185 if mf, ok := p.metrics[n]; ok && isSummaryOrHistogram(mf.typ) {
186 p.isSum = true
183 - p.currSeries[0].Value = n
187 + _ = setLabelValue(p.currSeries, labels.MetricName, n)
188 p.currMF = mf
189 return
190 }
@@ -188,20 +192,20 @@ func (p *promTextParser) setMetricFamilyBySeries() {
192 n := strings.TrimSuffix(name, countSuffix)
193 if mf, ok := p.metrics[n]; ok && isSummaryOrHistogram(mf.typ) {
194 p.isCount = true
191 - p.currSeries[0].Value = n
195 + _ = setLabelValue(p.currSeries, labels.MetricName, n)
196 p.currMF = mf
197 return
198 }
199 case strings.HasSuffix(name, bucketSuffix):
200 n := strings.TrimSuffix(name, bucketSuffix)
201 if mf, ok := p.metrics[n]; ok && isSummaryOrHistogram(mf.typ) {
198 - p.currSeries[0].Value = n
202 + _ = setLabelValue(p.currSeries, labels.MetricName, n)
203 p.setBucket()
204 p.currMF = mf
205 return
206 }
207 if p.currSeries.Has(bucketLabel) {
204 - p.currSeries[0].Value = n
208 + _ = setLabelValue(p.currSeries, labels.MetricName, n)
209 p.setBucket()
210 name = n
211 typ = model.MetricTypeHistogram
@@ -234,7 +238,7 @@ func (p *promTextParser) setBucket() {
238 }
239
240 func (p *promTextParser) addGauge(value float64) {
237 - p.currSeries = p.currSeries[1:] // remove "__name__"
241 + p.currSeries, _, _ = removeLabel(p.currSeries, labels.MetricName)
242
243 if v := len(p.currMF.metrics); v == cap(p.currMF.metrics) {
244 p.currMF.metrics = append(p.currMF.metrics, Metric{
@@ -253,7 +257,7 @@ func (p *promTextParser) addGauge(value float64) {
257 }
258
259 func (p *promTextParser) addCounter(value float64) {
256 - p.currSeries = p.currSeries[1:] // remove "__name__"
260 + p.currSeries, _, _ = removeLabel(p.currSeries, labels.MetricName)
261
262 if v := len(p.currMF.metrics); v == cap(p.currMF.metrics) {
263 p.currMF.metrics = append(p.currMF.metrics, Metric{
@@ -272,7 +276,7 @@ func (p *promTextParser) addCounter(value float64) {
276 }
277
278 func (p *promTextParser) addUnknown(value float64) {
275 - p.currSeries = p.currSeries[1:] // remove "__name__"
279 + p.currSeries, _, _ = removeLabel(p.currSeries, labels.MetricName)
280
281 if v := len(p.currMF.metrics); v == cap(p.currMF.metrics) {
282 p.currMF.metrics = append(p.currMF.metrics, Metric{
@@ -293,7 +297,7 @@ func (p *promTextParser) addUnknown(value float64) {
297 func (p *promTextParser) addSummary(value float64) {
298 hash := p.currSeries.Hash()
299
296 - p.currSeries = p.currSeries[1:] // remove "__name__"
300 + p.currSeries, _, _ = removeLabel(p.currSeries, labels.MetricName)
301
302 s, ok := p.summaries[hash]
303 if !ok {
@@ -332,7 +336,7 @@ func (p *promTextParser) addSummary(value float64) {
336 func (p *promTextParser) addHistogram(value float64) {
337 hash := p.currSeries.Hash()
338
335 - p.currSeries = p.currSeries[1:] // remove "__name__"
339 + p.currSeries, _, _ = removeLabel(p.currSeries, labels.MetricName)
340
341 h, ok := p.histograms[hash]
342 if !ok {
@@ -409,6 +413,25 @@ func removeLabel(lbs labels.Labels, name string) (labels.Labels, string, bool) {
413 return lbs, "", false
414 }
415
416 +func metricNameValue(lbs labels.Labels) (string, bool) {
417 + for _, v := range lbs {
418 + if v.Name == labels.MetricName {
419 + return v.Value, true
420 + }
421 + }
422 + return "", false
423 +}
424 +
425 +func setLabelValue(lbs labels.Labels, name, value string) bool {
426 + for i, v := range lbs {
427 + if v.Name == name {
428 + lbs[i].Value = value
429 + return true
430 + }
431 + }
432 + return false
433 +}
434 +
435 func isSummaryOrHistogram(typ model.MetricType) bool {
436 return typ == model.MetricTypeSummary || typ == model.MetricTypeHistogram
437 }
src/go/pkg/prometheus/parse_test.go
+55
@@ -1665,6 +1665,61 @@ test_gauge_metric_2{label1="value2"} 1
1665 assert.Equal(t, want, series)
1666 }
1667
1668 +func TestPromTextParser_parseToMetricFamilies_metricNameNotFirstLabel(t *testing.T) {
1669 + var p promTextParser
1670 +
1671 + txt := []byte(`
1672 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization
1673 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
1674 +DCGM_FI_DEV_GPU_UTIL{UUID="GPU-aaa",gpu="0"} 80
1675 +`)
1676 +
1677 + mfs, err := p.parseToMetricFamilies(txt)
1678 + require.NoError(t, err)
1679 +
1680 + require.Contains(t, mfs, "DCGM_FI_DEV_GPU_UTIL")
1681 + require.NotContains(t, mfs, "GPU-aaa")
1682 +
1683 + mf := mfs["DCGM_FI_DEV_GPU_UTIL"]
1684 + require.Len(t, mf.metrics, 1)
1685 + assert.Equal(t, model.MetricTypeGauge, mf.typ)
1686 + assert.Equal(t, 80.0, mf.metrics[0].gauge.value)
1687 + assert.EqualValues(t, labels.Labels{
1688 + {Name: "UUID", Value: "GPU-aaa"},
1689 + {Name: "gpu", Value: "0"},
1690 + }, mf.metrics[0].labels)
1691 +}
1692 +
1693 +func TestPromTextParser_parseToMetricFamilies_failsOnInvalidSeriesValue(t *testing.T) {
1694 + var p promTextParser
1695 +
1696 + txt := []byte(`
1697 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization
1698 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
1699 +DCGM_FI_DEV_GPU_UTIL{UUID="GPU-aaa",gpu="0"} 80
1700 +# HELP DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK Requested power profile mask
1701 +# TYPE DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK gauge
1702 +DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK{UUID="GPU-aaa",gpu="0"} ERROR - FAILED TO CONVERT TO STRING
1703 +`)
1704 +
1705 + _, err := p.parseToMetricFamilies(txt)
1706 + require.Error(t, err)
1707 + assert.Contains(t, err.Error(), "failed to parse prometheus metrics")
1708 +}
1709 +
1710 +func TestPromTextParser_parseToSeries_failsOnInvalidSeriesValue(t *testing.T) {
1711 + var p promTextParser
1712 +
1713 + txt := []byte(`
1714 +DCGM_FI_DEV_GPU_UTIL{UUID="GPU-aaa",gpu="0"} 80
1715 +DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK{UUID="GPU-aaa",gpu="0"} ERROR - FAILED TO CONVERT TO STRING
1716 +`)
1717 +
1718 + _, err := p.parseToSeries(txt)
1719 + require.Error(t, err)
1720 + assert.Contains(t, err.Error(), "failed to parse prometheus metrics")
1721 +}
1722 +
1723 func joinData(data ...[]byte) []byte {
1724 var buf bytes.Buffer
1725 for _, v := range data {
src/go/plugin/go.d/collector/dcgm/cache.go new
+96
@@ -0,0 +1,96 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dcgm
4 +
5 +import "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
6 +
7 +const (
8 + maxNotSeenCharts = 10
9 + maxNotSeenDims = 10
10 +)
11 +
12 +type (
13 + cache struct {
14 + charts map[string]*cacheChart
15 + }
16 +
17 + cacheChart struct {
18 + chart *module.Chart
19 + seen bool
20 + notSeenTimes int
21 + dims map[string]*cacheDim
22 + }
23 +
24 + cacheDim struct {
25 + seen bool
26 + notSeenTimes int
27 + }
28 +)
29 +
30 +func newCache() *cache {
31 + return &cache{charts: make(map[string]*cacheChart)}
32 +}
33 +
34 +func (c *cache) reset() {
35 + for _, ch := range c.charts {
36 + ch.seen = false
37 + for _, d := range ch.dims {
38 + d.seen = false
39 + }
40 + }
41 +}
42 +
43 +func (c *cache) getChart(key string) (*cacheChart, bool) {
44 + v, ok := c.charts[key]
45 + if !ok {
46 + return nil, false
47 + }
48 + v.seen = true
49 + v.notSeenTimes = 0
50 + return v, true
51 +}
52 +
53 +func (c *cache) putChart(key string, chart *module.Chart) *cacheChart {
54 + v := &cacheChart{chart: chart, seen: true, dims: make(map[string]*cacheDim)}
55 + c.charts[key] = v
56 + return v
57 +}
58 +
59 +func (ch *cacheChart) touchDim(dimID string) (exists bool) {
60 + if d, ok := ch.dims[dimID]; ok {
61 + d.seen = true
62 + d.notSeenTimes = 0
63 + return true
64 + }
65 + ch.dims[dimID] = &cacheDim{seen: true}
66 + return false
67 +}
68 +
69 +func (c *Collector) removeStaleChartsAndDims() {
70 + for key, ch := range c.cache.charts {
71 + if !ch.seen {
72 + ch.notSeenTimes++
73 + if ch.notSeenTimes >= maxNotSeenCharts {
74 + ch.chart.MarkRemove()
75 + ch.chart.MarkNotCreated()
76 + delete(c.cache.charts, key)
77 + }
78 + continue
79 + }
80 +
81 + for dimID, d := range ch.dims {
82 + if d.seen {
83 + d.notSeenTimes = 0
84 + continue
85 + }
86 + d.notSeenTimes++
87 + if d.notSeenTimes >= maxNotSeenDims {
88 + if err := ch.chart.MarkDimRemove(dimID, false); err != nil {
89 + c.Warning(err)
90 + }
91 + ch.chart.MarkNotCreated()
92 + delete(ch.dims, dimID)
93 + }
94 + }
95 + }
96 +}
src/go/plugin/go.d/collector/dcgm/catalog.go new
+661
@@ -0,0 +1,661 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dcgm
4 +
5 +import (
6 + "fmt"
7 + "strings"
8 +
9 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
10 +)
11 +
12 +type metricEntity string
13 +
14 +const (
15 + entityGPU metricEntity = "gpu"
16 + entityMIG metricEntity = "mig"
17 + entityNVLink metricEntity = "nvlink"
18 + entityNVSwitch metricEntity = "nvswitch"
19 + entityCPU metricEntity = "cpu"
20 + entityCPUCore metricEntity = "cpu_core"
21 + entityExporter metricEntity = "exporter"
22 +)
23 +
24 +type sampleKind uint8
25 +
26 +const (
27 + sampleGauge sampleKind = iota
28 + sampleCounter
29 + sampleUnsupported
30 +)
31 +
32 +type contextSpec struct {
33 + ID string
34 + Title string
35 + Units string
36 + Family string
37 + Type module.ChartType
38 + Priority int
39 +}
40 +
41 +type metricSpec struct {
42 + Context contextSpec
43 + DimName string
44 + Scale float64
45 +}
46 +
47 +type groupSpec struct {
48 + Suffix string
49 + Title string
50 + Units string
51 + Family string
52 + Type module.ChartType
53 +}
54 +
55 +var groupCatalog = []groupSpec{
56 + {Suffix: "compute.utilization", Title: "Compute Utilization", Units: "percentage", Family: "compute", Type: module.Line},
57 + {Suffix: "compute.activity", Title: "Compute Pipeline Activity", Units: "percentage", Family: "compute", Type: module.Line},
58 + {Suffix: "compute.tensor.activity", Title: "Tensor Core Activity by Precision", Units: "percentage", Family: "compute", Type: module.Line},
59 + {Suffix: "compute.media.activity", Title: "Media Engine Activity", Units: "percentage", Family: "compute", Type: module.Line},
60 + {Suffix: "compute.cache.activity", Title: "Memory Cache Hit/Miss", Units: "events/s", Family: "compute", Type: module.Line},
61 + {Suffix: "memory.utilization", Title: "Memory Utilization", Units: "percentage", Family: "memory", Type: module.Line},
62 + {Suffix: "memory.usage", Title: "Memory Usage", Units: "bytes", Family: "memory", Type: module.Stacked},
63 + {Suffix: "memory.capacity", Title: "Memory Capacity", Units: "bytes", Family: "memory", Type: module.Line},
64 + {Suffix: "memory.bar1_usage", Title: "BAR1 Memory Usage", Units: "bytes", Family: "memory", Type: module.Stacked},
65 + {Suffix: "memory.bar1_capacity", Title: "BAR1 Memory Capacity", Units: "bytes", Family: "memory", Type: module.Line},
66 + {Suffix: "memory.ecc_errors", Title: "ECC Errors", Units: "errors", Family: "memory", Type: module.Line},
67 + {Suffix: "memory.ecc_error_rate", Title: "ECC Error Rate", Units: "errors/s", Family: "memory", Type: module.Line},
68 + {Suffix: "memory.page_retirements", Title: "Retired Memory Pages", Units: "pages/s", Family: "memory", Type: module.Line},
69 + {Suffix: "reliability.row_remap_status", Title: "Row Remap Status", Units: "state", Family: "reliability", Type: module.Line},
70 + {Suffix: "reliability.row_remap_events", Title: "Row Remap Events", Units: "rows/s", Family: "reliability", Type: module.Line},
71 + {Suffix: "reliability.memory_health", Title: "Memory Health", Units: "state", Family: "reliability", Type: module.Line},
72 + {Suffix: "reliability.recovery_action", Title: "Recovery Action", Units: "state", Family: "reliability", Type: module.Line},
73 + {Suffix: "clock.frequency", Title: "Clock Frequency", Units: "MHz", Family: "clock", Type: module.Line},
74 + {Suffix: "throttle.reasons", Title: "Throttle Reasons", Units: "bitmask", Family: "throttle", Type: module.Line},
75 + {Suffix: "thermal.temperature", Title: "Temperature", Units: "Celsius", Family: "thermal", Type: module.Line},
76 + {Suffix: "thermal.fan_speed", Title: "Fan Speed", Units: "percentage", Family: "thermal", Type: module.Line},
77 + {Suffix: "power.usage", Title: "Power Usage", Units: "Watts", Family: "power", Type: module.Line},
78 + {Suffix: "power.energy", Title: "Energy Consumption Rate", Units: "mJ/s", Family: "power", Type: module.Line},
79 + {Suffix: "power.profiles", Title: "Power Profiles", Units: "state", Family: "power", Type: module.Line},
80 + {Suffix: "power.smoothing", Title: "Power Smoothing", Units: "value", Family: "power", Type: module.Line},
81 + {Suffix: "throttle.violations", Title: "Throttle Violation Duration", Units: "milliseconds/s", Family: "throttle", Type: module.Line},
82 + {Suffix: "interconnect.total.throughput", Title: "Interconnect Total Throughput", Units: "bytes/s", Family: "interconnect/overview", Type: module.Area},
83 + {Suffix: "interconnect.pcie.throughput", Title: "PCIe Throughput", Units: "bytes/s", Family: "interconnect/pcie", Type: module.Area},
84 + {Suffix: "interconnect.nvlink.throughput", Title: "NVLink Throughput", Units: "bytes/s", Family: "interconnect/nvlink", Type: module.Area},
85 + {Suffix: "interconnect.throughput", Title: "Interconnect Throughput", Units: "bytes/s", Family: "interconnect/overview", Type: module.Area},
86 + {Suffix: "interconnect.pcie.traffic", Title: "PCIe Traffic", Units: "events/s", Family: "interconnect/pcie", Type: module.Line},
87 + {Suffix: "interconnect.nvlink.traffic", Title: "NVLink Traffic", Units: "events/s", Family: "interconnect/nvlink", Type: module.Line},
88 + {Suffix: "interconnect.traffic", Title: "Interconnect Traffic", Units: "events/s", Family: "interconnect/overview", Type: module.Line},
89 + {Suffix: "interconnect.pcie.ber", Title: "PCIe Bit Error Rate", Units: "ratio", Family: "interconnect/pcie", Type: module.Line},
90 + {Suffix: "interconnect.nvlink.ber", Title: "NVLink Bit Error Rate", Units: "ratio", Family: "interconnect/nvlink", Type: module.Line},
91 + {Suffix: "interconnect.ber", Title: "Interconnect Bit Error Rate", Units: "ratio", Family: "interconnect/overview", Type: module.Line},
92 + {Suffix: "interconnect.pcie.link.generation", Title: "PCIe Link Generation", Units: "generation", Family: "interconnect/pcie", Type: module.Line},
93 + {Suffix: "interconnect.pcie.link.width", Title: "PCIe Link Width", Units: "lanes", Family: "interconnect/pcie", Type: module.Line},
94 + {Suffix: "interconnect.pcie.state", Title: "PCIe State", Units: "state", Family: "interconnect/pcie", Type: module.Line},
95 + {Suffix: "interconnect.nvlink.state", Title: "NVLink State", Units: "state", Family: "interconnect/nvlink", Type: module.Line},
96 + {Suffix: "interconnect.state", Title: "Interconnect State", Units: "state", Family: "interconnect/overview", Type: module.Line},
97 + {Suffix: "interconnect.nvlink.congestion", Title: "NVLink Congestion", Units: "events/s", Family: "interconnect/nvlink", Type: module.Line},
98 + {Suffix: "interconnect.congestion", Title: "Interconnect Congestion", Units: "events/s", Family: "interconnect/overview", Type: module.Line},
99 + {Suffix: "interconnect.fabric", Title: "Fabric State", Units: "state", Family: "interconnect/overview", Type: module.Line},
100 + {Suffix: "interconnect.pcie.errors", Title: "PCIe Errors", Units: "errors", Family: "interconnect/pcie", Type: module.Line},
101 + {Suffix: "interconnect.pcie.error_rate", Title: "PCIe Error Rate", Units: "errors/s", Family: "interconnect/pcie", Type: module.Line},
102 + {Suffix: "interconnect.nvlink.errors", Title: "NVLink Errors", Units: "errors", Family: "interconnect/nvlink", Type: module.Line},
103 + {Suffix: "interconnect.nvlink.error_rate", Title: "NVLink Error Rate", Units: "errors/s", Family: "interconnect/nvlink", Type: module.Line},
104 + {Suffix: "interconnect.errors", Title: "Interconnect Errors", Units: "errors", Family: "interconnect/overview", Type: module.Line},
105 + {Suffix: "interconnect.error_rate", Title: "Interconnect Error Rate", Units: "errors/s", Family: "interconnect/overview", Type: module.Line},
106 + {Suffix: "interconnect.connectx.status", Title: "ConnectX Status", Units: "state", Family: "interconnect/pcie", Type: module.Line},
107 + {Suffix: "interconnect.connectx.link", Title: "ConnectX Link", Units: "value", Family: "interconnect/pcie", Type: module.Line},
108 + {Suffix: "interconnect.connectx.temperature", Title: "ConnectX Temperature", Units: "Celsius", Family: "interconnect/pcie", Type: module.Line},
109 + {Suffix: "interconnect.connectx.errors", Title: "ConnectX Errors", Units: "errors/s", Family: "interconnect/pcie", Type: module.Line},
110 + {Suffix: "interconnect.nvswitch.status", Title: "NVSwitch Status", Units: "state", Family: "interconnect/nvlink", Type: module.Line},
111 + {Suffix: "interconnect.nvswitch.topology", Title: "NVSwitch Topology", Units: "value", Family: "interconnect/nvlink", Type: module.Line},
112 + {Suffix: "interconnect.nvswitch.throughput", Title: "NVSwitch Throughput", Units: "bytes/s", Family: "interconnect/nvlink", Type: module.Area},
113 + {Suffix: "interconnect.nvswitch.latency", Title: "NVSwitch Link Latency", Units: "events/s", Family: "interconnect/nvlink", Type: module.Line},
114 + {Suffix: "interconnect.nvswitch.errors", Title: "NVSwitch Errors", Units: "errors/s", Family: "interconnect/nvlink", Type: module.Line},
115 + {Suffix: "interconnect.nvswitch.temperature", Title: "NVSwitch Temperature", Units: "Celsius", Family: "interconnect/nvlink", Type: module.Line},
116 + {Suffix: "interconnect.nvswitch.power", Title: "NVSwitch Power", Units: "Watts", Family: "interconnect/nvlink", Type: module.Line},
117 + {Suffix: "interconnect.nvswitch.current", Title: "NVSwitch Current", Units: "value", Family: "interconnect/nvlink", Type: module.Line},
118 + {Suffix: "interconnect.nvswitch.voltage", Title: "NVSwitch Voltage", Units: "mV", Family: "interconnect/nvlink", Type: module.Line},
119 + {Suffix: "interconnect.connectx.error_status", Title: "ConnectX Error Status", Units: "state", Family: "interconnect/pcie", Type: module.Line},
120 + {Suffix: "reliability.xid", Title: "XID Errors", Units: "code", Family: "reliability", Type: module.Line},
121 + {Suffix: "health.status", Title: "Health Status", Units: "state", Family: "health", Type: module.Line},
122 + {Suffix: "state.performance", Title: "Performance State", Units: "state", Family: "state", Type: module.Line},
123 + {Suffix: "state.virtualization", Title: "Virtualization State", Units: "state", Family: "state", Type: module.Line},
124 + {Suffix: "state.configuration", Title: "Configuration State", Units: "state", Family: "state", Type: module.Line},
125 + {Suffix: "virtualization.vgpu.license", Title: "vGPU License", Units: "state", Family: "virtualization", Type: module.Line},
126 + {Suffix: "virtualization.vgpu.type", Title: "vGPU Type", Units: "value", Family: "virtualization", Type: module.Line},
127 + {Suffix: "virtualization.vgpu.instance", Title: "vGPU Instance", Units: "value", Family: "virtualization", Type: module.Line},
128 + {Suffix: "virtualization.vgpu.vm", Title: "vGPU VM", Units: "value", Family: "virtualization", Type: module.Line},
129 + {Suffix: "virtualization.vgpu.memory", Title: "vGPU Memory", Units: "bytes", Family: "virtualization", Type: module.Line},
130 + {Suffix: "virtualization.vgpu.frame_rate", Title: "vGPU Frame Rate", Units: "fps", Family: "virtualization", Type: module.Line},
131 + {Suffix: "virtualization.vgpu.utilization", Title: "vGPU Utilization", Units: "percentage", Family: "virtualization", Type: module.Line},
132 + {Suffix: "virtualization.vgpu.sessions", Title: "vGPU Sessions", Units: "value", Family: "virtualization", Type: module.Line},
133 + {Suffix: "virtualization.vgpu.software", Title: "vGPU Software", Units: "value", Family: "virtualization", Type: module.Line},
134 + {Suffix: "workload.sessions", Title: "Workload Sessions", Units: "value", Family: "workload", Type: module.Line},
135 + {Suffix: "cpu.utilization", Title: "CPU Utilization", Units: "percentage", Family: "cpu", Type: module.Line},
136 + {Suffix: "cpu.temperature", Title: "CPU Temperature", Units: "Celsius", Family: "cpu", Type: module.Line},
137 + {Suffix: "cpu.power", Title: "CPU Power", Units: "Watts", Family: "cpu", Type: module.Line},
138 + {Suffix: "cpu.info", Title: "CPU Information", Units: "value", Family: "cpu", Type: module.Line},
139 + {Suffix: "diagnostics.status", Title: "Diagnostics Status", Units: "state", Family: "diagnostics", Type: module.Line},
140 + {Suffix: "diagnostics.results", Title: "Diagnostics Results", Units: "state", Family: "diagnostics", Type: module.Line},
141 + {Suffix: "inventory.identity", Title: "Inventory Identity", Units: "value", Family: "inventory", Type: module.Line},
142 + {Suffix: "inventory.software", Title: "Software and Firmware", Units: "value", Family: "inventory", Type: module.Line},
143 + {Suffix: "inventory.platform", Title: "Platform Inventory", Units: "value", Family: "inventory", Type: module.Line},
144 + {Suffix: "topology.affinity", Title: "Topology and Affinity", Units: "value", Family: "topology", Type: module.Line},
145 + {Suffix: "capability.support", Title: "Capability Support", Units: "state", Family: "capability", Type: module.Line},
146 + {Suffix: "internal.boundary", Title: "Internal Boundary Fields", Units: "state", Family: "internal", Type: module.Line},
147 + {Suffix: "state", Title: "Device State", Units: "state", Family: "state", Type: module.Line},
148 + {Suffix: "other.gauge", Title: "Other Metrics", Units: "value", Family: "other", Type: module.Line},
149 + {Suffix: "other.counter", Title: "Other Metric Rate", Units: "events/s", Family: "other", Type: module.Line},
150 +}
151 +
152 +var contextCatalog = buildContextCatalog()
153 +
154 +func buildContextCatalog() map[string]contextSpec {
155 + entities := []metricEntity{entityGPU, entityMIG, entityNVLink, entityNVSwitch, entityCPU, entityCPUCore, entityExporter}
156 + catalog := make(map[string]contextSpec, len(entities)*len(groupCatalog))
157 +
158 + prio := module.Priority
159 + for _, entity := range entities {
160 + for _, group := range groupCatalog {
161 + ctx := fmt.Sprintf("dcgm.%s.%s", entity, group.Suffix)
162 + catalog[ctx] = contextSpec{
163 + ID: ctx,
164 + Title: fmt.Sprintf("%s %s", entityDisplayName(entity), group.Title),
165 + Units: group.Units,
166 + Family: fmt.Sprintf("%s %s", entityFamilyPrefix(entity), group.Family),
167 + Type: group.Type,
168 + Priority: prio,
169 + }
170 + prio += 10
171 + }
172 + }
173 +
174 + return catalog
175 +}
176 +
177 +func entityDisplayName(entity metricEntity) string {
178 + switch entity {
179 + case entityGPU:
180 + return "GPU"
181 + case entityMIG:
182 + return "MIG"
183 + case entityNVLink:
184 + return "NVLink"
185 + case entityNVSwitch:
186 + return "NVSwitch"
187 + case entityCPU:
188 + return "CPU"
189 + case entityCPUCore:
190 + return "CPU Core"
191 + default:
192 + return "Exporter"
193 + }
194 +}
195 +
196 +func entityFamilyPrefix(entity metricEntity) string {
197 + switch entity {
198 + case entityCPUCore:
199 + return "cpu core"
200 + default:
201 + return string(entity)
202 + }
203 +}
204 +
205 +func classifyMetric(entity metricEntity, metricName, help string, typ sampleKind) metricSpec {
206 + group := classifyMetricGroup(entity, metricName, typ)
207 + ctxID := fmt.Sprintf("dcgm.%s.%s", entity, group)
208 + spec, ok := contextCatalog[ctxID]
209 + if !ok {
210 + fallback := "other.gauge"
211 + if typ == sampleCounter {
212 + fallback = "other.counter"
213 + }
214 + spec = contextCatalog[fmt.Sprintf("dcgm.%s.%s", entity, fallback)]
215 + }
216 +
217 + return metricSpec{
218 + Context: spec,
219 + DimName: metricDimensionName(metricName),
220 + Scale: metricScale(metricName, help, spec),
221 + }
222 +}
223 +
224 +func classifyMetricGroup(entity metricEntity, metricName string, typ sampleKind) string {
225 + name := strings.ToUpper(metricName)
226 +
227 + switch {
228 + case strings.HasPrefix(name, "DCGM_FI_INTERNAL_FIELDS_"),
229 + strings.Contains(name, "FIRST_"),
230 + strings.Contains(name, "LAST_"):
231 + return "internal.boundary"
232 + case !strings.Contains(name, "VGPU_") && containsAny(name, "GPU_UTIL", "MEM_COPY_UTIL", "ENC_UTIL", "DEC_UTIL"):
233 + return "compute.utilization"
234 + case containsAny(name, "TENSOR_HMMA", "TENSOR_IMMA", "TENSOR_DFMA"):
235 + return "compute.tensor.activity"
236 + case containsAny(name, "NVDEC", "NVJPG", "NVOFA"):
237 + return "compute.media.activity"
238 + case containsAny(name, "HOSTMEM_CACHE", "PEERMEM_CACHE"):
239 + return "compute.cache.activity"
240 + case containsAny(name,
241 + "SM_ACTIVE",
242 + "SM_OCCUPANCY",
243 + "GR_ENGINE_ACTIVE",
244 + "PIPE_",
245 + "DRAM_ACTIVE",
246 + "TENSOR",
247 + "FP16",
248 + "FP32",
249 + "FP64",
250 + "INTEGER_ACTIVE",
251 + ):
252 + return "compute.activity"
253 + case strings.Contains(name, "FB_USED_PERCENT"):
254 + return "memory.utilization"
255 + case strings.Contains(name, "BAR1_TOTAL"):
256 + return "memory.bar1_capacity"
257 + case strings.Contains(name, "BAR1"):
258 + return "memory.bar1_usage"
259 + case strings.Contains(name, "FB_TOTAL"):
260 + return "memory.capacity"
261 + case containsAny(name, "FB_FREE", "FB_USED", "FB_RESERVED", "FRAME_BUFFER"):
262 + return "memory.usage"
263 + case strings.Contains(name, "ECC_"):
264 + if typ == sampleCounter {
265 + return "memory.ecc_error_rate"
266 + }
267 + return "memory.ecc_errors"
268 + case strings.Contains(name, "RETIRED_"):
269 + return "memory.page_retirements"
270 + case strings.Contains(name, "ROW_REMAP_FAILURE"):
271 + return "reliability.row_remap_status"
272 + case strings.Contains(name, "ROW_REMAP_PENDING"):
273 + return "reliability.row_remap_status"
274 + case strings.Contains(name, "REMAPPED_ROWS"):
275 + return "reliability.row_remap_events"
276 + case containsAny(name, "BANKS_REMAP", "MEMORY_UNREPAIRABLE_FLAG", "THRESHOLD_SRM"):
277 + return "reliability.memory_health"
278 + case strings.Contains(name, "GET_GPU_RECOVERY_ACTION"):
279 + return "reliability.recovery_action"
280 + case containsAny(name, "HEALTH_STATUS", "P2P_STATUS", "CLOCK_EVENTS_COUNT", "IMEX_DOMAIN_STATUS", "IMEX_DAEMON_STATUS", "BIND_UNBIND_EVENT"):
281 + return "health.status"
282 + case strings.Contains(name, "CLOCKS_EVENT_REASONS"):
283 + return "throttle.reasons"
284 + case strings.Contains(name, "CLOCKS_EVENT_REASON"):
285 + return "throttle.violations"
286 + case strings.Contains(name, "CLOCK_THROTTLE_REASONS"):
287 + return "throttle.reasons"
288 + case strings.Contains(name, "PSTATE"):
289 + return "state.performance"
290 + case containsAny(name, "VIRTUAL_MODE", "MIG_MODE"):
291 + return "state.virtualization"
292 + case containsAny(name, "COMPUTE_MODE", "PERSISTENCE_MODE", "AUTOBOOST", "SYNC_BOOST"):
293 + return "state.configuration"
294 + case containsAny(name, "GPU_TEMP", "MEMORY_TEMP", "SLOWDOWN_TEMP", "MAX_OP_TEMP", "SHUTDOWN_TEMP", "TEMPERATURE"):
295 + return "thermal.temperature"
296 + case strings.Contains(name, "FAN_SPEED"):
297 + return "thermal.fan_speed"
298 + case strings.Contains(name, "TOTAL_ENERGY"):
299 + return "power.energy"
300 + case containsAny(name, "POWER_PROFILE_MASK"):
301 + return "power.profiles"
302 + case containsAny(name, "PWR_SMOOTHING"):
303 + return "power.smoothing"
304 + case containsAny(name, "POWER_USAGE", "POWER_MGMT_LIMIT", "POWER_MANAGEMENT_LIMIT", "ENFORCED_POWER_LIMIT"):
305 + return "power.usage"
306 + case strings.Contains(name, "VIOLATION"):
307 + return "throttle.violations"
308 + case strings.Contains(name, "PCIE") && strings.Contains(name, "LINK_GEN"):
309 + return "interconnect.pcie.link.generation"
310 + case strings.Contains(name, "PCIE") && strings.Contains(name, "LINK_WIDTH"):
311 + return "interconnect.pcie.link.width"
312 + case strings.Contains(name, "NVSWITCH"):
313 + return classifyNVSwitchGroup(name, typ)
314 + case strings.Contains(name, "CONNECTX"):
315 + return classifyConnectXGroup(name, typ)
316 + case strings.Contains(name, "C2C_"):
317 + return classifyC2CGroup(name, typ)
318 + case strings.Contains(name, "PCIE") || strings.Contains(name, "NVLINK") || strings.Contains(name, "P2P_") || strings.Contains(name, "FABRIC_"):
319 + return classifyInterconnectGroup(entity, name, typ)
320 + case strings.Contains(name, "XID"):
321 + return "reliability.xid"
322 + case strings.Contains(name, "DIAG_"):
323 + if strings.HasSuffix(name, "_STATUS") {
324 + return "diagnostics.status"
325 + }
326 + return "diagnostics.results"
327 + case strings.Contains(name, "CPU_UTIL"):
328 + return "cpu.utilization"
329 + case strings.Contains(name, "CPU_TEMP"):
330 + return "cpu.temperature"
331 + case containsAny(name, "CPU_POWER", "MODULE_POWER", "SYSIO_POWER"):
332 + return "cpu.power"
333 + case containsAny(name, "CPU_VENDOR", "CPU_MODEL"):
334 + return "cpu.info"
335 + case containsAny(name, "CPU_AFFINITY", "MEM_AFFINITY", "GPU_TOPOLOGY", "PCI_BUSID", "PCI_COMBINED_ID", "PCI_SUBSYS_ID"):
336 + return "topology.affinity"
337 + case strings.Contains(name, "VGPU_"):
338 + return classifyVGPUGroup(name)
339 + case containsAny(name, "ACCOUNTING_DATA", "FBC_", "ENC_STATS"):
340 + return "workload.sessions"
341 + case containsAny(name, "SUPPORTED_", "CREATABLE_", "CUDA_COMPUTE_CAPABILITY", "CC_MODE", "GPM_SUPPORT", "MIG_ATTRIBUTES", "MIG_GI_INFO", "MIG_CI_INFO", "MIG_MAX_SLICES"):
342 + return "capability.support"
343 + case containsAny(name, "DRIVER_VERSION", "NVML_VERSION", "VBIOS_VERSION", "INFOROM", "OEM_INFOROM", "PROCESS_NAME"):
344 + return "inventory.software"
345 + case containsAny(name, "PLATFORM_", "CHASSIS", "HOST_ID", "TRAY_INDEX", "MODULE_ID", "INFINIBAND_GUID"):
346 + return "inventory.platform"
347 + case containsAny(name, "DEV_NAME", "BRAND", "SERIAL", "UUID", "MINOR_NUMBER", "NVML_INDEX", "CUDA_VISIBLE_DEVICES_STR", "DEV_COUNT"):
348 + return "inventory.identity"
349 + case strings.Contains(name, "CLOCK"):
350 + return "clock.frequency"
351 + default:
352 + if typ == sampleCounter {
353 + return "other.counter"
354 + }
355 + return "other.gauge"
356 + }
357 +}
358 +
359 +func classifyInterconnectGroup(entity metricEntity, name string, typ sampleKind) string {
360 + switch {
361 + case containsAny(name, "P2P_STATUS"):
362 + return "health.status"
363 + case containsAny(name, "FABRIC_"):
364 + return "interconnect.fabric"
365 + case strings.Contains(name, "PCIE") && containsAny(name, "BYTES", "THROUGHPUT", "BANDWIDTH"):
366 + return "interconnect.pcie.throughput"
367 + case strings.Contains(name, "NVLINK") && containsAny(name, "BYTES", "THROUGHPUT", "BANDWIDTH"):
368 + if entity == entityNVLink {
369 + return "interconnect.throughput"
370 + }
371 + return "interconnect.nvlink.throughput"
372 + case containsAny(name, "XMIT_WAIT"):
373 + if strings.Contains(name, "NVLINK") {
374 + if entity == entityNVLink {
375 + return "interconnect.congestion"
376 + }
377 + return "interconnect.nvlink.congestion"
378 + }
379 + return "interconnect.congestion"
380 + case containsAny(name, "BER"):
381 + if strings.Contains(name, "PCIE") {
382 + return "interconnect.pcie.ber"
383 + }
384 + if strings.Contains(name, "NVLINK") {
385 + if entity == entityNVLink {
386 + return "interconnect.ber"
387 + }
388 + return "interconnect.nvlink.ber"
389 + }
390 + return "interconnect.ber"
391 + case containsAny(name, "PACKETS", "CODES"):
392 + if strings.Contains(name, "PCIE") {
393 + return "interconnect.pcie.traffic"
394 + }
395 + if strings.Contains(name, "NVLINK") {
396 + if entity == entityNVLink {
397 + return "interconnect.traffic"
398 + }
399 + return "interconnect.nvlink.traffic"
400 + }
401 + return "interconnect.traffic"
402 + case containsAny(name, "BYTES", "THROUGHPUT", "BANDWIDTH"):
403 + return "interconnect.throughput"
404 + case containsAny(name, "ERROR", "CRC", "REPLAY", "RECOVERY", "DISCARD", "FEC", "UNCORRECTABLE", "INTEGRITY"):
405 + if strings.Contains(name, "PCIE") {
406 + if typ == sampleCounter {
407 + return "interconnect.pcie.error_rate"
408 + }
409 + return "interconnect.pcie.errors"
410 + }
411 + if strings.Contains(name, "NVLINK") {
412 + if entity == entityNVLink {
413 + if typ == sampleCounter {
414 + return "interconnect.error_rate"
415 + }
416 + return "interconnect.errors"
417 + }
418 + if typ == sampleCounter {
419 + return "interconnect.nvlink.error_rate"
420 + }
421 + return "interconnect.nvlink.errors"
422 + }
423 + if typ == sampleCounter {
424 + return "interconnect.error_rate"
425 + }
426 + return "interconnect.errors"
427 + case strings.Contains(name, "PCIE") && strings.Contains(name, "RESULT"):
428 + return "interconnect.pcie.state"
429 + case containsAny(name, "STATE", "STATUS", "POWER_STATE", "LINK_COUNT"):
430 + if strings.Contains(name, "PCIE") {
431 + return "interconnect.pcie.state"
432 + }
433 + if strings.Contains(name, "NVLINK") {
434 + if entity == entityNVLink {
435 + return "interconnect.state"
436 + }
437 + return "interconnect.nvlink.state"
438 + }
439 + return "interconnect.state"
440 + default:
441 + return "interconnect.state"
442 + }
443 +}
444 +
445 +func classifyNVSwitchGroup(name string, typ sampleKind) string {
446 + switch {
447 + case containsAny(name, "LINK_LATENCY_"):
448 + return "interconnect.nvswitch.latency"
449 + case containsAny(name, "THROUGHPUT"):
450 + return "interconnect.nvswitch.throughput"
451 + case containsAny(name, "ERROR", "FATAL", "NON_FATAL", "CRC", "REPLAY", "RECOVERY", "FLIT"):
452 + if typ == sampleCounter {
453 + return "interconnect.nvswitch.errors"
454 + }
455 + return "interconnect.nvswitch.errors"
456 + case containsAny(name, "TEMPERATURE"):
457 + return "interconnect.nvswitch.temperature"
458 + case containsAny(name, "POWER_"):
459 + return "interconnect.nvswitch.power"
460 + case containsAny(name, "CURRENT_"):
461 + return "interconnect.nvswitch.current"
462 + case containsAny(name, "VOLTAGE"):
463 + return "interconnect.nvswitch.voltage"
464 + case containsAny(name, "PCIE_", "PHYS_ID", "DEVICE_UUID", "LINK_ID", "LINK_SID", "DEVICE_LINK_ID", "REMOTE_PCIE_"):
465 + return "interconnect.nvswitch.topology"
466 + default:
467 + return "interconnect.nvswitch.status"
468 + }
469 +}
470 +
471 +func classifyConnectXGroup(name string, typ sampleKind) string {
472 + switch {
473 + case containsAny(name, "TEMPERATURE"):
474 + return "interconnect.connectx.temperature"
475 + case containsAny(name, "ERR", "ERROR"):
476 + if typ == sampleCounter {
477 + return "interconnect.connectx.errors"
478 + }
479 + return "interconnect.connectx.error_status"
480 + case containsAny(name, "LINK_SPEED", "LINK_WIDTH", "PCIE"):
481 + return "interconnect.connectx.link"
482 + default:
483 + return "interconnect.connectx.status"
484 + }
485 +}
486 +
487 +func classifyC2CGroup(name string, typ sampleKind) string {
488 + switch {
489 + case containsAny(name, "RX_", "TX_", "BANDWIDTH"):
490 + return "interconnect.throughput"
491 + case containsAny(name, "ERROR", "REPLAY", "FEC", "BER", "INTR", "DISCARD", "RECOVERY"):
492 + if typ == sampleCounter {
493 + return "interconnect.error_rate"
494 + }
495 + return "interconnect.errors"
496 + default:
497 + return "interconnect.state"
498 + }
499 +}
500 +
501 +func classifyVGPUGroup(name string) string {
502 + switch {
503 + case containsAny(name, "VGPU_LICENSE_STATUS", "TYPE_LICENSE", "INSTANCE_LICENSE_STATE"):
504 + return "virtualization.vgpu.license"
505 + case containsAny(name, "VGPU_TYPE", "TYPE_CLASS", "TYPE_INFO", "TYPE_NAME"):
506 + return "virtualization.vgpu.type"
507 + case containsAny(name, "VGPU_INSTANCE_IDS", "VGPU_UUID", "VGPU_PCI_ID"):
508 + return "virtualization.vgpu.instance"
509 + case containsAny(name, "VGPU_VM_"):
510 + return "virtualization.vgpu.vm"
511 + case containsAny(name, "VGPU_MEMORY_USAGE"):
512 + return "virtualization.vgpu.memory"
513 + case containsAny(name, "VGPU_FRAME_RATE_LIMIT"):
514 + return "virtualization.vgpu.frame_rate"
515 + case containsAny(name, "VGPU_UTILIZATIONS", "PER_PROCESS_UTILIZATION"):
516 + return "virtualization.vgpu.utilization"
517 + case containsAny(name, "VGPU_ENC_", "VGPU_FBC_"):
518 + return "virtualization.vgpu.sessions"
519 + case containsAny(name, "VGPU_DRIVER_VERSION"):
520 + return "virtualization.vgpu.software"
521 + default:
522 + return "virtualization.vgpu.instance"
523 + }
524 +}
525 +
526 +func metricScale(metricName, help string, spec contextSpec) float64 {
527 + name := strings.ToUpper(metricName)
528 + h := strings.ToLower(help)
529 +
530 + if spec.ID == "" {
531 + return 1
532 + }
533 +
534 + if strings.HasSuffix(spec.ID, ".compute.activity") ||
535 + strings.HasSuffix(spec.ID, ".compute.tensor.activity") ||
536 + strings.HasSuffix(spec.ID, ".compute.media.activity") {
537 + return 100
538 + }
539 +
540 + if strings.HasSuffix(spec.ID, ".memory.utilization") {
541 + return 100
542 + }
543 +
544 + if (strings.HasSuffix(spec.ID, ".memory.usage") ||
545 + strings.HasSuffix(spec.ID, ".memory.capacity") ||
546 + strings.HasSuffix(spec.ID, ".memory.bar1_usage") ||
547 + strings.HasSuffix(spec.ID, ".memory.bar1_capacity")) &&
548 + (containsAny(name, "FB_", "BAR1") || strings.Contains(h, "mib") || strings.Contains(h, " mb")) {
549 + return 1024 * 1024
550 + }
551 +
552 + if strings.HasSuffix(spec.ID, ".virtualization.vgpu.memory") &&
553 + (strings.Contains(h, "mib") || strings.Contains(h, " mb")) {
554 + return 1024 * 1024
555 + }
556 +
557 + if strings.HasSuffix(spec.ID, ".virtualization.vgpu.utilization") && strings.Contains(h, "ratio") {
558 + return 100
559 + }
560 +
561 + if strings.HasSuffix(spec.ID, ".throttle.violations") && strings.Contains(name, "VIOLATION") {
562 + return 1.0 / 1e6 // ns => ms
563 + }
564 + if strings.HasSuffix(spec.ID, ".throttle.violations") && strings.Contains(name, "CLOCKS_EVENT_REASON") {
565 + return 1.0 / 1e6 // ns => ms
566 + }
567 +
568 + return 1
569 +}
570 +
571 +func metricDimensionName(metricName string) string {
572 + name := strings.ToUpper(metricName)
573 + if v, ok := metricDimensionAliases[name]; ok {
574 + return v
575 + }
576 +
577 + for _, pfx := range []string{
578 + "DCGM_FI_DEV_",
579 + "DCGM_FI_PROF_",
580 + "DCGM_FI_",
581 + "DCGM_EXP_",
582 + "DCGM_",
583 + } {
584 + name = strings.TrimPrefix(name, pfx)
585 + }
586 +
587 + name = strings.TrimSuffix(name, "_TOTAL")
588 + name = strings.TrimSuffix(name, "_COUNT")
589 + name = strings.TrimSuffix(name, "_VALUE")
590 + name = strings.ToLower(name)
591 +
592 + return sanitizeID(name)
593 +}
594 +
595 +func containsAny(s string, values ...string) bool {
596 + for _, v := range values {
597 + if strings.Contains(s, v) {
598 + return true
599 + }
600 + }
601 + return false
602 +}
603 +
604 +var metricDimensionAliases = map[string]string{
605 + "DCGM_FI_DEV_GPU_UTIL": "gpu",
606 + "DCGM_FI_DEV_MEM_COPY_UTIL": "memory_copy",
607 + "DCGM_FI_DEV_ENC_UTIL": "encoder",
608 + "DCGM_FI_DEV_DEC_UTIL": "decoder",
609 + "DCGM_FI_DEV_GPU_TEMP": "gpu",
610 + "DCGM_FI_DEV_MEMORY_TEMP": "memory",
611 + "DCGM_FI_DEV_SM_CLOCK": "sm",
612 + "DCGM_FI_DEV_MEM_CLOCK": "memory",
613 + "DCGM_FI_DEV_POWER_USAGE": "draw",
614 + "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION": "total",
615 + "DCGM_FI_DEV_PCIE_REPLAY_COUNTER": "pcie_replay",
616 + "DCGM_FI_DEV_XID_ERRORS": "xid",
617 + "DCGM_FI_DEV_POWER_VIOLATION": "power_violation",
618 + "DCGM_FI_DEV_THERMAL_VIOLATION": "thermal_violation",
619 + "DCGM_FI_DEV_SYNC_BOOST_VIOLATION": "sync_boost_violation",
620 + "DCGM_FI_DEV_BOARD_LIMIT_VIOLATION": "board_limit_violation",
621 + "DCGM_FI_DEV_LOW_UTIL_VIOLATION": "low_utilization_violation",
622 + "DCGM_FI_DEV_RELIABILITY_VIOLATION": "reliability_violation",
623 + "DCGM_FI_DEV_FB_FREE": "free",
624 + "DCGM_FI_DEV_FB_TOTAL": "total",
625 + "DCGM_FI_DEV_FB_USED": "used",
626 + "DCGM_FI_DEV_FB_RESERVED": "reserved",
627 + "DCGM_FI_DEV_FB_USED_PERCENT": "used_percent",
628 + "DCGM_FI_DEV_BAR1_TOTAL": "total",
629 + "DCGM_FI_DEV_BAR1_USED": "used",
630 + "DCGM_FI_DEV_BAR1_FREE": "free",
631 + "DCGM_FI_DEV_FAN_SPEED": "fan_speed",
632 + "DCGM_FI_DEV_ENFORCED_POWER_LIMIT": "enforced_limit",
633 + "DCGM_FI_DEV_PCIE_LINK_GEN": "link_gen",
634 + "DCGM_FI_DEV_PCIE_MAX_LINK_GEN": "max_link_gen",
635 + "DCGM_FI_DEV_PCIE_LINK_WIDTH": "link_width",
636 + "DCGM_FI_DEV_PCIE_MAX_LINK_WIDTH": "max_link_width",
637 + "DCGM_FI_DEV_CLOCK_THROTTLE_REASONS": "reasons",
638 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS": "sw_power_cap",
639 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_THERM_SLOWDOWN_NS": "hw_therm_slowdown",
640 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_THERM_SLOWDOWN_NS": "sw_therm_slowdown",
641 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_POWER_BRAKE_SLOWDOWN_NS": "hw_power_brake_slowdown",
642 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST_NS": "sync_boost",
643 + "DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS": "uncorrectable_remapped_rows",
644 + "DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS": "correctable_remapped_rows",
645 + "DCGM_FI_DEV_ROW_REMAP_FAILURE": "row_remap_failure",
646 + "DCGM_FI_DEV_ROW_REMAP_PENDING": "row_remap_pending",
647 + "DCGM_FI_PROF_SM_ACTIVE": "sm_active",
648 + "DCGM_FI_PROF_SM_OCCUPANCY": "sm_occupancy",
649 + "DCGM_FI_PROF_GR_ENGINE_ACTIVE": "graphics_engine_active",
650 + "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE": "tensor",
651 + "DCGM_FI_PROF_DRAM_ACTIVE": "dram",
652 + "DCGM_FI_PROF_PIPE_FP64_ACTIVE": "fp64",
653 + "DCGM_FI_PROF_PIPE_FP32_ACTIVE": "fp32",
654 + "DCGM_FI_PROF_PIPE_FP16_ACTIVE": "fp16",
655 + "DCGM_FI_PROF_PCIE_TX_BYTES": "pcie_tx",
656 + "DCGM_FI_PROF_PCIE_RX_BYTES": "pcie_rx",
657 + "DCGM_FI_PROF_PIPE_INT_ACTIVE": "integer",
658 + "DCGM_FI_PROF_PIPE_TENSOR_DFMA_ACTIVE": "tensor_dfma",
659 + "DCGM_FI_PROF_PIPE_TENSOR_HMMA_ACTIVE": "tensor_hmma",
660 + "DCGM_FI_PROF_PIPE_TENSOR_IMMA_ACTIVE": "tensor_imma",
661 +}
src/go/plugin/go.d/collector/dcgm/collect.go new
+570
@@ -0,0 +1,570 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dcgm
4 +
5 +import (
6 + "fmt"
7 + "hash/fnv"
8 + "math"
9 + "sort"
10 + "strconv"
11 + "strings"
12 +
13 + "github.com/prometheus/common/model"
14 + promlabels "github.com/prometheus/prometheus/model/labels"
15 +
16 + "github.com/netdata/netdata/go/plugins/pkg/prometheus"
17 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
18 +)
19 +
20 +const precision = 1000.0
21 +
22 +type interconnectThroughputTotals struct {
23 + instance entityInstance
24 + pcie int64
25 + nvlink int64
26 + hasPcie bool
27 + hasNvlink bool
28 + hasExplicitNvlinkTt bool
29 +}
30 +
31 +func (c *Collector) collect() (map[string]int64, error) {
32 + mfs, err := c.prom.Scrape()
33 + if err != nil {
34 + return nil, err
35 + }
36 +
37 + if mfs.Len() == 0 {
38 + c.Warningf("endpoint '%s' returned 0 metric families", c.URL)
39 + return nil, nil
40 + }
41 +
42 + if c.checkMetrics && !hasDCGMMetricFamilies(mfs) {
43 + return nil, fmt.Errorf("'%s' metrics have no DCGM prefix", c.URL)
44 + }
45 + c.checkMetrics = false
46 +
47 + if c.MaxTS > 0 {
48 + if n := calcDCGMMetricSeries(mfs); n > c.MaxTS {
49 + return nil, fmt.Errorf("'%s' num of time series (%d) > limit (%d)", c.URL, n, c.MaxTS)
50 + }
51 + }
52 +
53 + mx := make(map[string]int64)
54 + totals := make(map[string]*interconnectThroughputTotals)
55 + c.cache.reset()
56 +
57 + for _, mf := range mfs {
58 + if !isDCGMMetricName(mf.Name()) {
59 + continue
60 + }
61 +
62 + if c.MaxTSPerMetric > 0 && len(mf.Metrics()) > c.MaxTSPerMetric {
63 + c.Debugf(
64 + "metric '%s' num of time series (%d) > limit (%d), skipping it",
65 + mf.Name(),
66 + len(mf.Metrics()),
67 + c.MaxTSPerMetric,
68 + )
69 + continue
70 + }
71 +
72 + typ := metricFamilyKind(mf)
73 + if typ == sampleUnsupported {
74 + c.Debugf("metric '%s' has unsupported Prometheus type '%s', skipping it", mf.Name(), mf.Type())
75 + continue
76 + }
77 + for _, metric := range mf.Metrics() {
78 + value, ok := metricValue(metric, typ)
79 + if !ok || isInvalidMetricValue(value) {
80 + continue
81 + }
82 +
83 + instance := resolveEntityInstance(metric.Labels())
84 + spec := classifyMetric(instance.entity, mf.Name(), mf.Help(), typ)
85 + skipPrimary := shouldSkipPrimarySeries(spec.Context.ID, mf.Name())
86 + scaled := int64(value * spec.Scale * precision)
87 + c.accumulateInterconnectTotals(totals, instance, spec.Context.ID, mf.Name(), scaled)
88 + if skipPrimary {
89 + continue
90 + }
91 +
92 + chartKey, chart := c.ensureChart(instance, spec.Context)
93 + dimID := c.ensureDim(chartKey, chart, spec, metric.Labels(), typ)
94 +
95 + mx[dimID] += scaled
96 + }
97 + }
98 +
99 + c.emitInterconnectTotals(mx, totals)
100 + c.removeStaleChartsAndDims()
101 +
102 + if len(mx) == 0 {
103 + return nil, nil
104 + }
105 +
106 + return mx, nil
107 +}
108 +
109 +func (c *Collector) accumulateInterconnectTotals(
110 + totals map[string]*interconnectThroughputTotals,
111 + instance entityInstance,
112 + contextID, metricName string,
113 + scaled int64,
114 +) {
115 + isPCIe := strings.HasSuffix(contextID, ".interconnect.pcie.throughput")
116 + isNVLink := strings.HasSuffix(contextID, ".interconnect.nvlink.throughput")
117 + if !isPCIe && !isNVLink {
118 + return
119 + }
120 +
121 + key := string(instance.entity) + "|" + instance.key
122 + tot, ok := totals[key]
123 + if !ok {
124 + tot = &interconnectThroughputTotals{instance: instance}
125 + totals[key] = tot
126 + }
127 +
128 + if isPCIe {
129 + tot.pcie += scaled
130 + tot.hasPcie = true
131 + return
132 + }
133 +
134 + if isNVLinkTotalMetricName(metricName) {
135 + if !tot.hasExplicitNvlinkTt {
136 + tot.nvlink = 0
137 + tot.hasExplicitNvlinkTt = true
138 + }
139 + tot.nvlink += scaled
140 + tot.hasNvlink = true
141 + return
142 + }
143 +
144 + if tot.hasExplicitNvlinkTt {
145 + return
146 + }
147 + tot.nvlink += scaled
148 + tot.hasNvlink = true
149 +}
150 +
151 +func (c *Collector) emitInterconnectTotals(mx map[string]int64, totals map[string]*interconnectThroughputTotals) {
152 + for _, tot := range totals {
153 + if !tot.hasPcie && !tot.hasNvlink {
154 + continue
155 + }
156 +
157 + ctxID := fmt.Sprintf("dcgm.%s.interconnect.total.throughput", tot.instance.entity)
158 + spec, ok := contextCatalog[ctxID]
159 + if !ok {
160 + continue
161 + }
162 +
163 + chartKey, chart := c.ensureChart(tot.instance, spec)
164 + if tot.hasPcie {
165 + dimID := c.ensureDim(chartKey, chart, metricSpec{Context: spec, DimName: "pcie", Scale: 1}, nil, sampleGauge)
166 + mx[dimID] += tot.pcie
167 + }
168 + if tot.hasNvlink {
169 + dimID := c.ensureDim(chartKey, chart, metricSpec{Context: spec, DimName: "nvlink", Scale: 1}, nil, sampleGauge)
170 + mx[dimID] += tot.nvlink
171 + }
172 + }
173 +}
174 +
175 +func (c *Collector) ensureChart(instance entityInstance, spec contextSpec) (string, *module.Chart) {
176 + chartKey := spec.ID + "|" + instance.key
177 + if ch, ok := c.cache.getChart(chartKey); ok {
178 + return chartKey, ch.chart
179 + }
180 +
181 + chart := &module.Chart{
182 + ID: makeID(spec.ID, instance.key),
183 + Title: spec.Title,
184 + Units: spec.Units,
185 + Fam: spec.Family,
186 + Ctx: spec.ID,
187 + Type: spec.Type,
188 + Priority: spec.Priority,
189 + Labels: append([]module.Label(nil), instance.chartLabels...),
190 + }
191 +
192 + if err := c.Charts().Add(chart); err != nil {
193 + c.Warning(err)
194 + }
195 +
196 + ch := c.cache.putChart(chartKey, chart)
197 + return chartKey, ch.chart
198 +}
199 +
200 +func (c *Collector) ensureDim(
201 + chartKey string,
202 + chart *module.Chart,
203 + spec metricSpec,
204 + lbls promlabels.Labels,
205 + typ sampleKind,
206 +) string {
207 + extra := ""
208 + if !strings.HasSuffix(spec.Context.ID, ".reliability.xid") {
209 + extra = semanticDimSuffix(lbls)
210 + }
211 + dimName := spec.DimName
212 + dimName = normalizeDimName(spec.Context.ID, dimName)
213 + if extra != "" {
214 + dimName = dimName + "_" + extra
215 + }
216 +
217 + dimID := makeID(chart.ID, dimName)
218 +
219 + ch, ok := c.cache.charts[chartKey]
220 + if !ok {
221 + return dimID
222 + }
223 +
224 + if exists := ch.touchDim(dimID); !exists {
225 + dim := &module.Dim{ID: dimID, Name: dimName, Div: int(precision)}
226 + switch typ {
227 + case sampleCounter:
228 + dim.Algo = module.Incremental
229 + default:
230 + dim.Algo = module.Absolute
231 + }
232 + if shouldHideDimensionByDefault(spec.Context.ID, dimName) {
233 + dim.Hidden = true
234 + }
235 +
236 + if err := chart.AddDim(dim); err != nil {
237 + c.Warning(err)
238 + } else {
239 + chart.MarkNotCreated()
240 + }
241 + }
242 +
243 + return dimID
244 +}
245 +
246 +func shouldSkipPrimarySeries(contextID, metricName string) bool {
247 + // Keep NVLink total-only bandwidth in the interconnect overview context.
248 + return strings.HasSuffix(contextID, ".interconnect.nvlink.throughput") &&
249 + isNVLinkTotalMetricName(metricName)
250 +}
251 +
252 +func normalizeDimName(contextID, dimName string) string {
253 + if isNVLinkThroughputContext(contextID) && strings.HasSuffix(dimName, "_bytes") {
254 + return strings.TrimSuffix(dimName, "_bytes")
255 + }
256 + return dimName
257 +}
258 +
259 +func isNVLinkThroughputContext(contextID string) bool {
260 + return strings.HasSuffix(contextID, ".interconnect.nvlink.throughput") ||
261 + strings.HasPrefix(contextID, "dcgm.nvlink.") && strings.HasSuffix(contextID, ".interconnect.throughput")
262 +}
263 +
264 +func metricFamilyKind(mf *prometheus.MetricFamily) sampleKind {
265 + switch mf.Type() {
266 + case model.MetricTypeCounter:
267 + return sampleCounter
268 + case model.MetricTypeGauge:
269 + return sampleGauge
270 + case model.MetricTypeHistogram, model.MetricTypeSummary:
271 + return sampleUnsupported
272 + default:
273 + if strings.HasSuffix(strings.ToLower(mf.Name()), "_total") {
274 + return sampleCounter
275 + }
276 + return sampleGauge
277 + }
278 +}
279 +
280 +func metricValue(metric prometheus.Metric, typ sampleKind) (float64, bool) {
281 + if typ == sampleCounter {
282 + if c := metric.Counter(); c != nil {
283 + return c.Value(), true
284 + }
285 + if u := metric.Untyped(); u != nil {
286 + return u.Value(), true
287 + }
288 + if g := metric.Gauge(); g != nil {
289 + return g.Value(), true
290 + }
291 + return 0, false
292 + }
293 +
294 + if g := metric.Gauge(); g != nil {
295 + return g.Value(), true
296 + }
297 + if u := metric.Untyped(); u != nil {
298 + return u.Value(), true
299 + }
300 + if c := metric.Counter(); c != nil {
301 + return c.Value(), true
302 + }
303 +
304 + return 0, false
305 +}
306 +
307 +func hasDCGMMetricFamilies(mfs prometheus.MetricFamilies) bool {
308 + for name := range mfs {
309 + if isDCGMMetricName(name) {
310 + return true
311 + }
312 + }
313 + return false
314 +}
315 +
316 +func isDCGMMetricName(name string) bool {
317 + return strings.HasPrefix(name, "DCGM_") || strings.HasPrefix(strings.ToLower(name), "dcgm_")
318 +}
319 +
320 +func calcDCGMMetricSeries(mfs prometheus.MetricFamilies) int {
321 + var total int
322 + for name, mf := range mfs {
323 + if !isDCGMMetricName(name) {
324 + continue
325 + }
326 + total += len(mf.Metrics())
327 + }
328 + return total
329 +}
330 +
331 +func isInvalidMetricValue(v float64) bool {
332 + if math.IsNaN(v) || math.IsInf(v, 0) {
333 + return true
334 + }
335 + // DCGM often uses large sentinel values for unsupported fields.
336 + if math.Abs(v) >= 9e18 {
337 + return true
338 + }
339 + return false
340 +}
341 +
342 +type entityInstance struct {
343 + entity metricEntity
344 + key string
345 + chartLabels []module.Label
346 +}
347 +
348 +func resolveEntityInstance(lbls promlabels.Labels) entityInstance {
349 + idx := make(map[string]string, len(lbls))
350 + for _, lbl := range lbls {
351 + if lbl.Name == "" || lbl.Value == "" {
352 + continue
353 + }
354 + idx[strings.ToLower(lbl.Name)] = lbl.Value
355 + }
356 +
357 + entity := detectEntity(idx)
358 + identityKeys := identityKeysForEntity(entity)
359 + parts := make([]string, 0, len(identityKeys))
360 +
361 + for _, key := range identityKeys {
362 + v, ok := idx[key]
363 + if !ok || v == "" {
364 + continue
365 + }
366 + parts = append(parts, key+"="+v)
367 + }
368 +
369 + if len(parts) == 0 {
370 + parts = append(parts, "global")
371 + }
372 +
373 + chartLabels := buildChartLabels(idx)
374 +
375 + return entityInstance{
376 + entity: entity,
377 + key: strings.Join(parts, "|"),
378 + chartLabels: chartLabels,
379 + }
380 +}
381 +
382 +func detectEntity(idx map[string]string) metricEntity {
383 + switch {
384 + case hasLabel(idx, "gpu_i_id") || hasLabel(idx, "gpu_instance_id"):
385 + return entityMIG
386 + case hasLabel(idx, "nvlink"):
387 + return entityNVLink
388 + case hasLabel(idx, "nvswitch"):
389 + return entityNVSwitch
390 + case hasLabel(idx, "cpucore"):
391 + return entityCPUCore
392 + case hasLabel(idx, "cpu"):
393 + return entityCPU
394 + case hasLabel(idx, "gpu") || hasLabel(idx, "uuid") || hasLabel(idx, "gpu_uuid"):
395 + return entityGPU
396 + default:
397 + return entityExporter
398 + }
399 +}
400 +
401 +func hasLabel(idx map[string]string, key string) bool {
402 + v, ok := idx[key]
403 + return ok && v != ""
404 +}
405 +
406 +func identityKeysForEntity(entity metricEntity) []string {
407 + workload := []string{"namespace", "pod", "container", "job", "hpc_job", "hpc_job_id"}
408 + switch entity {
409 + case entityGPU:
410 + return append([]string{"gpu", "uuid", "gpu_uuid"}, workload...)
411 + case entityMIG:
412 + return append([]string{"gpu", "uuid", "gpu_uuid", "gpu_i_id", "gpu_instance_id", "gpu_i_profile", "gpu_instance_profile"}, workload...)
413 + case entityNVLink:
414 + return append([]string{"nvswitch", "gpu", "gpu_uuid", "nvlink"}, workload...)
415 + case entityNVSwitch:
416 + return append([]string{"nvswitch"}, workload...)
417 + case entityCPU:
418 + return append([]string{"cpu"}, workload...)
419 + case entityCPUCore:
420 + return append([]string{"cpu", "cpucore"}, workload...)
421 + default:
422 + return append([]string{"hostname"}, workload...)
423 + }
424 +}
425 +
426 +func semanticDimSuffix(lbls promlabels.Labels) string {
427 + if len(lbls) == 0 {
428 + return ""
429 + }
430 +
431 + // Only keep dynamic, semantically meaningful labels that define distinct series
432 + // for a single DCGM field; avoid static identity/metadata labels in dim names.
433 + allowed := map[string]bool{
434 + "err_code": true,
435 + }
436 +
437 + tokens := make([]string, 0, len(lbls))
438 + for _, lbl := range lbls {
439 + k := strings.ToLower(lbl.Name)
440 + if lbl.Value == "" || !allowed[k] {
441 + continue
442 + }
443 + tokens = append(tokens, normalizeLabelKey(k)+"_"+sanitizeID(strings.ToLower(lbl.Value)))
444 + }
445 +
446 + if len(tokens) == 0 {
447 + return ""
448 + }
449 +
450 + sort.Strings(tokens)
451 + return strings.Join(tokens, "__")
452 +}
453 +
454 +func normalizeLabelKey(s string) string {
455 + s = strings.ToLower(s)
456 + return sanitizeID(s)
457 +}
458 +
459 +func buildChartLabels(idx map[string]string) []module.Label {
460 + ignore := map[string]bool{
461 + "hostname": true, // host identity is already part of Netdata host model
462 + "err_code": true, // used for dynamic XID dimension split, not chart label
463 + "le": true, // histogram bucket label
464 + "quantile": true, // summary label
465 + "__name__": true, // metric family name label, if present
466 + }
467 +
468 + keys := make([]string, 0, len(idx))
469 + for key, value := range idx {
470 + if value == "" || ignore[key] {
471 + continue
472 + }
473 + keys = append(keys, key)
474 + }
475 +
476 + sort.Strings(keys)
477 + labels := make([]module.Label, 0, len(keys))
478 + for _, key := range keys {
479 + labels = append(labels, module.Label{Key: normalizeLabelKey(key), Value: idx[key]})
480 + }
481 + return labels
482 +}
483 +
484 +func shouldHideDimensionByDefault(contextID, dimName string) bool {
485 + switch {
486 + case strings.HasSuffix(contextID, ".clock.frequency"):
487 + return containsAny(dimName,
488 + "app_mem_clock",
489 + "app_sm_clock",
490 + "max_mem_clock",
491 + "max_sm_clock",
492 + "max_video_clock",
493 + )
494 + case strings.HasSuffix(contextID, ".thermal.temperature"):
495 + return containsAny(dimName,
496 + "gpu_max_op_temp",
497 + "gpu_temp_limit",
498 + "mem_max_op_temp",
499 + "shutdown_temp",
500 + "slowdown_temp",
501 + )
502 + case strings.HasSuffix(contextID, ".power.usage"):
503 + return containsAny(dimName,
504 + "enforced_limit",
505 + "power_mgmt_limit",
506 + "power_mgmt_limit_def",
507 + "power_mgmt_limit_max",
508 + "power_mgmt_limit_min",
509 + )
510 + case strings.HasSuffix(contextID, ".interconnect.pcie.link.generation"):
511 + return containsAny(dimName, "max_link_gen")
512 + case strings.HasSuffix(contextID, ".interconnect.pcie.link.width"):
513 + return containsAny(dimName, "max_link_width")
514 + default:
515 + return false
516 + }
517 +}
518 +
519 +func isNVLinkTotalMetricName(name string) bool {
520 + n := strings.ToUpper(name)
521 + return strings.Contains(n, "NVLINK") && containsAny(n, "BANDWIDTH_TOTAL", "RX_BANDWIDTH_TOTAL", "TX_BANDWIDTH_TOTAL")
522 +}
523 +
524 +func makeID(parts ...string) string {
525 + raw := strings.Join(parts, "_")
526 + id := sanitizeID(raw)
527 + if len(id) <= 180 {
528 + return id
529 + }
530 +
531 + h := fnv.New64a()
532 + _, _ = h.Write([]byte(id))
533 + checksum := strconv.FormatUint(h.Sum64(), 36)
534 + return id[:140] + "_" + checksum
535 +}
536 +
537 +func sanitizeID(s string) string {
538 + if s == "" {
539 + return "unknown"
540 + }
541 +
542 + var b strings.Builder
543 + b.Grow(len(s))
544 + lastUnderscore := false
545 +
546 + for _, r := range s {
547 + isAlphaNum := (r >= 'a' && r <= 'z') ||
548 + (r >= 'A' && r <= 'Z') ||
549 + (r >= '0' && r <= '9')
550 + if isAlphaNum {
551 + b.WriteRune(r)
552 + lastUnderscore = false
553 + continue
554 + }
555 +
556 + if !lastUnderscore {
557 + b.WriteByte('_')
558 + lastUnderscore = true
559 + }
560 + }
561 +
562 + id := strings.Trim(b.String(), "_")
563 + if id == "" {
564 + id = "unknown"
565 + }
566 + if id[0] >= '0' && id[0] <= '9' {
567 + id = "n_" + id
568 + }
569 + return id
570 +}
src/go/plugin/go.d/collector/dcgm/collector.go new
+123
@@ -0,0 +1,123 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dcgm
4 +
5 +import (
6 + "context"
7 + _ "embed"
8 + "errors"
9 + "fmt"
10 + "time"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/confopt"
13 + "github.com/netdata/netdata/go/plugins/pkg/prometheus"
14 + "github.com/netdata/netdata/go/plugins/pkg/web"
15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
16 +)
17 +
18 +//go:embed "config_schema.json"
19 +var configSchema string
20 +
21 +func init() {
22 + module.Register("dcgm", module.Creator{
23 + JobConfigSchema: configSchema,
24 + Defaults: module.Defaults{
25 + UpdateEvery: 30,
26 + },
27 + Create: func() module.Module { return New() },
28 + Config: func() any { return &Config{} },
29 + })
30 +}
31 +
32 +func New() *Collector {
33 + return &Collector{
34 + Config: Config{
35 + HTTPConfig: web.HTTPConfig{
36 + RequestConfig: web.RequestConfig{
37 + URL: "http://127.0.0.1:9400/metrics",
38 + },
39 + ClientConfig: web.ClientConfig{
40 + Timeout: confopt.Duration(time.Second * 10),
41 + },
42 + },
43 + MaxTS: 2000,
44 + MaxTSPerMetric: 200,
45 + },
46 + charts: &module.Charts{},
47 + cache: newCache(),
48 + checkMetrics: true,
49 + }
50 +}
51 +
52 +type Config struct {
53 + Vnode string `yaml:"vnode,omitempty" json:"vnode"`
54 + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
55 + AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
56 + web.HTTPConfig `yaml:",inline" json:""`
57 + MaxTS int `yaml:"max_time_series" json:"max_time_series"`
58 + MaxTSPerMetric int `yaml:"max_time_series_per_metric" json:"max_time_series_per_metric"`
59 +}
60 +
61 +type Collector struct {
62 + module.Base
63 + Config `yaml:",inline" json:""`
64 +
65 + charts *module.Charts
66 + prom prometheus.Prometheus
67 +
68 + cache *cache
69 + checkMetrics bool
70 +}
71 +
72 +func (c *Collector) Configuration() any {
73 + return c.Config
74 +}
75 +
76 +func (c *Collector) Init(context.Context) error {
77 + if err := c.validateConfig(); err != nil {
78 + return fmt.Errorf("config validation: %v", err)
79 + }
80 +
81 + prom, err := c.initPrometheusClient()
82 + if err != nil {
83 + return fmt.Errorf("init prometheus client: %v", err)
84 + }
85 + c.prom = prom
86 +
87 + return nil
88 +}
89 +
90 +func (c *Collector) Check(context.Context) error {
91 + mx, err := c.collect()
92 + if err != nil {
93 + return err
94 + }
95 + if len(mx) == 0 {
96 + return errors.New("no metrics collected")
97 + }
98 + return nil
99 +}
100 +
101 +func (c *Collector) Charts() *module.Charts {
102 + return c.charts
103 +}
104 +
105 +func (c *Collector) Collect(context.Context) map[string]int64 {
106 + mx, err := c.collect()
107 + if err != nil {
108 + c.Error(err)
109 + return nil
110 + }
111 +
112 + if len(mx) == 0 {
113 + return nil
114 + }
115 +
116 + return mx
117 +}
118 +
119 +func (c *Collector) Cleanup(context.Context) {
120 + if c.prom != nil && c.prom.HTTPClient() != nil {
121 + c.prom.HTTPClient().CloseIdleConnections()
122 + }
123 +}
src/go/plugin/go.d/collector/dcgm/collector_test.go new
+749
@@ -0,0 +1,749 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dcgm
4 +
5 +import (
6 + "context"
7 + "net/http"
8 + "net/http/httptest"
9 + "os"
10 + "strings"
11 + "testing"
12 +
13 + "github.com/stretchr/testify/assert"
14 + "github.com/stretchr/testify/require"
15 +
16 + "github.com/netdata/netdata/go/plugins/pkg/web"
17 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
18 +)
19 +
20 +var (
21 + dataConfigJSON, _ = os.ReadFile("testdata/config.json")
22 + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")
23 +
24 + dataMetricsValid, _ = os.ReadFile("testdata/metrics_valid.prom")
25 + dataMetricsNonDCGM, _ = os.ReadFile("testdata/metrics_non_dcgm.prom")
26 + dataAllFieldsList, _ = os.ReadFile("testdata/all_fields_nonlabel.txt")
27 +)
28 +
29 +func Test_testDataIsValid(t *testing.T) {
30 + for name, data := range map[string][]byte{
31 + "dataConfigJSON": dataConfigJSON,
32 + "dataConfigYAML": dataConfigYAML,
33 + "dataMetricsValid": dataMetricsValid,
34 + "dataMetricsNonDCGM": dataMetricsNonDCGM,
35 + "dataAllFieldsList": dataAllFieldsList,
36 + } {
37 + require.NotNil(t, data, name)
38 + }
39 +}
40 +
41 +func TestCollector_ConfigurationSerialize(t *testing.T) {
42 + module.TestConfigurationSerialize(t, &Collector{}, dataConfigJSON, dataConfigYAML)
43 +}
44 +
45 +func TestCollector_Init(t *testing.T) {
46 + tests := map[string]struct {
47 + config Config
48 + wantFail bool
49 + }{
50 + "valid URL": {
51 + config: Config{
52 + HTTPConfig: web.HTTPConfig{
53 + RequestConfig: web.RequestConfig{URL: "http://127.0.0.1:9400/metrics"},
54 + },
55 + },
56 + wantFail: false,
57 + },
58 + "empty URL": {
59 + config: Config{},
60 + wantFail: true,
61 + },
62 + }
63 +
64 + for name, test := range tests {
65 + t.Run(name, func(t *testing.T) {
66 + collr := New()
67 + collr.Config = test.config
68 +
69 + if test.wantFail {
70 + assert.Error(t, collr.Init(context.Background()))
71 + } else {
72 + assert.NoError(t, collr.Init(context.Background()))
73 + }
74 + })
75 + }
76 +}
77 +
78 +func TestCollector_Check(t *testing.T) {
79 + tests := map[string]struct {
80 + metrics []byte
81 + prepare func(*Collector)
82 + wantFail bool
83 + }{
84 + "success valid dcgm metrics": {
85 + metrics: dataMetricsValid,
86 + wantFail: false,
87 + },
88 + "fail if endpoint has no dcgm metric prefix": {
89 + metrics: dataMetricsNonDCGM,
90 + wantFail: true,
91 + },
92 + "success when global limit counts only dcgm series": {
93 + metrics: []byte(`
94 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
95 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
96 +DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa"} 80
97 +# HELP go_memstats_alloc_bytes Number of bytes allocated in heap.
98 +# TYPE go_memstats_alloc_bytes gauge
99 +go_memstats_alloc_bytes 12
100 +`),
101 + prepare: func(c *Collector) { c.MaxTS = 1 },
102 + wantFail: false,
103 + },
104 + "fail when per-metric series limit is exceeded": {
105 + metrics: []byte(`
106 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
107 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
108 +DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa"} 80
109 +DCGM_FI_DEV_GPU_UTIL{gpu="1",UUID="GPU-bbb"} 70
110 +`),
111 + prepare: func(c *Collector) { c.MaxTSPerMetric = 1 },
112 + wantFail: true,
113 + },
114 + }
115 +
116 + for name, test := range tests {
117 + t.Run(name, func(t *testing.T) {
118 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
119 + _, _ = w.Write(test.metrics)
120 + }))
121 + defer srv.Close()
122 +
123 + collr := New()
124 + collr.URL = srv.URL
125 + if test.prepare != nil {
126 + test.prepare(collr)
127 + }
128 +
129 + require.NoError(t, collr.Init(context.Background()))
130 +
131 + if test.wantFail {
132 + assert.Error(t, collr.Check(context.Background()))
133 + } else {
134 + assert.NoError(t, collr.Check(context.Background()))
135 + }
136 + })
137 + }
138 +}
139 +
140 +func TestCollector_Collect(t *testing.T) {
141 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
142 + _, _ = w.Write(dataMetricsValid)
143 + }))
144 + defer srv.Close()
145 +
146 + collr := New()
147 + collr.URL = srv.URL
148 + require.NoError(t, collr.Init(context.Background()))
149 +
150 + mx := collr.Collect(context.Background())
151 + require.NotNil(t, mx)
152 +
153 + gpuKey := "gpu=0|uuid=GPU-aaa"
154 + migKey := "gpu=0|uuid=GPU-aaa|gpu_i_id=2|gpu_i_profile=1g.10gb"
155 + linkKey := "gpu=0|gpu_uuid=GPU-aaa|nvlink=1"
156 +
157 + expect := map[string]int64{
158 + makeID(makeID("dcgm.gpu.compute.utilization", gpuKey), "gpu"): 80000,
159 + makeID(makeID("dcgm.mig.compute.utilization", migKey), "gpu"): 60000,
160 + makeID(makeID("dcgm.gpu.memory.usage", gpuKey), "used"): 1073741824000,
161 + makeID(makeID("dcgm.gpu.reliability.xid", gpuKey), "xid"): 31000,
162 + makeID(makeID("dcgm.gpu.reliability.row_remap_status", gpuKey), "row_remap_failure"): 1000,
163 + makeID(makeID("dcgm.gpu.reliability.row_remap_events", gpuKey), "uncorrectable_remapped_rows"): 7000,
164 + makeID(makeID("dcgm.gpu.throttle.violations", gpuKey), "power_violation"): 2000,
165 + makeID(makeID("dcgm.gpu.throttle.violations", gpuKey), "thermal_violation"): 5000,
166 + makeID(makeID("dcgm.gpu.interconnect.pcie.throughput", gpuKey), "pcie_tx"): 123456000,
167 + makeID(makeID("dcgm.gpu.interconnect.total.throughput", gpuKey), "pcie"): 123456000,
168 + makeID(makeID("dcgm.nvlink.interconnect.error_rate", linkKey), "nvlink_replay_error"): 4000,
169 + }
170 +
171 + assert.Len(t, mx, len(expect))
172 + for dimID, want := range expect {
173 + assert.Equal(t, want, mx[dimID], dimID)
174 + }
175 +
176 + assert.Len(t, *collr.Charts(), 10)
177 +
178 + seenCtx := make(map[string]bool)
179 + for _, ch := range *collr.Charts() {
180 + seenCtx[ch.Ctx] = true
181 + assert.NotContains(t, ch.Title, "(gpu:")
182 + assert.NotContains(t, ch.Title, "(uuid:")
183 + for _, lbl := range ch.Labels {
184 + assert.NotEqual(t, "hostname", lbl.Key)
185 + }
186 + }
187 +
188 + assert.True(t, seenCtx["dcgm.gpu.compute.utilization"])
189 + assert.True(t, seenCtx["dcgm.mig.compute.utilization"])
190 + assert.True(t, seenCtx["dcgm.gpu.memory.usage"])
191 + assert.True(t, seenCtx["dcgm.gpu.reliability.xid"])
192 + assert.True(t, seenCtx["dcgm.gpu.reliability.row_remap_status"])
193 + assert.True(t, seenCtx["dcgm.gpu.reliability.row_remap_events"])
194 + assert.True(t, seenCtx["dcgm.gpu.throttle.violations"])
195 + assert.True(t, seenCtx["dcgm.gpu.interconnect.pcie.throughput"])
196 + assert.True(t, seenCtx["dcgm.gpu.interconnect.total.throughput"])
197 + assert.True(t, seenCtx["dcgm.nvlink.interconnect.error_rate"])
198 + assert.False(t, seenCtx["dcgm.gpu.thermal.temperature"])
199 +}
200 +
201 +func TestCollector_Collect_NVLinkTotalOnlyInOverviewAndCleanDimNames(t *testing.T) {
202 + metrics := []byte(`
203 +# HELP DCGM_FI_PROF_NVLINK_RX_BYTES NVLink RX bytes.
204 +# TYPE DCGM_FI_PROF_NVLINK_RX_BYTES gauge
205 +DCGM_FI_PROF_NVLINK_RX_BYTES{gpu="0",UUID="GPU-aaa"} 10
206 +# HELP DCGM_FI_PROF_NVLINK_TX_BYTES NVLink TX bytes.
207 +# TYPE DCGM_FI_PROF_NVLINK_TX_BYTES gauge
208 +DCGM_FI_PROF_NVLINK_TX_BYTES{gpu="0",UUID="GPU-aaa"} 20
209 +# HELP DCGM_FI_PROF_NVLINK_RX_BYTES NVLink RX bytes.
210 +# TYPE DCGM_FI_PROF_NVLINK_RX_BYTES gauge
211 +DCGM_FI_PROF_NVLINK_RX_BYTES{gpu="0",gpu_uuid="GPU-aaa",nvlink="1"} 11
212 +# HELP DCGM_FI_PROF_NVLINK_TX_BYTES NVLink TX bytes.
213 +# TYPE DCGM_FI_PROF_NVLINK_TX_BYTES gauge
214 +DCGM_FI_PROF_NVLINK_TX_BYTES{gpu="0",gpu_uuid="GPU-aaa",nvlink="1"} 22
215 +# HELP DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL Total NVLink bandwidth.
216 +# TYPE DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL counter
217 +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL{gpu="0",UUID="GPU-aaa"} 400
218 +`)
219 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
220 + _, _ = w.Write(metrics)
221 + }))
222 + defer srv.Close()
223 +
224 + collr := New()
225 + collr.URL = srv.URL
226 + require.NoError(t, collr.Init(context.Background()))
227 +
228 + mx := collr.Collect(context.Background())
229 + require.NotNil(t, mx)
230 +
231 + gpuKey := "gpu=0|uuid=GPU-aaa"
232 + linkKey := "gpu=0|gpu_uuid=GPU-aaa|nvlink=1"
233 + nvlinkCtxID := "dcgm.gpu.interconnect.nvlink.throughput"
234 + nvlinkEntityCtxID := "dcgm.nvlink.interconnect.throughput"
235 + totalCtxID := "dcgm.gpu.interconnect.total.throughput"
236 +
237 + assert.Equal(t, int64(10000), mx[makeID(makeID(nvlinkCtxID, gpuKey), "nvlink_rx")])
238 + assert.Equal(t, int64(20000), mx[makeID(makeID(nvlinkCtxID, gpuKey), "nvlink_tx")])
239 + assert.NotContains(t, mx, makeID(makeID(nvlinkCtxID, gpuKey), "nvlink_bandwidth"))
240 + assert.Equal(t, int64(11000), mx[makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_rx")])
241 + assert.Equal(t, int64(22000), mx[makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_tx")])
242 + assert.NotContains(t, mx, makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_rx_bytes"))
243 + assert.NotContains(t, mx, makeID(makeID(nvlinkEntityCtxID, linkKey), "nvlink_tx_bytes"))
244 + // Explicit NVLink total metric should win over rx+tx aggregation for overview.
245 + assert.Equal(t, int64(400000), mx[makeID(makeID(totalCtxID, gpuKey), "nvlink")])
246 +}
247 +
248 +func TestCollector_Collect_XIDErrorCodeCreatesCleanDimensions(t *testing.T) {
249 + metrics := []byte(`
250 +# HELP DCGM_FI_DEV_XID_ERRORS Value of the last XID error encountered.
251 +# TYPE DCGM_FI_DEV_XID_ERRORS gauge
252 +DCGM_FI_DEV_XID_ERRORS{gpu="0",UUID="GPU-aaa",err_code="31",err_msg="MMU fault",DCGM_FI_DEV_BRAND="GeForce"} 31
253 +`)
254 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
255 + _, _ = w.Write(metrics)
256 + }))
257 + defer srv.Close()
258 +
259 + collr := New()
260 + collr.URL = srv.URL
261 + require.NoError(t, collr.Init(context.Background()))
262 +
263 + mx := collr.Collect(context.Background())
264 + require.NotNil(t, mx)
265 + require.Len(t, mx, 1)
266 +
267 + chartID := makeID("dcgm.gpu.reliability.xid", "gpu=0|uuid=GPU-aaa")
268 + dimID := makeID(chartID, "xid")
269 + assert.Equal(t, int64(31000), mx[dimID], dimID)
270 +
271 + for dimID := range mx {
272 + assert.False(t, strings.Contains(dimID, "err_code"), dimID)
273 + assert.False(t, strings.Contains(dimID, "dcgm_fi_dev_brand"), dimID)
274 + }
275 +}
276 +
277 +func TestCollector_Collect_ExposesDatasetLabelsExceptHostname(t *testing.T) {
278 + metrics := []byte(`
279 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
280 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
281 +DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa",Hostname="host1",DCGM_FI_PROCESS_NAME="/usr/bin/nv-hostengine",DCGM_FI_DRIVER_VERSION="590.48.01"} 80
282 +`)
283 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
284 + _, _ = w.Write(metrics)
285 + }))
286 + defer srv.Close()
287 +
288 + collr := New()
289 + collr.URL = srv.URL
290 + require.NoError(t, collr.Init(context.Background()))
291 +
292 + mx := collr.Collect(context.Background())
293 + require.NotNil(t, mx)
294 +
295 + charts := *collr.Charts()
296 + require.NotEmpty(t, charts)
297 +
298 + var labels []module.Label
299 + for _, ch := range charts {
300 + if ch.Ctx == "dcgm.gpu.compute.utilization" {
301 + labels = ch.Labels
302 + break
303 + }
304 + }
305 + require.NotNil(t, labels)
306 +
307 + assertChartHasLabel(t, labels, "dcgm_fi_process_name")
308 + assertChartHasLabel(t, labels, "dcgm_fi_driver_version")
309 + assertChartHasNoLabel(t, labels, "hostname")
310 +}
311 +
312 +func TestCollector_Collect_RatioAndBar1Classification(t *testing.T) {
313 + metrics := []byte(`
314 +# HELP DCGM_FI_PROF_SM_ACTIVE Ratio of cycles an SM has at least 1 warp assigned.
315 +# TYPE DCGM_FI_PROF_SM_ACTIVE gauge
316 +DCGM_FI_PROF_SM_ACTIVE{gpu="0",UUID="GPU-aaa"} 0.25
317 +# HELP DCGM_FI_DEV_FB_USED_PERCENT Framebuffer memory used percent.
318 +# TYPE DCGM_FI_DEV_FB_USED_PERCENT gauge
319 +DCGM_FI_DEV_FB_USED_PERCENT{gpu="0",UUID="GPU-aaa"} 0.921353
320 +# HELP DCGM_FI_DEV_FB_USED Framebuffer memory used (in MiB).
321 +# TYPE DCGM_FI_DEV_FB_USED gauge
322 +DCGM_FI_DEV_FB_USED{gpu="0",UUID="GPU-aaa"} 1024
323 +# HELP DCGM_FI_DEV_FB_TOTAL Framebuffer memory total (in MiB).
324 +# TYPE DCGM_FI_DEV_FB_TOTAL gauge
325 +DCGM_FI_DEV_FB_TOTAL{gpu="0",UUID="GPU-aaa"} 32768
326 +# HELP DCGM_FI_DEV_BAR1_USED BAR1 used (in MiB).
327 +# TYPE DCGM_FI_DEV_BAR1_USED gauge
328 +DCGM_FI_DEV_BAR1_USED{gpu="0",UUID="GPU-aaa"} 162
329 +# HELP DCGM_FI_DEV_BAR1_TOTAL BAR1 total (in MiB).
330 +# TYPE DCGM_FI_DEV_BAR1_TOTAL gauge
331 +DCGM_FI_DEV_BAR1_TOTAL{gpu="0",UUID="GPU-aaa"} 256
332 +`)
333 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
334 + _, _ = w.Write(metrics)
335 + }))
336 + defer srv.Close()
337 +
338 + collr := New()
339 + collr.URL = srv.URL
340 + require.NoError(t, collr.Init(context.Background()))
341 +
342 + mx := collr.Collect(context.Background())
343 + require.NotNil(t, mx)
344 +
345 + gpuKey := "gpu=0|uuid=GPU-aaa"
346 + expect := map[string]int64{
347 + makeID(makeID("dcgm.gpu.compute.activity", gpuKey), "sm_active"): 25000,
348 + makeID(makeID("dcgm.gpu.memory.utilization", gpuKey), "used_percent"): 92135,
349 + makeID(makeID("dcgm.gpu.memory.usage", gpuKey), "used"): 1073741824000,
350 + makeID(makeID("dcgm.gpu.memory.capacity", gpuKey), "total"): 34359738368000,
351 + makeID(makeID("dcgm.gpu.memory.bar1_usage", gpuKey), "used"): 169869312000,
352 + makeID(makeID("dcgm.gpu.memory.bar1_capacity", gpuKey), "total"): 268435456000,
353 + }
354 +
355 + assert.Len(t, mx, len(expect))
356 + for dimID, want := range expect {
357 + assert.Equal(t, want, mx[dimID], dimID)
358 + }
359 +
360 + seenUnits := make(map[string]string)
361 + for _, ch := range *collr.Charts() {
362 + seenUnits[ch.Ctx] = ch.Units
363 + }
364 + assert.Equal(t, "percentage", seenUnits["dcgm.gpu.compute.activity"])
365 + assert.Equal(t, "percentage", seenUnits["dcgm.gpu.memory.utilization"])
366 + assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.usage"])
367 + assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.capacity"])
368 + assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.bar1_usage"])
369 + assert.Equal(t, "bytes", seenUnits["dcgm.gpu.memory.bar1_capacity"])
370 +}
371 +
372 +func TestCollector_Collect_AvoidsOtherContextsForKnownMetrics(t *testing.T) {
373 + metrics := []byte(`
374 +# HELP DCGM_FI_DEV_PSTATE Performance state.
375 +# TYPE DCGM_FI_DEV_PSTATE gauge
376 +DCGM_FI_DEV_PSTATE{gpu="0",UUID="GPU-aaa"} 0
377 +# HELP DCGM_FI_DEV_VGPU_LICENSE_STATUS vGPU license status.
378 +# TYPE DCGM_FI_DEV_VGPU_LICENSE_STATUS gauge
379 +DCGM_FI_DEV_VGPU_LICENSE_STATUS{gpu="0",UUID="GPU-aaa"} 1
380 +# HELP DCGM_FI_DEV_VIRTUAL_MODE GPU virtualization mode.
381 +# TYPE DCGM_FI_DEV_VIRTUAL_MODE gauge
382 +DCGM_FI_DEV_VIRTUAL_MODE{gpu="0",UUID="GPU-aaa"} 0
383 +# HELP DCGM_FI_DEV_CLOCK_THROTTLE_REASONS Clock throttle reasons.
384 +# TYPE DCGM_FI_DEV_CLOCK_THROTTLE_REASONS gauge
385 +DCGM_FI_DEV_CLOCK_THROTTLE_REASONS{gpu="0",UUID="GPU-aaa"} 0
386 +# HELP DCGM_FI_DEV_FAN_SPEED Fan speed (in %).
387 +# TYPE DCGM_FI_DEV_FAN_SPEED gauge
388 +DCGM_FI_DEV_FAN_SPEED{gpu="0",UUID="GPU-aaa"} 30
389 +# HELP DCGM_FI_DEV_ENFORCED_POWER_LIMIT Enforced power limit (in W).
390 +# TYPE DCGM_FI_DEV_ENFORCED_POWER_LIMIT gauge
391 +DCGM_FI_DEV_ENFORCED_POWER_LIMIT{gpu="0",UUID="GPU-aaa"} 600
392 +# HELP DCGM_FI_DEV_PCIE_LINK_GEN PCIe current link generation.
393 +# TYPE DCGM_FI_DEV_PCIE_LINK_GEN gauge
394 +DCGM_FI_DEV_PCIE_LINK_GEN{gpu="0",UUID="GPU-aaa"} 5
395 +# HELP DCGM_FI_DEV_PCIE_LINK_WIDTH PCIe current link width.
396 +# TYPE DCGM_FI_DEV_PCIE_LINK_WIDTH gauge
397 +DCGM_FI_DEV_PCIE_LINK_WIDTH{gpu="0",UUID="GPU-aaa"} 16
398 +# HELP DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS Time throttled by SW power cap (in ns).
399 +# TYPE DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS counter
400 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS{gpu="0",UUID="GPU-aaa"} 1000000
401 +`)
402 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
403 + _, _ = w.Write(metrics)
404 + }))
405 + defer srv.Close()
406 +
407 + collr := New()
408 + collr.URL = srv.URL
409 + require.NoError(t, collr.Init(context.Background()))
410 +
411 + mx := collr.Collect(context.Background())
412 + require.NotNil(t, mx)
413 +
414 + seenCtx := make(map[string]bool)
415 + for _, ch := range *collr.Charts() {
416 + seenCtx[ch.Ctx] = true
417 + }
418 +
419 + assert.True(t, seenCtx["dcgm.gpu.state.performance"])
420 + assert.True(t, seenCtx["dcgm.gpu.state.virtualization"])
421 + assert.True(t, seenCtx["dcgm.gpu.virtualization.vgpu.license"])
422 + assert.True(t, seenCtx["dcgm.gpu.throttle.reasons"])
423 + assert.True(t, seenCtx["dcgm.gpu.thermal.fan_speed"])
424 + assert.True(t, seenCtx["dcgm.gpu.power.usage"])
425 + assert.True(t, seenCtx["dcgm.gpu.interconnect.pcie.link.generation"])
426 + assert.True(t, seenCtx["dcgm.gpu.interconnect.pcie.link.width"])
427 + assert.True(t, seenCtx["dcgm.gpu.throttle.violations"])
428 + assert.False(t, seenCtx["dcgm.gpu.other.gauge"])
429 + assert.False(t, seenCtx["dcgm.gpu.other.counter"])
430 +}
431 +
432 +func TestCollector_Collect_HidesThresholdDimensionsByDefault(t *testing.T) {
433 + metrics := []byte(`
434 +# HELP DCGM_FI_DEV_SM_CLOCK SM clock in MHz.
435 +# TYPE DCGM_FI_DEV_SM_CLOCK gauge
436 +DCGM_FI_DEV_SM_CLOCK{gpu="0",UUID="GPU-aaa"} 2100
437 +# HELP DCGM_FI_DEV_MAX_SM_CLOCK Max SM clock in MHz.
438 +# TYPE DCGM_FI_DEV_MAX_SM_CLOCK gauge
439 +DCGM_FI_DEV_MAX_SM_CLOCK{gpu="0",UUID="GPU-aaa"} 3000
440 +# HELP DCGM_FI_DEV_APP_SM_CLOCK App SM clock in MHz.
441 +# TYPE DCGM_FI_DEV_APP_SM_CLOCK gauge
442 +DCGM_FI_DEV_APP_SM_CLOCK{gpu="0",UUID="GPU-aaa"} 2800
443 +# HELP DCGM_FI_DEV_GPU_TEMP GPU temperature in C.
444 +# TYPE DCGM_FI_DEV_GPU_TEMP gauge
445 +DCGM_FI_DEV_GPU_TEMP{gpu="0",UUID="GPU-aaa"} 55
446 +# HELP DCGM_FI_DEV_GPU_TEMP_LIMIT GPU temperature limit in C.
447 +# TYPE DCGM_FI_DEV_GPU_TEMP_LIMIT gauge
448 +DCGM_FI_DEV_GPU_TEMP_LIMIT{gpu="0",UUID="GPU-aaa"} 90
449 +# HELP DCGM_FI_DEV_SHUTDOWN_TEMP Shutdown temperature in C.
450 +# TYPE DCGM_FI_DEV_SHUTDOWN_TEMP gauge
451 +DCGM_FI_DEV_SHUTDOWN_TEMP{gpu="0",UUID="GPU-aaa"} 95
452 +# HELP DCGM_FI_DEV_POWER_USAGE Power draw in W.
453 +# TYPE DCGM_FI_DEV_POWER_USAGE gauge
454 +DCGM_FI_DEV_POWER_USAGE{gpu="0",UUID="GPU-aaa"} 320
455 +# HELP DCGM_FI_DEV_POWER_USAGE_INSTANT Instant power draw in W.
456 +# TYPE DCGM_FI_DEV_POWER_USAGE_INSTANT gauge
457 +DCGM_FI_DEV_POWER_USAGE_INSTANT{gpu="0",UUID="GPU-aaa"} 330
458 +# HELP DCGM_FI_DEV_ENFORCED_POWER_LIMIT Enforced power limit in W.
459 +# TYPE DCGM_FI_DEV_ENFORCED_POWER_LIMIT gauge
460 +DCGM_FI_DEV_ENFORCED_POWER_LIMIT{gpu="0",UUID="GPU-aaa"} 600
461 +# HELP DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX Maximum power limit in W.
462 +# TYPE DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX gauge
463 +DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX{gpu="0",UUID="GPU-aaa"} 650
464 +`)
465 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
466 + _, _ = w.Write(metrics)
467 + }))
468 + defer srv.Close()
469 +
470 + collr := New()
471 + collr.URL = srv.URL
472 + require.NoError(t, collr.Init(context.Background()))
473 +
474 + mx := collr.Collect(context.Background())
475 + require.NotNil(t, mx)
476 +
477 + findChartByCtx := func(ctx string) *module.Chart {
478 + for _, ch := range *collr.Charts() {
479 + if ch.Ctx == ctx {
480 + return ch
481 + }
482 + }
483 + return nil
484 + }
485 + dimHiddenByName := func(ch *module.Chart, name string) (bool, bool) {
486 + for _, d := range ch.Dims {
487 + if d.Name == name {
488 + return d.Hidden, true
489 + }
490 + }
491 + return false, false
492 + }
493 +
494 + clock := findChartByCtx("dcgm.gpu.clock.frequency")
495 + require.NotNil(t, clock)
496 + hidden, ok := dimHiddenByName(clock, "sm")
497 + require.True(t, ok)
498 + assert.False(t, hidden)
499 + hidden, ok = dimHiddenByName(clock, "max_sm_clock")
500 + require.True(t, ok)
501 + assert.True(t, hidden)
502 + hidden, ok = dimHiddenByName(clock, "app_sm_clock")
503 + require.True(t, ok)
504 + assert.True(t, hidden)
505 +
506 + thermal := findChartByCtx("dcgm.gpu.thermal.temperature")
507 + require.NotNil(t, thermal)
508 + hidden, ok = dimHiddenByName(thermal, "gpu")
509 + require.True(t, ok)
510 + assert.False(t, hidden)
511 + hidden, ok = dimHiddenByName(thermal, "gpu_temp_limit")
512 + require.True(t, ok)
513 + assert.True(t, hidden)
514 + hidden, ok = dimHiddenByName(thermal, "shutdown_temp")
515 + require.True(t, ok)
516 + assert.True(t, hidden)
517 +
518 + power := findChartByCtx("dcgm.gpu.power.usage")
519 + require.NotNil(t, power)
520 + hidden, ok = dimHiddenByName(power, "draw")
521 + require.True(t, ok)
522 + assert.False(t, hidden)
523 + hidden, ok = dimHiddenByName(power, "power_usage_instant")
524 + require.True(t, ok)
525 + assert.False(t, hidden)
526 + hidden, ok = dimHiddenByName(power, "enforced_limit")
527 + require.True(t, ok)
528 + assert.True(t, hidden)
529 + hidden, ok = dimHiddenByName(power, "power_mgmt_limit_max")
530 + require.True(t, ok)
531 + assert.True(t, hidden)
532 +}
533 +
534 +func TestCollector_Collect_UsesNVSwitchEntityContextToken(t *testing.T) {
535 + metrics := []byte(`
536 +# HELP DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX NVSwitch RX throughput.
537 +# TYPE DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX counter
538 +DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX{nvswitch="0"} 42
539 +`)
540 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
541 + _, _ = w.Write(metrics)
542 + }))
543 + defer srv.Close()
544 +
545 + collr := New()
546 + collr.URL = srv.URL
547 + require.NoError(t, collr.Init(context.Background()))
548 +
549 + mx := collr.Collect(context.Background())
550 + require.NotNil(t, mx)
551 +
552 + seenCtx := make(map[string]bool)
553 + for _, ch := range *collr.Charts() {
554 + seenCtx[ch.Ctx] = true
555 + }
556 +
557 + assert.True(t, seenCtx["dcgm.nvswitch.interconnect.nvswitch.throughput"])
558 + assert.False(t, seenCtx["dcgm.switch.interconnect.nvswitch.throughput"])
559 +}
560 +
561 +func TestCollector_Collect_SkipsUnsupportedSummaryHistogramFamilies(t *testing.T) {
562 + metrics := []byte(`
563 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
564 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
565 +DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa"} 77
566 +# HELP DCGM_FI_DEV_FAKE_SUMMARY synthetic summary for test.
567 +# TYPE DCGM_FI_DEV_FAKE_SUMMARY summary
568 +DCGM_FI_DEV_FAKE_SUMMARY{gpu="0",UUID="GPU-aaa",quantile="0.5"} 1
569 +DCGM_FI_DEV_FAKE_SUMMARY_sum{gpu="0",UUID="GPU-aaa"} 2
570 +DCGM_FI_DEV_FAKE_SUMMARY_count{gpu="0",UUID="GPU-aaa"} 3
571 +# HELP DCGM_FI_DEV_FAKE_HIST synthetic histogram for test.
572 +# TYPE DCGM_FI_DEV_FAKE_HIST histogram
573 +DCGM_FI_DEV_FAKE_HIST_bucket{gpu="0",UUID="GPU-aaa",le="1"} 4
574 +DCGM_FI_DEV_FAKE_HIST_sum{gpu="0",UUID="GPU-aaa"} 5
575 +DCGM_FI_DEV_FAKE_HIST_count{gpu="0",UUID="GPU-aaa"} 6
576 +`)
577 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
578 + _, _ = w.Write(metrics)
579 + }))
580 + defer srv.Close()
581 +
582 + collr := New()
583 + collr.URL = srv.URL
584 + require.NoError(t, collr.Init(context.Background()))
585 +
586 + mx := collr.Collect(context.Background())
587 + require.NotNil(t, mx)
588 +
589 + gpuKey := "gpu=0|uuid=GPU-aaa"
590 + utilDimID := makeID(makeID("dcgm.gpu.compute.utilization", gpuKey), "gpu")
591 + assert.Equal(t, int64(77000), mx[utilDimID], utilDimID)
592 + assert.Len(t, mx, 1)
593 +
594 + for _, ch := range *collr.Charts() {
595 + assert.NotContains(t, ch.Ctx, "fake_summary")
596 + assert.NotContains(t, ch.Ctx, "fake_hist")
597 + }
598 +}
599 +
600 +func TestClassifier_StrictNIDLSplitsForRareFamilies(t *testing.T) {
601 + tests := []struct {
602 + name string
603 + typ sampleKind
604 + group string
605 + }{
606 + {name: "DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL", typ: sampleCounter, group: "interconnect.nvlink.throughput"},
607 + {name: "DCGM_FI_DEV_NVLINK_COUNT_TX_PACKETS", typ: sampleCounter, group: "interconnect.nvlink.traffic"},
608 + {name: "DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER", typ: sampleGauge, group: "interconnect.nvlink.ber"},
609 + {name: "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC0", typ: sampleCounter, group: "interconnect.nvswitch.latency"},
610 + {name: "DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_BUS", typ: sampleGauge, group: "interconnect.nvswitch.topology"},
611 + {name: "DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_STATUS", typ: sampleGauge, group: "interconnect.connectx.error_status"},
612 + {name: "DCGM_FI_DEV_CLOCKS_EVENT_REASONS", typ: sampleGauge, group: "throttle.reasons"},
613 + {name: "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST_NS", typ: sampleCounter, group: "throttle.violations"},
614 + {name: "DCGM_FI_DEV_VGPU_MEMORY_USAGE", typ: sampleGauge, group: "virtualization.vgpu.memory"},
615 + {name: "DCGM_FI_DEV_VGPU_FRAME_RATE_LIMIT", typ: sampleGauge, group: "virtualization.vgpu.frame_rate"},
616 + {name: "DCGM_FI_DEV_VGPU_TYPE_NAME", typ: sampleGauge, group: "virtualization.vgpu.type"},
617 + {name: "DCGM_FI_DEV_VGPU_VM_NAME", typ: sampleGauge, group: "virtualization.vgpu.vm"},
618 + {name: "DCGM_FI_DEV_VGPU_INSTANCE_IDS", typ: sampleGauge, group: "virtualization.vgpu.instance"},
619 + {name: "DCGM_FI_DEV_VGPU_LICENSE_STATUS", typ: sampleGauge, group: "virtualization.vgpu.license"},
620 + {name: "DCGM_FI_DEV_VGPU_UTILIZATIONS", typ: sampleGauge, group: "virtualization.vgpu.utilization"},
621 + {name: "DCGM_FI_DEV_VGPU_ENC_SESSIONS_INFO", typ: sampleGauge, group: "virtualization.vgpu.sessions"},
622 + {name: "DCGM_FI_DEV_FB_TOTAL", typ: sampleGauge, group: "memory.capacity"},
623 + {name: "DCGM_FI_DEV_BAR1_TOTAL", typ: sampleGauge, group: "memory.bar1_capacity"},
624 + }
625 +
626 + for _, tc := range tests {
627 + got := classifyMetricGroup(entityGPU, tc.name, tc.typ)
628 + assert.Equal(t, tc.group, got, tc.name)
629 + }
630 +}
631 +
632 +func TestClassifier_AllKnownFieldsAvoidOtherContexts(t *testing.T) {
633 + lines := strings.Split(string(dataAllFieldsList), "\n")
634 + var unmapped []string
635 + for _, line := range lines {
636 + name := strings.TrimSpace(line)
637 + if name == "" || strings.HasPrefix(name, "#") {
638 + continue
639 + }
640 + for _, kind := range []sampleKind{sampleGauge, sampleCounter} {
641 + group := classifyMetricGroup(entityGPU, name, kind)
642 + if group == "other.gauge" || group == "other.counter" {
643 + unmapped = append(unmapped, name)
644 + break
645 + }
646 + }
647 + }
648 +
649 + assert.Empty(t, unmapped, "unmapped DCGM fields fell into other contexts")
650 +}
651 +
652 +func TestClassifier_NIDLInterconnectAndVGPUSplits(t *testing.T) {
653 + lines := strings.Split(string(dataAllFieldsList), "\n")
654 +
655 + for _, line := range lines {
656 + name := strings.TrimSpace(line)
657 + if name == "" || strings.HasPrefix(name, "#") {
658 + continue
659 + }
660 +
661 + for _, kind := range []sampleKind{sampleGauge, sampleCounter} {
662 + group := classifyMetricGroup(entityGPU, name, kind)
663 +
664 + if group == "interconnect.throughput" {
665 + assert.True(t,
666 + containsAny(name, "C2C_"),
667 + "generic throughput grouping should only contain C2C-style throughput fields: %s", name)
668 + }
669 +
670 + if group == "interconnect.pcie.throughput" {
671 + assert.True(t,
672 + strings.Contains(name, "PCIE") && containsAny(name, "BYTES", "THROUGHPUT", "BANDWIDTH"),
673 + "pcie throughput grouping got non-PCIe throughput field: %s", name)
674 + }
675 +
676 + if group == "interconnect.nvlink.throughput" {
677 + assert.True(t,
678 + strings.Contains(name, "NVLINK") &&
679 + containsAny(name, "BYTES", "THROUGHPUT", "BANDWIDTH"),
680 + "nvlink throughput grouping got non-NVLink throughput field: %s", name)
681 + }
682 +
683 + if group == "interconnect.pcie.traffic" || group == "interconnect.nvlink.traffic" || group == "interconnect.traffic" {
684 + assert.True(t, containsAny(name, "PACKETS", "CODES"), "traffic grouping got non-traffic field: %s", name)
685 + }
686 +
687 + if group == "interconnect.pcie.ber" || group == "interconnect.nvlink.ber" || group == "interconnect.ber" {
688 + assert.True(t, containsAny(name, "BER"), "BER grouping got non-BER field: %s", name)
689 + }
690 +
691 + if group == "virtualization.vgpu.utilization" {
692 + assert.True(t,
693 + containsAny(name, "UTILIZATION"),
694 + "vGPU utilization grouping got non-utilization field: %s", name)
695 + }
696 +
697 + if group == "virtualization.vgpu.memory" {
698 + assert.True(t, containsAny(name, "MEMORY_USAGE"), "vGPU memory grouping got non-memory field: %s", name)
699 + }
700 + }
701 + }
702 +}
703 +
704 +func TestCatalog_GPUInterconnectFamiliesOnlyThreeVariants(t *testing.T) {
705 + got := make(map[string]struct{})
706 + for _, g := range groupCatalog {
707 + if !strings.HasPrefix(g.Suffix, "interconnect.") {
708 + continue
709 + }
710 + got["gpu "+g.Family] = struct{}{}
711 + }
712 +
713 + want := map[string]struct{}{
714 + "gpu interconnect/overview": {},
715 + "gpu interconnect/pcie": {},
716 + "gpu interconnect/nvlink": {},
717 + }
718 +
719 + assert.Equal(t, want, got)
720 +}
721 +
722 +func TestCollector_Cleanup(t *testing.T) {
723 + assert.NotPanics(t, func() { New().Cleanup(context.Background()) })
724 +
725 + collr := New()
726 + collr.URL = "http://127.0.0.1:9400/metrics"
727 + require.NoError(t, collr.Init(context.Background()))
728 + assert.NotPanics(t, func() { collr.Cleanup(context.Background()) })
729 +}
730 +
731 +func assertChartHasLabel(t *testing.T, labels []module.Label, key string) {
732 + t.Helper()
733 + for _, lbl := range labels {
734 + if lbl.Key == key {
735 + return
736 + }
737 + }
738 + assert.Failf(t, "missing label", "expected chart label %q", key)
739 +}
740 +
741 +func assertChartHasNoLabel(t *testing.T, labels []module.Label, key string) {
742 + t.Helper()
743 + for _, lbl := range labels {
744 + if lbl.Key == key {
745 + assert.Failf(t, "unexpected label", "did not expect chart label %q", key)
746 + return
747 + }
748 + }
749 +}
src/go/plugin/go.d/collector/dcgm/config_schema.json new
+207
@@ -0,0 +1,207 @@
1 +{
2 + "jsonSchema": {
3 + "$schema": "http://json-schema.org/draft-07/schema#",
4 + "title": "DCGM exporter collector configuration.",
5 + "type": "object",
6 + "properties": {
7 + "update_every": {
8 + "title": "Update every",
9 + "description": "Data collection interval, measured in seconds.",
10 + "type": "integer",
11 + "minimum": 1,
12 + "default": 30
13 + },
14 + "autodetection_retry": {
15 + "title": "Detection retry",
16 + "description": "Recheck interval in seconds. Zero means no recheck will be scheduled.",
17 + "type": "integer",
18 + "minimum": 0,
19 + "default": 0
20 + },
21 + "url": {
22 + "title": "URL",
23 + "description": "The URL of the dcgm-exporter Prometheus endpoint.",
24 + "type": "string",
25 + "format": "uri",
26 + "default": "http://127.0.0.1:9400/metrics"
27 + },
28 + "timeout": {
29 + "title": "Timeout",
30 + "description": "The timeout in seconds for the HTTP request.",
31 + "type": "number",
32 + "minimum": 0.5,
33 + "default": 10
34 + },
35 + "max_time_series": {
36 + "title": "Time series limit",
37 + "description": "If an endpoint returns more time series than this limit, the data is not processed. Set to 0 for no limit.",
38 + "type": "integer",
39 + "minimum": 0,
40 + "default": 2000
41 + },
42 + "max_time_series_per_metric": {
43 + "title": "Time series per metric limit",
44 + "description": "Metrics with more time series than this limit are skipped. Set to 0 for no limit.",
45 + "type": "integer",
46 + "minimum": 0,
47 + "default": 200
48 + },
49 + "vnode": {
50 + "title": "Vnode",
51 + "description": "Associates this data collection job with a Virtual Node.",
52 + "type": "string"
53 + },
54 + "username": {
55 + "title": "Username",
56 + "description": "The username for basic authentication.",
57 + "type": "string",
58 + "sensitive": true
59 + },
60 + "password": {
61 + "title": "Password",
62 + "description": "The password for basic authentication.",
63 + "type": "string",
64 + "sensitive": true
65 + },
66 + "bearer_token_file": {
67 + "title": "Bearer token file",
68 + "description": "The path to the file with Bearer token.",
69 + "type": "string"
70 + },
71 + "proxy_url": {
72 + "title": "Proxy URL",
73 + "description": "The URL of the proxy server.",
74 + "type": "string"
75 + },
76 + "proxy_username": {
77 + "title": "Proxy username",
78 + "description": "The username for proxy authentication.",
79 + "type": "string",
80 + "sensitive": true
81 + },
82 + "proxy_password": {
83 + "title": "Proxy password",
84 + "description": "The password for proxy authentication.",
85 + "type": "string",
86 + "sensitive": true
87 + },
88 + "headers": {
89 + "title": "Headers",
90 + "description": "Additional HTTP headers to include in the request.",
91 + "type": [
92 + "object",
93 + "null"
94 + ],
95 + "additionalProperties": {
96 + "type": "string"
97 + }
98 + },
99 + "tls_skip_verify": {
100 + "title": "Skip TLS verification",
101 + "description": "If set, TLS certificate verification will be skipped.",
102 + "type": "boolean"
103 + },
104 + "tls_ca": {
105 + "title": "TLS CA",
106 + "description": "The path to the CA certificate file for TLS verification.",
107 + "type": "string",
108 + "pattern": "^$|^/"
109 + },
110 + "tls_cert": {
111 + "title": "TLS certificate",
112 + "description": "The path to the client certificate file for TLS authentication.",
113 + "type": "string",
114 + "pattern": "^$|^/"
115 + },
116 + "tls_key": {
117 + "title": "TLS key",
118 + "description": "The path to the client key file for TLS authentication.",
119 + "type": "string",
120 + "pattern": "^$|^/"
121 + },
122 + "body": {
123 + "title": "Body",
124 + "type": "string"
125 + },
126 + "method": {
127 + "title": "Method",
128 + "type": "string"
129 + },
130 + "not_follow_redirects": {
131 + "title": "Not follow redirects",
132 + "description": "If set, the client will not follow HTTP redirects automatically.",
133 + "type": "boolean"
134 + },
135 + "force_http2": {
136 + "title": "Force HTTP2",
137 + "description": "If set, forces the use of HTTP/2 protocol for all requests, even over plain TCP (h2c).",
138 + "type": "boolean"
139 + }
140 + },
141 + "required": [
142 + "url"
143 + ]
144 + },
145 + "uiSchema": {
146 + "uiOptions": {
147 + "fullPage": true
148 + },
149 + "ui:flavour": "tabs",
150 + "ui:options": {
151 + "tabs": [
152 + {
153 + "title": "Base",
154 + "fields": [
155 + "update_every",
156 + "autodetection_retry",
157 + "url",
158 + "timeout",
159 + "vnode"
160 + ]
161 + },
162 + {
163 + "title": "Limits",
164 + "fields": [
165 + "max_time_series",
166 + "max_time_series_per_metric"
167 + ]
168 + },
169 + {
170 + "title": "Auth",
171 + "fields": [
172 + "username",
173 + "password",
174 + "bearer_token_file"
175 + ]
176 + },
177 + {
178 + "title": "TLS",
179 + "fields": [
180 + "tls_skip_verify",
181 + "tls_ca",
182 + "tls_cert",
183 + "tls_key"
184 + ]
185 + },
186 + {
187 + "title": "Proxy",
188 + "fields": [
189 + "proxy_url",
190 + "proxy_username",
191 + "proxy_password"
192 + ]
193 + },
194 + {
195 + "title": "Request",
196 + "fields": [
197 + "headers",
198 + "method",
199 + "body",
200 + "not_follow_redirects",
201 + "force_http2"
202 + ]
203 + }
204 + ]
205 + }
206 + }
207 +}
src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv new
+1272
@@ -0,0 +1,1272 @@
1 +# Netdata recommended dcgm-exporter profile.
2 +#
3 +# Usage:
4 +# dcgm-exporter -f /path/to/dcgm-exporter-netdata.csv
5 +#
6 +# Keep exporter interval aligned with Netdata update_every.
7 +# Example:
8 +# dcgm-exporter -f /path/to/dcgm-exporter-netdata.csv -c 30000
9 +#
10 +# This file documents all known DCGM fields (623 in current source dataset).
11 +# Exactly 123 fields are enabled by default below (127 max per field group).
12 +# To enable a different field, uncomment one line and comment another enabled line.
13 +#
14 +# IMPORTANT: Fields with DCGM_FT_BINARY type (structs/blobs) MUST remain disabled.
15 +# They return non-numeric values that break Prometheus metric parsing.
16 +# Known BINARY fields: ACCOUNTING_DATA, COMPUTE_PIDS, CREATABLE_VGPU_TYPE_IDS,
17 +# ENC_STATS, FBC_SESSIONS_INFO, FBC_STATS, GRAPHICS_PIDS, SUPPORTED_CLOCKS,
18 +# SUPPORTED_TYPE_INFO, VGPU_ENC_SESSIONS_INFO, VGPU_ENC_STATS,
19 +# VGPU_FBC_SESSIONS_INFO, VGPU_FBC_STATS, VGPU_INSTANCE_IDS,
20 +# VGPU_PER_PROCESS_UTILIZATION, VGPU_UTILIZATIONS, GPU_TOPOLOGY_AFFINITY,
21 +# GPU_TOPOLOGY_NVLINK, GPU_TOPOLOGY_PCI, SYNC_BOOST
22 +#
23 +# Format:
24 +# DCGM FIELD, Prometheus metric type, help message
25 +
26 +# context=dcgm.exporter.health.status family=exporter health dimension=bind_unbind_event
27 +# DCGM_FI_BIND_UNBIND_EVENT, gauge, Bind unbind event
28 +# context=dcgm.exporter.inventory.software family=exporter inventory dimension=cuda_driver_version
29 +DCGM_FI_CUDA_DRIVER_VERSION, label, Cuda driver version
30 +# context=dcgm.gpu.workload.sessions family=gpu workload dimension=accounting_data
31 +# DCGM_FI_DEV_ACCOUNTING_DATA, gauge, Accounting data
32 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=app_mem_clock
33 +DCGM_FI_DEV_APP_MEM_CLOCK, gauge, App mem clock
34 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=app_sm_clock
35 +DCGM_FI_DEV_APP_SM_CLOCK, gauge, App sm clock
36 +# context=dcgm.gpu.state.configuration family=gpu state dimension=autoboost
37 +# DCGM_FI_DEV_AUTOBOOST, gauge, Autoboost
38 +# context=dcgm.gpu.reliability.memory_health family=gpu reliability dimension=banks_remap_rows_avail_high
39 +# DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_HIGH, gauge, Banks remap rows avail high
40 +# context=dcgm.gpu.reliability.memory_health family=gpu reliability dimension=banks_remap_rows_avail_low
41 +# DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_LOW, gauge, Banks remap rows avail low
42 +# context=dcgm.gpu.reliability.memory_health family=gpu reliability dimension=banks_remap_rows_avail_max
43 +# DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_MAX, gauge, Banks remap rows avail max
44 +# context=dcgm.gpu.reliability.memory_health family=gpu reliability dimension=banks_remap_rows_avail_none
45 +# DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_NONE, gauge, Banks remap rows avail none
46 +# context=dcgm.gpu.reliability.memory_health family=gpu reliability dimension=banks_remap_rows_avail_partial
47 +# DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_PARTIAL, gauge, Banks remap rows avail partial
48 +# context=dcgm.gpu.memory.bar1_usage family=gpu memory dimension=free
49 +DCGM_FI_DEV_BAR1_FREE, gauge, Bar1 free
50 +# context=dcgm.gpu.memory.bar1_capacity family=gpu memory dimension=total
51 +DCGM_FI_DEV_BAR1_TOTAL, gauge, Bar1 total
52 +# context=dcgm.gpu.memory.bar1_usage family=gpu memory dimension=used
53 +DCGM_FI_DEV_BAR1_USED, gauge, Bar1 used
54 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=board_limit_violation
55 +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Board limit violation
56 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=brand
57 +DCGM_FI_DEV_BRAND, label, Brand
58 +# context=dcgm.gpu.interconnect.state family=gpu interconnect/overview dimension=c2c_link
59 +# DCGM_FI_DEV_C2C_LINK_COUNT, counter, C2c link count
60 +# context=dcgm.gpu.interconnect.error_rate family=gpu interconnect/overview dimension=c2c_link_error_intr
61 +# DCGM_FI_DEV_C2C_LINK_ERROR_INTR, counter, C2c link error intr
62 +# context=dcgm.gpu.interconnect.error_rate family=gpu interconnect/overview dimension=c2c_link_error_replay
63 +# DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY, counter, C2c link error replay
64 +# context=dcgm.gpu.interconnect.error_rate family=gpu interconnect/overview dimension=c2c_link_error_replay_b2b
65 +# DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY_B2B, counter, C2c link error replay b2b
66 +# context=dcgm.gpu.interconnect.state family=gpu interconnect/overview dimension=c2c_link_power_state
67 +# DCGM_FI_DEV_C2C_LINK_POWER_STATE, gauge, C2c link power state
68 +# context=dcgm.gpu.interconnect.state family=gpu interconnect/overview dimension=c2c_link_status
69 +# DCGM_FI_DEV_C2C_LINK_STATUS, gauge, C2c link status
70 +# context=dcgm.gpu.interconnect.throughput family=gpu interconnect/overview dimension=c2c_max_bandwidth
71 +# DCGM_FI_DEV_C2C_MAX_BANDWIDTH, gauge, C2c max bandwidth
72 +# context=dcgm.gpu.capability.support family=gpu capability dimension=cc_mode
73 +# DCGM_FI_DEV_CC_MODE, gauge, Cc mode
74 +# Legacy name in older stacks: DCGM_FI_DEV_CLOCK_THROTTLE_REASONS.
75 +# context=dcgm.gpu.throttle.reasons family=gpu throttle dimension=clocks_event_reasons
76 +DCGM_FI_DEV_CLOCKS_EVENT_REASONS, gauge, Clocks event reasons
77 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=hw_power_brake_slowdown
78 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_POWER_BRAKE_SLOWDOWN_NS, counter, Clocks event reason hw power brake slowdown ns
79 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=hw_therm_slowdown
80 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_THERM_SLOWDOWN_NS, counter, Clocks event reason hw therm slowdown ns
81 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=sw_power_cap
82 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS, counter, Clocks event reason sw power cap ns
83 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=sw_therm_slowdown
84 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_THERM_SLOWDOWN_NS, counter, Clocks event reason sw therm slowdown ns
85 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=sync_boost
86 +# DCGM_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST_NS, counter, Clocks event reason sync boost ns
87 +# context=dcgm.gpu.state.configuration family=gpu state dimension=compute_mode
88 +DCGM_FI_DEV_COMPUTE_MODE, gauge, Compute mode
89 +# context=dcgm.gpu.interconnect.connectx.link family=gpu interconnect/pcie dimension=connectx_active_pcie_link_speed
90 +# DCGM_FI_DEV_CONNECTX_ACTIVE_PCIE_LINK_SPEED, gauge, Connectx active pcie link speed
91 +# context=dcgm.gpu.interconnect.pcie.link.width family=gpu interconnect/pcie dimension=connectx_active_pcie_link_width
92 +# DCGM_FI_DEV_CONNECTX_ACTIVE_PCIE_LINK_WIDTH, gauge, Connectx active pcie link width
93 +# context=dcgm.gpu.interconnect.connectx.error_status family=gpu interconnect/pcie dimension=connectx_correctable_err_mask
94 +# DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_MASK, gauge, Connectx correctable err mask
95 +# context=dcgm.gpu.interconnect.connectx.error_status family=gpu interconnect/pcie dimension=connectx_correctable_err_status
96 +# DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_STATUS, gauge, Connectx correctable err status
97 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=connectx_device_temperature
98 +# DCGM_FI_DEV_CONNECTX_DEVICE_TEMPERATURE, gauge, Connectx device temperature
99 +# context=dcgm.gpu.interconnect.connectx.link family=gpu interconnect/pcie dimension=connectx_expect_pcie_link_speed
100 +# DCGM_FI_DEV_CONNECTX_EXPECT_PCIE_LINK_SPEED, gauge, Connectx expect pcie link speed
101 +# context=dcgm.gpu.interconnect.pcie.link.width family=gpu interconnect/pcie dimension=connectx_expect_pcie_link_width
102 +# DCGM_FI_DEV_CONNECTX_EXPECT_PCIE_LINK_WIDTH, gauge, Connectx expect pcie link width
103 +# context=dcgm.gpu.interconnect.connectx.status family=gpu interconnect/pcie dimension=connectx_health
104 +# DCGM_FI_DEV_CONNECTX_HEALTH, gauge, Connectx health
105 +# context=dcgm.gpu.interconnect.connectx.error_status family=gpu interconnect/pcie dimension=connectx_uncorrectable_err_mask
106 +# DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_MASK, gauge, Connectx uncorrectable err mask
107 +# context=dcgm.gpu.interconnect.connectx.error_status family=gpu interconnect/pcie dimension=connectx_uncorrectable_err_severity
108 +# DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_SEVERITY, gauge, Connectx uncorrectable err severity
109 +# context=dcgm.gpu.interconnect.connectx.error_status family=gpu interconnect/pcie dimension=connectx_uncorrectable_err_status
110 +# DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_STATUS, gauge, Connectx uncorrectable err status
111 +# context=dcgm.gpu.reliability.row_remap_events family=gpu reliability dimension=correctable_remapped_rows
112 +DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Correctable remapped rows
113 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=count
114 +DCGM_FI_DEV_COUNT, gauge, Count
115 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=cpu_affinity_0
116 +# DCGM_FI_DEV_CPU_AFFINITY_0, gauge, Cpu affinity 0
117 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=cpu_affinity_1
118 +# DCGM_FI_DEV_CPU_AFFINITY_1, gauge, Cpu affinity 1
119 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=cpu_affinity_2
120 +# DCGM_FI_DEV_CPU_AFFINITY_2, gauge, Cpu affinity 2
121 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=cpu_affinity_3
122 +# DCGM_FI_DEV_CPU_AFFINITY_3, gauge, Cpu affinity 3
123 +# context=dcgm.cpu.clock.frequency family=cpu clock dimension=cpu_clock_current
124 +# DCGM_FI_DEV_CPU_CLOCK_CURRENT, gauge, Cpu clock current
125 +# context=dcgm.gpu.cpu.info family=gpu cpu dimension=cpu_model
126 +# DCGM_FI_DEV_CPU_MODEL, label, Cpu model
127 +# context=dcgm.cpu.cpu.power family=cpu cpu dimension=cpu_power_limit
128 +# DCGM_FI_DEV_CPU_POWER_LIMIT, gauge, Cpu power limit
129 +# context=dcgm.cpu.cpu.power family=cpu cpu dimension=cpu_power_util_current
130 +# DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT, gauge, Cpu power util current
131 +# context=dcgm.cpu.cpu.temperature family=cpu cpu dimension=cpu_temp_critical
132 +# DCGM_FI_DEV_CPU_TEMP_CRITICAL, gauge, Cpu temp critical
133 +# context=dcgm.cpu.cpu.temperature family=cpu cpu dimension=cpu_temp_current
134 +# DCGM_FI_DEV_CPU_TEMP_CURRENT, gauge, Cpu temp current
135 +# context=dcgm.cpu.cpu.temperature family=cpu cpu dimension=cpu_temp_warning
136 +# DCGM_FI_DEV_CPU_TEMP_WARNING, gauge, Cpu temp warning
137 +# context=dcgm.cpu.cpu.utilization family=cpu cpu dimension=cpu_util_irq
138 +# DCGM_FI_DEV_CPU_UTIL_IRQ, gauge, Cpu util irq
139 +# context=dcgm.cpu.cpu.utilization family=cpu cpu dimension=cpu_util_nice
140 +# DCGM_FI_DEV_CPU_UTIL_NICE, gauge, Cpu util nice
141 +# context=dcgm.cpu.cpu.utilization family=cpu cpu dimension=cpu_util_sys
142 +# DCGM_FI_DEV_CPU_UTIL_SYS, gauge, Cpu util sys
143 +# context=dcgm.cpu.cpu.utilization family=cpu cpu dimension=cpu_util
144 +# DCGM_FI_DEV_CPU_UTIL_TOTAL, gauge, Cpu util total
145 +# context=dcgm.cpu.cpu.utilization family=cpu cpu dimension=cpu_util_user
146 +# DCGM_FI_DEV_CPU_UTIL_USER, gauge, Cpu util user
147 +# context=dcgm.gpu.cpu.info family=gpu cpu dimension=cpu_vendor
148 +# DCGM_FI_DEV_CPU_VENDOR, label, Cpu vendor
149 +# context=dcgm.gpu.virtualization.vgpu.type family=gpu virtualization dimension=creatable_vgpu_type_ids
150 +# DCGM_FI_DEV_CREATABLE_VGPU_TYPE_IDS, gauge, Creatable vgpu type ids
151 +# context=dcgm.gpu.capability.support family=gpu capability dimension=cuda_compute_capability
152 +# DCGM_FI_DEV_CUDA_COMPUTE_CAPABILITY, gauge, Cuda compute capability
153 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=cuda_visible_devices_str
154 +# DCGM_FI_DEV_CUDA_VISIBLE_DEVICES_STR, label, Cuda visible devices str
155 +# context=dcgm.gpu.compute.utilization family=gpu compute dimension=decoder
156 +DCGM_FI_DEV_DEC_UTIL, gauge, Dec util
157 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_cpu_eud_result
158 +# DCGM_FI_DEV_DIAG_CPU_EUD_RESULT, gauge, Diag cpu eud result
159 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_diagnostic_result
160 +# DCGM_FI_DEV_DIAG_DIAGNOSTIC_RESULT, gauge, Diag diagnostic result
161 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_eud_result
162 +# DCGM_FI_DEV_DIAG_EUD_RESULT, gauge, Diag eud result
163 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_memory_bandwidth_result
164 +# DCGM_FI_DEV_DIAG_MEMORY_BANDWIDTH_RESULT, gauge, Diag memory bandwidth result
165 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_memory_result
166 +# DCGM_FI_DEV_DIAG_MEMORY_RESULT, gauge, Diag memory result
167 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_memtest_result
168 +# DCGM_FI_DEV_DIAG_MEMTEST_RESULT, gauge, Diag memtest result
169 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_nccl_tests_result
170 +# DCGM_FI_DEV_DIAG_NCCL_TESTS_RESULT, gauge, Diag nccl tests result
171 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_nvbandwidth_result
172 +# DCGM_FI_DEV_DIAG_NVBANDWIDTH_RESULT, gauge, Diag nvbandwidth result
173 +# context=dcgm.gpu.interconnect.pcie.state family=gpu interconnect/pcie dimension=diag_pcie_result
174 +# DCGM_FI_DEV_DIAG_PCIE_RESULT, gauge, Diag pcie result
175 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_pulse_test_result
176 +# DCGM_FI_DEV_DIAG_PULSE_TEST_RESULT, gauge, Diag pulse test result
177 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_software_result
178 +# DCGM_FI_DEV_DIAG_SOFTWARE_RESULT, gauge, Diag software result
179 +# context=dcgm.gpu.diagnostics.status family=gpu diagnostics dimension=diag_status
180 +# DCGM_FI_DEV_DIAG_STATUS, gauge, Diag status
181 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_targeted_power_result
182 +# DCGM_FI_DEV_DIAG_TARGETED_POWER_RESULT, gauge, Diag targeted power result
183 +# context=dcgm.gpu.diagnostics.results family=gpu diagnostics dimension=diag_targeted_stress_result
184 +# DCGM_FI_DEV_DIAG_TARGETED_STRESS_RESULT, gauge, Diag targeted stress result
185 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_current
186 +DCGM_FI_DEV_ECC_CURRENT, gauge, Ecc current
187 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_cbu
188 +# DCGM_FI_DEV_ECC_DBE_AGG_CBU, gauge, Ecc dbe agg cbu
189 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_dev
190 +# DCGM_FI_DEV_ECC_DBE_AGG_DEV, gauge, Ecc dbe agg dev
191 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_l1
192 +# DCGM_FI_DEV_ECC_DBE_AGG_L1, gauge, Ecc dbe agg l1
193 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_l2
194 +DCGM_FI_DEV_ECC_DBE_AGG_L2, gauge, Ecc dbe agg l2
195 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_reg
196 +# DCGM_FI_DEV_ECC_DBE_AGG_REG, gauge, Ecc dbe agg reg
197 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_shm
198 +# DCGM_FI_DEV_ECC_DBE_AGG_SHM, gauge, Ecc dbe agg shm
199 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_srm
200 +# DCGM_FI_DEV_ECC_DBE_AGG_SRM, gauge, Ecc dbe agg srm
201 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_agg_tex
202 +# DCGM_FI_DEV_ECC_DBE_AGG_TEX, gauge, Ecc dbe agg tex
203 +# context=dcgm.gpu.memory.ecc_error_rate family=gpu memory dimension=ecc_dbe_agg
204 +DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Ecc dbe agg total
205 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_cbu
206 +# DCGM_FI_DEV_ECC_DBE_VOL_CBU, gauge, Ecc dbe vol cbu
207 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_dev
208 +DCGM_FI_DEV_ECC_DBE_VOL_DEV, gauge, Ecc dbe vol dev
209 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_l1
210 +# DCGM_FI_DEV_ECC_DBE_VOL_L1, gauge, Ecc dbe vol l1
211 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_l2
212 +DCGM_FI_DEV_ECC_DBE_VOL_L2, gauge, Ecc dbe vol l2
213 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_reg
214 +# DCGM_FI_DEV_ECC_DBE_VOL_REG, gauge, Ecc dbe vol reg
215 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_shm
216 +# DCGM_FI_DEV_ECC_DBE_VOL_SHM, gauge, Ecc dbe vol shm
217 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_srm
218 +# DCGM_FI_DEV_ECC_DBE_VOL_SRM, gauge, Ecc dbe vol srm
219 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_dbe_vol_tex
220 +# DCGM_FI_DEV_ECC_DBE_VOL_TEX, gauge, Ecc dbe vol tex
221 +# context=dcgm.gpu.memory.ecc_error_rate family=gpu memory dimension=ecc_dbe_vol
222 +DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Ecc dbe vol total
223 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_inforom_ver
224 +# DCGM_FI_DEV_ECC_INFOROM_VER, label, Ecc inforom ver
225 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_pending
226 +DCGM_FI_DEV_ECC_PENDING, gauge, Ecc pending
227 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_cbu
228 +# DCGM_FI_DEV_ECC_SBE_AGG_CBU, gauge, Ecc sbe agg cbu
229 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_dev
230 +# DCGM_FI_DEV_ECC_SBE_AGG_DEV, gauge, Ecc sbe agg dev
231 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_l1
232 +# DCGM_FI_DEV_ECC_SBE_AGG_L1, gauge, Ecc sbe agg l1
233 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_l2
234 +DCGM_FI_DEV_ECC_SBE_AGG_L2, gauge, Ecc sbe agg l2
235 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_reg
236 +# DCGM_FI_DEV_ECC_SBE_AGG_REG, gauge, Ecc sbe agg reg
237 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_shm
238 +# DCGM_FI_DEV_ECC_SBE_AGG_SHM, gauge, Ecc sbe agg shm
239 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_srm
240 +# DCGM_FI_DEV_ECC_SBE_AGG_SRM, gauge, Ecc sbe agg srm
241 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_agg_tex
242 +# DCGM_FI_DEV_ECC_SBE_AGG_TEX, gauge, Ecc sbe agg tex
243 +# context=dcgm.gpu.memory.ecc_error_rate family=gpu memory dimension=ecc_sbe_agg
244 +DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Ecc sbe agg total
245 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_cbu
246 +# DCGM_FI_DEV_ECC_SBE_VOL_CBU, gauge, Ecc sbe vol cbu
247 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_dev
248 +DCGM_FI_DEV_ECC_SBE_VOL_DEV, gauge, Ecc sbe vol dev
249 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_l1
250 +# DCGM_FI_DEV_ECC_SBE_VOL_L1, gauge, Ecc sbe vol l1
251 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_l2
252 +DCGM_FI_DEV_ECC_SBE_VOL_L2, gauge, Ecc sbe vol l2
253 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_reg
254 +# DCGM_FI_DEV_ECC_SBE_VOL_REG, gauge, Ecc sbe vol reg
255 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_shm
256 +# DCGM_FI_DEV_ECC_SBE_VOL_SHM, gauge, Ecc sbe vol shm
257 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_srm
258 +# DCGM_FI_DEV_ECC_SBE_VOL_SRM, gauge, Ecc sbe vol srm
259 +# context=dcgm.gpu.memory.ecc_errors family=gpu memory dimension=ecc_sbe_vol_tex
260 +# DCGM_FI_DEV_ECC_SBE_VOL_TEX, gauge, Ecc sbe vol tex
261 +# context=dcgm.gpu.memory.ecc_error_rate family=gpu memory dimension=ecc_sbe_vol
262 +DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Ecc sbe vol total
263 +# context=dcgm.gpu.workload.sessions family=gpu workload dimension=enc_stats
264 +# DCGM_FI_DEV_ENC_STATS, gauge, Enc stats
265 +# context=dcgm.gpu.compute.utilization family=gpu compute dimension=encoder
266 +DCGM_FI_DEV_ENC_UTIL, gauge, Enc util
267 +# context=dcgm.gpu.power.usage family=gpu power dimension=enforced_limit
268 +DCGM_FI_DEV_ENFORCED_POWER_LIMIT, gauge, Enforced power limit
269 +# context=dcgm.gpu.power.profiles family=gpu power dimension=enforced_power_profile_mask
270 +# DCGM_FI_DEV_ENFORCED_POWER_PROFILE_MASK, counter, Enforced power profile mask
271 +# context=dcgm.gpu.interconnect.fabric family=gpu interconnect/overview dimension=fabric_clique_id
272 +# DCGM_FI_DEV_FABRIC_CLIQUE_ID, gauge, Fabric clique id
273 +# context=dcgm.gpu.interconnect.fabric family=gpu interconnect/overview dimension=fabric_cluster_uuid
274 +# DCGM_FI_DEV_FABRIC_CLUSTER_UUID, label, Fabric cluster uuid
275 +# context=dcgm.gpu.interconnect.fabric family=gpu interconnect/overview dimension=fabric_health_mask
276 +# DCGM_FI_DEV_FABRIC_HEALTH_MASK, gauge, Fabric health mask
277 +# context=dcgm.gpu.interconnect.fabric family=gpu interconnect/overview dimension=fabric_manager_error_code
278 +DCGM_FI_DEV_FABRIC_MANAGER_ERROR_CODE, counter, Fabric manager error code
279 +# context=dcgm.gpu.interconnect.fabric family=gpu interconnect/overview dimension=fabric_manager_status
280 +DCGM_FI_DEV_FABRIC_MANAGER_STATUS, gauge, Fabric manager status
281 +# context=dcgm.gpu.thermal.fan_speed family=gpu thermal dimension=fan_speed
282 +DCGM_FI_DEV_FAN_SPEED, gauge, Fan speed
283 +# context=dcgm.gpu.workload.sessions family=gpu workload dimension=fbc_sessions_info
284 +# DCGM_FI_DEV_FBC_SESSIONS_INFO, gauge, Fbc sessions info
285 +# context=dcgm.gpu.workload.sessions family=gpu workload dimension=fbc_stats
286 +# DCGM_FI_DEV_FBC_STATS, gauge, Fbc stats
287 +# context=dcgm.gpu.memory.usage family=gpu memory dimension=free
288 +DCGM_FI_DEV_FB_FREE, gauge, Fb free
289 +# context=dcgm.gpu.memory.usage family=gpu memory dimension=reserved
290 +DCGM_FI_DEV_FB_RESERVED, gauge, Fb reserved
291 +# context=dcgm.gpu.memory.capacity family=gpu memory dimension=total
292 +DCGM_FI_DEV_FB_TOTAL, gauge, Fb total
293 +# context=dcgm.gpu.memory.usage family=gpu memory dimension=used
294 +DCGM_FI_DEV_FB_USED, gauge, Fb used
295 +# context=dcgm.gpu.memory.utilization family=gpu memory dimension=used_percent
296 +DCGM_FI_DEV_FB_USED_PERCENT, gauge, Fb used percent
297 +# context=dcgm.gpu.internal.boundary family=gpu internal dimension=first_connectx_field_id
298 +# DCGM_FI_DEV_FIRST_CONNECTX_FIELD_ID, gauge, First connectx field id
299 +# context=dcgm.gpu.reliability.recovery_action family=gpu reliability dimension=get_gpu_recovery_action
300 +# DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION, counter, Get gpu recovery action
301 +# context=dcgm.gpu.capability.support family=gpu capability dimension=gpm_support
302 +# DCGM_FI_DEV_GPM_SUPPORT, gauge, Gpm support
303 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=gpu_max_op_temp
304 +DCGM_FI_DEV_GPU_MAX_OP_TEMP, gauge, Gpu max op temp
305 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=gpu_nvlink_errors
306 +DCGM_FI_DEV_GPU_NVLINK_ERRORS, counter, Gpu nvlink errors
307 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=gpu
308 +DCGM_FI_DEV_GPU_TEMP, gauge, Gpu temp
309 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=gpu_temp_limit
310 +DCGM_FI_DEV_GPU_TEMP_LIMIT, gauge, Gpu temp limit
311 +# context=dcgm.gpu.compute.utilization family=gpu compute dimension=gpu
312 +DCGM_FI_DEV_GPU_UTIL, gauge, Gpu util
313 +# context=dcgm.gpu.inventory.software family=gpu inventory dimension=inforom_config_check
314 +# DCGM_FI_DEV_INFOROM_CONFIG_CHECK, label, Inforom config check
315 +# context=dcgm.gpu.inventory.software family=gpu inventory dimension=inforom_config_valid
316 +# DCGM_FI_DEV_INFOROM_CONFIG_VALID, label, Inforom config valid
317 +# context=dcgm.gpu.inventory.software family=gpu inventory dimension=inforom_image_ver
318 +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image ver
319 +# context=dcgm.gpu.internal.boundary family=gpu internal dimension=last_connectx_field_id
320 +# DCGM_FI_DEV_LAST_CONNECTX_FIELD_ID, gauge, Last connectx field id
321 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=low_utilization_violation
322 +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, gauge, Low util violation
323 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=max_mem_clock
324 +DCGM_FI_DEV_MAX_MEM_CLOCK, gauge, Max mem clock
325 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=max_sm_clock
326 +DCGM_FI_DEV_MAX_SM_CLOCK, gauge, Max sm clock
327 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=max_video_clock
328 +DCGM_FI_DEV_MAX_VIDEO_CLOCK, gauge, Max video clock
329 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=memory
330 +DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temp
331 +# context=dcgm.gpu.reliability.memory_health family=gpu reliability dimension=memory_unrepairable_flag
332 +# DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG, gauge, Memory unrepairable flag
333 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=mem_affinity_0
334 +# DCGM_FI_DEV_MEM_AFFINITY_0, gauge, Mem affinity 0
335 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=mem_affinity_1
336 +# DCGM_FI_DEV_MEM_AFFINITY_1, gauge, Mem affinity 1
337 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=mem_affinity_2
338 +# DCGM_FI_DEV_MEM_AFFINITY_2, gauge, Mem affinity 2
339 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=mem_affinity_3
340 +# DCGM_FI_DEV_MEM_AFFINITY_3, gauge, Mem affinity 3
341 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=memory
342 +DCGM_FI_DEV_MEM_CLOCK, gauge, Mem clock
343 +# context=dcgm.gpu.compute.utilization family=gpu compute dimension=memory_copy
344 +DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Mem copy util
345 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=mem_max_op_temp
346 +DCGM_FI_DEV_MEM_MAX_OP_TEMP, gauge, Mem max op temp
347 +# context=dcgm.gpu.capability.support family=gpu capability dimension=mig_attributes
348 +# DCGM_FI_DEV_MIG_ATTRIBUTES, gauge, Mig attributes
349 +# context=dcgm.gpu.capability.support family=gpu capability dimension=mig_ci_info
350 +# DCGM_FI_DEV_MIG_CI_INFO, gauge, Mig ci info
351 +# context=dcgm.gpu.capability.support family=gpu capability dimension=mig_gi_info
352 +# DCGM_FI_DEV_MIG_GI_INFO, gauge, Mig gi info
353 +# context=dcgm.gpu.capability.support family=gpu capability dimension=mig_max_slices
354 +DCGM_FI_DEV_MIG_MAX_SLICES, gauge, Mig max slices
355 +# context=dcgm.gpu.state.virtualization family=gpu state dimension=mig_mode
356 +DCGM_FI_DEV_MIG_MODE, gauge, Mig mode
357 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=minor_number
358 +# DCGM_FI_DEV_MINOR_NUMBER, label, Minor number
359 +# context=dcgm.gpu.cpu.power family=gpu cpu dimension=module_power_util_current
360 +# DCGM_FI_DEV_MODULE_POWER_UTIL_CURRENT, gauge, Module power util current
361 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=name
362 +# DCGM_FI_DEV_NAME, label, Name
363 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l0
364 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, Nvlink bandwidth l0
365 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l1
366 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L1, counter, Nvlink bandwidth l1
367 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l10
368 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L10, counter, Nvlink bandwidth l10
369 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l11
370 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L11, counter, Nvlink bandwidth l11
371 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l12
372 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L12, counter, Nvlink bandwidth l12
373 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l13
374 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L13, counter, Nvlink bandwidth l13
375 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l14
376 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L14, counter, Nvlink bandwidth l14
377 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l15
378 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L15, counter, Nvlink bandwidth l15
379 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l16
380 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L16, counter, Nvlink bandwidth l16
381 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l17
382 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L17, counter, Nvlink bandwidth l17
383 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l2
384 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L2, counter, Nvlink bandwidth l2
385 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l3
386 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L3, counter, Nvlink bandwidth l3
387 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l4
388 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L4, counter, Nvlink bandwidth l4
389 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l5
390 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L5, counter, Nvlink bandwidth l5
391 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l6
392 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L6, counter, Nvlink bandwidth l6
393 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l7
394 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L7, counter, Nvlink bandwidth l7
395 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l8
396 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L8, counter, Nvlink bandwidth l8
397 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth_l9
398 +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L9, counter, Nvlink bandwidth l9
399 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_bandwidth
400 +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Nvlink bandwidth total
401 +# context=dcgm.nvlink.interconnect.ber family=nvlink interconnect dimension=nvlink_count_effective_ber
402 +# DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER, counter, Nvlink count effective ber
403 +# context=dcgm.nvlink.interconnect.ber family=nvlink interconnect dimension=nvlink_count_effective_ber_float
404 +# DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER_FLOAT, counter, Nvlink count effective ber float
405 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_effective_errors
406 +# DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS, counter, Nvlink count effective errors
407 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_0
408 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_0, counter, Nvlink count fec history 0
409 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_1
410 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_1, counter, Nvlink count fec history 1
411 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_10
412 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_10, counter, Nvlink count fec history 10
413 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_11
414 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_11, counter, Nvlink count fec history 11
415 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_12
416 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_12, counter, Nvlink count fec history 12
417 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_13
418 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_13, counter, Nvlink count fec history 13
419 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_14
420 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_14, counter, Nvlink count fec history 14
421 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_15
422 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_15, counter, Nvlink count fec history 15
423 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_2
424 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_2, counter, Nvlink count fec history 2
425 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_3
426 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_3, counter, Nvlink count fec history 3
427 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_4
428 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_4, counter, Nvlink count fec history 4
429 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_5
430 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_5, counter, Nvlink count fec history 5
431 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_6
432 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_6, counter, Nvlink count fec history 6
433 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_7
434 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_7, counter, Nvlink count fec history 7
435 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_8
436 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_8, counter, Nvlink count fec history 8
437 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_fec_history_9
438 +# DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_9, counter, Nvlink count fec history 9
439 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_link_recovery_events
440 +# DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS, counter, Nvlink count link recovery events
441 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_link_recovery_failed_events
442 +# DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS, counter, Nvlink count link recovery failed events
443 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_link_recovery_successful_events
444 +# DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_SUCCESSFUL_EVENTS, counter, Nvlink count link recovery successful events
445 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_local_link_integrity_errors
446 +# DCGM_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS, counter, Nvlink count local link integrity errors
447 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_rx_buffer_overrun_errors
448 +# DCGM_FI_DEV_NVLINK_COUNT_RX_BUFFER_OVERRUN_ERRORS, counter, Nvlink count rx buffer overrun errors
449 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_count_rx
450 +DCGM_FI_DEV_NVLINK_COUNT_RX_BYTES, counter, Nvlink count rx bytes
451 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_rx_errors
452 +# DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS, counter, Nvlink count rx errors
453 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_rx_general_errors
454 +# DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS, counter, Nvlink count rx general errors
455 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_rx_malformed_packet_errors
456 +# DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS, counter, Nvlink count rx malformed packet errors
457 +# context=dcgm.nvlink.interconnect.traffic family=nvlink interconnect dimension=nvlink_count_rx_packets
458 +# DCGM_FI_DEV_NVLINK_COUNT_RX_PACKETS, counter, Nvlink count rx packets
459 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_rx_remote_errors
460 +# DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS, counter, Nvlink count rx remote errors
461 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_rx_symbol_errors
462 +# DCGM_FI_DEV_NVLINK_COUNT_RX_SYMBOL_ERRORS, counter, Nvlink count rx symbol errors
463 +# context=dcgm.nvlink.interconnect.ber family=nvlink interconnect dimension=nvlink_count_symbol_ber
464 +# DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER, counter, Nvlink count symbol ber
465 +# context=dcgm.nvlink.interconnect.ber family=nvlink interconnect dimension=nvlink_count_symbol_ber_float
466 +# DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER_FLOAT, counter, Nvlink count symbol ber float
467 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_count_tx
468 +DCGM_FI_DEV_NVLINK_COUNT_TX_BYTES, counter, Nvlink count tx bytes
469 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_count_tx_discards
470 +# DCGM_FI_DEV_NVLINK_COUNT_TX_DISCARDS, counter, Nvlink count tx discards
471 +# context=dcgm.nvlink.interconnect.traffic family=nvlink interconnect dimension=nvlink_count_tx_packets
472 +# DCGM_FI_DEV_NVLINK_COUNT_TX_PACKETS, counter, Nvlink count tx packets
473 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l0
474 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0, counter, Nvlink crc data error count l0
475 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l1
476 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1, counter, Nvlink crc data error count l1
477 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l10
478 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10, counter, Nvlink crc data error count l10
479 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l11
480 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11, counter, Nvlink crc data error count l11
481 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l12
482 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L12, counter, Nvlink crc data error count l12
483 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l13
484 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L13, counter, Nvlink crc data error count l13
485 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l14
486 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L14, counter, Nvlink crc data error count l14
487 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l15
488 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L15, counter, Nvlink crc data error count l15
489 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l16
490 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L16, counter, Nvlink crc data error count l16
491 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l17
492 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L17, counter, Nvlink crc data error count l17
493 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l2
494 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2, counter, Nvlink crc data error count l2
495 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l3
496 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3, counter, Nvlink crc data error count l3
497 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l4
498 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4, counter, Nvlink crc data error count l4
499 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l5
500 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5, counter, Nvlink crc data error count l5
501 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l6
502 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6, counter, Nvlink crc data error count l6
503 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l7
504 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7, counter, Nvlink crc data error count l7
505 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l8
506 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8, counter, Nvlink crc data error count l8
507 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error_count_l9
508 +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9, counter, Nvlink crc data error count l9
509 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_data_error
510 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Nvlink crc data error count total
511 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l0
512 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0, counter, Nvlink crc flit error count l0
513 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l1
514 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1, counter, Nvlink crc flit error count l1
515 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l10
516 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10, counter, Nvlink crc flit error count l10
517 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l11
518 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11, counter, Nvlink crc flit error count l11
519 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l12
520 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L12, counter, Nvlink crc flit error count l12
521 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l13
522 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L13, counter, Nvlink crc flit error count l13
523 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l14
524 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L14, counter, Nvlink crc flit error count l14
525 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l15
526 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L15, counter, Nvlink crc flit error count l15
527 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l16
528 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L16, counter, Nvlink crc flit error count l16
529 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l17
530 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L17, counter, Nvlink crc flit error count l17
531 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l2
532 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2, counter, Nvlink crc flit error count l2
533 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l3
534 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3, counter, Nvlink crc flit error count l3
535 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l4
536 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4, counter, Nvlink crc flit error count l4
537 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l5
538 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5, counter, Nvlink crc flit error count l5
539 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l6
540 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6, counter, Nvlink crc flit error count l6
541 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l7
542 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7, counter, Nvlink crc flit error count l7
543 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l8
544 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8, counter, Nvlink crc flit error count l8
545 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error_count_l9
546 +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9, counter, Nvlink crc flit error count l9
547 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_crc_flit_error
548 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Nvlink crc flit error count total
549 +# context=dcgm.nvlink.memory.ecc_error_rate family=nvlink memory dimension=nvlink_ecc_data_error
550 +# DCGM_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL, counter, Nvlink ecc data error count total
551 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_error_dl_crc
552 +DCGM_FI_DEV_NVLINK_ERROR_DL_CRC, counter, Nvlink error dl crc
553 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_error_dl_recovery
554 +DCGM_FI_DEV_NVLINK_ERROR_DL_RECOVERY, counter, Nvlink error dl recovery
555 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_error_dl_replay
556 +DCGM_FI_DEV_NVLINK_ERROR_DL_REPLAY, counter, Nvlink error dl replay
557 +# context=dcgm.nvlink.interconnect.state family=nvlink interconnect dimension=nvlink_get_state
558 +# DCGM_FI_DEV_NVLINK_GET_STATE, gauge, Nvlink get state
559 +# context=dcgm.nvlink.interconnect.congestion family=nvlink interconnect dimension=nvlink_ppcnt_ibpc_port_xmit_wait
560 +# DCGM_FI_DEV_NVLINK_PPCNT_IBPC_PORT_XMIT_WAIT, gauge, Nvlink ppcnt ibpc port xmit wait
561 +# context=dcgm.nvlink.interconnect.state family=nvlink interconnect dimension=nvlink_ppcnt_physical_link_down_counter
562 +# DCGM_FI_DEV_NVLINK_PPCNT_PHYSICAL_LINK_DOWN_COUNTER, counter, Nvlink ppcnt physical link down counter
563 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_ppcnt_physical_successful_recovery_events
564 +# DCGM_FI_DEV_NVLINK_PPCNT_PHYSICAL_SUCCESSFUL_RECOVERY_EVENTS, counter, Nvlink ppcnt physical successful recovery events
565 +# context=dcgm.nvlink.interconnect.traffic family=nvlink interconnect dimension=nvlink_ppcnt_plr_rcv_codes
566 +# DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_CODES, counter, Nvlink ppcnt plr rcv codes
567 +# context=dcgm.nvlink.interconnect.state family=nvlink interconnect dimension=nvlink_ppcnt_plr_rcv_code_err
568 +# DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_CODE_ERR, gauge, Nvlink ppcnt plr rcv code err
569 +# context=dcgm.nvlink.interconnect.errors family=nvlink interconnect dimension=nvlink_ppcnt_plr_rcv_uncorrectable_code
570 +# DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_UNCORRECTABLE_CODE, gauge, Nvlink ppcnt plr rcv uncorrectable code
571 +# context=dcgm.nvlink.interconnect.state family=nvlink interconnect dimension=nvlink_ppcnt_plr_sync_events
572 +# DCGM_FI_DEV_NVLINK_PPCNT_PLR_SYNC_EVENTS, gauge, Nvlink ppcnt plr sync events
573 +# context=dcgm.nvlink.interconnect.traffic family=nvlink interconnect dimension=nvlink_ppcnt_plr_xmit_codes
574 +# DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_CODES, counter, Nvlink ppcnt plr xmit codes
575 +# context=dcgm.nvlink.interconnect.traffic family=nvlink interconnect dimension=nvlink_ppcnt_plr_xmit_retry_codes
576 +# DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_RETRY_CODES, counter, Nvlink ppcnt plr xmit retry codes
577 +# context=dcgm.nvlink.interconnect.state family=nvlink interconnect dimension=nvlink_ppcnt_plr_xmit_retry_events
578 +# DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_RETRY_EVENTS, gauge, Nvlink ppcnt plr xmit retry events
579 +# context=dcgm.nvlink.internal.boundary family=nvlink internal dimension=nvlink_ppcnt_recovery_time_between_last_two
580 +# DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TIME_BETWEEN_LAST_TWO, counter, Nvlink ppcnt recovery time between last two
581 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_ppcnt_recovery_time_since_last
582 +# DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TIME_SINCE_LAST, counter, Nvlink ppcnt recovery time since last
583 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_ppcnt_recovery_total_successful_events
584 +# DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TOTAL_SUCCESSFUL_EVENTS, counter, Nvlink ppcnt recovery total successful events
585 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_pprm_oper_recovery
586 +# DCGM_FI_DEV_NVLINK_PPRM_OPER_RECOVERY, counter, Nvlink pprm oper recovery
587 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l0
588 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0, counter, Nvlink recovery error count l0
589 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l1
590 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1, counter, Nvlink recovery error count l1
591 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l10
592 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10, counter, Nvlink recovery error count l10
593 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l11
594 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11, counter, Nvlink recovery error count l11
595 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l12
596 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L12, counter, Nvlink recovery error count l12
597 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l13
598 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L13, counter, Nvlink recovery error count l13
599 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l14
600 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L14, counter, Nvlink recovery error count l14
601 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l15
602 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L15, counter, Nvlink recovery error count l15
603 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l16
604 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L16, counter, Nvlink recovery error count l16
605 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l17
606 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L17, counter, Nvlink recovery error count l17
607 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l2
608 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2, counter, Nvlink recovery error count l2
609 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l3
610 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3, counter, Nvlink recovery error count l3
611 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l4
612 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4, counter, Nvlink recovery error count l4
613 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l5
614 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5, counter, Nvlink recovery error count l5
615 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l6
616 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6, counter, Nvlink recovery error count l6
617 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l7
618 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7, counter, Nvlink recovery error count l7
619 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l8
620 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8, counter, Nvlink recovery error count l8
621 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error_count_l9
622 +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9, counter, Nvlink recovery error count l9
623 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_recovery_error
624 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Nvlink recovery error count total
625 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l0
626 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0, counter, Nvlink replay error count l0
627 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l1
628 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1, counter, Nvlink replay error count l1
629 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l10
630 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10, counter, Nvlink replay error count l10
631 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l11
632 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11, counter, Nvlink replay error count l11
633 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l12
634 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L12, counter, Nvlink replay error count l12
635 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l13
636 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L13, counter, Nvlink replay error count l13
637 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l14
638 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L14, counter, Nvlink replay error count l14
639 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l15
640 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L15, counter, Nvlink replay error count l15
641 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l16
642 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L16, counter, Nvlink replay error count l16
643 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l17
644 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L17, counter, Nvlink replay error count l17
645 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l2
646 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2, counter, Nvlink replay error count l2
647 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l3
648 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3, counter, Nvlink replay error count l3
649 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l4
650 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4, counter, Nvlink replay error count l4
651 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l5
652 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5, counter, Nvlink replay error count l5
653 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l6
654 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6, counter, Nvlink replay error count l6
655 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l7
656 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7, counter, Nvlink replay error count l7
657 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l8
658 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8, counter, Nvlink replay error count l8
659 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error_count_l9
660 +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9, counter, Nvlink replay error count l9
661 +# context=dcgm.nvlink.interconnect.error_rate family=nvlink interconnect dimension=nvlink_replay_error
662 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Nvlink replay error count total
663 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l0
664 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L0, counter, Nvlink rx bandwidth l0
665 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l1
666 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L1, counter, Nvlink rx bandwidth l1
667 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l10
668 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L10, counter, Nvlink rx bandwidth l10
669 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l11
670 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L11, counter, Nvlink rx bandwidth l11
671 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l12
672 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L12, counter, Nvlink rx bandwidth l12
673 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l13
674 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L13, counter, Nvlink rx bandwidth l13
675 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l14
676 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L14, counter, Nvlink rx bandwidth l14
677 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l15
678 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L15, counter, Nvlink rx bandwidth l15
679 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l16
680 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L16, counter, Nvlink rx bandwidth l16
681 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l17
682 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L17, counter, Nvlink rx bandwidth l17
683 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l2
684 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L2, counter, Nvlink rx bandwidth l2
685 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l3
686 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L3, counter, Nvlink rx bandwidth l3
687 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l4
688 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L4, counter, Nvlink rx bandwidth l4
689 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l5
690 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L5, counter, Nvlink rx bandwidth l5
691 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l6
692 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L6, counter, Nvlink rx bandwidth l6
693 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l7
694 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L7, counter, Nvlink rx bandwidth l7
695 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l8
696 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L8, counter, Nvlink rx bandwidth l8
697 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth_l9
698 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L9, counter, Nvlink rx bandwidth l9
699 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx_bandwidth
700 +# DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_TOTAL, counter, Nvlink rx bandwidth total
701 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l0
702 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L0, counter, Nvlink tx bandwidth l0
703 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l1
704 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L1, counter, Nvlink tx bandwidth l1
705 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l10
706 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L10, counter, Nvlink tx bandwidth l10
707 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l11
708 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L11, counter, Nvlink tx bandwidth l11
709 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l12
710 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L12, counter, Nvlink tx bandwidth l12
711 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l13
712 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L13, counter, Nvlink tx bandwidth l13
713 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l14
714 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L14, counter, Nvlink tx bandwidth l14
715 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l15
716 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L15, counter, Nvlink tx bandwidth l15
717 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l16
718 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L16, counter, Nvlink tx bandwidth l16
719 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l17
720 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L17, counter, Nvlink tx bandwidth l17
721 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l2
722 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L2, counter, Nvlink tx bandwidth l2
723 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l3
724 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L3, counter, Nvlink tx bandwidth l3
725 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l4
726 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L4, counter, Nvlink tx bandwidth l4
727 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l5
728 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L5, counter, Nvlink tx bandwidth l5
729 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l6
730 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L6, counter, Nvlink tx bandwidth l6
731 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l7
732 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L7, counter, Nvlink tx bandwidth l7
733 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l8
734 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L8, counter, Nvlink tx bandwidth l8
735 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth_l9
736 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L9, counter, Nvlink tx bandwidth l9
737 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx_bandwidth
738 +# DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_TOTAL, counter, Nvlink tx bandwidth total
739 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=nvml_index
740 +# DCGM_FI_DEV_NVML_INDEX, gauge, Nvml index
741 +# context=dcgm.nvswitch.interconnect.nvswitch.current family=nvswitch interconnect dimension=nvswitch_current_iddq
742 +# DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ, gauge, Nvswitch current iddq
743 +# context=dcgm.nvswitch.interconnect.nvswitch.current family=nvswitch interconnect dimension=nvswitch_current_iddq_dvdd
744 +# DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ_DVDD, gauge, Nvswitch current iddq dvdd
745 +# context=dcgm.nvswitch.interconnect.nvswitch.current family=nvswitch interconnect dimension=nvswitch_current_iddq_rev
746 +# DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ_REV, gauge, Nvswitch current iddq rev
747 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_device_uuid
748 +# DCGM_FI_DEV_NVSWITCH_DEVICE_UUID, label, Nvswitch device uuid
749 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_fatal_errors
750 +DCGM_FI_DEV_NVSWITCH_FATAL_ERRORS, counter, Nvswitch fatal errors
751 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors
752 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS, counter, Nvswitch link crc errors
753 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane0
754 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE0, counter, Nvswitch link crc errors lane0
755 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane1
756 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE1, counter, Nvswitch link crc errors lane1
757 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane2
758 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE2, counter, Nvswitch link crc errors lane2
759 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane3
760 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE3, counter, Nvswitch link crc errors lane3
761 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane4
762 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE4, counter, Nvswitch link crc errors lane4
763 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane5
764 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE5, counter, Nvswitch link crc errors lane5
765 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane6
766 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE6, counter, Nvswitch link crc errors lane6
767 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_crc_errors_lane7
768 +# DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE7, counter, Nvswitch link crc errors lane7
769 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_link_device_link_id
770 +# DCGM_FI_DEV_NVSWITCH_LINK_DEVICE_LINK_ID, gauge, Nvswitch link device link id
771 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_link_device_link_sid
772 +# DCGM_FI_DEV_NVSWITCH_LINK_DEVICE_LINK_SID, gauge, Nvswitch link device link sid
773 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors
774 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS, counter, Nvswitch link ecc errors
775 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane0
776 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE0, counter, Nvswitch link ecc errors lane0
777 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane1
778 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE1, counter, Nvswitch link ecc errors lane1
779 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane2
780 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE2, counter, Nvswitch link ecc errors lane2
781 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane3
782 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE3, counter, Nvswitch link ecc errors lane3
783 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane4
784 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE4, counter, Nvswitch link ecc errors lane4
785 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane5
786 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE5, counter, Nvswitch link ecc errors lane5
787 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane6
788 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE6, counter, Nvswitch link ecc errors lane6
789 +# context=dcgm.nvswitch.memory.ecc_error_rate family=nvswitch memory dimension=nvswitch_link_ecc_errors_lane7
790 +# DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE7, counter, Nvswitch link ecc errors lane7
791 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_fatal_errors
792 +DCGM_FI_DEV_NVSWITCH_LINK_FATAL_ERRORS, counter, Nvswitch link fatal errors
793 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_flit_errors
794 +# DCGM_FI_DEV_NVSWITCH_LINK_FLIT_ERRORS, counter, Nvswitch link flit errors
795 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_link_id
796 +# DCGM_FI_DEV_NVSWITCH_LINK_ID, gauge, Nvswitch link id
797 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_count_vc0
798 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC0, counter, Nvswitch link latency count vc0
799 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_count_vc1
800 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC1, counter, Nvswitch link latency count vc1
801 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_count_vc2
802 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC2, counter, Nvswitch link latency count vc2
803 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_count_vc3
804 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC3, counter, Nvswitch link latency count vc3
805 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_high_vc0
806 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC0, counter, Nvswitch link latency high vc0
807 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_high_vc1
808 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC1, counter, Nvswitch link latency high vc1
809 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_high_vc2
810 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC2, counter, Nvswitch link latency high vc2
811 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_high_vc3
812 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC3, counter, Nvswitch link latency high vc3
813 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_low_vc0
814 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC0, counter, Nvswitch link latency low vc0
815 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_low_vc1
816 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC1, counter, Nvswitch link latency low vc1
817 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_low_vc2
818 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC2, counter, Nvswitch link latency low vc2
819 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_low_vc3
820 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC3, counter, Nvswitch link latency low vc3
821 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_medium_vc0
822 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC0, counter, Nvswitch link latency medium vc0
823 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_medium_vc1
824 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC1, counter, Nvswitch link latency medium vc1
825 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_medium_vc2
826 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC2, counter, Nvswitch link latency medium vc2
827 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_medium_vc3
828 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC3, counter, Nvswitch link latency medium vc3
829 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_panic_vc0
830 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC0, counter, Nvswitch link latency panic vc0
831 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_panic_vc1
832 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC1, counter, Nvswitch link latency panic vc1
833 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_panic_vc2
834 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC2, counter, Nvswitch link latency panic vc2
835 +# context=dcgm.nvswitch.interconnect.nvswitch.latency family=nvswitch interconnect dimension=nvswitch_link_latency_panic_vc3
836 +# DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC3, counter, Nvswitch link latency panic vc3
837 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_non_fatal_errors
838 +DCGM_FI_DEV_NVSWITCH_LINK_NON_FATAL_ERRORS, counter, Nvswitch link non fatal errors
839 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_recovery_errors
840 +# DCGM_FI_DEV_NVSWITCH_LINK_RECOVERY_ERRORS, counter, Nvswitch link recovery errors
841 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_link_remote_pcie_bus
842 +# DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_BUS, gauge, Nvswitch link remote pcie bus
843 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_link_remote_pcie_device
844 +# DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_DEVICE, gauge, Nvswitch link remote pcie device
845 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_link_remote_pcie_domain
846 +# DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_DOMAIN, gauge, Nvswitch link remote pcie domain
847 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_link_remote_pcie_function
848 +# DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_FUNCTION, gauge, Nvswitch link remote pcie function
849 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_link_replay_errors
850 +DCGM_FI_DEV_NVSWITCH_LINK_REPLAY_ERRORS, counter, Nvswitch link replay errors
851 +# context=dcgm.nvswitch.interconnect.nvswitch.status family=nvswitch interconnect dimension=nvswitch_link_status
852 +# DCGM_FI_DEV_NVSWITCH_LINK_STATUS, gauge, Nvswitch link status
853 +# context=dcgm.nvswitch.interconnect.nvswitch.throughput family=nvswitch interconnect dimension=nvswitch_link_throughput_rx
854 +DCGM_FI_DEV_NVSWITCH_LINK_THROUGHPUT_RX, counter, Nvswitch link throughput rx
855 +# context=dcgm.nvswitch.interconnect.nvswitch.throughput family=nvswitch interconnect dimension=nvswitch_link_throughput_tx
856 +DCGM_FI_DEV_NVSWITCH_LINK_THROUGHPUT_TX, counter, Nvswitch link throughput tx
857 +# context=dcgm.nvswitch.interconnect.nvswitch.status family=nvswitch interconnect dimension=nvswitch_link_type
858 +# DCGM_FI_DEV_NVSWITCH_LINK_TYPE, gauge, Nvswitch link type
859 +# context=dcgm.nvswitch.interconnect.nvswitch.errors family=nvswitch interconnect dimension=nvswitch_non_fatal_errors
860 +DCGM_FI_DEV_NVSWITCH_NON_FATAL_ERRORS, counter, Nvswitch non fatal errors
861 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_pcie_bus
862 +# DCGM_FI_DEV_NVSWITCH_PCIE_BUS, gauge, Nvswitch pcie bus
863 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_pcie_device
864 +# DCGM_FI_DEV_NVSWITCH_PCIE_DEVICE, gauge, Nvswitch pcie device
865 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_pcie_domain
866 +# DCGM_FI_DEV_NVSWITCH_PCIE_DOMAIN, gauge, Nvswitch pcie domain
867 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_pcie_function
868 +# DCGM_FI_DEV_NVSWITCH_PCIE_FUNCTION, gauge, Nvswitch pcie function
869 +# context=dcgm.nvswitch.interconnect.nvswitch.topology family=nvswitch interconnect dimension=nvswitch_phys_id
870 +# DCGM_FI_DEV_NVSWITCH_PHYS_ID, gauge, Nvswitch phys id
871 +# context=dcgm.nvswitch.interconnect.nvswitch.power family=nvswitch interconnect dimension=nvswitch_power_dvdd
872 +# DCGM_FI_DEV_NVSWITCH_POWER_DVDD, gauge, Nvswitch power dvdd
873 +# context=dcgm.nvswitch.interconnect.nvswitch.power family=nvswitch interconnect dimension=nvswitch_power_hvdd
874 +# DCGM_FI_DEV_NVSWITCH_POWER_HVDD, gauge, Nvswitch power hvdd
875 +# context=dcgm.nvswitch.interconnect.nvswitch.power family=nvswitch interconnect dimension=nvswitch_power_vdd
876 +# DCGM_FI_DEV_NVSWITCH_POWER_VDD, gauge, Nvswitch power vdd
877 +# context=dcgm.nvswitch.interconnect.nvswitch.status family=nvswitch interconnect dimension=nvswitch_reset_required
878 +# DCGM_FI_DEV_NVSWITCH_RESET_REQUIRED, gauge, Nvswitch reset required
879 +# context=dcgm.nvswitch.thermal.temperature family=nvswitch thermal dimension=nvswitch_temperature_current
880 +DCGM_FI_DEV_NVSWITCH_TEMPERATURE_CURRENT, gauge, Nvswitch temperature current
881 +# context=dcgm.nvswitch.thermal.temperature family=nvswitch thermal dimension=nvswitch_temperature_limit_shutdown
882 +# DCGM_FI_DEV_NVSWITCH_TEMPERATURE_LIMIT_SHUTDOWN, gauge, Nvswitch temperature limit shutdown
883 +# context=dcgm.nvswitch.thermal.temperature family=nvswitch thermal dimension=nvswitch_temperature_limit_slowdown
884 +# DCGM_FI_DEV_NVSWITCH_TEMPERATURE_LIMIT_SLOWDOWN, gauge, Nvswitch temperature limit slowdown
885 +# context=dcgm.nvswitch.interconnect.nvswitch.throughput family=nvswitch interconnect dimension=nvswitch_throughput_rx
886 +# DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX, counter, Nvswitch throughput rx
887 +# context=dcgm.nvswitch.interconnect.nvswitch.throughput family=nvswitch interconnect dimension=nvswitch_throughput_tx
888 +# DCGM_FI_DEV_NVSWITCH_THROUGHPUT_TX, counter, Nvswitch throughput tx
889 +# context=dcgm.nvswitch.interconnect.nvswitch.voltage family=nvswitch interconnect dimension=nvswitch_voltage_mvolt
890 +# DCGM_FI_DEV_NVSWITCH_VOLTAGE_MVOLT, gauge, Nvswitch voltage mvolt
891 +# context=dcgm.gpu.inventory.software family=gpu inventory dimension=oem_inforom_ver
892 +# DCGM_FI_DEV_OEM_INFOROM_VER, label, Oem inforom ver
893 +# context=dcgm.nvlink.interconnect.state family=nvlink interconnect dimension=p2p_nvlink_status
894 +# DCGM_FI_DEV_P2P_NVLINK_STATUS, gauge, P2p nvlink status
895 +# context=dcgm.gpu.interconnect.pcie.error_rate family=gpu interconnect/pcie dimension=pcie_count_correctable_errors
896 +# DCGM_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS, counter, Pcie count correctable errors
897 +# context=dcgm.gpu.interconnect.pcie.link.generation family=gpu interconnect/pcie dimension=link_gen
898 +DCGM_FI_DEV_PCIE_LINK_GEN, gauge, Pcie link gen
899 +# context=dcgm.gpu.interconnect.pcie.link.width family=gpu interconnect/pcie dimension=link_width
900 +DCGM_FI_DEV_PCIE_LINK_WIDTH, gauge, Pcie link width
901 +# context=dcgm.gpu.interconnect.pcie.link.generation family=gpu interconnect/pcie dimension=max_link_gen
902 +# DCGM_FI_DEV_PCIE_MAX_LINK_GEN, gauge, Pcie max link gen
903 +# context=dcgm.gpu.interconnect.pcie.link.width family=gpu interconnect/pcie dimension=max_link_width
904 +# DCGM_FI_DEV_PCIE_MAX_LINK_WIDTH, gauge, Pcie max link width
905 +# context=dcgm.gpu.interconnect.pcie.error_rate family=gpu interconnect/pcie dimension=pcie_replay
906 +DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Pcie replay counter
907 +# context=dcgm.gpu.interconnect.pcie.throughput family=gpu interconnect/pcie dimension=pcie_rx_throughput
908 +DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Pcie rx throughput
909 +# context=dcgm.gpu.interconnect.pcie.throughput family=gpu interconnect/pcie dimension=pcie_tx_throughput
910 +DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Pcie tx throughput
911 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=pci_busid
912 +# DCGM_FI_DEV_PCI_BUSID, label, Pci busid
913 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=pci_combined_id
914 +# DCGM_FI_DEV_PCI_COMBINED_ID, gauge, Pci combined id
915 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=pci_subsys_id
916 +# DCGM_FI_DEV_PCI_SUBSYS_ID, gauge, Pci subsys id
917 +# context=dcgm.gpu.state.configuration family=gpu state dimension=persistence_mode
918 +DCGM_FI_DEV_PERSISTENCE_MODE, gauge, Persistence mode
919 +# context=dcgm.gpu.inventory.platform family=gpu inventory dimension=platform_chassis_serial_number
920 +# DCGM_FI_DEV_PLATFORM_CHASSIS_SERIAL_NUMBER, label, Platform chassis serial number
921 +# context=dcgm.gpu.inventory.platform family=gpu inventory dimension=platform_chassis_slot_number
922 +# DCGM_FI_DEV_PLATFORM_CHASSIS_SLOT_NUMBER, gauge, Platform chassis slot number
923 +# context=dcgm.gpu.inventory.platform family=gpu inventory dimension=platform_host_id
924 +# DCGM_FI_DEV_PLATFORM_HOST_ID, gauge, Platform host id
925 +# context=dcgm.gpu.inventory.platform family=gpu inventory dimension=platform_infiniband_guid
926 +# DCGM_FI_DEV_PLATFORM_INFINIBAND_GUID, gauge, Platform infiniband guid
927 +# context=dcgm.gpu.inventory.platform family=gpu inventory dimension=platform_module_id
928 +# DCGM_FI_DEV_PLATFORM_MODULE_ID, gauge, Platform module id
929 +# context=dcgm.gpu.inventory.platform family=gpu inventory dimension=platform_peer_type
930 +# DCGM_FI_DEV_PLATFORM_PEER_TYPE, gauge, Platform peer type
931 +# context=dcgm.gpu.inventory.platform family=gpu inventory dimension=platform_tray_index
932 +# DCGM_FI_DEV_PLATFORM_TRAY_INDEX, gauge, Platform tray index
933 +# context=dcgm.gpu.inventory.software family=gpu inventory dimension=power_inforom_ver
934 +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power inforom ver
935 +# context=dcgm.gpu.power.usage family=gpu power dimension=power_mgmt_limit
936 +DCGM_FI_DEV_POWER_MGMT_LIMIT, gauge, Power mgmt limit
937 +# context=dcgm.gpu.power.usage family=gpu power dimension=power_mgmt_limit_def
938 +DCGM_FI_DEV_POWER_MGMT_LIMIT_DEF, gauge, Power mgmt limit def
939 +# context=dcgm.gpu.power.usage family=gpu power dimension=power_mgmt_limit_max
940 +DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX, gauge, Power mgmt limit max
941 +# context=dcgm.gpu.power.usage family=gpu power dimension=power_mgmt_limit_min
942 +DCGM_FI_DEV_POWER_MGMT_LIMIT_MIN, gauge, Power mgmt limit min
943 +# context=dcgm.gpu.power.usage family=gpu power dimension=draw
944 +DCGM_FI_DEV_POWER_USAGE, gauge, Power usage
945 +# context=dcgm.gpu.power.usage family=gpu power dimension=power_usage_instant
946 +DCGM_FI_DEV_POWER_USAGE_INSTANT, gauge, Power usage instant
947 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=power_violation
948 +DCGM_FI_DEV_POWER_VIOLATION, counter, Power violation
949 +# context=dcgm.gpu.state.performance family=gpu state dimension=pstate
950 +DCGM_FI_DEV_PSTATE, gauge, Pstate
951 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_active_preset_profile
952 +# DCGM_FI_DEV_PWR_SMOOTHING_ACTIVE_PRESET_PROFILE, gauge, Pwr smoothing active preset profile
953 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_admin_override_percent_tmp_floor
954 +# DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_PERCENT_TMP_FLOOR, gauge, Pwr smoothing admin override percent tmp floor
955 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_admin_override_ramp_down_hyst_val
956 +# DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_HYST_VAL, gauge, Pwr smoothing admin override ramp down hyst val
957 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_admin_override_ramp_down_rate
958 +# DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_RATE, gauge, Pwr smoothing admin override ramp down rate
959 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_admin_override_ramp_up_rate
960 +# DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_UP_RATE, gauge, Pwr smoothing admin override ramp up rate
961 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_applied_tmp_ceil
962 +# DCGM_FI_DEV_PWR_SMOOTHING_APPLIED_TMP_CEIL, gauge, Pwr smoothing applied tmp ceil
963 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_applied_tmp_floor
964 +# DCGM_FI_DEV_PWR_SMOOTHING_APPLIED_TMP_FLOOR, gauge, Pwr smoothing applied tmp floor
965 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_enabled
966 +# DCGM_FI_DEV_PWR_SMOOTHING_ENABLED, gauge, Pwr smoothing enabled
967 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_hw_circuitry_percent_lifetime_remaining
968 +# DCGM_FI_DEV_PWR_SMOOTHING_HW_CIRCUITRY_PERCENT_LIFETIME_REMAINING, gauge, Pwr smoothing hw circuitry percent lifetime remaining
969 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_imm_ramp_down_enabled
970 +# DCGM_FI_DEV_PWR_SMOOTHING_IMM_RAMP_DOWN_ENABLED, gauge, Pwr smoothing imm ramp down enabled
971 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_max_num_preset_profiles
972 +# DCGM_FI_DEV_PWR_SMOOTHING_MAX_NUM_PRESET_PROFILES, gauge, Pwr smoothing max num preset profiles
973 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_max_percent_tmp_floor_setting
974 +# DCGM_FI_DEV_PWR_SMOOTHING_MAX_PERCENT_TMP_FLOOR_SETTING, gauge, Pwr smoothing max percent tmp floor setting
975 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_min_percent_tmp_floor_setting
976 +# DCGM_FI_DEV_PWR_SMOOTHING_MIN_PERCENT_TMP_FLOOR_SETTING, gauge, Pwr smoothing min percent tmp floor setting
977 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_priv_lvl
978 +# DCGM_FI_DEV_PWR_SMOOTHING_PRIV_LVL, gauge, Pwr smoothing priv lvl
979 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_profile_percent_tmp_floor
980 +# DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_PERCENT_TMP_FLOOR, gauge, Pwr smoothing profile percent tmp floor
981 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_profile_ramp_down_hyst_val
982 +# DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_DOWN_HYST_VAL, gauge, Pwr smoothing profile ramp down hyst val
983 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_profile_ramp_down_rate
984 +# DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_DOWN_RATE, gauge, Pwr smoothing profile ramp down rate
985 +# context=dcgm.gpu.power.smoothing family=gpu power dimension=pwr_smoothing_profile_ramp_up_rate
986 +# DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_UP_RATE, gauge, Pwr smoothing profile ramp up rate
987 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=reliability_violation
988 +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Reliability violation
989 +# context=dcgm.gpu.power.profiles family=gpu power dimension=requested_power_profile_mask
990 +# DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK, counter, Requested power profile mask
991 +# context=dcgm.gpu.memory.page_retirements family=gpu memory dimension=retired_dbe
992 +DCGM_FI_DEV_RETIRED_DBE, counter, Retired dbe
993 +# context=dcgm.gpu.memory.page_retirements family=gpu memory dimension=retired_pending
994 +DCGM_FI_DEV_RETIRED_PENDING, counter, Retired pending
995 +# context=dcgm.gpu.memory.page_retirements family=gpu memory dimension=retired_sbe
996 +DCGM_FI_DEV_RETIRED_SBE, counter, Retired sbe
997 +# context=dcgm.gpu.reliability.row_remap_status family=gpu reliability dimension=row_remap_failure
998 +DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Row remap failure
999 +# context=dcgm.gpu.reliability.row_remap_status family=gpu reliability dimension=row_remap_pending
1000 +DCGM_FI_DEV_ROW_REMAP_PENDING, gauge, Row remap pending
1001 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=serial
1002 +DCGM_FI_DEV_SERIAL, label, Serial
1003 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=shutdown_temp
1004 +DCGM_FI_DEV_SHUTDOWN_TEMP, gauge, Shutdown temp
1005 +# context=dcgm.gpu.thermal.temperature family=gpu thermal dimension=slowdown_temp
1006 +DCGM_FI_DEV_SLOWDOWN_TEMP, gauge, Slowdown temp
1007 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=sm
1008 +DCGM_FI_DEV_SM_CLOCK, gauge, Sm clock
1009 +# context=dcgm.gpu.capability.support family=gpu capability dimension=supported_clocks
1010 +# DCGM_FI_DEV_SUPPORTED_CLOCKS, gauge, Supported clocks
1011 +# context=dcgm.gpu.capability.support family=gpu capability dimension=supported_type_info
1012 +# DCGM_FI_DEV_SUPPORTED_TYPE_INFO, label, Supported type info
1013 +# context=dcgm.gpu.virtualization.vgpu.type family=gpu virtualization dimension=supported_vgpu_type_ids
1014 +# DCGM_FI_DEV_SUPPORTED_VGPU_TYPE_IDS, gauge, Supported vgpu type ids
1015 +# context=dcgm.gpu.state.configuration family=gpu state dimension=sync_boost_violation
1016 +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Sync boost violation
1017 +# context=dcgm.gpu.cpu.power family=gpu cpu dimension=sysio_power_util_current
1018 +# DCGM_FI_DEV_SYSIO_POWER_UTIL_CURRENT, gauge, Sysio power util current
1019 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=thermal_violation
1020 +DCGM_FI_DEV_THERMAL_VIOLATION, counter, Thermal violation
1021 +# context=dcgm.gpu.reliability.memory_health family=gpu reliability dimension=threshold_srm
1022 +DCGM_FI_DEV_THRESHOLD_SRM, gauge, Threshold srm
1023 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=total_app_clocks_violation
1024 +# DCGM_FI_DEV_TOTAL_APP_CLOCKS_VIOLATION, counter, Total app clocks violation
1025 +# context=dcgm.gpu.throttle.violations family=gpu throttle dimension=total_base_clocks_violation
1026 +# DCGM_FI_DEV_TOTAL_BASE_CLOCKS_VIOLATION, counter, Total base clocks violation
1027 +# context=dcgm.gpu.power.energy family=gpu power dimension=total
1028 +DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption
1029 +# context=dcgm.gpu.reliability.row_remap_events family=gpu reliability dimension=uncorrectable_remapped_rows
1030 +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Uncorrectable remapped rows
1031 +# context=dcgm.gpu.inventory.identity family=gpu inventory dimension=uuid
1032 +# DCGM_FI_DEV_UUID, label, Uuid
1033 +# context=dcgm.gpu.power.profiles family=gpu power dimension=valid_power_profile_mask
1034 +# DCGM_FI_DEV_VALID_POWER_PROFILE_MASK, counter, Valid power profile mask
1035 +# context=dcgm.gpu.inventory.software family=gpu inventory dimension=vbios_version
1036 +# DCGM_FI_DEV_VBIOS_VERSION, label, Vbios version
1037 +# context=dcgm.gpu.virtualization.vgpu.software family=gpu virtualization dimension=vgpu_driver_version
1038 +# DCGM_FI_DEV_VGPU_DRIVER_VERSION, label, Vgpu driver version
1039 +# context=dcgm.gpu.virtualization.vgpu.sessions family=gpu virtualization dimension=vgpu_enc_sessions_info
1040 +# DCGM_FI_DEV_VGPU_ENC_SESSIONS_INFO, gauge, Vgpu enc sessions info
1041 +# context=dcgm.gpu.virtualization.vgpu.sessions family=gpu virtualization dimension=vgpu_enc_stats
1042 +# DCGM_FI_DEV_VGPU_ENC_STATS, gauge, Vgpu enc stats
1043 +# context=dcgm.gpu.virtualization.vgpu.sessions family=gpu virtualization dimension=vgpu_fbc_sessions_info
1044 +# DCGM_FI_DEV_VGPU_FBC_SESSIONS_INFO, gauge, Vgpu fbc sessions info
1045 +# context=dcgm.gpu.virtualization.vgpu.sessions family=gpu virtualization dimension=vgpu_fbc_stats
1046 +# DCGM_FI_DEV_VGPU_FBC_STATS, gauge, Vgpu fbc stats
1047 +# context=dcgm.gpu.virtualization.vgpu.frame_rate family=gpu virtualization dimension=vgpu_frame_rate_limit
1048 +# DCGM_FI_DEV_VGPU_FRAME_RATE_LIMIT, gauge, Vgpu frame rate limit
1049 +# context=dcgm.gpu.virtualization.vgpu.instance family=gpu virtualization dimension=vgpu_instance_ids
1050 +# DCGM_FI_DEV_VGPU_INSTANCE_IDS, gauge, Vgpu instance ids
1051 +# context=dcgm.gpu.virtualization.vgpu.license family=gpu virtualization dimension=vgpu_instance_license_state
1052 +# DCGM_FI_DEV_VGPU_INSTANCE_LICENSE_STATE, gauge, Vgpu instance license state
1053 +# context=dcgm.gpu.virtualization.vgpu.license family=gpu virtualization dimension=vgpu_license_status
1054 +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, Vgpu license status
1055 +# context=dcgm.gpu.virtualization.vgpu.memory family=gpu virtualization dimension=vgpu_memory_usage
1056 +DCGM_FI_DEV_VGPU_MEMORY_USAGE, gauge, Vgpu memory usage
1057 +# context=dcgm.gpu.virtualization.vgpu.instance family=gpu virtualization dimension=vgpu_pci_id
1058 +# DCGM_FI_DEV_VGPU_PCI_ID, gauge, Vgpu pci id
1059 +# context=dcgm.gpu.virtualization.vgpu.utilization family=gpu virtualization dimension=vgpu_per_process_utilization
1060 +# DCGM_FI_DEV_VGPU_PER_PROCESS_UTILIZATION, gauge, Vgpu per process utilization
1061 +# context=dcgm.gpu.virtualization.vgpu.type family=gpu virtualization dimension=vgpu_type
1062 +# DCGM_FI_DEV_VGPU_TYPE, gauge, Vgpu type
1063 +# context=dcgm.gpu.virtualization.vgpu.type family=gpu virtualization dimension=vgpu_type_class
1064 +# DCGM_FI_DEV_VGPU_TYPE_CLASS, gauge, Vgpu type class
1065 +# context=dcgm.gpu.virtualization.vgpu.type family=gpu virtualization dimension=vgpu_type_info
1066 +# DCGM_FI_DEV_VGPU_TYPE_INFO, label, Vgpu type info
1067 +# context=dcgm.gpu.virtualization.vgpu.license family=gpu virtualization dimension=vgpu_type_license
1068 +# DCGM_FI_DEV_VGPU_TYPE_LICENSE, gauge, Vgpu type license
1069 +# context=dcgm.gpu.virtualization.vgpu.type family=gpu virtualization dimension=vgpu_type_name
1070 +# DCGM_FI_DEV_VGPU_TYPE_NAME, label, Vgpu type name
1071 +# context=dcgm.gpu.virtualization.vgpu.utilization family=gpu virtualization dimension=vgpu_utilizations
1072 +# DCGM_FI_DEV_VGPU_UTILIZATIONS, gauge, Vgpu utilizations
1073 +# context=dcgm.gpu.virtualization.vgpu.instance family=gpu virtualization dimension=vgpu_uuid
1074 +# DCGM_FI_DEV_VGPU_UUID, label, Vgpu uuid
1075 +# context=dcgm.gpu.virtualization.vgpu.vm family=gpu virtualization dimension=vgpu_vm_gpu_instance_id
1076 +# DCGM_FI_DEV_VGPU_VM_GPU_INSTANCE_ID, gauge, Vgpu vm gpu instance id
1077 +# context=dcgm.gpu.virtualization.vgpu.vm family=gpu virtualization dimension=vgpu_vm_id
1078 +# DCGM_FI_DEV_VGPU_VM_ID, label, Vgpu vm id
1079 +# context=dcgm.gpu.virtualization.vgpu.vm family=gpu virtualization dimension=vgpu_vm_name
1080 +# DCGM_FI_DEV_VGPU_VM_NAME, label, Vgpu vm name
1081 +# context=dcgm.gpu.clock.frequency family=gpu clock dimension=video_clock
1082 +DCGM_FI_DEV_VIDEO_CLOCK, gauge, Video clock
1083 +# context=dcgm.gpu.state.virtualization family=gpu state dimension=virtual_mode
1084 +DCGM_FI_DEV_VIRTUAL_MODE, gauge, Virtual mode
1085 +# context=dcgm.gpu.reliability.xid family=gpu reliability dimension=xid
1086 +DCGM_FI_DEV_XID_ERRORS, counter, Xid errors
1087 +# context=dcgm.exporter.inventory.software family=exporter inventory dimension=driver_version
1088 +DCGM_FI_DRIVER_VERSION, label, Driver version
1089 +# context=dcgm.nvswitch.internal.boundary family=nvswitch internal dimension=first_nvswitch_field_id
1090 +# DCGM_FI_FIRST_NVSWITCH_FIELD_ID, gauge, First nvswitch field id
1091 +# context=dcgm.gpu.internal.boundary family=gpu internal dimension=first_vgpu_field_id
1092 +# DCGM_FI_FIRST_VGPU_FIELD_ID, gauge, First vgpu field id
1093 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=gpu_topology_affinity
1094 +# DCGM_FI_GPU_TOPOLOGY_AFFINITY, gauge, Gpu topology affinity
1095 +# context=dcgm.nvlink.interconnect.state family=nvlink interconnect dimension=gpu_topology_nvlink
1096 +# DCGM_FI_GPU_TOPOLOGY_NVLINK, gauge, Gpu topology nvlink
1097 +# context=dcgm.gpu.topology.affinity family=gpu topology dimension=gpu_topology_pci
1098 +# DCGM_FI_GPU_TOPOLOGY_PCI, gauge, Gpu topology pci
1099 +# context=dcgm.gpu.health.status family=gpu health dimension=imex_daemon_status
1100 +# DCGM_FI_IMEX_DAEMON_STATUS, gauge, Imex daemon status
1101 +# context=dcgm.gpu.health.status family=gpu health dimension=imex_domain_status
1102 +# DCGM_FI_IMEX_DOMAIN_STATUS, gauge, Imex domain status
1103 +# context=dcgm.gpu.internal.boundary family=gpu internal dimension=internal_fields_0_end
1104 +# DCGM_FI_INTERNAL_FIELDS_0_END, gauge, Internal fields 0 end
1105 +# context=dcgm.gpu.internal.boundary family=gpu internal dimension=internal_fields_0_start
1106 +# DCGM_FI_INTERNAL_FIELDS_0_START, gauge, Internal fields 0 start
1107 +# context=dcgm.nvswitch.internal.boundary family=nvswitch internal dimension=last_nvswitch_field_id
1108 +# DCGM_FI_LAST_NVSWITCH_FIELD_ID, gauge, Last nvswitch field id
1109 +# context=dcgm.gpu.internal.boundary family=gpu internal dimension=last_vgpu_field_id
1110 +# DCGM_FI_LAST_VGPU_FIELD_ID, gauge, Last vgpu field id
1111 +# context=dcgm.exporter.inventory.software family=exporter inventory dimension=nvml_version
1112 +# DCGM_FI_NVML_VERSION, label, Nvml version
1113 +# context=dcgm.gpu.inventory.software family=gpu inventory dimension=process_name
1114 +# DCGM_FI_PROCESS_NAME, label, Process name
1115 +# context=dcgm.gpu.interconnect.throughput family=gpu interconnect/overview dimension=c2c_rx_all_bytes
1116 +# DCGM_FI_PROF_C2C_RX_ALL_BYTES, gauge, C2c rx all bytes
1117 +# context=dcgm.gpu.interconnect.throughput family=gpu interconnect/overview dimension=c2c_rx_data_bytes
1118 +# DCGM_FI_PROF_C2C_RX_DATA_BYTES, gauge, C2c rx data bytes
1119 +# context=dcgm.gpu.interconnect.throughput family=gpu interconnect/overview dimension=c2c_tx_all_bytes
1120 +# DCGM_FI_PROF_C2C_TX_ALL_BYTES, gauge, C2c tx all bytes
1121 +# context=dcgm.gpu.interconnect.throughput family=gpu interconnect/overview dimension=c2c_tx_data_bytes
1122 +# DCGM_FI_PROF_C2C_TX_DATA_BYTES, gauge, C2c tx data bytes
1123 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=dram
1124 +DCGM_FI_PROF_DRAM_ACTIVE, gauge, Dram active
1125 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=graphics_engine_active
1126 +DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Gr engine active
1127 +# context=dcgm.gpu.compute.cache.activity family=gpu compute dimension=hostmem_cache_hit
1128 +DCGM_FI_PROF_HOSTMEM_CACHE_HIT, gauge, Hostmem cache hit
1129 +# context=dcgm.gpu.compute.cache.activity family=gpu compute dimension=hostmem_cache_miss
1130 +DCGM_FI_PROF_HOSTMEM_CACHE_MISS, gauge, Hostmem cache miss
1131 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec0_active
1132 +# DCGM_FI_PROF_NVDEC0_ACTIVE, gauge, Nvdec0 active
1133 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec1_active
1134 +# DCGM_FI_PROF_NVDEC1_ACTIVE, gauge, Nvdec1 active
1135 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec2_active
1136 +# DCGM_FI_PROF_NVDEC2_ACTIVE, gauge, Nvdec2 active
1137 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec3_active
1138 +# DCGM_FI_PROF_NVDEC3_ACTIVE, gauge, Nvdec3 active
1139 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec4_active
1140 +# DCGM_FI_PROF_NVDEC4_ACTIVE, gauge, Nvdec4 active
1141 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec5_active
1142 +# DCGM_FI_PROF_NVDEC5_ACTIVE, gauge, Nvdec5 active
1143 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec6_active
1144 +# DCGM_FI_PROF_NVDEC6_ACTIVE, gauge, Nvdec6 active
1145 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvdec7_active
1146 +# DCGM_FI_PROF_NVDEC7_ACTIVE, gauge, Nvdec7 active
1147 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg0_active
1148 +# DCGM_FI_PROF_NVJPG0_ACTIVE, gauge, Nvjpg0 active
1149 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg1_active
1150 +# DCGM_FI_PROF_NVJPG1_ACTIVE, gauge, Nvjpg1 active
1151 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg2_active
1152 +# DCGM_FI_PROF_NVJPG2_ACTIVE, gauge, Nvjpg2 active
1153 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg3_active
1154 +# DCGM_FI_PROF_NVJPG3_ACTIVE, gauge, Nvjpg3 active
1155 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg4_active
1156 +# DCGM_FI_PROF_NVJPG4_ACTIVE, gauge, Nvjpg4 active
1157 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg5_active
1158 +# DCGM_FI_PROF_NVJPG5_ACTIVE, gauge, Nvjpg5 active
1159 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg6_active
1160 +# DCGM_FI_PROF_NVJPG6_ACTIVE, gauge, Nvjpg6 active
1161 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvjpg7_active
1162 +# DCGM_FI_PROF_NVJPG7_ACTIVE, gauge, Nvjpg7 active
1163 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l0_rx
1164 +# DCGM_FI_PROF_NVLINK_L0_RX_BYTES, gauge, Nvlink l0 rx bytes
1165 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l0_tx
1166 +# DCGM_FI_PROF_NVLINK_L0_TX_BYTES, gauge, Nvlink l0 tx bytes
1167 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l10_rx
1168 +# DCGM_FI_PROF_NVLINK_L10_RX_BYTES, gauge, Nvlink l10 rx bytes
1169 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l10_tx
1170 +# DCGM_FI_PROF_NVLINK_L10_TX_BYTES, gauge, Nvlink l10 tx bytes
1171 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l11_rx
1172 +# DCGM_FI_PROF_NVLINK_L11_RX_BYTES, gauge, Nvlink l11 rx bytes
1173 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l11_tx
1174 +# DCGM_FI_PROF_NVLINK_L11_TX_BYTES, gauge, Nvlink l11 tx bytes
1175 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l12_rx
1176 +# DCGM_FI_PROF_NVLINK_L12_RX_BYTES, gauge, Nvlink l12 rx bytes
1177 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l12_tx
1178 +# DCGM_FI_PROF_NVLINK_L12_TX_BYTES, gauge, Nvlink l12 tx bytes
1179 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l13_rx
1180 +# DCGM_FI_PROF_NVLINK_L13_RX_BYTES, gauge, Nvlink l13 rx bytes
1181 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l13_tx
1182 +# DCGM_FI_PROF_NVLINK_L13_TX_BYTES, gauge, Nvlink l13 tx bytes
1183 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l14_rx
1184 +# DCGM_FI_PROF_NVLINK_L14_RX_BYTES, gauge, Nvlink l14 rx bytes
1185 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l14_tx
1186 +# DCGM_FI_PROF_NVLINK_L14_TX_BYTES, gauge, Nvlink l14 tx bytes
1187 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l15_rx
1188 +# DCGM_FI_PROF_NVLINK_L15_RX_BYTES, gauge, Nvlink l15 rx bytes
1189 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l15_tx
1190 +# DCGM_FI_PROF_NVLINK_L15_TX_BYTES, gauge, Nvlink l15 tx bytes
1191 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l16_rx
1192 +# DCGM_FI_PROF_NVLINK_L16_RX_BYTES, gauge, Nvlink l16 rx bytes
1193 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l16_tx
1194 +# DCGM_FI_PROF_NVLINK_L16_TX_BYTES, gauge, Nvlink l16 tx bytes
1195 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l17_rx
1196 +# DCGM_FI_PROF_NVLINK_L17_RX_BYTES, gauge, Nvlink l17 rx bytes
1197 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l17_tx
1198 +# DCGM_FI_PROF_NVLINK_L17_TX_BYTES, gauge, Nvlink l17 tx bytes
1199 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l1_rx
1200 +# DCGM_FI_PROF_NVLINK_L1_RX_BYTES, gauge, Nvlink l1 rx bytes
1201 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l1_tx
1202 +# DCGM_FI_PROF_NVLINK_L1_TX_BYTES, gauge, Nvlink l1 tx bytes
1203 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l2_rx
1204 +# DCGM_FI_PROF_NVLINK_L2_RX_BYTES, gauge, Nvlink l2 rx bytes
1205 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l2_tx
1206 +# DCGM_FI_PROF_NVLINK_L2_TX_BYTES, gauge, Nvlink l2 tx bytes
1207 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l3_rx
1208 +# DCGM_FI_PROF_NVLINK_L3_RX_BYTES, gauge, Nvlink l3 rx bytes
1209 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l3_tx
1210 +# DCGM_FI_PROF_NVLINK_L3_TX_BYTES, gauge, Nvlink l3 tx bytes
1211 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l4_rx
1212 +# DCGM_FI_PROF_NVLINK_L4_RX_BYTES, gauge, Nvlink l4 rx bytes
1213 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l4_tx
1214 +# DCGM_FI_PROF_NVLINK_L4_TX_BYTES, gauge, Nvlink l4 tx bytes
1215 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l5_rx
1216 +# DCGM_FI_PROF_NVLINK_L5_RX_BYTES, gauge, Nvlink l5 rx bytes
1217 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l5_tx
1218 +# DCGM_FI_PROF_NVLINK_L5_TX_BYTES, gauge, Nvlink l5 tx bytes
1219 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l6_rx
1220 +# DCGM_FI_PROF_NVLINK_L6_RX_BYTES, gauge, Nvlink l6 rx bytes
1221 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l6_tx
1222 +# DCGM_FI_PROF_NVLINK_L6_TX_BYTES, gauge, Nvlink l6 tx bytes
1223 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l7_rx
1224 +# DCGM_FI_PROF_NVLINK_L7_RX_BYTES, gauge, Nvlink l7 rx bytes
1225 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l7_tx
1226 +# DCGM_FI_PROF_NVLINK_L7_TX_BYTES, gauge, Nvlink l7 tx bytes
1227 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l8_rx
1228 +# DCGM_FI_PROF_NVLINK_L8_RX_BYTES, gauge, Nvlink l8 rx bytes
1229 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l8_tx
1230 +# DCGM_FI_PROF_NVLINK_L8_TX_BYTES, gauge, Nvlink l8 tx bytes
1231 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l9_rx
1232 +# DCGM_FI_PROF_NVLINK_L9_RX_BYTES, gauge, Nvlink l9 rx bytes
1233 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_l9_tx
1234 +# DCGM_FI_PROF_NVLINK_L9_TX_BYTES, gauge, Nvlink l9 tx bytes
1235 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_rx
1236 +DCGM_FI_PROF_NVLINK_RX_BYTES, gauge, Nvlink rx bytes
1237 +# context=dcgm.nvlink.interconnect.throughput family=nvlink interconnect dimension=nvlink_tx
1238 +DCGM_FI_PROF_NVLINK_TX_BYTES, gauge, Nvlink tx bytes
1239 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvofa0_active
1240 +# DCGM_FI_PROF_NVOFA0_ACTIVE, gauge, Nvofa0 active
1241 +# context=dcgm.gpu.compute.media.activity family=gpu compute dimension=nvofa1_active
1242 +# DCGM_FI_PROF_NVOFA1_ACTIVE, gauge, Nvofa1 active
1243 +# context=dcgm.gpu.interconnect.pcie.throughput family=gpu interconnect/pcie dimension=pcie_rx
1244 +DCGM_FI_PROF_PCIE_RX_BYTES, gauge, Pcie rx bytes
1245 +# context=dcgm.gpu.interconnect.pcie.throughput family=gpu interconnect/pcie dimension=pcie_tx
1246 +DCGM_FI_PROF_PCIE_TX_BYTES, gauge, Pcie tx bytes
1247 +# context=dcgm.gpu.compute.cache.activity family=gpu compute dimension=peermem_cache_hit
1248 +DCGM_FI_PROF_PEERMEM_CACHE_HIT, gauge, Peermem cache hit
1249 +# context=dcgm.gpu.compute.cache.activity family=gpu compute dimension=peermem_cache_miss
1250 +DCGM_FI_PROF_PEERMEM_CACHE_MISS, gauge, Peermem cache miss
1251 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=fp16
1252 +DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Pipe fp16 active
1253 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=fp32
1254 +DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Pipe fp32 active
1255 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=fp64
1256 +DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Pipe fp64 active
1257 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=integer
1258 +DCGM_FI_PROF_PIPE_INT_ACTIVE, gauge, Pipe int active
1259 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=tensor
1260 +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Pipe tensor active
1261 +# context=dcgm.gpu.compute.tensor.activity family=gpu compute dimension=tensor_dfma
1262 +DCGM_FI_PROF_PIPE_TENSOR_DFMA_ACTIVE, gauge, Pipe tensor dfma active
1263 +# context=dcgm.gpu.compute.tensor.activity family=gpu compute dimension=tensor_hmma
1264 +DCGM_FI_PROF_PIPE_TENSOR_HMMA_ACTIVE, gauge, Pipe tensor hmma active
1265 +# context=dcgm.gpu.compute.tensor.activity family=gpu compute dimension=tensor_imma
1266 +DCGM_FI_PROF_PIPE_TENSOR_IMMA_ACTIVE, gauge, Pipe tensor imma active
1267 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=sm_active
1268 +DCGM_FI_PROF_SM_ACTIVE, gauge, Sm active
1269 +# context=dcgm.gpu.compute.activity family=gpu compute dimension=sm_occupancy
1270 +DCGM_FI_PROF_SM_OCCUPANCY, gauge, Sm occupancy
1271 +# context=dcgm.gpu.state.configuration family=gpu state dimension=sync_boost
1272 +# DCGM_FI_SYNC_BOOST, gauge, Sync boost
src/go/plugin/go.d/collector/dcgm/init.go new
+29
@@ -0,0 +1,29 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package dcgm
4 +
5 +import (
6 + "errors"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/prometheus"
10 + "github.com/netdata/netdata/go/plugins/pkg/web"
11 +)
12 +
13 +func (c *Collector) validateConfig() error {
14 + if c.URL == "" {
15 + return errors.New("'url' can not be empty")
16 + }
17 + return nil
18 +}
19 +
20 +func (c *Collector) initPrometheusClient() (prometheus.Prometheus, error) {
21 + httpClient, err := web.NewHTTPClient(c.ClientConfig)
22 + if err != nil {
23 + return nil, fmt.Errorf("init HTTP client: %v", err)
24 + }
25 +
26 + req := c.RequestConfig.Copy()
27 +
28 + return prometheus.New(httpClient, req), nil
29 +}
src/go/plugin/go.d/collector/dcgm/integrations/nvidia_dcgm_exporter.md new
+77
@@ -0,0 +1,77 @@
1 +# NVIDIA DCGM Exporter
2 +
3 +Plugin: `go.d.plugin`
4 +Module: `dcgm`
5 +
6 +## Overview
7 +
8 +The `dcgm` collector scrapes NVIDIA `dcgm-exporter` Prometheus metrics (default `http://127.0.0.1:9400/metrics`) and maps them to static Netdata-native contexts.
9 +
10 +- All numeric fields exported by `dcgm-exporter` are supported.
11 +- Contexts are created lazily: only contexts with collected metrics are instantiated.
12 +- v1 uses manual job configuration (no autodiscovery).
13 +
14 +## Prerequisites
15 +
16 +- NVIDIA driver + DCGM installed.
17 +- `dcgm-exporter` running and reachable.
18 +- Exporter field CSV configured with the fields you want to collect.
19 +- Profiling fields may require additional capabilities/privileges in your runtime.
20 +
21 +## Interval Coupling
22 +
23 +Keep Netdata `update_every` aligned with `dcgm-exporter` collection interval.
24 +
25 +- Exporter default collection interval: `30s`.
26 +- Collector default `update_every`: `30`.
27 +- If you change one side, change the other side too.
28 +
29 +## Configuration
30 +
31 +Example `go.d/dcgm.conf`:
32 +
33 +```yaml
34 +jobs:
35 + - name: local
36 + url: http://127.0.0.1:9400/metrics
37 + update_every: 30
38 +```
39 +
40 +## Field Profiles
41 +
42 +`dcgm-exporter` ships with a small default field set. For production, use an explicit field CSV profile.
43 +Netdata provides a recommended exporter profile file:
44 +[`dcgm-exporter-netdata.csv`](https://github.com/netdata/netdata/blob/master/src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv)
45 +(raw: `https://raw.githubusercontent.com/netdata/netdata/master/src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv`).
46 +
47 +Example:
48 +`dcgm-exporter -f /path/to/dcgm-exporter-netdata.csv`
49 +
50 +The Netdata profile enables 127 high-value fields by default and keeps all other known DCGM fields in the same file as commented entries for easy customization.
51 +
52 +Runtime validation artifact:
53 +- `src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.md`
54 +- `src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.json`
55 +
56 +Validation results are primarily version-scoped (NVIDIA driver + DCGM/dcgm-exporter versions). Use the artifact as a concrete baseline, not as a universal compatibility guarantee.
57 +
58 +Each field line includes a comment with:
59 +- Netdata context
60 +- Netdata family
61 +- Netdata dimension
62 +
63 +When customizing:
64 +- Uncomment the field you need.
65 +- Comment one currently enabled field.
66 +
67 +## Alerts
68 +
69 +Default alerts included for universally actionable conditions:
70 +
71 +- XID errors
72 +- Row remap failure
73 +- New uncorrectable remapped rows
74 +- Power violation duration
75 +- Thermal violation duration
76 +
77 +See: `src/health/health.d/dcgm.conf`.
src/go/plugin/go.d/collector/dcgm/metadata.yaml new
+2224
@@ -0,0 +1,2224 @@
1 +plugin_name: go.d.plugin
2 +modules:
3 + - meta:
4 + id: collector-go.d.plugin-dcgm
5 + plugin_name: go.d.plugin
6 + module_name: dcgm
7 + monitored_instance:
8 + name: NVIDIA DCGM Exporter
9 + link: https://github.com/NVIDIA/dcgm-exporter
10 + icon_filename: nvidia.svg
11 + categories:
12 + - data-collection.hardware-devices-and-sensors
13 + keywords:
14 + - nvidia
15 + - gpu
16 + - dcgm
17 + - dcgm-exporter
18 + related_resources:
19 + integrations:
20 + list: []
21 + info_provided_to_referring_integrations:
22 + description: ""
23 + most_popular: false
24 + overview:
25 + data_collection:
26 + metrics_description: |
27 + This collector gathers NVIDIA GPU telemetry from a `dcgm-exporter` endpoint.
28 + It supports all numeric fields exposed by the exporter and maps them into Netdata-native contexts.
29 + method_description: |
30 + It collects metrics by periodically scraping the exporter Prometheus endpoint over HTTP.
31 + supported_platforms:
32 + include: []
33 + exclude: []
34 + multi_instance: true
35 + additional_permissions:
36 + description: ""
37 + default_behavior:
38 + auto_detection:
39 + description: This integration does not support auto-detection in v1.
40 + limits:
41 + description: |
42 + The collector applies global and per-metric time series limits to prevent excessive cardinality.
43 + performance_impact:
44 + description: The impact depends on dcgm-exporter field selection and resulting series cardinality.
45 + setup:
46 + prerequisites:
47 + list:
48 + - title: Run dcgm-exporter
49 + description: Install DCGM and run `dcgm-exporter` so that a Prometheus endpoint is available (default `:9400/metrics`).
50 + - title: Configure exporter field list
51 + description: |
52 + The default exporter profile exposes a small subset of fields.
53 + Use the Netdata recommended profile:
54 + [`dcgm-exporter-netdata.csv`](https://github.com/netdata/netdata/blob/master/src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv)
55 + (raw download: `https://raw.githubusercontent.com/netdata/netdata/master/src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv`).
56 +
57 + The Netdata profile enables 127 fields by default and documents all remaining known DCGM fields as commented entries.
58 + To customize beyond the baseline, uncomment the field you need and comment one currently enabled field.
59 +
60 + Runtime validation artifact:
61 + `src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.md`
62 + and
63 + `src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.json`
64 +
65 + Validation is primarily version-scoped (NVIDIA driver + DCGM/DCGM-exporter versions), so treat it as a strong baseline rather than universal compatibility.
66 +
67 + Example:
68 + `dcgm-exporter -f /path/to/dcgm-exporter-netdata.csv`
69 + - title: Keep collection intervals aligned
70 + description: |
71 + Set Netdata `update_every` to the same value as dcgm-exporter collection interval (default 30 seconds).
72 + Example exporter interval: `dcgm-exporter -c 30000` and Netdata `update_every: 30`.
73 + - title: Enable profiling capabilities (optional)
74 + description: Profiling fields may require additional privileges/capabilities in your runtime environment.
75 + configuration:
76 + file:
77 + name: go.d/dcgm.conf
78 + options:
79 + description: |
80 + The following options can be defined globally: update_every, autodetection_retry.
81 + folding:
82 + title: Config options
83 + enabled: true
84 + list:
85 + - name: update_every
86 + description: Data collection interval (seconds). Keep this aligned with dcgm-exporter collection interval.
87 + default_value: 30
88 + required: false
89 + group: Collection
90 + - name: autodetection_retry
91 + description: Autodetection retry interval (seconds). Set 0 to disable.
92 + default_value: 0
93 + required: false
94 + group: Collection
95 + - name: url
96 + description: DCGM exporter metrics endpoint URL.
97 + default_value: http://127.0.0.1:9400/metrics
98 + required: true
99 + group: Target
100 + - name: timeout
101 + description: HTTP request timeout (seconds).
102 + default_value: 10
103 + required: false
104 + group: Target
105 + - name: max_time_series
106 + description: Global time series limit. If exceeded, collection is skipped for this cycle.
107 + default_value: 2000
108 + required: false
109 + group: Limits
110 + - name: max_time_series_per_metric
111 + description: Per-metric time series limit. Metrics above this limit are skipped.
112 + default_value: 200
113 + required: false
114 + group: Limits
115 + - name: username
116 + description: Username for Basic HTTP authentication.
117 + default_value: ""
118 + required: false
119 + group: HTTP Auth
120 + - name: password
121 + description: Password for Basic HTTP authentication.
122 + default_value: ""
123 + required: false
124 + group: HTTP Auth
125 + - name: bearer_token_file
126 + description: Path to a file containing a bearer token.
127 + default_value: ""
128 + required: false
129 + group: HTTP Auth
130 + - name: tls_skip_verify
131 + description: Skip TLS certificate and hostname verification (insecure).
132 + default_value: no
133 + required: false
134 + group: TLS
135 + - name: tls_ca
136 + description: Path to CA bundle used to validate the server certificate.
137 + default_value: ""
138 + required: false
139 + group: TLS
140 + - name: tls_cert
141 + description: Path to client TLS certificate (for mTLS).
142 + default_value: ""
143 + required: false
144 + group: TLS
145 + - name: tls_key
146 + description: Path to client TLS private key (for mTLS).
147 + default_value: ""
148 + required: false
149 + group: TLS
150 + - name: proxy_url
151 + description: HTTP proxy URL.
152 + default_value: ""
153 + required: false
154 + group: Proxy
155 + - name: proxy_username
156 + description: Username for proxy authentication.
157 + default_value: ""
158 + required: false
159 + group: Proxy
160 + - name: proxy_password
161 + description: Password for proxy authentication.
162 + default_value: ""
163 + required: false
164 + group: Proxy
165 + - name: headers
166 + description: Additional HTTP headers to include in the request.
167 + default_value: ""
168 + required: false
169 + group: Request
170 + - name: method
171 + description: HTTP method.
172 + default_value: GET
173 + required: false
174 + group: Request
175 + - name: body
176 + description: HTTP request body.
177 + default_value: ""
178 + required: false
179 + group: Request
180 + - name: not_follow_redirects
181 + description: Do not follow HTTP redirects.
182 + default_value: no
183 + required: false
184 + group: Request
185 + - name: force_http2
186 + description: Force HTTP/2 (including h2c over TCP).
187 + default_value: no
188 + required: false
189 + group: Request
190 + - name: vnode
191 + description: Associate this job with a Virtual Node.
192 + default_value: ""
193 + required: false
194 + group: Virtual Node
195 + examples:
196 + folding:
197 + title: Config
198 + enabled: true
199 + list:
200 + - name: Local exporter
201 + description: Collect metrics from a local dcgm-exporter endpoint.
202 + config: |
203 + jobs:
204 + - name: local
205 + url: http://127.0.0.1:9400/metrics
206 + update_every: 30
207 + - name: TLS endpoint
208 + description: Collect metrics over HTTPS with custom CA certificate.
209 + config: |
210 + jobs:
211 + - name: secure
212 + url: https://dcgm-exporter.example.com:9400/metrics
213 + update_every: 30
214 + tls_ca: /etc/netdata/certs/dcgm-ca.crt
215 + - name: Increased cardinality limits
216 + description: Increase limits when collecting large field sets and multiple entities.
217 + config: |
218 + jobs:
219 + - name: dcgm_large
220 + url: http://127.0.0.1:9400/metrics
221 + update_every: 30
222 + max_time_series: 10000
223 + max_time_series_per_metric: 2000
224 + troubleshooting:
225 + problems:
226 + list: []
227 + alerts:
228 + - name: dcgm_gpu_xid_errors
229 + metric: dcgm.gpu.reliability.xid
230 + info: NVIDIA driver reported GPU XID error on GPU ${label:gpu}
231 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
232 + - name: dcgm_gpu_row_remap_failure
233 + metric: dcgm.gpu.reliability.row_remap_status
234 + info: GPU row remapping failed on GPU ${label:gpu}
235 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
236 + - name: dcgm_gpu_uncorrectable_remapped_rows
237 + metric: dcgm.gpu.reliability.row_remap_events
238 + info: Uncorrectable remapped rows increased on GPU ${label:gpu}
239 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
240 + - name: dcgm_gpu_power_violation
241 + metric: dcgm.gpu.throttle.violations
242 + info: Power throttling detected on GPU ${label:gpu}
243 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
244 + - name: dcgm_gpu_thermal_violation
245 + metric: dcgm.gpu.throttle.violations
246 + info: Thermal throttling detected on GPU ${label:gpu}
247 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
248 + metrics:
249 + folding:
250 + title: Metrics
251 + enabled: false
252 + description: |
253 + Metrics are grouped into static Netdata contexts. Contexts are created only when matching DCGM fields are present in the exporter output.
254 + availability: []
255 + scopes:
256 + - name: gpu
257 + description: These metrics refer to GPU device instances.
258 + labels:
259 + - name: gpu
260 + description: gpu label from exporter metrics.
261 + - name: uuid
262 + description: uuid label from exporter metrics.
263 + metrics:
264 + - name: dcgm.gpu.capability.support
265 + description: GPU Capability Support metrics.
266 + unit: state
267 + chart_type: line
268 + dimensions:
269 + - name: cc_mode
270 + - name: cuda_compute_capability
271 + - name: gpm_support
272 + - name: mig_attributes
273 + - name: mig_ci_info
274 + - name: mig_gi_info
275 + - name: mig_max_slices
276 + - name: supported_clocks
277 + - name: supported_type_info
278 + - name: dcgm.gpu.clock.frequency
279 + description: GPU Clock Frequency metrics.
280 + unit: MHz
281 + chart_type: line
282 + dimensions:
283 + - name: app_mem_clock
284 + visibility: hidden
285 + - name: app_sm_clock
286 + visibility: hidden
287 + - name: max_mem_clock
288 + visibility: hidden
289 + - name: max_sm_clock
290 + visibility: hidden
291 + - name: max_video_clock
292 + visibility: hidden
293 + - name: memory
294 + - name: sm
295 + - name: video_clock
296 + - name: dcgm.gpu.compute.activity
297 + description: GPU Compute Pipeline Activity metrics.
298 + unit: '%'
299 + chart_type: line
300 + dimensions:
301 + - name: dram
302 + - name: fp16
303 + - name: fp32
304 + - name: fp64
305 + - name: graphics_engine_active
306 + - name: integer
307 + - name: sm_active
308 + - name: sm_occupancy
309 + - name: tensor
310 + - name: dcgm.gpu.compute.tensor.activity
311 + description: GPU Tensor Core Activity by precision type.
312 + unit: '%'
313 + chart_type: line
314 + dimensions:
315 + - name: tensor_dfma
316 + - name: tensor_hmma
317 + - name: tensor_imma
318 + - name: dcgm.gpu.compute.media.activity
319 + description: GPU Media Engine Activity metrics.
320 + unit: '%'
321 + chart_type: line
322 + dimensions:
323 + - name: nvdec0_active
324 + - name: nvdec1_active
325 + - name: nvdec2_active
326 + - name: nvdec3_active
327 + - name: nvdec4_active
328 + - name: nvdec5_active
329 + - name: nvdec6_active
330 + - name: nvdec7_active
331 + - name: nvjpg0_active
332 + - name: nvjpg1_active
333 + - name: nvjpg2_active
334 + - name: nvjpg3_active
335 + - name: nvjpg4_active
336 + - name: nvjpg5_active
337 + - name: nvjpg6_active
338 + - name: nvjpg7_active
339 + - name: nvofa0_active
340 + - name: nvofa1_active
341 + - name: dcgm.gpu.compute.cache.activity
342 + description: GPU Memory Cache Hit/Miss metrics.
343 + unit: 'events/s'
344 + chart_type: line
345 + dimensions:
346 + - name: hostmem_cache_hit
347 + - name: hostmem_cache_miss
348 + - name: peermem_cache_hit
349 + - name: peermem_cache_miss
350 + - name: dcgm.gpu.compute.utilization
351 + description: GPU Compute Utilization metrics.
352 + unit: '%'
353 + chart_type: line
354 + dimensions:
355 + - name: decoder
356 + - name: encoder
357 + - name: gpu
358 + - name: memory_copy
359 + - name: dcgm.gpu.cpu.power
360 + description: GPU CPU Power metrics.
361 + unit: Watts
362 + chart_type: line
363 + dimensions:
364 + - name: module_power_util_current
365 + - name: sysio_power_util_current
366 + - name: dcgm.gpu.cpu.info
367 + description: GPU CPU Information metrics.
368 + unit: value
369 + chart_type: line
370 + dimensions:
371 + - name: cpu_model
372 + - name: cpu_vendor
373 + - name: dcgm.gpu.diagnostics.results
374 + description: GPU Diagnostics Results metrics.
375 + unit: state
376 + chart_type: line
377 + dimensions:
378 + - name: diag_diagnostic_result
379 + - name: diag_eud_result
380 + - name: diag_memory_bandwidth_result
381 + - name: diag_memory_result
382 + - name: diag_memtest_result
383 + - name: diag_nccl_tests_result
384 + - name: diag_nvbandwidth_result
385 + - name: diag_pulse_test_result
386 + - name: diag_software_result
387 + - name: diag_targeted_power_result
388 + - name: diag_targeted_stress_result
389 + - name: dcgm.gpu.diagnostics.status
390 + description: GPU Diagnostics Status metrics.
391 + unit: state
392 + chart_type: line
393 + dimensions:
394 + - name: diag_status
395 + - name: dcgm.gpu.health.status
396 + description: GPU Health Status metrics.
397 + unit: state
398 + chart_type: line
399 + dimensions:
400 + - name: imex_daemon_status
401 + - name: imex_domain_status
402 + - name: dcgm.gpu.interconnect.connectx.error_status
403 + description: GPU ConnectX Error Status metrics.
404 + unit: state
405 + chart_type: line
406 + dimensions:
407 + - name: connectx_correctable_err_mask
408 + - name: connectx_correctable_err_status
409 + - name: connectx_uncorrectable_err_mask
410 + - name: connectx_uncorrectable_err_severity
411 + - name: connectx_uncorrectable_err_status
412 + - name: dcgm.gpu.interconnect.connectx.errors
413 + description: GPU ConnectX Errors metrics.
414 + unit: errors/s
415 + chart_type: line
416 + dimensions:
417 + - name: connectx_correctable_err_mask
418 + - name: connectx_correctable_err_status
419 + - name: connectx_uncorrectable_err_mask
420 + - name: connectx_uncorrectable_err_severity
421 + - name: connectx_uncorrectable_err_status
422 + - name: dcgm.gpu.interconnect.connectx.link
423 + description: GPU ConnectX Link metrics.
424 + unit: value
425 + chart_type: line
426 + dimensions:
427 + - name: connectx_active_pcie_link_speed
428 + - name: connectx_expect_pcie_link_speed
429 + - name: dcgm.gpu.interconnect.connectx.status
430 + description: GPU ConnectX Status metrics.
431 + unit: state
432 + chart_type: line
433 + dimensions:
434 + - name: connectx_health
435 + - name: dcgm.gpu.interconnect.error_rate
436 + description: GPU Interconnect Error Rate metrics.
437 + unit: errors/s
438 + chart_type: line
439 + dimensions:
440 + - name: c2c_link_error_intr
441 + - name: c2c_link_error_replay
442 + - name: c2c_link_error_replay_b2b
443 + - name: dcgm.gpu.interconnect.fabric
444 + description: GPU Fabric State metrics.
445 + unit: state
446 + chart_type: line
447 + dimensions:
448 + - name: fabric_clique_id
449 + - name: fabric_cluster_uuid
450 + - name: fabric_health_mask
451 + - name: fabric_manager_error_code
452 + - name: fabric_manager_status
453 + - name: dcgm.gpu.interconnect.nvlink.error_rate
454 + description: GPU NVLink Error Rate metrics.
455 + unit: errors/s
456 + chart_type: line
457 + dimensions:
458 + - name: gpu_nvlink_errors
459 + - name: dcgm.gpu.interconnect.pcie.error_rate
460 + description: GPU PCIe Error Rate metrics.
461 + unit: errors/s
462 + chart_type: line
463 + dimensions:
464 + - name: pcie_count_correctable_errors
465 + - name: pcie_replay
466 + - name: dcgm.gpu.interconnect.pcie.link.generation
467 + description: GPU PCIe Link Generation metrics.
468 + unit: generation
469 + chart_type: line
470 + dimensions:
471 + - name: link_gen
472 + - name: max_link_gen
473 + visibility: hidden
474 + - name: dcgm.gpu.interconnect.pcie.link.width
475 + description: GPU PCIe Link Width metrics.
476 + unit: lanes
477 + chart_type: line
478 + dimensions:
479 + - name: connectx_active_pcie_link_width
480 + - name: connectx_expect_pcie_link_width
481 + - name: link_width
482 + - name: max_link_width
483 + visibility: hidden
484 + - name: dcgm.gpu.interconnect.state
485 + description: GPU Interconnect State metrics.
486 + unit: state
487 + chart_type: line
488 + dimensions:
489 + - name: c2c_link
490 + - name: c2c_link_power_state
491 + - name: c2c_link_status
492 + - name: dcgm.gpu.interconnect.pcie.state
493 + description: GPU PCIe State metrics.
494 + unit: state
495 + chart_type: line
496 + dimensions:
497 + - name: diag_pcie_result
498 + - name: dcgm.gpu.interconnect.throughput
499 + description: GPU Interconnect Throughput metrics.
500 + unit: B/s
501 + chart_type: area
502 + dimensions:
503 + - name: c2c_max_bandwidth
504 + - name: c2c_rx_all_bytes
505 + - name: c2c_rx_data_bytes
506 + - name: c2c_tx_all_bytes
507 + - name: c2c_tx_data_bytes
508 + - name: dcgm.gpu.interconnect.pcie.throughput
509 + description: GPU PCIe Throughput metrics.
510 + unit: B/s
511 + chart_type: area
512 + dimensions:
513 + - name: pcie_rx
514 + - name: pcie_rx_throughput
515 + - name: pcie_tx
516 + - name: pcie_tx_throughput
517 + - name: dcgm.gpu.interconnect.nvlink.throughput
518 + description: GPU NVLink Throughput metrics.
519 + unit: B/s
520 + chart_type: area
521 + dimensions:
522 + - name: nvlink_rx
523 + - name: nvlink_tx
524 + - name: dcgm.gpu.interconnect.total.throughput
525 + description: GPU Interconnect Total Throughput metrics.
526 + unit: B/s
527 + chart_type: area
528 + dimensions:
529 + - name: pcie
530 + - name: nvlink
531 + - name: dcgm.gpu.internal.boundary
532 + description: GPU Internal Boundary Fields metrics.
533 + unit: state
534 + chart_type: line
535 + dimensions:
536 + - name: first_connectx_field_id
537 + - name: first_vgpu_field_id
538 + - name: internal_fields_0_end
539 + - name: internal_fields_0_start
540 + - name: last_connectx_field_id
541 + - name: last_vgpu_field_id
542 + - name: dcgm.gpu.inventory.identity
543 + description: GPU Inventory Identity metrics.
544 + unit: value
545 + chart_type: line
546 + dimensions:
547 + - name: brand
548 + - name: count
549 + - name: cuda_visible_devices_str
550 + - name: minor_number
551 + - name: name
552 + - name: nvml_index
553 + - name: serial
554 + - name: uuid
555 + - name: dcgm.gpu.inventory.platform
556 + description: GPU Platform Inventory metrics.
557 + unit: value
558 + chart_type: line
559 + dimensions:
560 + - name: platform_chassis_serial_number
561 + - name: platform_chassis_slot_number
562 + - name: platform_host_id
563 + - name: platform_infiniband_guid
564 + - name: platform_module_id
565 + - name: platform_peer_type
566 + - name: platform_tray_index
567 + - name: dcgm.gpu.inventory.software
568 + description: GPU Software and Firmware metrics.
569 + unit: value
570 + chart_type: line
571 + dimensions:
572 + - name: inforom_config_check
573 + - name: inforom_config_valid
574 + - name: inforom_image_ver
575 + - name: oem_inforom_ver
576 + - name: power_inforom_ver
577 + - name: process_name
578 + - name: vbios_version
579 + - name: dcgm.gpu.memory.bar1_usage
580 + description: GPU BAR1 Memory Usage metrics.
581 + unit: B
582 + chart_type: stacked
583 + dimensions:
584 + - name: free
585 + - name: used
586 + - name: dcgm.gpu.memory.bar1_capacity
587 + description: GPU BAR1 Memory Capacity metrics.
588 + unit: B
589 + chart_type: line
590 + dimensions:
591 + - name: total
592 + - name: dcgm.gpu.memory.ecc_error_rate
593 + description: GPU ECC Error Rate metrics.
594 + unit: errors/s
595 + chart_type: line
596 + dimensions:
597 + - name: ecc_current
598 + - name: ecc_dbe_agg
599 + - name: ecc_dbe_agg_cbu
600 + - name: ecc_dbe_agg_dev
601 + - name: ecc_dbe_agg_l1
602 + - name: ecc_dbe_agg_l2
603 + - name: ecc_dbe_agg_reg
604 + - name: ecc_dbe_agg_shm
605 + - name: ecc_dbe_agg_srm
606 + - name: ecc_dbe_agg_tex
607 + - name: ecc_dbe_vol
608 + - name: ecc_dbe_vol_cbu
609 + - name: ecc_dbe_vol_dev
610 + - name: ecc_dbe_vol_l1
611 + - name: ecc_dbe_vol_l2
612 + - name: ecc_dbe_vol_reg
613 + - name: ecc_dbe_vol_shm
614 + - name: ecc_dbe_vol_srm
615 + - name: ecc_dbe_vol_tex
616 + - name: ecc_pending
617 + - name: ecc_sbe_agg
618 + - name: ecc_sbe_agg_cbu
619 + - name: ecc_sbe_agg_dev
620 + - name: ecc_sbe_agg_l1
621 + - name: ecc_sbe_agg_l2
622 + - name: ecc_sbe_agg_reg
623 + - name: ecc_sbe_agg_shm
624 + - name: ecc_sbe_agg_srm
625 + - name: ecc_sbe_agg_tex
626 + - name: ecc_sbe_vol
627 + - name: ecc_sbe_vol_cbu
628 + - name: ecc_sbe_vol_dev
629 + - name: ecc_sbe_vol_l1
630 + - name: ecc_sbe_vol_l2
631 + - name: ecc_sbe_vol_reg
632 + - name: ecc_sbe_vol_shm
633 + - name: ecc_sbe_vol_srm
634 + - name: ecc_sbe_vol_tex
635 + - name: dcgm.gpu.memory.ecc_errors
636 + description: GPU ECC Errors metrics.
637 + unit: errors
638 + chart_type: line
639 + dimensions:
640 + - name: ecc_current
641 + - name: ecc_dbe_agg_cbu
642 + - name: ecc_dbe_agg_dev
643 + - name: ecc_dbe_agg_l1
644 + - name: ecc_dbe_agg_l2
645 + - name: ecc_dbe_agg_reg
646 + - name: ecc_dbe_agg_shm
647 + - name: ecc_dbe_agg_srm
648 + - name: ecc_dbe_agg_tex
649 + - name: ecc_dbe_vol_cbu
650 + - name: ecc_dbe_vol_dev
651 + - name: ecc_dbe_vol_l1
652 + - name: ecc_dbe_vol_l2
653 + - name: ecc_dbe_vol_reg
654 + - name: ecc_dbe_vol_shm
655 + - name: ecc_dbe_vol_srm
656 + - name: ecc_dbe_vol_tex
657 + - name: ecc_inforom_ver
658 + - name: ecc_pending
659 + - name: ecc_sbe_agg_cbu
660 + - name: ecc_sbe_agg_dev
661 + - name: ecc_sbe_agg_l1
662 + - name: ecc_sbe_agg_l2
663 + - name: ecc_sbe_agg_reg
664 + - name: ecc_sbe_agg_shm
665 + - name: ecc_sbe_agg_srm
666 + - name: ecc_sbe_agg_tex
667 + - name: ecc_sbe_vol_cbu
668 + - name: ecc_sbe_vol_dev
669 + - name: ecc_sbe_vol_l1
670 + - name: ecc_sbe_vol_l2
671 + - name: ecc_sbe_vol_reg
672 + - name: ecc_sbe_vol_shm
673 + - name: ecc_sbe_vol_srm
674 + - name: ecc_sbe_vol_tex
675 + - name: dcgm.gpu.memory.page_retirements
676 + description: GPU Retired Memory Pages metrics.
677 + unit: pages/s
678 + chart_type: line
679 + dimensions:
680 + - name: retired_dbe
681 + - name: retired_pending
682 + - name: retired_sbe
683 + - name: dcgm.gpu.memory.usage
684 + description: GPU Memory Usage metrics.
685 + unit: B
686 + chart_type: stacked
687 + dimensions:
688 + - name: free
689 + - name: reserved
690 + - name: used
691 + - name: dcgm.gpu.memory.capacity
692 + description: GPU Memory Capacity metrics.
693 + unit: B
694 + chart_type: line
695 + dimensions:
696 + - name: total
697 + - name: dcgm.gpu.memory.utilization
698 + description: GPU Memory Utilization metrics.
699 + unit: '%'
700 + chart_type: line
701 + dimensions:
702 + - name: used_percent
703 + - name: dcgm.gpu.power.energy
704 + description: GPU Energy Consumption Rate metrics.
705 + unit: mJ/s
706 + chart_type: line
707 + dimensions:
708 + - name: total
709 + - name: dcgm.gpu.power.profiles
710 + description: GPU Power Profiles metrics.
711 + unit: state
712 + chart_type: line
713 + dimensions:
714 + - name: enforced_power_profile_mask
715 + - name: requested_power_profile_mask
716 + - name: valid_power_profile_mask
717 + - name: dcgm.gpu.power.smoothing
718 + description: GPU Power Smoothing metrics.
719 + unit: value
720 + chart_type: line
721 + dimensions:
722 + - name: pwr_smoothing_active_preset_profile
723 + - name: pwr_smoothing_admin_override_percent_tmp_floor
724 + - name: pwr_smoothing_admin_override_ramp_down_hyst_val
725 + - name: pwr_smoothing_admin_override_ramp_down_rate
726 + - name: pwr_smoothing_admin_override_ramp_up_rate
727 + - name: pwr_smoothing_applied_tmp_ceil
728 + - name: pwr_smoothing_applied_tmp_floor
729 + - name: pwr_smoothing_enabled
730 + - name: pwr_smoothing_hw_circuitry_percent_lifetime_remaining
731 + - name: pwr_smoothing_imm_ramp_down_enabled
732 + - name: pwr_smoothing_max_num_preset_profiles
733 + - name: pwr_smoothing_max_percent_tmp_floor_setting
734 + - name: pwr_smoothing_min_percent_tmp_floor_setting
735 + - name: pwr_smoothing_priv_lvl
736 + - name: pwr_smoothing_profile_percent_tmp_floor
737 + - name: pwr_smoothing_profile_ramp_down_hyst_val
738 + - name: pwr_smoothing_profile_ramp_down_rate
739 + - name: pwr_smoothing_profile_ramp_up_rate
740 + - name: dcgm.gpu.power.usage
741 + description: GPU Power Usage metrics.
742 + unit: Watts
743 + chart_type: line
744 + dimensions:
745 + - name: draw
746 + - name: enforced_limit
747 + visibility: hidden
748 + - name: power_mgmt_limit
749 + visibility: hidden
750 + - name: power_mgmt_limit_def
751 + visibility: hidden
752 + - name: power_mgmt_limit_max
753 + visibility: hidden
754 + - name: power_mgmt_limit_min
755 + visibility: hidden
756 + - name: power_usage_instant
757 + - name: dcgm.gpu.reliability.memory_health
758 + description: GPU Memory Health metrics.
759 + unit: state
760 + chart_type: line
761 + dimensions:
762 + - name: banks_remap_rows_avail_high
763 + - name: banks_remap_rows_avail_low
764 + - name: banks_remap_rows_avail_max
765 + - name: banks_remap_rows_avail_none
766 + - name: banks_remap_rows_avail_partial
767 + - name: memory_unrepairable_flag
768 + - name: threshold_srm
769 + - name: dcgm.gpu.reliability.recovery_action
770 + description: GPU Recovery Action metrics.
771 + unit: state
772 + chart_type: line
773 + dimensions:
774 + - name: get_gpu_recovery_action
775 + - name: dcgm.gpu.reliability.row_remap_events
776 + description: GPU Row Remap Events metrics.
777 + unit: rows/s
778 + chart_type: line
779 + dimensions:
780 + - name: correctable_remapped_rows
781 + - name: uncorrectable_remapped_rows
782 + - name: dcgm.gpu.reliability.row_remap_status
783 + description: GPU Row Remap Status metrics.
784 + unit: state
785 + chart_type: line
786 + dimensions:
787 + - name: row_remap_failure
788 + - name: row_remap_pending
789 + - name: dcgm.gpu.reliability.xid
790 + description: GPU XID Errors metrics.
791 + unit: code
792 + chart_type: line
793 + dimensions:
794 + - name: xid
795 + - name: dcgm.gpu.state.configuration
796 + description: GPU Configuration State metrics.
797 + unit: state
798 + chart_type: line
799 + dimensions:
800 + - name: autoboost
801 + - name: compute_mode
802 + - name: persistence_mode
803 + - name: sync_boost
804 + - name: sync_boost_violation
805 + - name: dcgm.gpu.state.performance
806 + description: GPU Performance State metrics.
807 + unit: state
808 + chart_type: line
809 + dimensions:
810 + - name: pstate
811 + - name: dcgm.gpu.state.virtualization
812 + description: GPU Virtualization State metrics.
813 + unit: state
814 + chart_type: line
815 + dimensions:
816 + - name: mig_mode
817 + - name: virtual_mode
818 + - name: dcgm.gpu.thermal.fan_speed
819 + description: GPU Fan Speed metrics.
820 + unit: '%'
821 + chart_type: line
822 + dimensions:
823 + - name: fan_speed
824 + - name: dcgm.gpu.thermal.temperature
825 + description: GPU Temperature metrics.
826 + unit: Celsius
827 + chart_type: line
828 + dimensions:
829 + - name: connectx_device_temperature
830 + - name: gpu
831 + - name: gpu_max_op_temp
832 + visibility: hidden
833 + - name: gpu_temp_limit
834 + visibility: hidden
835 + - name: mem_max_op_temp
836 + visibility: hidden
837 + - name: memory
838 + - name: shutdown_temp
839 + visibility: hidden
840 + - name: slowdown_temp
841 + visibility: hidden
842 + - name: dcgm.gpu.throttle.reasons
843 + description: GPU Throttle Reasons metrics.
844 + unit: bitmask
845 + chart_type: line
846 + dimensions:
847 + - name: clocks_event_reasons
848 + - name: dcgm.gpu.throttle.violations
849 + description: GPU Throttle Violation Duration metrics.
850 + unit: milliseconds/s
851 + chart_type: line
852 + dimensions:
853 + - name: board_limit_violation
854 + - name: hw_power_brake_slowdown
855 + - name: hw_therm_slowdown
856 + - name: low_utilization_violation
857 + - name: power_violation
858 + - name: reliability_violation
859 + - name: sw_power_cap
860 + - name: sw_therm_slowdown
861 + - name: sync_boost
862 + - name: thermal_violation
863 + - name: total_app_clocks_violation
864 + - name: total_base_clocks_violation
865 + - name: dcgm.gpu.topology.affinity
866 + description: GPU Topology and Affinity metrics.
867 + unit: value
868 + chart_type: line
869 + dimensions:
870 + - name: cpu_affinity_0
871 + - name: cpu_affinity_1
872 + - name: cpu_affinity_2
873 + - name: cpu_affinity_3
874 + - name: gpu_topology_affinity
875 + - name: gpu_topology_pci
876 + - name: mem_affinity_0
877 + - name: mem_affinity_1
878 + - name: mem_affinity_2
879 + - name: mem_affinity_3
880 + - name: pci_busid
881 + - name: pci_combined_id
882 + - name: pci_subsys_id
883 + - name: dcgm.gpu.virtualization.vgpu.frame_rate
884 + description: GPU vGPU Frame Rate metrics.
885 + unit: fps
886 + chart_type: line
887 + dimensions:
888 + - name: vgpu_frame_rate_limit
889 + - name: dcgm.gpu.virtualization.vgpu.instance
890 + description: GPU vGPU Instance metrics.
891 + unit: value
892 + chart_type: line
893 + dimensions:
894 + - name: vgpu_instance_ids
895 + - name: vgpu_pci_id
896 + - name: vgpu_uuid
897 + - name: dcgm.gpu.virtualization.vgpu.license
898 + description: GPU vGPU License metrics.
899 + unit: state
900 + chart_type: line
901 + dimensions:
902 + - name: vgpu_instance_license_state
903 + - name: vgpu_license_status
904 + - name: vgpu_type_license
905 + - name: dcgm.gpu.virtualization.vgpu.memory
906 + description: GPU vGPU Memory metrics.
907 + unit: B
908 + chart_type: line
909 + dimensions:
910 + - name: vgpu_memory_usage
911 + - name: dcgm.gpu.virtualization.vgpu.sessions
912 + description: GPU vGPU Sessions metrics.
913 + unit: value
914 + chart_type: line
915 + dimensions:
916 + - name: vgpu_enc_sessions_info
917 + - name: vgpu_enc_stats
918 + - name: vgpu_fbc_sessions_info
919 + - name: vgpu_fbc_stats
920 + - name: dcgm.gpu.virtualization.vgpu.software
921 + description: GPU vGPU Software metrics.
922 + unit: value
923 + chart_type: line
924 + dimensions:
925 + - name: vgpu_driver_version
926 + - name: dcgm.gpu.virtualization.vgpu.type
927 + description: GPU vGPU Type metrics.
928 + unit: value
929 + chart_type: line
930 + dimensions:
931 + - name: creatable_vgpu_type_ids
932 + - name: supported_vgpu_type_ids
933 + - name: vgpu_type
934 + - name: vgpu_type_class
935 + - name: vgpu_type_info
936 + - name: vgpu_type_name
937 + - name: dcgm.gpu.virtualization.vgpu.utilization
938 + description: GPU vGPU Utilization metrics.
939 + unit: '%'
940 + chart_type: line
941 + dimensions:
942 + - name: vgpu_per_process_utilization
943 + - name: dcgm.gpu.virtualization.vgpu.vm
944 + description: GPU vGPU VM metrics.
945 + unit: value
946 + chart_type: line
947 + dimensions:
948 + - name: vgpu_vm_gpu_instance_id
949 + - name: vgpu_vm_id
950 + - name: vgpu_vm_name
951 + - name: dcgm.gpu.workload.sessions
952 + description: GPU Workload Sessions metrics.
953 + unit: value
954 + chart_type: line
955 + dimensions:
956 + - name: accounting_data
957 + - name: enc_stats
958 + - name: fbc_sessions_info
959 + - name: fbc_stats
960 + - name: mig
961 + description: These metrics refer to MIG instances.
962 + labels:
963 + - name: gpu
964 + description: gpu label from exporter metrics.
965 + - name: gpu_i_id
966 + description: gpu_i_id label from exporter metrics.
967 + - name: gpu_i_profile
968 + description: gpu_i_profile label from exporter metrics.
969 + metrics:
970 + - name: dcgm.mig.clock.frequency
971 + description: MIG Clock Frequency metrics.
972 + unit: MHz
973 + chart_type: line
974 + dimensions:
975 + - name: app_mem_clock
976 + visibility: hidden
977 + - name: app_sm_clock
978 + visibility: hidden
979 + - name: max_mem_clock
980 + visibility: hidden
981 + - name: max_sm_clock
982 + visibility: hidden
983 + - name: max_video_clock
984 + visibility: hidden
985 + - name: memory
986 + - name: sm
987 + - name: video_clock
988 + - name: dcgm.mig.compute.activity
989 + description: MIG Compute Pipeline Activity metrics.
990 + unit: '%'
991 + chart_type: line
992 + dimensions:
993 + - name: dram
994 + - name: fp16
995 + - name: fp32
996 + - name: fp64
997 + - name: graphics_engine_active
998 + - name: integer
999 + - name: sm_active
1000 + - name: sm_occupancy
1001 + - name: tensor
1002 + - name: dcgm.mig.compute.tensor.activity
1003 + description: MIG Tensor Core Activity by precision type.
1004 + unit: '%'
1005 + chart_type: line
1006 + dimensions:
1007 + - name: tensor_dfma
1008 + - name: tensor_hmma
1009 + - name: tensor_imma
1010 + - name: dcgm.mig.compute.media.activity
1011 + description: MIG Media Engine Activity metrics.
1012 + unit: '%'
1013 + chart_type: line
1014 + dimensions:
1015 + - name: nvdec0_active
1016 + - name: nvdec1_active
1017 + - name: nvdec2_active
1018 + - name: nvdec3_active
1019 + - name: nvdec4_active
1020 + - name: nvdec5_active
1021 + - name: nvdec6_active
1022 + - name: nvdec7_active
1023 + - name: nvjpg0_active
1024 + - name: nvjpg1_active
1025 + - name: nvjpg2_active
1026 + - name: nvjpg3_active
1027 + - name: nvjpg4_active
1028 + - name: nvjpg5_active
1029 + - name: nvjpg6_active
1030 + - name: nvjpg7_active
1031 + - name: nvofa0_active
1032 + - name: nvofa1_active
1033 + - name: dcgm.mig.compute.cache.activity
1034 + description: MIG Memory Cache Hit/Miss metrics.
1035 + unit: 'events/s'
1036 + chart_type: line
1037 + dimensions:
1038 + - name: hostmem_cache_hit
1039 + - name: hostmem_cache_miss
1040 + - name: peermem_cache_hit
1041 + - name: peermem_cache_miss
1042 + - name: dcgm.mig.compute.utilization
1043 + description: MIG Compute Utilization metrics.
1044 + unit: '%'
1045 + chart_type: line
1046 + dimensions:
1047 + - name: decoder
1048 + - name: encoder
1049 + - name: gpu
1050 + - name: memory_copy
1051 + - name: dcgm.mig.interconnect.nvlink.ber
1052 + description: MIG NVLink Bit Error Rate metrics.
1053 + unit: ratio
1054 + chart_type: line
1055 + dimensions:
1056 + - name: nvlink_count_effective_ber
1057 + - name: nvlink_count_effective_ber_float
1058 + - name: nvlink_count_symbol_ber
1059 + - name: nvlink_count_symbol_ber_float
1060 + - name: dcgm.mig.interconnect.nvlink.congestion
1061 + description: MIG NVLink Congestion metrics.
1062 + unit: events/s
1063 + chart_type: line
1064 + dimensions:
1065 + - name: nvlink_ppcnt_ibpc_port_xmit_wait
1066 + - name: dcgm.mig.interconnect.error_rate
1067 + description: MIG Interconnect Error Rate metrics.
1068 + unit: errors/s
1069 + chart_type: line
1070 + dimensions:
1071 + - name: c2c_link_error_intr
1072 + - name: c2c_link_error_replay
1073 + - name: c2c_link_error_replay_b2b
1074 + - name: dcgm.mig.interconnect.nvlink.error_rate
1075 + description: MIG NVLink Error Rate metrics.
1076 + unit: errors/s
1077 + chart_type: line
1078 + dimensions:
1079 + - name: gpu_nvlink_errors
1080 + - name: nvlink_count_effective_errors
1081 + - name: nvlink_count_fec_history_0
1082 + - name: nvlink_count_fec_history_1
1083 + - name: nvlink_count_fec_history_10
1084 + - name: nvlink_count_fec_history_11
1085 + - name: nvlink_count_fec_history_12
1086 + - name: nvlink_count_fec_history_13
1087 + - name: nvlink_count_fec_history_14
1088 + - name: nvlink_count_fec_history_15
1089 + - name: nvlink_count_fec_history_2
1090 + - name: nvlink_count_fec_history_3
1091 + - name: nvlink_count_fec_history_4
1092 + - name: nvlink_count_fec_history_5
1093 + - name: nvlink_count_fec_history_6
1094 + - name: nvlink_count_fec_history_7
1095 + - name: nvlink_count_fec_history_8
1096 + - name: nvlink_count_fec_history_9
1097 + - name: nvlink_count_link_recovery_events
1098 + - name: nvlink_count_link_recovery_failed_events
1099 + - name: nvlink_count_link_recovery_successful_events
1100 + - name: nvlink_count_local_link_integrity_errors
1101 + - name: nvlink_count_rx_buffer_overrun_errors
1102 + - name: nvlink_count_rx_errors
1103 + - name: nvlink_count_rx_general_errors
1104 + - name: nvlink_count_rx_malformed_packet_errors
1105 + - name: nvlink_count_rx_remote_errors
1106 + - name: nvlink_count_rx_symbol_errors
1107 + - name: nvlink_count_tx_discards
1108 + - name: nvlink_crc_data_error
1109 + - name: nvlink_crc_data_error_count_l0
1110 + - name: nvlink_crc_data_error_count_l1
1111 + - name: nvlink_crc_data_error_count_l10
1112 + - name: nvlink_crc_data_error_count_l11
1113 + - name: nvlink_crc_data_error_count_l12
1114 + - name: nvlink_crc_data_error_count_l13
1115 + - name: nvlink_crc_data_error_count_l14
1116 + - name: nvlink_crc_data_error_count_l15
1117 + - name: nvlink_crc_data_error_count_l16
1118 + - name: nvlink_crc_data_error_count_l17
1119 + - name: nvlink_crc_data_error_count_l2
1120 + - name: nvlink_crc_data_error_count_l3
1121 + - name: nvlink_crc_data_error_count_l4
1122 + - name: nvlink_crc_data_error_count_l5
1123 + - name: nvlink_crc_data_error_count_l6
1124 + - name: nvlink_crc_data_error_count_l7
1125 + - name: nvlink_crc_data_error_count_l8
1126 + - name: nvlink_crc_data_error_count_l9
1127 + - name: nvlink_crc_flit_error
1128 + - name: nvlink_crc_flit_error_count_l0
1129 + - name: nvlink_crc_flit_error_count_l1
1130 + - name: nvlink_crc_flit_error_count_l10
1131 + - name: nvlink_crc_flit_error_count_l11
1132 + - name: nvlink_crc_flit_error_count_l12
1133 + - name: nvlink_crc_flit_error_count_l13
1134 + - name: nvlink_crc_flit_error_count_l14
1135 + - name: nvlink_crc_flit_error_count_l15
1136 + - name: nvlink_crc_flit_error_count_l16
1137 + - name: nvlink_crc_flit_error_count_l17
1138 + - name: nvlink_crc_flit_error_count_l2
1139 + - name: nvlink_crc_flit_error_count_l3
1140 + - name: nvlink_crc_flit_error_count_l4
1141 + - name: nvlink_crc_flit_error_count_l5
1142 + - name: nvlink_crc_flit_error_count_l6
1143 + - name: nvlink_crc_flit_error_count_l7
1144 + - name: nvlink_crc_flit_error_count_l8
1145 + - name: nvlink_crc_flit_error_count_l9
1146 + - name: nvlink_error_dl_crc
1147 + - name: nvlink_error_dl_recovery
1148 + - name: nvlink_error_dl_replay
1149 + - name: nvlink_ppcnt_physical_successful_recovery_events
1150 + - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1151 + - name: nvlink_ppcnt_recovery_time_since_last
1152 + - name: nvlink_ppcnt_recovery_total_successful_events
1153 + - name: nvlink_pprm_oper_recovery
1154 + - name: nvlink_recovery_error
1155 + - name: nvlink_recovery_error_count_l0
1156 + - name: nvlink_recovery_error_count_l1
1157 + - name: nvlink_recovery_error_count_l10
1158 + - name: nvlink_recovery_error_count_l11
1159 + - name: nvlink_recovery_error_count_l12
1160 + - name: nvlink_recovery_error_count_l13
1161 + - name: nvlink_recovery_error_count_l14
1162 + - name: nvlink_recovery_error_count_l15
1163 + - name: nvlink_recovery_error_count_l16
1164 + - name: nvlink_recovery_error_count_l17
1165 + - name: nvlink_recovery_error_count_l2
1166 + - name: nvlink_recovery_error_count_l3
1167 + - name: nvlink_recovery_error_count_l4
1168 + - name: nvlink_recovery_error_count_l5
1169 + - name: nvlink_recovery_error_count_l6
1170 + - name: nvlink_recovery_error_count_l7
1171 + - name: nvlink_recovery_error_count_l8
1172 + - name: nvlink_recovery_error_count_l9
1173 + - name: nvlink_replay_error
1174 + - name: nvlink_replay_error_count_l0
1175 + - name: nvlink_replay_error_count_l1
1176 + - name: nvlink_replay_error_count_l10
1177 + - name: nvlink_replay_error_count_l11
1178 + - name: nvlink_replay_error_count_l12
1179 + - name: nvlink_replay_error_count_l13
1180 + - name: nvlink_replay_error_count_l14
1181 + - name: nvlink_replay_error_count_l15
1182 + - name: nvlink_replay_error_count_l16
1183 + - name: nvlink_replay_error_count_l17
1184 + - name: nvlink_replay_error_count_l2
1185 + - name: nvlink_replay_error_count_l3
1186 + - name: nvlink_replay_error_count_l4
1187 + - name: nvlink_replay_error_count_l5
1188 + - name: nvlink_replay_error_count_l6
1189 + - name: nvlink_replay_error_count_l7
1190 + - name: nvlink_replay_error_count_l8
1191 + - name: nvlink_replay_error_count_l9
1192 + - name: dcgm.mig.interconnect.pcie.error_rate
1193 + description: MIG PCIe Error Rate metrics.
1194 + unit: errors/s
1195 + chart_type: line
1196 + dimensions:
1197 + - name: pcie_count_correctable_errors
1198 + - name: pcie_replay
1199 + - name: dcgm.mig.interconnect.nvlink.errors
1200 + description: MIG NVLink Errors metrics.
1201 + unit: errors
1202 + chart_type: line
1203 + dimensions:
1204 + - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1205 + - name: dcgm.mig.interconnect.fabric
1206 + description: MIG Fabric State metrics.
1207 + unit: state
1208 + chart_type: line
1209 + dimensions:
1210 + - name: fabric_clique_id
1211 + - name: fabric_cluster_uuid
1212 + - name: fabric_health_mask
1213 + - name: fabric_manager_error_code
1214 + - name: fabric_manager_status
1215 + - name: dcgm.mig.interconnect.pcie.link.generation
1216 + description: MIG PCIe Link Generation metrics.
1217 + unit: generation
1218 + chart_type: line
1219 + dimensions:
1220 + - name: link_gen
1221 + - name: max_link_gen
1222 + visibility: hidden
1223 + - name: dcgm.mig.interconnect.pcie.link.width
1224 + description: MIG PCIe Link Width metrics.
1225 + unit: lanes
1226 + chart_type: line
1227 + dimensions:
1228 + - name: link_width
1229 + - name: max_link_width
1230 + visibility: hidden
1231 + - name: dcgm.mig.interconnect.state
1232 + description: MIG Interconnect State metrics.
1233 + unit: state
1234 + chart_type: line
1235 + dimensions:
1236 + - name: c2c_link
1237 + - name: c2c_link_power_state
1238 + - name: c2c_link_status
1239 + - name: dcgm.mig.interconnect.pcie.state
1240 + description: MIG PCIe State metrics.
1241 + unit: state
1242 + chart_type: line
1243 + dimensions:
1244 + - name: diag_pcie_result
1245 + - name: dcgm.mig.interconnect.nvlink.state
1246 + description: MIG NVLink State metrics.
1247 + unit: state
1248 + chart_type: line
1249 + dimensions:
1250 + - name: gpu_topology_nvlink
1251 + - name: nvlink_get_state
1252 + - name: nvlink_ppcnt_physical_link_down_counter
1253 + - name: nvlink_ppcnt_plr_rcv_code_err
1254 + - name: nvlink_ppcnt_plr_sync_events
1255 + - name: nvlink_ppcnt_plr_xmit_retry_events
1256 + - name: p2p_nvlink_status
1257 + - name: dcgm.mig.interconnect.throughput
1258 + description: MIG Interconnect Throughput metrics.
1259 + unit: B/s
1260 + chart_type: area
1261 + dimensions:
1262 + - name: c2c_max_bandwidth
1263 + - name: c2c_rx_all_bytes
1264 + - name: c2c_rx_data_bytes
1265 + - name: c2c_tx_all_bytes
1266 + - name: c2c_tx_data_bytes
1267 + - name: dcgm.mig.interconnect.nvlink.throughput
1268 + description: MIG NVLink Throughput metrics.
1269 + unit: B/s
1270 + chart_type: area
1271 + dimensions:
1272 + - name: nvlink_bandwidth_l0
1273 + - name: nvlink_bandwidth_l1
1274 + - name: nvlink_bandwidth_l10
1275 + - name: nvlink_bandwidth_l11
1276 + - name: nvlink_bandwidth_l12
1277 + - name: nvlink_bandwidth_l13
1278 + - name: nvlink_bandwidth_l14
1279 + - name: nvlink_bandwidth_l15
1280 + - name: nvlink_bandwidth_l16
1281 + - name: nvlink_bandwidth_l17
1282 + - name: nvlink_bandwidth_l2
1283 + - name: nvlink_bandwidth_l3
1284 + - name: nvlink_bandwidth_l4
1285 + - name: nvlink_bandwidth_l5
1286 + - name: nvlink_bandwidth_l6
1287 + - name: nvlink_bandwidth_l7
1288 + - name: nvlink_bandwidth_l8
1289 + - name: nvlink_bandwidth_l9
1290 + - name: nvlink_count_rx
1291 + - name: nvlink_count_tx
1292 + - name: nvlink_l0_rx
1293 + - name: nvlink_l0_tx
1294 + - name: nvlink_l10_rx
1295 + - name: nvlink_l10_tx
1296 + - name: nvlink_l11_rx
1297 + - name: nvlink_l11_tx
1298 + - name: nvlink_l12_rx
1299 + - name: nvlink_l12_tx
1300 + - name: nvlink_l13_rx
1301 + - name: nvlink_l13_tx
1302 + - name: nvlink_l14_rx
1303 + - name: nvlink_l14_tx
1304 + - name: nvlink_l15_rx
1305 + - name: nvlink_l15_tx
1306 + - name: nvlink_l16_rx
1307 + - name: nvlink_l16_tx
1308 + - name: nvlink_l17_rx
1309 + - name: nvlink_l17_tx
1310 + - name: nvlink_l1_rx
1311 + - name: nvlink_l1_tx
1312 + - name: nvlink_l2_rx
1313 + - name: nvlink_l2_tx
1314 + - name: nvlink_l3_rx
1315 + - name: nvlink_l3_tx
1316 + - name: nvlink_l4_rx
1317 + - name: nvlink_l4_tx
1318 + - name: nvlink_l5_rx
1319 + - name: nvlink_l5_tx
1320 + - name: nvlink_l6_rx
1321 + - name: nvlink_l6_tx
1322 + - name: nvlink_l7_rx
1323 + - name: nvlink_l7_tx
1324 + - name: nvlink_l8_rx
1325 + - name: nvlink_l8_tx
1326 + - name: nvlink_l9_rx
1327 + - name: nvlink_l9_tx
1328 + - name: nvlink_rx_bandwidth
1329 + - name: nvlink_rx_bandwidth_l0
1330 + - name: nvlink_rx_bandwidth_l1
1331 + - name: nvlink_rx_bandwidth_l10
1332 + - name: nvlink_rx_bandwidth_l11
1333 + - name: nvlink_rx_bandwidth_l12
1334 + - name: nvlink_rx_bandwidth_l13
1335 + - name: nvlink_rx_bandwidth_l14
1336 + - name: nvlink_rx_bandwidth_l15
1337 + - name: nvlink_rx_bandwidth_l16
1338 + - name: nvlink_rx_bandwidth_l17
1339 + - name: nvlink_rx_bandwidth_l2
1340 + - name: nvlink_rx_bandwidth_l3
1341 + - name: nvlink_rx_bandwidth_l4
1342 + - name: nvlink_rx_bandwidth_l5
1343 + - name: nvlink_rx_bandwidth_l6
1344 + - name: nvlink_rx_bandwidth_l7
1345 + - name: nvlink_rx_bandwidth_l8
1346 + - name: nvlink_rx_bandwidth_l9
1347 + - name: nvlink_rx
1348 + - name: nvlink_tx_bandwidth
1349 + - name: nvlink_tx_bandwidth_l0
1350 + - name: nvlink_tx_bandwidth_l1
1351 + - name: nvlink_tx_bandwidth_l10
1352 + - name: nvlink_tx_bandwidth_l11
1353 + - name: nvlink_tx_bandwidth_l12
1354 + - name: nvlink_tx_bandwidth_l13
1355 + - name: nvlink_tx_bandwidth_l14
1356 + - name: nvlink_tx_bandwidth_l15
1357 + - name: nvlink_tx_bandwidth_l16
1358 + - name: nvlink_tx_bandwidth_l17
1359 + - name: nvlink_tx_bandwidth_l2
1360 + - name: nvlink_tx_bandwidth_l3
1361 + - name: nvlink_tx_bandwidth_l4
1362 + - name: nvlink_tx_bandwidth_l5
1363 + - name: nvlink_tx_bandwidth_l6
1364 + - name: nvlink_tx_bandwidth_l7
1365 + - name: nvlink_tx_bandwidth_l8
1366 + - name: nvlink_tx_bandwidth_l9
1367 + - name: nvlink_tx
1368 + - name: dcgm.mig.interconnect.pcie.throughput
1369 + description: MIG PCIe Throughput metrics.
1370 + unit: B/s
1371 + chart_type: area
1372 + dimensions:
1373 + - name: pcie_rx
1374 + - name: pcie_rx_throughput
1375 + - name: pcie_tx
1376 + - name: pcie_tx_throughput
1377 + - name: dcgm.mig.interconnect.total.throughput
1378 + description: MIG Interconnect Total Throughput metrics.
1379 + unit: B/s
1380 + chart_type: area
1381 + dimensions:
1382 + - name: pcie
1383 + - name: nvlink
1384 + - name: dcgm.mig.interconnect.nvlink.traffic
1385 + description: MIG NVLink Traffic metrics.
1386 + unit: events/s
1387 + chart_type: line
1388 + dimensions:
1389 + - name: nvlink_count_rx_packets
1390 + - name: nvlink_count_tx_packets
1391 + - name: nvlink_ppcnt_plr_rcv_codes
1392 + - name: nvlink_ppcnt_plr_xmit_codes
1393 + - name: nvlink_ppcnt_plr_xmit_retry_codes
1394 + - name: dcgm.mig.memory.bar1_usage
1395 + description: MIG BAR1 Memory Usage metrics.
1396 + unit: B
1397 + chart_type: stacked
1398 + dimensions:
1399 + - name: free
1400 + - name: used
1401 + - name: dcgm.mig.memory.bar1_capacity
1402 + description: MIG BAR1 Memory Capacity metrics.
1403 + unit: B
1404 + chart_type: line
1405 + dimensions:
1406 + - name: total
1407 + - name: dcgm.mig.memory.ecc_error_rate
1408 + description: MIG ECC Error Rate metrics.
1409 + unit: errors/s
1410 + chart_type: line
1411 + dimensions:
1412 + - name: ecc_current
1413 + - name: ecc_dbe_agg
1414 + - name: ecc_dbe_agg_cbu
1415 + - name: ecc_dbe_agg_dev
1416 + - name: ecc_dbe_agg_l1
1417 + - name: ecc_dbe_agg_l2
1418 + - name: ecc_dbe_agg_reg
1419 + - name: ecc_dbe_agg_shm
1420 + - name: ecc_dbe_agg_srm
1421 + - name: ecc_dbe_agg_tex
1422 + - name: ecc_dbe_vol
1423 + - name: ecc_dbe_vol_cbu
1424 + - name: ecc_dbe_vol_dev
1425 + - name: ecc_dbe_vol_l1
1426 + - name: ecc_dbe_vol_l2
1427 + - name: ecc_dbe_vol_reg
1428 + - name: ecc_dbe_vol_shm
1429 + - name: ecc_dbe_vol_srm
1430 + - name: ecc_dbe_vol_tex
1431 + - name: ecc_pending
1432 + - name: ecc_sbe_agg
1433 + - name: ecc_sbe_agg_cbu
1434 + - name: ecc_sbe_agg_dev
1435 + - name: ecc_sbe_agg_l1
1436 + - name: ecc_sbe_agg_l2
1437 + - name: ecc_sbe_agg_reg
1438 + - name: ecc_sbe_agg_shm
1439 + - name: ecc_sbe_agg_srm
1440 + - name: ecc_sbe_agg_tex
1441 + - name: ecc_sbe_vol
1442 + - name: ecc_sbe_vol_cbu
1443 + - name: ecc_sbe_vol_dev
1444 + - name: ecc_sbe_vol_l1
1445 + - name: ecc_sbe_vol_l2
1446 + - name: ecc_sbe_vol_reg
1447 + - name: ecc_sbe_vol_shm
1448 + - name: ecc_sbe_vol_srm
1449 + - name: ecc_sbe_vol_tex
1450 + - name: nvlink_ecc_data_error
1451 + - name: dcgm.mig.memory.ecc_errors
1452 + description: MIG ECC Errors metrics.
1453 + unit: errors
1454 + chart_type: line
1455 + dimensions:
1456 + - name: ecc_current
1457 + - name: ecc_dbe_agg_cbu
1458 + - name: ecc_dbe_agg_dev
1459 + - name: ecc_dbe_agg_l1
1460 + - name: ecc_dbe_agg_l2
1461 + - name: ecc_dbe_agg_reg
1462 + - name: ecc_dbe_agg_shm
1463 + - name: ecc_dbe_agg_srm
1464 + - name: ecc_dbe_agg_tex
1465 + - name: ecc_dbe_vol_cbu
1466 + - name: ecc_dbe_vol_dev
1467 + - name: ecc_dbe_vol_l1
1468 + - name: ecc_dbe_vol_l2
1469 + - name: ecc_dbe_vol_reg
1470 + - name: ecc_dbe_vol_shm
1471 + - name: ecc_dbe_vol_srm
1472 + - name: ecc_dbe_vol_tex
1473 + - name: ecc_inforom_ver
1474 + - name: ecc_pending
1475 + - name: ecc_sbe_agg_cbu
1476 + - name: ecc_sbe_agg_dev
1477 + - name: ecc_sbe_agg_l1
1478 + - name: ecc_sbe_agg_l2
1479 + - name: ecc_sbe_agg_reg
1480 + - name: ecc_sbe_agg_shm
1481 + - name: ecc_sbe_agg_srm
1482 + - name: ecc_sbe_agg_tex
1483 + - name: ecc_sbe_vol_cbu
1484 + - name: ecc_sbe_vol_dev
1485 + - name: ecc_sbe_vol_l1
1486 + - name: ecc_sbe_vol_l2
1487 + - name: ecc_sbe_vol_reg
1488 + - name: ecc_sbe_vol_shm
1489 + - name: ecc_sbe_vol_srm
1490 + - name: ecc_sbe_vol_tex
1491 + - name: dcgm.mig.memory.page_retirements
1492 + description: MIG Retired Memory Pages metrics.
1493 + unit: pages/s
1494 + chart_type: line
1495 + dimensions:
1496 + - name: retired_dbe
1497 + - name: retired_pending
1498 + - name: retired_sbe
1499 + - name: dcgm.mig.memory.usage
1500 + description: MIG Memory Usage metrics.
1501 + unit: B
1502 + chart_type: stacked
1503 + dimensions:
1504 + - name: free
1505 + - name: reserved
1506 + - name: used
1507 + - name: dcgm.mig.memory.capacity
1508 + description: MIG Memory Capacity metrics.
1509 + unit: B
1510 + chart_type: line
1511 + dimensions:
1512 + - name: total
1513 + - name: dcgm.mig.memory.utilization
1514 + description: MIG Memory Utilization metrics.
1515 + unit: '%'
1516 + chart_type: line
1517 + dimensions:
1518 + - name: used_percent
1519 + - name: dcgm.mig.power.energy
1520 + description: MIG Energy Consumption Rate metrics.
1521 + unit: mJ/s
1522 + chart_type: line
1523 + dimensions:
1524 + - name: total
1525 + - name: dcgm.mig.power.profiles
1526 + description: MIG Power Profiles metrics.
1527 + unit: state
1528 + chart_type: line
1529 + dimensions:
1530 + - name: enforced_power_profile_mask
1531 + - name: requested_power_profile_mask
1532 + - name: valid_power_profile_mask
1533 + - name: dcgm.mig.power.smoothing
1534 + description: MIG Power Smoothing metrics.
1535 + unit: value
1536 + chart_type: line
1537 + dimensions:
1538 + - name: pwr_smoothing_active_preset_profile
1539 + - name: pwr_smoothing_admin_override_percent_tmp_floor
1540 + - name: pwr_smoothing_admin_override_ramp_down_hyst_val
1541 + - name: pwr_smoothing_admin_override_ramp_down_rate
1542 + - name: pwr_smoothing_admin_override_ramp_up_rate
1543 + - name: pwr_smoothing_applied_tmp_ceil
1544 + - name: pwr_smoothing_applied_tmp_floor
1545 + - name: pwr_smoothing_enabled
1546 + - name: pwr_smoothing_hw_circuitry_percent_lifetime_remaining
1547 + - name: pwr_smoothing_imm_ramp_down_enabled
1548 + - name: pwr_smoothing_max_num_preset_profiles
1549 + - name: pwr_smoothing_max_percent_tmp_floor_setting
1550 + - name: pwr_smoothing_min_percent_tmp_floor_setting
1551 + - name: pwr_smoothing_priv_lvl
1552 + - name: pwr_smoothing_profile_percent_tmp_floor
1553 + - name: pwr_smoothing_profile_ramp_down_hyst_val
1554 + - name: pwr_smoothing_profile_ramp_down_rate
1555 + - name: pwr_smoothing_profile_ramp_up_rate
1556 + - name: dcgm.mig.power.usage
1557 + description: MIG Power Usage metrics.
1558 + unit: Watts
1559 + chart_type: line
1560 + dimensions:
1561 + - name: draw
1562 + - name: enforced_limit
1563 + visibility: hidden
1564 + - name: power_mgmt_limit
1565 + visibility: hidden
1566 + - name: power_mgmt_limit_def
1567 + visibility: hidden
1568 + - name: power_mgmt_limit_max
1569 + visibility: hidden
1570 + - name: power_mgmt_limit_min
1571 + visibility: hidden
1572 + - name: power_usage_instant
1573 + - name: dcgm.mig.reliability.memory_health
1574 + description: MIG Memory Health metrics.
1575 + unit: state
1576 + chart_type: line
1577 + dimensions:
1578 + - name: banks_remap_rows_avail_high
1579 + - name: banks_remap_rows_avail_low
1580 + - name: banks_remap_rows_avail_max
1581 + - name: banks_remap_rows_avail_none
1582 + - name: banks_remap_rows_avail_partial
1583 + - name: memory_unrepairable_flag
1584 + - name: threshold_srm
1585 + - name: dcgm.mig.reliability.recovery_action
1586 + description: MIG Recovery Action metrics.
1587 + unit: state
1588 + chart_type: line
1589 + dimensions:
1590 + - name: get_gpu_recovery_action
1591 + - name: dcgm.mig.reliability.row_remap_events
1592 + description: MIG Row Remap Events metrics.
1593 + unit: rows/s
1594 + chart_type: line
1595 + dimensions:
1596 + - name: correctable_remapped_rows
1597 + - name: uncorrectable_remapped_rows
1598 + - name: dcgm.mig.reliability.row_remap_status
1599 + description: MIG Row Remap Status metrics.
1600 + unit: state
1601 + chart_type: line
1602 + dimensions:
1603 + - name: row_remap_failure
1604 + - name: row_remap_pending
1605 + - name: dcgm.mig.reliability.xid
1606 + description: MIG XID Errors metrics.
1607 + unit: code
1608 + chart_type: line
1609 + dimensions:
1610 + - name: xid
1611 + - name: dcgm.mig.state.configuration
1612 + description: MIG Configuration State metrics.
1613 + unit: state
1614 + chart_type: line
1615 + dimensions:
1616 + - name: autoboost
1617 + - name: compute_mode
1618 + - name: persistence_mode
1619 + - name: sync_boost
1620 + - name: sync_boost_violation
1621 + - name: dcgm.mig.state.performance
1622 + description: MIG Performance State metrics.
1623 + unit: state
1624 + chart_type: line
1625 + dimensions:
1626 + - name: pstate
1627 + - name: dcgm.mig.state.virtualization
1628 + description: MIG Virtualization State metrics.
1629 + unit: state
1630 + chart_type: line
1631 + dimensions:
1632 + - name: mig_mode
1633 + - name: virtual_mode
1634 + - name: dcgm.mig.thermal.fan_speed
1635 + description: MIG Fan Speed metrics.
1636 + unit: '%'
1637 + chart_type: line
1638 + dimensions:
1639 + - name: fan_speed
1640 + - name: dcgm.mig.thermal.temperature
1641 + description: MIG Temperature metrics.
1642 + unit: Celsius
1643 + chart_type: line
1644 + dimensions:
1645 + - name: gpu
1646 + - name: gpu_max_op_temp
1647 + visibility: hidden
1648 + - name: gpu_temp_limit
1649 + visibility: hidden
1650 + - name: mem_max_op_temp
1651 + visibility: hidden
1652 + - name: memory
1653 + - name: shutdown_temp
1654 + visibility: hidden
1655 + - name: slowdown_temp
1656 + visibility: hidden
1657 + - name: dcgm.mig.throttle.reasons
1658 + description: MIG Throttle Reasons metrics.
1659 + unit: bitmask
1660 + chart_type: line
1661 + dimensions:
1662 + - name: clocks_event_reasons
1663 + - name: dcgm.mig.throttle.violations
1664 + description: MIG Throttle Violation Duration metrics.
1665 + unit: milliseconds/s
1666 + chart_type: line
1667 + dimensions:
1668 + - name: board_limit_violation
1669 + - name: hw_power_brake_slowdown
1670 + - name: hw_therm_slowdown
1671 + - name: low_utilization_violation
1672 + - name: power_violation
1673 + - name: reliability_violation
1674 + - name: sw_power_cap
1675 + - name: sw_therm_slowdown
1676 + - name: sync_boost
1677 + - name: thermal_violation
1678 + - name: total_app_clocks_violation
1679 + - name: total_base_clocks_violation
1680 + - name: nvlink
1681 + description: These metrics refer to NVLink link instances.
1682 + labels:
1683 + - name: gpu
1684 + description: gpu label from exporter metrics.
1685 + - name: gpu_uuid
1686 + description: gpu_uuid label from exporter metrics.
1687 + - name: nvlink
1688 + description: nvlink label from exporter metrics.
1689 + metrics:
1690 + - name: dcgm.nvlink.interconnect.ber
1691 + description: NVLink Interconnect Bit Error Rate metrics.
1692 + unit: ratio
1693 + chart_type: line
1694 + dimensions:
1695 + - name: nvlink_count_effective_ber
1696 + - name: nvlink_count_effective_ber_float
1697 + - name: nvlink_count_symbol_ber
1698 + - name: nvlink_count_symbol_ber_float
1699 + - name: dcgm.nvlink.interconnect.congestion
1700 + description: NVLink Interconnect Congestion metrics.
1701 + unit: events/s
1702 + chart_type: line
1703 + dimensions:
1704 + - name: nvlink_ppcnt_ibpc_port_xmit_wait
1705 + - name: dcgm.nvlink.interconnect.error_rate
1706 + description: NVLink Interconnect Error Rate metrics.
1707 + unit: errors/s
1708 + chart_type: line
1709 + dimensions:
1710 + - name: gpu_nvlink_errors
1711 + - name: nvlink_count_effective_errors
1712 + - name: nvlink_count_fec_history_0
1713 + - name: nvlink_count_fec_history_1
1714 + - name: nvlink_count_fec_history_10
1715 + - name: nvlink_count_fec_history_11
1716 + - name: nvlink_count_fec_history_12
1717 + - name: nvlink_count_fec_history_13
1718 + - name: nvlink_count_fec_history_14
1719 + - name: nvlink_count_fec_history_15
1720 + - name: nvlink_count_fec_history_2
1721 + - name: nvlink_count_fec_history_3
1722 + - name: nvlink_count_fec_history_4
1723 + - name: nvlink_count_fec_history_5
1724 + - name: nvlink_count_fec_history_6
1725 + - name: nvlink_count_fec_history_7
1726 + - name: nvlink_count_fec_history_8
1727 + - name: nvlink_count_fec_history_9
1728 + - name: nvlink_count_link_recovery_events
1729 + - name: nvlink_count_link_recovery_failed_events
1730 + - name: nvlink_count_link_recovery_successful_events
1731 + - name: nvlink_count_local_link_integrity_errors
1732 + - name: nvlink_count_rx_buffer_overrun_errors
1733 + - name: nvlink_count_rx_errors
1734 + - name: nvlink_count_rx_general_errors
1735 + - name: nvlink_count_rx_malformed_packet_errors
1736 + - name: nvlink_count_rx_remote_errors
1737 + - name: nvlink_count_rx_symbol_errors
1738 + - name: nvlink_count_tx_discards
1739 + - name: nvlink_crc_data_error
1740 + - name: nvlink_crc_data_error_count_l0
1741 + - name: nvlink_crc_data_error_count_l1
1742 + - name: nvlink_crc_data_error_count_l10
1743 + - name: nvlink_crc_data_error_count_l11
1744 + - name: nvlink_crc_data_error_count_l12
1745 + - name: nvlink_crc_data_error_count_l13
1746 + - name: nvlink_crc_data_error_count_l14
1747 + - name: nvlink_crc_data_error_count_l15
1748 + - name: nvlink_crc_data_error_count_l16
1749 + - name: nvlink_crc_data_error_count_l17
1750 + - name: nvlink_crc_data_error_count_l2
1751 + - name: nvlink_crc_data_error_count_l3
1752 + - name: nvlink_crc_data_error_count_l4
1753 + - name: nvlink_crc_data_error_count_l5
1754 + - name: nvlink_crc_data_error_count_l6
1755 + - name: nvlink_crc_data_error_count_l7
1756 + - name: nvlink_crc_data_error_count_l8
1757 + - name: nvlink_crc_data_error_count_l9
1758 + - name: nvlink_crc_flit_error
1759 + - name: nvlink_crc_flit_error_count_l0
1760 + - name: nvlink_crc_flit_error_count_l1
1761 + - name: nvlink_crc_flit_error_count_l10
1762 + - name: nvlink_crc_flit_error_count_l11
1763 + - name: nvlink_crc_flit_error_count_l12
1764 + - name: nvlink_crc_flit_error_count_l13
1765 + - name: nvlink_crc_flit_error_count_l14
1766 + - name: nvlink_crc_flit_error_count_l15
1767 + - name: nvlink_crc_flit_error_count_l16
1768 + - name: nvlink_crc_flit_error_count_l17
1769 + - name: nvlink_crc_flit_error_count_l2
1770 + - name: nvlink_crc_flit_error_count_l3
1771 + - name: nvlink_crc_flit_error_count_l4
1772 + - name: nvlink_crc_flit_error_count_l5
1773 + - name: nvlink_crc_flit_error_count_l6
1774 + - name: nvlink_crc_flit_error_count_l7
1775 + - name: nvlink_crc_flit_error_count_l8
1776 + - name: nvlink_crc_flit_error_count_l9
1777 + - name: nvlink_error_dl_crc
1778 + - name: nvlink_error_dl_recovery
1779 + - name: nvlink_error_dl_replay
1780 + - name: nvlink_ppcnt_physical_successful_recovery_events
1781 + - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1782 + - name: nvlink_ppcnt_recovery_time_since_last
1783 + - name: nvlink_ppcnt_recovery_total_successful_events
1784 + - name: nvlink_pprm_oper_recovery
1785 + - name: nvlink_recovery_error
1786 + - name: nvlink_recovery_error_count_l0
1787 + - name: nvlink_recovery_error_count_l1
1788 + - name: nvlink_recovery_error_count_l10
1789 + - name: nvlink_recovery_error_count_l11
1790 + - name: nvlink_recovery_error_count_l12
1791 + - name: nvlink_recovery_error_count_l13
1792 + - name: nvlink_recovery_error_count_l14
1793 + - name: nvlink_recovery_error_count_l15
1794 + - name: nvlink_recovery_error_count_l16
1795 + - name: nvlink_recovery_error_count_l17
1796 + - name: nvlink_recovery_error_count_l2
1797 + - name: nvlink_recovery_error_count_l3
1798 + - name: nvlink_recovery_error_count_l4
1799 + - name: nvlink_recovery_error_count_l5
1800 + - name: nvlink_recovery_error_count_l6
1801 + - name: nvlink_recovery_error_count_l7
1802 + - name: nvlink_recovery_error_count_l8
1803 + - name: nvlink_recovery_error_count_l9
1804 + - name: nvlink_replay_error
1805 + - name: nvlink_replay_error_count_l0
1806 + - name: nvlink_replay_error_count_l1
1807 + - name: nvlink_replay_error_count_l10
1808 + - name: nvlink_replay_error_count_l11
1809 + - name: nvlink_replay_error_count_l12
1810 + - name: nvlink_replay_error_count_l13
1811 + - name: nvlink_replay_error_count_l14
1812 + - name: nvlink_replay_error_count_l15
1813 + - name: nvlink_replay_error_count_l16
1814 + - name: nvlink_replay_error_count_l17
1815 + - name: nvlink_replay_error_count_l2
1816 + - name: nvlink_replay_error_count_l3
1817 + - name: nvlink_replay_error_count_l4
1818 + - name: nvlink_replay_error_count_l5
1819 + - name: nvlink_replay_error_count_l6
1820 + - name: nvlink_replay_error_count_l7
1821 + - name: nvlink_replay_error_count_l8
1822 + - name: nvlink_replay_error_count_l9
1823 + - name: dcgm.nvlink.interconnect.errors
1824 + description: NVLink Interconnect Errors metrics.
1825 + unit: errors
1826 + chart_type: line
1827 + dimensions:
1828 + - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1829 + - name: dcgm.nvlink.interconnect.state
1830 + description: NVLink Interconnect State metrics.
1831 + unit: state
1832 + chart_type: line
1833 + dimensions:
1834 + - name: gpu_topology_nvlink
1835 + - name: nvlink_get_state
1836 + - name: nvlink_ppcnt_physical_link_down_counter
1837 + - name: nvlink_ppcnt_plr_rcv_code_err
1838 + - name: nvlink_ppcnt_plr_sync_events
1839 + - name: nvlink_ppcnt_plr_xmit_retry_events
1840 + - name: p2p_nvlink_status
1841 + - name: dcgm.nvlink.interconnect.throughput
1842 + description: NVLink Interconnect Throughput metrics.
1843 + unit: B/s
1844 + chart_type: area
1845 + dimensions:
1846 + - name: nvlink_bandwidth
1847 + - name: nvlink_bandwidth_l0
1848 + - name: nvlink_bandwidth_l1
1849 + - name: nvlink_bandwidth_l10
1850 + - name: nvlink_bandwidth_l11
1851 + - name: nvlink_bandwidth_l12
1852 + - name: nvlink_bandwidth_l13
1853 + - name: nvlink_bandwidth_l14
1854 + - name: nvlink_bandwidth_l15
1855 + - name: nvlink_bandwidth_l16
1856 + - name: nvlink_bandwidth_l17
1857 + - name: nvlink_bandwidth_l2
1858 + - name: nvlink_bandwidth_l3
1859 + - name: nvlink_bandwidth_l4
1860 + - name: nvlink_bandwidth_l5
1861 + - name: nvlink_bandwidth_l6
1862 + - name: nvlink_bandwidth_l7
1863 + - name: nvlink_bandwidth_l8
1864 + - name: nvlink_bandwidth_l9
1865 + - name: nvlink_count_rx
1866 + - name: nvlink_count_tx
1867 + - name: nvlink_l0_rx
1868 + - name: nvlink_l0_tx
1869 + - name: nvlink_l10_rx
1870 + - name: nvlink_l10_tx
1871 + - name: nvlink_l11_rx
1872 + - name: nvlink_l11_tx
1873 + - name: nvlink_l12_rx
1874 + - name: nvlink_l12_tx
1875 + - name: nvlink_l13_rx
1876 + - name: nvlink_l13_tx
1877 + - name: nvlink_l14_rx
1878 + - name: nvlink_l14_tx
1879 + - name: nvlink_l15_rx
1880 + - name: nvlink_l15_tx
1881 + - name: nvlink_l16_rx
1882 + - name: nvlink_l16_tx
1883 + - name: nvlink_l17_rx
1884 + - name: nvlink_l17_tx
1885 + - name: nvlink_l1_rx
1886 + - name: nvlink_l1_tx
1887 + - name: nvlink_l2_rx
1888 + - name: nvlink_l2_tx
1889 + - name: nvlink_l3_rx
1890 + - name: nvlink_l3_tx
1891 + - name: nvlink_l4_rx
1892 + - name: nvlink_l4_tx
1893 + - name: nvlink_l5_rx
1894 + - name: nvlink_l5_tx
1895 + - name: nvlink_l6_rx
1896 + - name: nvlink_l6_tx
1897 + - name: nvlink_l7_rx
1898 + - name: nvlink_l7_tx
1899 + - name: nvlink_l8_rx
1900 + - name: nvlink_l8_tx
1901 + - name: nvlink_l9_rx
1902 + - name: nvlink_l9_tx
1903 + - name: nvlink_rx_bandwidth
1904 + - name: nvlink_rx_bandwidth_l0
1905 + - name: nvlink_rx_bandwidth_l1
1906 + - name: nvlink_rx_bandwidth_l10
1907 + - name: nvlink_rx_bandwidth_l11
1908 + - name: nvlink_rx_bandwidth_l12
1909 + - name: nvlink_rx_bandwidth_l13
1910 + - name: nvlink_rx_bandwidth_l14
1911 + - name: nvlink_rx_bandwidth_l15
1912 + - name: nvlink_rx_bandwidth_l16
1913 + - name: nvlink_rx_bandwidth_l17
1914 + - name: nvlink_rx_bandwidth_l2
1915 + - name: nvlink_rx_bandwidth_l3
1916 + - name: nvlink_rx_bandwidth_l4
1917 + - name: nvlink_rx_bandwidth_l5
1918 + - name: nvlink_rx_bandwidth_l6
1919 + - name: nvlink_rx_bandwidth_l7
1920 + - name: nvlink_rx_bandwidth_l8
1921 + - name: nvlink_rx_bandwidth_l9
1922 + - name: nvlink_rx
1923 + - name: nvlink_tx_bandwidth
1924 + - name: nvlink_tx_bandwidth_l0
1925 + - name: nvlink_tx_bandwidth_l1
1926 + - name: nvlink_tx_bandwidth_l10
1927 + - name: nvlink_tx_bandwidth_l11
1928 + - name: nvlink_tx_bandwidth_l12
1929 + - name: nvlink_tx_bandwidth_l13
1930 + - name: nvlink_tx_bandwidth_l14
1931 + - name: nvlink_tx_bandwidth_l15
1932 + - name: nvlink_tx_bandwidth_l16
1933 + - name: nvlink_tx_bandwidth_l17
1934 + - name: nvlink_tx_bandwidth_l2
1935 + - name: nvlink_tx_bandwidth_l3
1936 + - name: nvlink_tx_bandwidth_l4
1937 + - name: nvlink_tx_bandwidth_l5
1938 + - name: nvlink_tx_bandwidth_l6
1939 + - name: nvlink_tx_bandwidth_l7
1940 + - name: nvlink_tx_bandwidth_l8
1941 + - name: nvlink_tx_bandwidth_l9
1942 + - name: nvlink_tx
1943 + - name: dcgm.nvlink.interconnect.traffic
1944 + description: NVLink Interconnect Traffic metrics.
1945 + unit: events/s
1946 + chart_type: line
1947 + dimensions:
1948 + - name: nvlink_count_rx_packets
1949 + - name: nvlink_count_tx_packets
1950 + - name: nvlink_ppcnt_plr_rcv_codes
1951 + - name: nvlink_ppcnt_plr_xmit_codes
1952 + - name: nvlink_ppcnt_plr_xmit_retry_codes
1953 + - name: dcgm.nvlink.internal.boundary
1954 + description: NVLink Internal Boundary Fields metrics.
1955 + unit: state
1956 + chart_type: line
1957 + dimensions:
1958 + - name: nvlink_ppcnt_recovery_time_between_last_two
1959 + - name: dcgm.nvlink.memory.ecc_error_rate
1960 + description: NVLink ECC Error Rate metrics.
1961 + unit: errors/s
1962 + chart_type: line
1963 + dimensions:
1964 + - name: nvlink_ecc_data_error
1965 + - name: nvswitch
1966 + description: These metrics refer to NVSwitch instances.
1967 + labels:
1968 + - name: nvswitch
1969 + description: nvswitch label from exporter metrics.
1970 + metrics:
1971 + - name: dcgm.nvswitch.interconnect.nvswitch.current
1972 + description: NVSwitch Current metrics.
1973 + unit: value
1974 + chart_type: line
1975 + dimensions:
1976 + - name: nvswitch_current_iddq
1977 + - name: nvswitch_current_iddq_dvdd
1978 + - name: nvswitch_current_iddq_rev
1979 + - name: dcgm.nvswitch.interconnect.nvswitch.errors
1980 + description: NVSwitch NVSwitch Errors metrics.
1981 + unit: errors/s
1982 + chart_type: line
1983 + dimensions:
1984 + - name: nvswitch_fatal_errors
1985 + - name: nvswitch_link_crc_errors
1986 + - name: nvswitch_link_crc_errors_lane0
1987 + - name: nvswitch_link_crc_errors_lane1
1988 + - name: nvswitch_link_crc_errors_lane2
1989 + - name: nvswitch_link_crc_errors_lane3
1990 + - name: nvswitch_link_crc_errors_lane4
1991 + - name: nvswitch_link_crc_errors_lane5
1992 + - name: nvswitch_link_crc_errors_lane6
1993 + - name: nvswitch_link_crc_errors_lane7
1994 + - name: nvswitch_link_fatal_errors
1995 + - name: nvswitch_link_flit_errors
1996 + - name: nvswitch_link_non_fatal_errors
1997 + - name: nvswitch_link_recovery_errors
1998 + - name: nvswitch_link_replay_errors
1999 + - name: nvswitch_non_fatal_errors
2000 + - name: dcgm.nvswitch.interconnect.nvswitch.latency
2001 + description: NVSwitch NVSwitch Link Latency metrics.
2002 + unit: events/s
2003 + chart_type: line
2004 + dimensions:
2005 + - name: nvswitch_link_latency_count_vc0
2006 + - name: nvswitch_link_latency_count_vc1
2007 + - name: nvswitch_link_latency_count_vc2
2008 + - name: nvswitch_link_latency_count_vc3
2009 + - name: nvswitch_link_latency_high_vc0
2010 + - name: nvswitch_link_latency_high_vc1
2011 + - name: nvswitch_link_latency_high_vc2
2012 + - name: nvswitch_link_latency_high_vc3
2013 + - name: nvswitch_link_latency_low_vc0
2014 + - name: nvswitch_link_latency_low_vc1
2015 + - name: nvswitch_link_latency_low_vc2
2016 + - name: nvswitch_link_latency_low_vc3
2017 + - name: nvswitch_link_latency_medium_vc0
2018 + - name: nvswitch_link_latency_medium_vc1
2019 + - name: nvswitch_link_latency_medium_vc2
2020 + - name: nvswitch_link_latency_medium_vc3
2021 + - name: nvswitch_link_latency_panic_vc0
2022 + - name: nvswitch_link_latency_panic_vc1
2023 + - name: nvswitch_link_latency_panic_vc2
2024 + - name: nvswitch_link_latency_panic_vc3
2025 + - name: dcgm.nvswitch.interconnect.nvswitch.power
2026 + description: NVSwitch NVSwitch Power metrics.
2027 + unit: Watts
2028 + chart_type: line
2029 + dimensions:
2030 + - name: nvswitch_power_dvdd
2031 + - name: nvswitch_power_hvdd
2032 + - name: nvswitch_power_vdd
2033 + - name: dcgm.nvswitch.interconnect.nvswitch.status
2034 + description: NVSwitch NVSwitch Status metrics.
2035 + unit: state
2036 + chart_type: line
2037 + dimensions:
2038 + - name: nvswitch_link_status
2039 + - name: nvswitch_link_type
2040 + - name: nvswitch_reset_required
2041 + - name: dcgm.nvswitch.interconnect.nvswitch.throughput
2042 + description: NVSwitch NVSwitch Throughput metrics.
2043 + unit: B/s
2044 + chart_type: area
2045 + dimensions:
2046 + - name: nvswitch_link_throughput_rx
2047 + - name: nvswitch_link_throughput_tx
2048 + - name: nvswitch_throughput_rx
2049 + - name: nvswitch_throughput_tx
2050 + - name: dcgm.nvswitch.interconnect.nvswitch.topology
2051 + description: NVSwitch NVSwitch Topology metrics.
2052 + unit: value
2053 + chart_type: line
2054 + dimensions:
2055 + - name: nvswitch_device_uuid
2056 + - name: nvswitch_link_device_link_id
2057 + - name: nvswitch_link_device_link_sid
2058 + - name: nvswitch_link_id
2059 + - name: nvswitch_link_remote_pcie_bus
2060 + - name: nvswitch_link_remote_pcie_device
2061 + - name: nvswitch_link_remote_pcie_domain
2062 + - name: nvswitch_link_remote_pcie_function
2063 + - name: nvswitch_pcie_bus
2064 + - name: nvswitch_pcie_device
2065 + - name: nvswitch_pcie_domain
2066 + - name: nvswitch_pcie_function
2067 + - name: nvswitch_phys_id
2068 + - name: dcgm.nvswitch.interconnect.nvswitch.voltage
2069 + description: NVSwitch NVSwitch Voltage metrics.
2070 + unit: mV
2071 + chart_type: line
2072 + dimensions:
2073 + - name: nvswitch_voltage_mvolt
2074 + - name: dcgm.nvswitch.internal.boundary
2075 + description: NVSwitch Internal Boundary Fields metrics.
2076 + unit: state
2077 + chart_type: line
2078 + dimensions:
2079 + - name: first_nvswitch_field_id
2080 + - name: last_nvswitch_field_id
2081 + - name: dcgm.nvswitch.memory.ecc_error_rate
2082 + description: NVSwitch ECC Error Rate metrics.
2083 + unit: errors/s
2084 + chart_type: line
2085 + dimensions:
2086 + - name: nvswitch_link_ecc_errors
2087 + - name: nvswitch_link_ecc_errors_lane0
2088 + - name: nvswitch_link_ecc_errors_lane1
2089 + - name: nvswitch_link_ecc_errors_lane2
2090 + - name: nvswitch_link_ecc_errors_lane3
2091 + - name: nvswitch_link_ecc_errors_lane4
2092 + - name: nvswitch_link_ecc_errors_lane5
2093 + - name: nvswitch_link_ecc_errors_lane6
2094 + - name: nvswitch_link_ecc_errors_lane7
2095 + - name: dcgm.nvswitch.thermal.temperature
2096 + description: NVSwitch Temperature metrics.
2097 + unit: Celsius
2098 + chart_type: line
2099 + dimensions:
2100 + - name: nvswitch_temperature_current
2101 + - name: nvswitch_temperature_limit_shutdown
2102 + - name: nvswitch_temperature_limit_slowdown
2103 + - name: cpu
2104 + description: These metrics refer to host CPU instances.
2105 + labels:
2106 + - name: cpu
2107 + description: cpu label from exporter metrics.
2108 + metrics:
2109 + - name: dcgm.cpu.clock.frequency
2110 + description: CPU Clock Frequency metrics.
2111 + unit: MHz
2112 + chart_type: line
2113 + dimensions:
2114 + - name: cpu_clock_current
2115 + - name: dcgm.cpu.cpu.info
2116 + description: CPU CPU Information metrics.
2117 + unit: value
2118 + chart_type: line
2119 + dimensions:
2120 + - name: cpu_model
2121 + - name: cpu_vendor
2122 + - name: dcgm.cpu.cpu.power
2123 + description: CPU CPU Power metrics.
2124 + unit: Watts
2125 + chart_type: line
2126 + dimensions:
2127 + - name: cpu_power_limit
2128 + - name: cpu_power_util_current
2129 + - name: dcgm.cpu.cpu.temperature
2130 + description: CPU CPU Temperature metrics.
2131 + unit: Celsius
2132 + chart_type: line
2133 + dimensions:
2134 + - name: cpu_temp_critical
2135 + - name: cpu_temp_current
2136 + - name: cpu_temp_warning
2137 + - name: dcgm.cpu.cpu.utilization
2138 + description: CPU CPU Utilization metrics.
2139 + unit: '%'
2140 + chart_type: line
2141 + dimensions:
2142 + - name: cpu_util
2143 + - name: cpu_util_irq
2144 + - name: cpu_util_nice
2145 + - name: cpu_util_sys
2146 + - name: cpu_util_user
2147 + - name: dcgm.cpu.diagnostics.results
2148 + description: CPU Diagnostics Results metrics.
2149 + unit: state
2150 + chart_type: line
2151 + dimensions:
2152 + - name: diag_cpu_eud_result
2153 + - name: cpu_core
2154 + description: These metrics refer to host CPU core instances.
2155 + labels:
2156 + - name: cpu
2157 + description: cpu label from exporter metrics.
2158 + - name: cpucore
2159 + description: cpucore label from exporter metrics.
2160 + metrics:
2161 + - name: dcgm.cpu_core.clock.frequency
2162 + description: CPU Core Clock Frequency metrics.
2163 + unit: MHz
2164 + chart_type: line
2165 + dimensions:
2166 + - name: cpu_clock_current
2167 + - name: dcgm.cpu_core.cpu.info
2168 + description: CPU Core CPU Information metrics.
2169 + unit: value
2170 + chart_type: line
2171 + dimensions:
2172 + - name: cpu_model
2173 + - name: cpu_vendor
2174 + - name: dcgm.cpu_core.cpu.power
2175 + description: CPU Core CPU Power metrics.
2176 + unit: Watts
2177 + chart_type: line
2178 + dimensions:
2179 + - name: cpu_power_limit
2180 + - name: cpu_power_util_current
2181 + - name: dcgm.cpu_core.cpu.temperature
2182 + description: CPU Core CPU Temperature metrics.
2183 + unit: Celsius
2184 + chart_type: line
2185 + dimensions:
2186 + - name: cpu_temp_critical
2187 + - name: cpu_temp_current
2188 + - name: cpu_temp_warning
2189 + - name: dcgm.cpu_core.cpu.utilization
2190 + description: CPU Core CPU Utilization metrics.
2191 + unit: '%'
2192 + chart_type: line
2193 + dimensions:
2194 + - name: cpu_util
2195 + - name: cpu_util_irq
2196 + - name: cpu_util_nice
2197 + - name: cpu_util_sys
2198 + - name: cpu_util_user
2199 + - name: dcgm.cpu_core.diagnostics.results
2200 + description: CPU Core Diagnostics Results metrics.
2201 + unit: state
2202 + chart_type: line
2203 + dimensions:
2204 + - name: diag_cpu_eud_result
2205 + - name: exporter
2206 + description: These metrics refer to exporter/global instances.
2207 + labels:
2208 + - name: job
2209 + description: job label from exporter metrics.
2210 + metrics:
2211 + - name: dcgm.exporter.health.status
2212 + description: Exporter Health Status metrics.
2213 + unit: state
2214 + chart_type: line
2215 + dimensions:
2216 + - name: bind_unbind_event
2217 + - name: dcgm.exporter.inventory.software
2218 + description: Exporter Software and Firmware metrics.
2219 + unit: value
2220 + chart_type: line
2221 + dimensions:
2222 + - name: cuda_driver_version
2223 + - name: driver_version
2224 + - name: nvml_version
src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.json new
+764
@@ -0,0 +1,764 @@
1 +{
2 + "generated_at": "2026-02-07T22:04:51Z",
3 + "method": "direct-dcgm-exporter-process-on-port-19400",
4 + "total_fields": 623,
5 + "profiles": [
6 + {
7 + "profile": 1,
8 + "original_fields_count": 127,
9 + "startup_fail_fields_count": 2,
10 + "startup_fail_fields": [
11 + "DCGM_FI_BIND_UNBIND_EVENT",
12 + "DCGM_FI_LAST_NVSWITCH_FIELD_ID"
13 + ],
14 + "non_numeric_fields": [
15 + "DCGM_FI_DEV_MIG_GI_INFO",
16 + "DCGM_FI_DEV_VGPU_PER_PROCESS_UTILIZATION"
17 + ],
18 + "numeric_seen_count": 53,
19 + "remaining_after_startup_filter": 125
20 + },
21 + {
22 + "profile": 2,
23 + "original_fields_count": 127,
24 + "startup_fail_fields_count": 4,
25 + "startup_fail_fields": [
26 + "DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT",
27 + "DCGM_FI_DEV_CPU_TEMP_CRITICAL",
28 + "DCGM_FI_DEV_NVLINK_GET_STATE",
29 + "DCGM_FI_LAST_VGPU_FIELD_ID"
30 + ],
31 + "non_numeric_fields": [
32 + "DCGM_FI_DEV_CREATABLE_VGPU_TYPE_IDS",
33 + "DCGM_FI_DEV_ENFORCED_POWER_PROFILE_MASK",
34 + "DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK",
35 + "DCGM_FI_DEV_SUPPORTED_VGPU_TYPE_IDS",
36 + "DCGM_FI_DEV_VGPU_INSTANCE_IDS"
37 + ],
38 + "numeric_seen_count": 46,
39 + "remaining_after_startup_filter": 123
40 + },
41 + {
42 + "profile": 3,
43 + "original_fields_count": 127,
44 + "startup_fail_fields_count": 6,
45 + "startup_fail_fields": [
46 + "DCGM_FI_DEV_DIAG_NCCL_TESTS_RESULT",
47 + "DCGM_FI_DEV_LAST_CONNECTX_FIELD_ID",
48 + "DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG",
49 + "DCGM_FI_DEV_NVLINK_PPCNT_IBPC_PORT_XMIT_WAIT",
50 + "DCGM_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS",
51 + "DCGM_FI_IMEX_DAEMON_STATUS"
52 + ],
53 + "non_numeric_fields": [
54 + "DCGM_FI_DEV_CREATABLE_VGPU_TYPE_IDS",
55 + "DCGM_FI_DEV_FBC_SESSIONS_INFO",
56 + "DCGM_FI_DEV_VGPU_TYPE_CLASS"
57 + ],
58 + "numeric_seen_count": 52,
59 + "remaining_after_startup_filter": 121
60 + },
61 + {
62 + "profile": 4,
63 + "original_fields_count": 127,
64 + "startup_fail_fields_count": 5,
65 + "startup_fail_fields": [
66 + "DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT",
67 + "DCGM_FI_DEV_FIRST_CONNECTX_FIELD_ID",
68 + "DCGM_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL",
69 + "DCGM_FI_FIRST_NVSWITCH_FIELD_ID",
70 + "DCGM_FI_IMEX_DOMAIN_STATUS"
71 + ],
72 + "non_numeric_fields": [
73 + "DCGM_FI_DEV_FBC_STATS"
74 + ],
75 + "numeric_seen_count": 34,
76 + "remaining_after_startup_filter": 122
77 + },
78 + {
79 + "profile": 5,
80 + "original_fields_count": 127,
81 + "startup_fail_fields_count": 5,
82 + "startup_fail_fields": [
83 + "DCGM_FI_DEV_CPU_TEMP_CRITICAL",
84 + "DCGM_FI_DEV_FABRIC_HEALTH_MASK",
85 + "DCGM_FI_DEV_FIRST_CONNECTX_FIELD_ID",
86 + "DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION",
87 + "DCGM_FI_INTERNAL_FIELDS_0_END"
88 + ],
89 + "non_numeric_fields": [
90 + "DCGM_FI_DEV_ENC_STATS",
91 + "DCGM_FI_DEV_ENFORCED_POWER_PROFILE_MASK",
92 + "DCGM_FI_DEV_FBC_SESSIONS_INFO",
93 + "DCGM_FI_DEV_FBC_STATS",
94 + "DCGM_FI_DEV_MIG_ATTRIBUTES",
95 + "DCGM_FI_DEV_VGPU_TYPE_LICENSE"
96 + ],
97 + "numeric_seen_count": 48,
98 + "remaining_after_startup_filter": 122
99 + },
100 + {
101 + "profile": 6,
102 + "original_fields_count": 127,
103 + "startup_fail_fields_count": 4,
104 + "startup_fail_fields": [
105 + "DCGM_FI_DEV_DIAG_NCCL_TESTS_RESULT",
106 + "DCGM_FI_DEV_LAST_CONNECTX_FIELD_ID",
107 + "DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG",
108 + "DCGM_FI_INTERNAL_FIELDS_0_START"
109 + ],
110 + "non_numeric_fields": [
111 + "DCGM_FI_DEV_MIG_ATTRIBUTES",
112 + "DCGM_FI_DEV_MIG_CI_INFO",
113 + "DCGM_FI_DEV_MIG_GI_INFO",
114 + "DCGM_FI_DEV_PLATFORM_INFINIBAND_GUID",
115 + "DCGM_FI_DEV_SUPPORTED_CLOCKS",
116 + "DCGM_FI_DEV_VALID_POWER_PROFILE_MASK",
117 + "DCGM_FI_GPU_TOPOLOGY_AFFINITY"
118 + ],
119 + "numeric_seen_count": 51,
120 + "remaining_after_startup_filter": 123
121 + }
122 + ],
123 + "classification": {
124 + "startup_fail_fields": [
125 + "DCGM_FI_BIND_UNBIND_EVENT",
126 + "DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT",
127 + "DCGM_FI_DEV_CPU_TEMP_CRITICAL",
128 + "DCGM_FI_DEV_DIAG_NCCL_TESTS_RESULT",
129 + "DCGM_FI_DEV_FABRIC_HEALTH_MASK",
130 + "DCGM_FI_DEV_FIRST_CONNECTX_FIELD_ID",
131 + "DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION",
132 + "DCGM_FI_DEV_LAST_CONNECTX_FIELD_ID",
133 + "DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG",
134 + "DCGM_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL",
135 + "DCGM_FI_DEV_NVLINK_GET_STATE",
136 + "DCGM_FI_DEV_NVLINK_PPCNT_IBPC_PORT_XMIT_WAIT",
137 + "DCGM_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS",
138 + "DCGM_FI_FIRST_NVSWITCH_FIELD_ID",
139 + "DCGM_FI_IMEX_DAEMON_STATUS",
140 + "DCGM_FI_IMEX_DOMAIN_STATUS",
141 + "DCGM_FI_INTERNAL_FIELDS_0_END",
142 + "DCGM_FI_INTERNAL_FIELDS_0_START",
143 + "DCGM_FI_LAST_NVSWITCH_FIELD_ID",
144 + "DCGM_FI_LAST_VGPU_FIELD_ID"
145 + ],
146 + "non_numeric_fields": [
147 + "DCGM_FI_DEV_CREATABLE_VGPU_TYPE_IDS",
148 + "DCGM_FI_DEV_ENC_STATS",
149 + "DCGM_FI_DEV_ENFORCED_POWER_PROFILE_MASK",
150 + "DCGM_FI_DEV_FBC_SESSIONS_INFO",
151 + "DCGM_FI_DEV_FBC_STATS",
152 + "DCGM_FI_DEV_MIG_ATTRIBUTES",
153 + "DCGM_FI_DEV_MIG_CI_INFO",
154 + "DCGM_FI_DEV_MIG_GI_INFO",
155 + "DCGM_FI_DEV_PLATFORM_INFINIBAND_GUID",
156 + "DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK",
157 + "DCGM_FI_DEV_SUPPORTED_CLOCKS",
158 + "DCGM_FI_DEV_SUPPORTED_VGPU_TYPE_IDS",
159 + "DCGM_FI_DEV_VALID_POWER_PROFILE_MASK",
160 + "DCGM_FI_DEV_VGPU_INSTANCE_IDS",
161 + "DCGM_FI_DEV_VGPU_PER_PROCESS_UTILIZATION",
162 + "DCGM_FI_DEV_VGPU_TYPE_CLASS",
163 + "DCGM_FI_DEV_VGPU_TYPE_LICENSE",
164 + "DCGM_FI_GPU_TOPOLOGY_AFFINITY"
165 + ],
166 + "numeric_seen_fields": [
167 + "DCGM_FI_DEV_ACCOUNTING_DATA",
168 + "DCGM_FI_DEV_BAR1_FREE",
169 + "DCGM_FI_DEV_BAR1_TOTAL",
170 + "DCGM_FI_DEV_BAR1_USED",
171 + "DCGM_FI_DEV_BOARD_LIMIT_VIOLATION",
172 + "DCGM_FI_DEV_C2C_LINK_ERROR_INTR",
173 + "DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY",
174 + "DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY_B2B",
175 + "DCGM_FI_DEV_C2C_LINK_POWER_STATE",
176 + "DCGM_FI_DEV_CC_MODE",
177 + "DCGM_FI_DEV_CLOCKS_EVENT_REASONS",
178 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_POWER_BRAKE_SLOWDOWN_NS",
179 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_THERM_SLOWDOWN_NS",
180 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS",
181 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_THERM_SLOWDOWN_NS",
182 + "DCGM_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST_NS",
183 + "DCGM_FI_DEV_COMPUTE_MODE",
184 + "DCGM_FI_DEV_COUNT",
185 + "DCGM_FI_DEV_CPU_AFFINITY_0",
186 + "DCGM_FI_DEV_CPU_AFFINITY_1",
187 + "DCGM_FI_DEV_CPU_AFFINITY_2",
188 + "DCGM_FI_DEV_CPU_AFFINITY_3",
189 + "DCGM_FI_DEV_CUDA_COMPUTE_CAPABILITY",
190 + "DCGM_FI_DEV_DEC_UTIL",
191 + "DCGM_FI_DEV_DIAG_CPU_EUD_RESULT",
192 + "DCGM_FI_DEV_DIAG_DIAGNOSTIC_RESULT",
193 + "DCGM_FI_DEV_DIAG_EUD_RESULT",
194 + "DCGM_FI_DEV_DIAG_MEMORY_BANDWIDTH_RESULT",
195 + "DCGM_FI_DEV_DIAG_MEMORY_RESULT",
196 + "DCGM_FI_DEV_DIAG_MEMTEST_RESULT",
197 + "DCGM_FI_DEV_DIAG_NVBANDWIDTH_RESULT",
198 + "DCGM_FI_DEV_DIAG_PCIE_RESULT",
199 + "DCGM_FI_DEV_DIAG_PULSE_TEST_RESULT",
200 + "DCGM_FI_DEV_DIAG_SOFTWARE_RESULT",
201 + "DCGM_FI_DEV_DIAG_STATUS",
202 + "DCGM_FI_DEV_DIAG_TARGETED_POWER_RESULT",
203 + "DCGM_FI_DEV_DIAG_TARGETED_STRESS_RESULT",
204 + "DCGM_FI_DEV_ECC_DBE_AGG_TOTAL",
205 + "DCGM_FI_DEV_ECC_DBE_VOL_TOTAL",
206 + "DCGM_FI_DEV_ECC_SBE_AGG_TOTAL",
207 + "DCGM_FI_DEV_ECC_SBE_VOL_TOTAL",
208 + "DCGM_FI_DEV_ENC_UTIL",
209 + "DCGM_FI_DEV_ENFORCED_POWER_LIMIT",
210 + "DCGM_FI_DEV_FABRIC_CLIQUE_ID",
211 + "DCGM_FI_DEV_FABRIC_MANAGER_STATUS",
212 + "DCGM_FI_DEV_FAN_SPEED",
213 + "DCGM_FI_DEV_FB_FREE",
214 + "DCGM_FI_DEV_FB_RESERVED",
215 + "DCGM_FI_DEV_FB_TOTAL",
216 + "DCGM_FI_DEV_FB_USED",
217 + "DCGM_FI_DEV_FB_USED_PERCENT",
218 + "DCGM_FI_DEV_GPM_SUPPORT",
219 + "DCGM_FI_DEV_GPU_MAX_OP_TEMP",
220 + "DCGM_FI_DEV_GPU_NVLINK_ERRORS",
221 + "DCGM_FI_DEV_GPU_TEMP",
222 + "DCGM_FI_DEV_GPU_TEMP_LIMIT",
223 + "DCGM_FI_DEV_GPU_UTIL",
224 + "DCGM_FI_DEV_LOW_UTIL_VIOLATION",
225 + "DCGM_FI_DEV_MAX_MEM_CLOCK",
226 + "DCGM_FI_DEV_MAX_SM_CLOCK",
227 + "DCGM_FI_DEV_MAX_VIDEO_CLOCK",
228 + "DCGM_FI_DEV_MEMORY_TEMP",
229 + "DCGM_FI_DEV_MEM_AFFINITY_0",
230 + "DCGM_FI_DEV_MEM_AFFINITY_1",
231 + "DCGM_FI_DEV_MEM_AFFINITY_2",
232 + "DCGM_FI_DEV_MEM_AFFINITY_3",
233 + "DCGM_FI_DEV_MEM_CLOCK",
234 + "DCGM_FI_DEV_MEM_COPY_UTIL",
235 + "DCGM_FI_DEV_MIG_MAX_SLICES",
236 + "DCGM_FI_DEV_MIG_MODE",
237 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L0",
238 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L1",
239 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L10",
240 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L11",
241 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L12",
242 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L13",
243 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L14",
244 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L15",
245 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L16",
246 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L17",
247 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L2",
248 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L3",
249 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L4",
250 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L5",
251 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L6",
252 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L7",
253 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L8",
254 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_L9",
255 + "DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL",
256 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_0",
257 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_1",
258 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_10",
259 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_11",
260 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_12",
261 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_13",
262 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_14",
263 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_15",
264 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_2",
265 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_3",
266 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_4",
267 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_5",
268 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_6",
269 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_7",
270 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_8",
271 + "DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_9",
272 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L0",
273 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L1",
274 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L10",
275 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L11",
276 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L12",
277 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L13",
278 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L14",
279 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L15",
280 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L16",
281 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L17",
282 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L2",
283 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L3",
284 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L4",
285 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L5",
286 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L6",
287 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L7",
288 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L8",
289 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L9",
290 + "DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_TOTAL",
291 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L0",
292 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L1",
293 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L10",
294 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L11",
295 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L12",
296 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L13",
297 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L14",
298 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L15",
299 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L16",
300 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L17",
301 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L2",
302 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L3",
303 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L4",
304 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L5",
305 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L6",
306 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L7",
307 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L8",
308 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L9",
309 + "DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_TOTAL",
310 + "DCGM_FI_DEV_NVML_INDEX",
311 + "DCGM_FI_DEV_P2P_NVLINK_STATUS",
312 + "DCGM_FI_DEV_PCIE_LINK_GEN",
313 + "DCGM_FI_DEV_PCIE_LINK_WIDTH",
314 + "DCGM_FI_DEV_PCIE_MAX_LINK_GEN",
315 + "DCGM_FI_DEV_PCIE_MAX_LINK_WIDTH",
316 + "DCGM_FI_DEV_PCIE_REPLAY_COUNTER",
317 + "DCGM_FI_DEV_PCI_COMBINED_ID",
318 + "DCGM_FI_DEV_PCI_SUBSYS_ID",
319 + "DCGM_FI_DEV_PERSISTENCE_MODE",
320 + "DCGM_FI_DEV_PLATFORM_CHASSIS_SLOT_NUMBER",
321 + "DCGM_FI_DEV_PLATFORM_HOST_ID",
322 + "DCGM_FI_DEV_PLATFORM_MODULE_ID",
323 + "DCGM_FI_DEV_PLATFORM_PEER_TYPE",
324 + "DCGM_FI_DEV_PLATFORM_TRAY_INDEX",
325 + "DCGM_FI_DEV_POWER_MGMT_LIMIT",
326 + "DCGM_FI_DEV_POWER_MGMT_LIMIT_DEF",
327 + "DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX",
328 + "DCGM_FI_DEV_POWER_MGMT_LIMIT_MIN",
329 + "DCGM_FI_DEV_POWER_USAGE",
330 + "DCGM_FI_DEV_POWER_USAGE_INSTANT",
331 + "DCGM_FI_DEV_POWER_VIOLATION",
332 + "DCGM_FI_DEV_PSTATE",
333 + "DCGM_FI_DEV_PWR_SMOOTHING_ACTIVE_PRESET_PROFILE",
334 + "DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_PERCENT_TMP_FLOOR",
335 + "DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_HYST_VAL",
336 + "DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_RATE",
337 + "DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_UP_RATE",
338 + "DCGM_FI_DEV_PWR_SMOOTHING_APPLIED_TMP_CEIL",
339 + "DCGM_FI_DEV_PWR_SMOOTHING_APPLIED_TMP_FLOOR",
340 + "DCGM_FI_DEV_PWR_SMOOTHING_ENABLED",
341 + "DCGM_FI_DEV_PWR_SMOOTHING_HW_CIRCUITRY_PERCENT_LIFETIME_REMAINING",
342 + "DCGM_FI_DEV_PWR_SMOOTHING_IMM_RAMP_DOWN_ENABLED",
343 + "DCGM_FI_DEV_PWR_SMOOTHING_MAX_NUM_PRESET_PROFILES",
344 + "DCGM_FI_DEV_PWR_SMOOTHING_MAX_PERCENT_TMP_FLOOR_SETTING",
345 + "DCGM_FI_DEV_PWR_SMOOTHING_MIN_PERCENT_TMP_FLOOR_SETTING",
346 + "DCGM_FI_DEV_PWR_SMOOTHING_PRIV_LVL",
347 + "DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_PERCENT_TMP_FLOOR",
348 + "DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_DOWN_HYST_VAL",
349 + "DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_DOWN_RATE",
350 + "DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_UP_RATE",
351 + "DCGM_FI_DEV_RELIABILITY_VIOLATION",
352 + "DCGM_FI_DEV_SHUTDOWN_TEMP",
353 + "DCGM_FI_DEV_SLOWDOWN_TEMP",
354 + "DCGM_FI_DEV_SM_CLOCK",
355 + "DCGM_FI_DEV_SYNC_BOOST_VIOLATION",
356 + "DCGM_FI_DEV_THERMAL_VIOLATION",
357 + "DCGM_FI_DEV_TOTAL_APP_CLOCKS_VIOLATION",
358 + "DCGM_FI_DEV_TOTAL_BASE_CLOCKS_VIOLATION",
359 + "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION",
360 + "DCGM_FI_DEV_VGPU_ENC_SESSIONS_INFO",
361 + "DCGM_FI_DEV_VGPU_ENC_STATS",
362 + "DCGM_FI_DEV_VGPU_FBC_SESSIONS_INFO",
363 + "DCGM_FI_DEV_VGPU_FBC_STATS",
364 + "DCGM_FI_DEV_VGPU_FRAME_RATE_LIMIT",
365 + "DCGM_FI_DEV_VGPU_INSTANCE_LICENSE_STATE",
366 + "DCGM_FI_DEV_VGPU_LICENSE_STATUS",
367 + "DCGM_FI_DEV_VGPU_MEMORY_USAGE",
368 + "DCGM_FI_DEV_VGPU_PCI_ID",
369 + "DCGM_FI_DEV_VGPU_TYPE",
370 + "DCGM_FI_DEV_VGPU_UTILIZATIONS",
371 + "DCGM_FI_DEV_VGPU_VM_GPU_INSTANCE_ID",
372 + "DCGM_FI_DEV_VIDEO_CLOCK",
373 + "DCGM_FI_DEV_VIRTUAL_MODE",
374 + "DCGM_FI_DEV_XID_ERRORS",
375 + "DCGM_FI_FIRST_VGPU_FIELD_ID",
376 + "DCGM_FI_GPU_TOPOLOGY_NVLINK",
377 + "DCGM_FI_GPU_TOPOLOGY_PCI",
378 + "DCGM_FI_PROF_DRAM_ACTIVE",
379 + "DCGM_FI_PROF_GR_ENGINE_ACTIVE",
380 + "DCGM_FI_PROF_NVLINK_RX_BYTES",
381 + "DCGM_FI_PROF_NVLINK_TX_BYTES",
382 + "DCGM_FI_PROF_PCIE_RX_BYTES",
383 + "DCGM_FI_PROF_PCIE_TX_BYTES",
384 + "DCGM_FI_PROF_PIPE_FP16_ACTIVE",
385 + "DCGM_FI_PROF_PIPE_FP32_ACTIVE",
386 + "DCGM_FI_PROF_PIPE_FP64_ACTIVE",
387 + "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE",
388 + "DCGM_FI_PROF_SM_ACTIVE",
389 + "DCGM_FI_PROF_SM_OCCUPANCY",
390 + "DCGM_FI_SYNC_BOOST"
391 + ],
392 + "unseen_fields": [
393 + "DCGM_FI_CUDA_DRIVER_VERSION",
394 + "DCGM_FI_DEV_APP_MEM_CLOCK",
395 + "DCGM_FI_DEV_APP_SM_CLOCK",
396 + "DCGM_FI_DEV_AUTOBOOST",
397 + "DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_HIGH",
398 + "DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_LOW",
399 + "DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_MAX",
400 + "DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_NONE",
401 + "DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_PARTIAL",
402 + "DCGM_FI_DEV_BRAND",
403 + "DCGM_FI_DEV_C2C_LINK_COUNT",
404 + "DCGM_FI_DEV_C2C_LINK_STATUS",
405 + "DCGM_FI_DEV_C2C_MAX_BANDWIDTH",
406 + "DCGM_FI_DEV_CONNECTX_ACTIVE_PCIE_LINK_SPEED",
407 + "DCGM_FI_DEV_CONNECTX_ACTIVE_PCIE_LINK_WIDTH",
408 + "DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_MASK",
409 + "DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_STATUS",
410 + "DCGM_FI_DEV_CONNECTX_DEVICE_TEMPERATURE",
411 + "DCGM_FI_DEV_CONNECTX_EXPECT_PCIE_LINK_SPEED",
412 + "DCGM_FI_DEV_CONNECTX_EXPECT_PCIE_LINK_WIDTH",
413 + "DCGM_FI_DEV_CONNECTX_HEALTH",
414 + "DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_MASK",
415 + "DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_SEVERITY",
416 + "DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_STATUS",
417 + "DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS",
418 + "DCGM_FI_DEV_CPU_CLOCK_CURRENT",
419 + "DCGM_FI_DEV_CPU_MODEL",
420 + "DCGM_FI_DEV_CPU_POWER_LIMIT",
421 + "DCGM_FI_DEV_CPU_TEMP_CURRENT",
422 + "DCGM_FI_DEV_CPU_TEMP_WARNING",
423 + "DCGM_FI_DEV_CPU_UTIL_IRQ",
424 + "DCGM_FI_DEV_CPU_UTIL_NICE",
425 + "DCGM_FI_DEV_CPU_UTIL_SYS",
426 + "DCGM_FI_DEV_CPU_UTIL_TOTAL",
427 + "DCGM_FI_DEV_CPU_UTIL_USER",
428 + "DCGM_FI_DEV_CPU_VENDOR",
429 + "DCGM_FI_DEV_CUDA_VISIBLE_DEVICES_STR",
430 + "DCGM_FI_DEV_ECC_CURRENT",
431 + "DCGM_FI_DEV_ECC_DBE_AGG_CBU",
432 + "DCGM_FI_DEV_ECC_DBE_AGG_DEV",
433 + "DCGM_FI_DEV_ECC_DBE_AGG_L1",
434 + "DCGM_FI_DEV_ECC_DBE_AGG_L2",
435 + "DCGM_FI_DEV_ECC_DBE_AGG_REG",
436 + "DCGM_FI_DEV_ECC_DBE_AGG_SHM",
437 + "DCGM_FI_DEV_ECC_DBE_AGG_SRM",
438 + "DCGM_FI_DEV_ECC_DBE_AGG_TEX",
439 + "DCGM_FI_DEV_ECC_DBE_VOL_CBU",
440 + "DCGM_FI_DEV_ECC_DBE_VOL_DEV",
441 + "DCGM_FI_DEV_ECC_DBE_VOL_L1",
442 + "DCGM_FI_DEV_ECC_DBE_VOL_L2",
443 + "DCGM_FI_DEV_ECC_DBE_VOL_REG",
444 + "DCGM_FI_DEV_ECC_DBE_VOL_SHM",
445 + "DCGM_FI_DEV_ECC_DBE_VOL_SRM",
446 + "DCGM_FI_DEV_ECC_DBE_VOL_TEX",
447 + "DCGM_FI_DEV_ECC_INFOROM_VER",
448 + "DCGM_FI_DEV_ECC_PENDING",
449 + "DCGM_FI_DEV_ECC_SBE_AGG_CBU",
450 + "DCGM_FI_DEV_ECC_SBE_AGG_DEV",
451 + "DCGM_FI_DEV_ECC_SBE_AGG_L1",
452 + "DCGM_FI_DEV_ECC_SBE_AGG_L2",
453 + "DCGM_FI_DEV_ECC_SBE_AGG_REG",
454 + "DCGM_FI_DEV_ECC_SBE_AGG_SHM",
455 + "DCGM_FI_DEV_ECC_SBE_AGG_SRM",
456 + "DCGM_FI_DEV_ECC_SBE_AGG_TEX",
457 + "DCGM_FI_DEV_ECC_SBE_VOL_CBU",
458 + "DCGM_FI_DEV_ECC_SBE_VOL_DEV",
459 + "DCGM_FI_DEV_ECC_SBE_VOL_L1",
460 + "DCGM_FI_DEV_ECC_SBE_VOL_L2",
461 + "DCGM_FI_DEV_ECC_SBE_VOL_REG",
462 + "DCGM_FI_DEV_ECC_SBE_VOL_SHM",
463 + "DCGM_FI_DEV_ECC_SBE_VOL_SRM",
464 + "DCGM_FI_DEV_ECC_SBE_VOL_TEX",
465 + "DCGM_FI_DEV_FABRIC_CLUSTER_UUID",
466 + "DCGM_FI_DEV_FABRIC_MANAGER_ERROR_CODE",
467 + "DCGM_FI_DEV_INFOROM_CONFIG_CHECK",
468 + "DCGM_FI_DEV_INFOROM_CONFIG_VALID",
469 + "DCGM_FI_DEV_INFOROM_IMAGE_VER",
470 + "DCGM_FI_DEV_MEM_MAX_OP_TEMP",
471 + "DCGM_FI_DEV_MINOR_NUMBER",
472 + "DCGM_FI_DEV_MODULE_POWER_UTIL_CURRENT",
473 + "DCGM_FI_DEV_NAME",
474 + "DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER",
475 + "DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER_FLOAT",
476 + "DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS",
477 + "DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS",
478 + "DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS",
479 + "DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_SUCCESSFUL_EVENTS",
480 + "DCGM_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS",
481 + "DCGM_FI_DEV_NVLINK_COUNT_RX_BUFFER_OVERRUN_ERRORS",
482 + "DCGM_FI_DEV_NVLINK_COUNT_RX_BYTES",
483 + "DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS",
484 + "DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS",
485 + "DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS",
486 + "DCGM_FI_DEV_NVLINK_COUNT_RX_PACKETS",
487 + "DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS",
488 + "DCGM_FI_DEV_NVLINK_COUNT_RX_SYMBOL_ERRORS",
489 + "DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER",
490 + "DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER_FLOAT",
491 + "DCGM_FI_DEV_NVLINK_COUNT_TX_BYTES",
492 + "DCGM_FI_DEV_NVLINK_COUNT_TX_DISCARDS",
493 + "DCGM_FI_DEV_NVLINK_COUNT_TX_PACKETS",
494 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0",
495 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1",
496 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10",
497 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11",
498 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L12",
499 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L13",
500 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L14",
501 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L15",
502 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L16",
503 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L17",
504 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2",
505 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3",
506 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4",
507 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5",
508 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6",
509 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7",
510 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8",
511 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9",
512 + "DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL",
513 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0",
514 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1",
515 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10",
516 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11",
517 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L12",
518 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L13",
519 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L14",
520 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L15",
521 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L16",
522 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L17",
523 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2",
524 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3",
525 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4",
526 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5",
527 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6",
528 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7",
529 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8",
530 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9",
531 + "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL",
532 + "DCGM_FI_DEV_NVLINK_ERROR_DL_CRC",
533 + "DCGM_FI_DEV_NVLINK_ERROR_DL_RECOVERY",
534 + "DCGM_FI_DEV_NVLINK_ERROR_DL_REPLAY",
535 + "DCGM_FI_DEV_NVLINK_PPCNT_PHYSICAL_LINK_DOWN_COUNTER",
536 + "DCGM_FI_DEV_NVLINK_PPCNT_PHYSICAL_SUCCESSFUL_RECOVERY_EVENTS",
537 + "DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_CODES",
538 + "DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_CODE_ERR",
539 + "DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_UNCORRECTABLE_CODE",
540 + "DCGM_FI_DEV_NVLINK_PPCNT_PLR_SYNC_EVENTS",
541 + "DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_CODES",
542 + "DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_RETRY_CODES",
543 + "DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_RETRY_EVENTS",
544 + "DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TIME_BETWEEN_LAST_TWO",
545 + "DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TIME_SINCE_LAST",
546 + "DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TOTAL_SUCCESSFUL_EVENTS",
547 + "DCGM_FI_DEV_NVLINK_PPRM_OPER_RECOVERY",
548 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0",
549 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1",
550 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10",
551 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11",
552 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L12",
553 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L13",
554 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L14",
555 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L15",
556 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L16",
557 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L17",
558 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2",
559 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3",
560 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4",
561 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5",
562 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6",
563 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7",
564 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8",
565 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9",
566 + "DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL",
567 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0",
568 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1",
569 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10",
570 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11",
571 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L12",
572 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L13",
573 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L14",
574 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L15",
575 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L16",
576 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L17",
577 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2",
578 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3",
579 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4",
580 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5",
581 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6",
582 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7",
583 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8",
584 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9",
585 + "DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL",
586 + "DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ",
587 + "DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ_DVDD",
588 + "DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ_REV",
589 + "DCGM_FI_DEV_NVSWITCH_DEVICE_UUID",
590 + "DCGM_FI_DEV_NVSWITCH_FATAL_ERRORS",
591 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS",
592 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE0",
593 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE1",
594 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE2",
595 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE3",
596 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE4",
597 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE5",
598 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE6",
599 + "DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE7",
600 + "DCGM_FI_DEV_NVSWITCH_LINK_DEVICE_LINK_ID",
601 + "DCGM_FI_DEV_NVSWITCH_LINK_DEVICE_LINK_SID",
602 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS",
603 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE0",
604 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE1",
605 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE2",
606 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE3",
607 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE4",
608 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE5",
609 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE6",
610 + "DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE7",
611 + "DCGM_FI_DEV_NVSWITCH_LINK_FATAL_ERRORS",
612 + "DCGM_FI_DEV_NVSWITCH_LINK_FLIT_ERRORS",
613 + "DCGM_FI_DEV_NVSWITCH_LINK_ID",
614 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC0",
615 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC1",
616 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC2",
617 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC3",
618 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC0",
619 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC1",
620 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC2",
621 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC3",
622 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC0",
623 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC1",
624 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC2",
625 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC3",
626 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC0",
627 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC1",
628 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC2",
629 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC3",
630 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC0",
631 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC1",
632 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC2",
633 + "DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC3",
634 + "DCGM_FI_DEV_NVSWITCH_LINK_NON_FATAL_ERRORS",
635 + "DCGM_FI_DEV_NVSWITCH_LINK_RECOVERY_ERRORS",
636 + "DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_BUS",
637 + "DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_DEVICE",
638 + "DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_DOMAIN",
639 + "DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_FUNCTION",
640 + "DCGM_FI_DEV_NVSWITCH_LINK_REPLAY_ERRORS",
641 + "DCGM_FI_DEV_NVSWITCH_LINK_STATUS",
642 + "DCGM_FI_DEV_NVSWITCH_LINK_THROUGHPUT_RX",
643 + "DCGM_FI_DEV_NVSWITCH_LINK_THROUGHPUT_TX",
644 + "DCGM_FI_DEV_NVSWITCH_LINK_TYPE",
645 + "DCGM_FI_DEV_NVSWITCH_NON_FATAL_ERRORS",
646 + "DCGM_FI_DEV_NVSWITCH_PCIE_BUS",
647 + "DCGM_FI_DEV_NVSWITCH_PCIE_DEVICE",
648 + "DCGM_FI_DEV_NVSWITCH_PCIE_DOMAIN",
649 + "DCGM_FI_DEV_NVSWITCH_PCIE_FUNCTION",
650 + "DCGM_FI_DEV_NVSWITCH_PHYS_ID",
651 + "DCGM_FI_DEV_NVSWITCH_POWER_DVDD",
652 + "DCGM_FI_DEV_NVSWITCH_POWER_HVDD",
653 + "DCGM_FI_DEV_NVSWITCH_POWER_VDD",
654 + "DCGM_FI_DEV_NVSWITCH_RESET_REQUIRED",
655 + "DCGM_FI_DEV_NVSWITCH_TEMPERATURE_CURRENT",
656 + "DCGM_FI_DEV_NVSWITCH_TEMPERATURE_LIMIT_SHUTDOWN",
657 + "DCGM_FI_DEV_NVSWITCH_TEMPERATURE_LIMIT_SLOWDOWN",
658 + "DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX",
659 + "DCGM_FI_DEV_NVSWITCH_THROUGHPUT_TX",
660 + "DCGM_FI_DEV_NVSWITCH_VOLTAGE_MVOLT",
661 + "DCGM_FI_DEV_OEM_INFOROM_VER",
662 + "DCGM_FI_DEV_PCIE_RX_THROUGHPUT",
663 + "DCGM_FI_DEV_PCIE_TX_THROUGHPUT",
664 + "DCGM_FI_DEV_PCI_BUSID",
665 + "DCGM_FI_DEV_PLATFORM_CHASSIS_SERIAL_NUMBER",
666 + "DCGM_FI_DEV_POWER_INFOROM_VER",
667 + "DCGM_FI_DEV_RETIRED_DBE",
668 + "DCGM_FI_DEV_RETIRED_PENDING",
669 + "DCGM_FI_DEV_RETIRED_SBE",
670 + "DCGM_FI_DEV_ROW_REMAP_FAILURE",
671 + "DCGM_FI_DEV_ROW_REMAP_PENDING",
672 + "DCGM_FI_DEV_SERIAL",
673 + "DCGM_FI_DEV_SUPPORTED_TYPE_INFO",
674 + "DCGM_FI_DEV_SYSIO_POWER_UTIL_CURRENT",
675 + "DCGM_FI_DEV_THRESHOLD_SRM",
676 + "DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS",
677 + "DCGM_FI_DEV_UUID",
678 + "DCGM_FI_DEV_VBIOS_VERSION",
679 + "DCGM_FI_DEV_VGPU_DRIVER_VERSION",
680 + "DCGM_FI_DEV_VGPU_TYPE_INFO",
681 + "DCGM_FI_DEV_VGPU_TYPE_NAME",
682 + "DCGM_FI_DEV_VGPU_UUID",
683 + "DCGM_FI_DEV_VGPU_VM_ID",
684 + "DCGM_FI_DEV_VGPU_VM_NAME",
685 + "DCGM_FI_DRIVER_VERSION",
686 + "DCGM_FI_NVML_VERSION",
687 + "DCGM_FI_PROCESS_NAME",
688 + "DCGM_FI_PROF_C2C_RX_ALL_BYTES",
689 + "DCGM_FI_PROF_C2C_RX_DATA_BYTES",
690 + "DCGM_FI_PROF_C2C_TX_ALL_BYTES",
691 + "DCGM_FI_PROF_C2C_TX_DATA_BYTES",
692 + "DCGM_FI_PROF_HOSTMEM_CACHE_HIT",
693 + "DCGM_FI_PROF_HOSTMEM_CACHE_MISS",
694 + "DCGM_FI_PROF_NVDEC0_ACTIVE",
695 + "DCGM_FI_PROF_NVDEC1_ACTIVE",
696 + "DCGM_FI_PROF_NVDEC2_ACTIVE",
697 + "DCGM_FI_PROF_NVDEC3_ACTIVE",
698 + "DCGM_FI_PROF_NVDEC4_ACTIVE",
699 + "DCGM_FI_PROF_NVDEC5_ACTIVE",
700 + "DCGM_FI_PROF_NVDEC6_ACTIVE",
701 + "DCGM_FI_PROF_NVDEC7_ACTIVE",
702 + "DCGM_FI_PROF_NVJPG0_ACTIVE",
703 + "DCGM_FI_PROF_NVJPG1_ACTIVE",
704 + "DCGM_FI_PROF_NVJPG2_ACTIVE",
705 + "DCGM_FI_PROF_NVJPG3_ACTIVE",
706 + "DCGM_FI_PROF_NVJPG4_ACTIVE",
707 + "DCGM_FI_PROF_NVJPG5_ACTIVE",
708 + "DCGM_FI_PROF_NVJPG6_ACTIVE",
709 + "DCGM_FI_PROF_NVJPG7_ACTIVE",
710 + "DCGM_FI_PROF_NVLINK_L0_RX_BYTES",
711 + "DCGM_FI_PROF_NVLINK_L0_TX_BYTES",
712 + "DCGM_FI_PROF_NVLINK_L10_RX_BYTES",
713 + "DCGM_FI_PROF_NVLINK_L10_TX_BYTES",
714 + "DCGM_FI_PROF_NVLINK_L11_RX_BYTES",
715 + "DCGM_FI_PROF_NVLINK_L11_TX_BYTES",
716 + "DCGM_FI_PROF_NVLINK_L12_RX_BYTES",
717 + "DCGM_FI_PROF_NVLINK_L12_TX_BYTES",
718 + "DCGM_FI_PROF_NVLINK_L13_RX_BYTES",
719 + "DCGM_FI_PROF_NVLINK_L13_TX_BYTES",
720 + "DCGM_FI_PROF_NVLINK_L14_RX_BYTES",
721 + "DCGM_FI_PROF_NVLINK_L14_TX_BYTES",
722 + "DCGM_FI_PROF_NVLINK_L15_RX_BYTES",
723 + "DCGM_FI_PROF_NVLINK_L15_TX_BYTES",
724 + "DCGM_FI_PROF_NVLINK_L16_RX_BYTES",
725 + "DCGM_FI_PROF_NVLINK_L16_TX_BYTES",
726 + "DCGM_FI_PROF_NVLINK_L17_RX_BYTES",
727 + "DCGM_FI_PROF_NVLINK_L17_TX_BYTES",
728 + "DCGM_FI_PROF_NVLINK_L1_RX_BYTES",
729 + "DCGM_FI_PROF_NVLINK_L1_TX_BYTES",
730 + "DCGM_FI_PROF_NVLINK_L2_RX_BYTES",
731 + "DCGM_FI_PROF_NVLINK_L2_TX_BYTES",
732 + "DCGM_FI_PROF_NVLINK_L3_RX_BYTES",
733 + "DCGM_FI_PROF_NVLINK_L3_TX_BYTES",
734 + "DCGM_FI_PROF_NVLINK_L4_RX_BYTES",
735 + "DCGM_FI_PROF_NVLINK_L4_TX_BYTES",
736 + "DCGM_FI_PROF_NVLINK_L5_RX_BYTES",
737 + "DCGM_FI_PROF_NVLINK_L5_TX_BYTES",
738 + "DCGM_FI_PROF_NVLINK_L6_RX_BYTES",
739 + "DCGM_FI_PROF_NVLINK_L6_TX_BYTES",
740 + "DCGM_FI_PROF_NVLINK_L7_RX_BYTES",
741 + "DCGM_FI_PROF_NVLINK_L7_TX_BYTES",
742 + "DCGM_FI_PROF_NVLINK_L8_RX_BYTES",
743 + "DCGM_FI_PROF_NVLINK_L8_TX_BYTES",
744 + "DCGM_FI_PROF_NVLINK_L9_RX_BYTES",
745 + "DCGM_FI_PROF_NVLINK_L9_TX_BYTES",
746 + "DCGM_FI_PROF_NVOFA0_ACTIVE",
747 + "DCGM_FI_PROF_NVOFA1_ACTIVE",
748 + "DCGM_FI_PROF_PEERMEM_CACHE_HIT",
749 + "DCGM_FI_PROF_PEERMEM_CACHE_MISS",
750 + "DCGM_FI_PROF_PIPE_INT_ACTIVE",
751 + "DCGM_FI_PROF_PIPE_TENSOR_DFMA_ACTIVE",
752 + "DCGM_FI_PROF_PIPE_TENSOR_HMMA_ACTIVE",
753 + "DCGM_FI_PROF_PIPE_TENSOR_IMMA_ACTIVE"
754 + ]
755 + },
756 + "unresolved_profiles": [],
757 + "validated_on": "2026-02-08",
758 + "environment": {
759 + "gpu_model": "NVIDIA GeForce RTX 5090",
760 + "driver_version": "590.48.01",
761 + "dcgm_exporter_version": "4.4.1-4.5.2",
762 + "test_method": "direct dcgm-exporter process on port 19400, fieldtest.csv in /var/snap/dcgm/common"
763 + }
764 +}
\ No newline at end of file
src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.md new
+69
@@ -0,0 +1,69 @@
1 +# DCGM Field Runtime Validation (Driver 590.48.01 / dcgm-exporter 4.4.1-4.5.2)
2 +
3 +This file documents runtime validation of the Netdata DCGM field dataset against a live exporter.
4 +
5 +## Environment
6 +
7 +- Driver: `590.48.01`
8 +- dcgm-exporter: `4.4.1-4.5.2`
9 +- Validation scope: `driver version + dcgm-exporter version`
10 +- Test host GPU (informational): `NVIDIA GeForce RTX 5090`
11 +- Date: `2026-02-08`
12 +- Method: start `dcgm-exporter` directly on port `19400` with generated 127-field profiles.
13 +- Source dataset: `src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv` (`623` documented fields)
14 +
15 +## Results
16 +
17 +- Profiles executed: `6` (each configured with `127` fields)
18 +- Startup-fail fields: `20`
19 +- Non-numeric output fields: `18`
20 +- Numeric-seen fields: `224`
21 +- Unseen fields: `361`
22 +
23 +## Startup-Fail Fields
24 +
25 +- `DCGM_FI_BIND_UNBIND_EVENT`
26 +- `DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT`
27 +- `DCGM_FI_DEV_CPU_TEMP_CRITICAL`
28 +- `DCGM_FI_DEV_DIAG_NCCL_TESTS_RESULT`
29 +- `DCGM_FI_DEV_FABRIC_HEALTH_MASK`
30 +- `DCGM_FI_DEV_FIRST_CONNECTX_FIELD_ID`
31 +- `DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION`
32 +- `DCGM_FI_DEV_LAST_CONNECTX_FIELD_ID`
33 +- `DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG`
34 +- `DCGM_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL`
35 +- `DCGM_FI_DEV_NVLINK_GET_STATE`
36 +- `DCGM_FI_DEV_NVLINK_PPCNT_IBPC_PORT_XMIT_WAIT`
37 +- `DCGM_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS`
38 +- `DCGM_FI_FIRST_NVSWITCH_FIELD_ID`
39 +- `DCGM_FI_IMEX_DAEMON_STATUS`
40 +- `DCGM_FI_IMEX_DOMAIN_STATUS`
41 +- `DCGM_FI_INTERNAL_FIELDS_0_END`
42 +- `DCGM_FI_INTERNAL_FIELDS_0_START`
43 +- `DCGM_FI_LAST_NVSWITCH_FIELD_ID`
44 +- `DCGM_FI_LAST_VGPU_FIELD_ID`
45 +
46 +## Non-Numeric Output Fields
47 +
48 +- `DCGM_FI_DEV_CREATABLE_VGPU_TYPE_IDS`
49 +- `DCGM_FI_DEV_ENC_STATS`
50 +- `DCGM_FI_DEV_ENFORCED_POWER_PROFILE_MASK`
51 +- `DCGM_FI_DEV_FBC_SESSIONS_INFO`
52 +- `DCGM_FI_DEV_FBC_STATS`
53 +- `DCGM_FI_DEV_MIG_ATTRIBUTES`
54 +- `DCGM_FI_DEV_MIG_CI_INFO`
55 +- `DCGM_FI_DEV_MIG_GI_INFO`
56 +- `DCGM_FI_DEV_PLATFORM_INFINIBAND_GUID`
57 +- `DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK`
58 +- `DCGM_FI_DEV_SUPPORTED_CLOCKS`
59 +- `DCGM_FI_DEV_SUPPORTED_VGPU_TYPE_IDS`
60 +- `DCGM_FI_DEV_VALID_POWER_PROFILE_MASK`
61 +- `DCGM_FI_DEV_VGPU_INSTANCE_IDS`
62 +- `DCGM_FI_DEV_VGPU_PER_PROCESS_UTILIZATION`
63 +- `DCGM_FI_DEV_VGPU_TYPE_CLASS`
64 +- `DCGM_FI_DEV_VGPU_TYPE_LICENSE`
65 +- `DCGM_FI_GPU_TOPOLOGY_AFFINITY`
66 +
67 +## Full Data
68 +
69 +- Full machine-readable report: `src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.json`
src/go/plugin/go.d/collector/dcgm/testdata/all_fields_nonlabel.txt new
+623
@@ -0,0 +1,623 @@
1 +DCGM_FI_BIND_UNBIND_EVENT
2 +DCGM_FI_CUDA_DRIVER_VERSION
3 +DCGM_FI_DEV_ACCOUNTING_DATA
4 +DCGM_FI_DEV_APP_MEM_CLOCK
5 +DCGM_FI_DEV_APP_SM_CLOCK
6 +DCGM_FI_DEV_AUTOBOOST
7 +DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_HIGH
8 +DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_LOW
9 +DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_MAX
10 +DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_NONE
11 +DCGM_FI_DEV_BANKS_REMAP_ROWS_AVAIL_PARTIAL
12 +DCGM_FI_DEV_BAR1_FREE
13 +DCGM_FI_DEV_BAR1_TOTAL
14 +DCGM_FI_DEV_BAR1_USED
15 +DCGM_FI_DEV_BOARD_LIMIT_VIOLATION
16 +DCGM_FI_DEV_BRAND
17 +DCGM_FI_DEV_C2C_LINK_COUNT
18 +DCGM_FI_DEV_C2C_LINK_ERROR_INTR
19 +DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY
20 +DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY_B2B
21 +DCGM_FI_DEV_C2C_LINK_POWER_STATE
22 +DCGM_FI_DEV_C2C_LINK_STATUS
23 +DCGM_FI_DEV_C2C_MAX_BANDWIDTH
24 +DCGM_FI_DEV_CC_MODE
25 +DCGM_FI_DEV_CLOCKS_EVENT_REASONS
26 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_POWER_BRAKE_SLOWDOWN_NS
27 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_HW_THERM_SLOWDOWN_NS
28 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP_NS
29 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_SW_THERM_SLOWDOWN_NS
30 +DCGM_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST_NS
31 +DCGM_FI_DEV_COMPUTE_MODE
32 +DCGM_FI_DEV_CONNECTX_ACTIVE_PCIE_LINK_SPEED
33 +DCGM_FI_DEV_CONNECTX_ACTIVE_PCIE_LINK_WIDTH
34 +DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_MASK
35 +DCGM_FI_DEV_CONNECTX_CORRECTABLE_ERR_STATUS
36 +DCGM_FI_DEV_CONNECTX_DEVICE_TEMPERATURE
37 +DCGM_FI_DEV_CONNECTX_EXPECT_PCIE_LINK_SPEED
38 +DCGM_FI_DEV_CONNECTX_EXPECT_PCIE_LINK_WIDTH
39 +DCGM_FI_DEV_CONNECTX_HEALTH
40 +DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_MASK
41 +DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_SEVERITY
42 +DCGM_FI_DEV_CONNECTX_UNCORRECTABLE_ERR_STATUS
43 +DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS
44 +DCGM_FI_DEV_COUNT
45 +DCGM_FI_DEV_CPU_AFFINITY_0
46 +DCGM_FI_DEV_CPU_AFFINITY_1
47 +DCGM_FI_DEV_CPU_AFFINITY_2
48 +DCGM_FI_DEV_CPU_AFFINITY_3
49 +DCGM_FI_DEV_CPU_CLOCK_CURRENT
50 +DCGM_FI_DEV_CPU_MODEL
51 +DCGM_FI_DEV_CPU_POWER_LIMIT
52 +DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT
53 +DCGM_FI_DEV_CPU_TEMP_CRITICAL
54 +DCGM_FI_DEV_CPU_TEMP_CURRENT
55 +DCGM_FI_DEV_CPU_TEMP_WARNING
56 +DCGM_FI_DEV_CPU_UTIL_IRQ
57 +DCGM_FI_DEV_CPU_UTIL_NICE
58 +DCGM_FI_DEV_CPU_UTIL_SYS
59 +DCGM_FI_DEV_CPU_UTIL_TOTAL
60 +DCGM_FI_DEV_CPU_UTIL_USER
61 +DCGM_FI_DEV_CPU_VENDOR
62 +DCGM_FI_DEV_CREATABLE_VGPU_TYPE_IDS
63 +DCGM_FI_DEV_CUDA_COMPUTE_CAPABILITY
64 +DCGM_FI_DEV_CUDA_VISIBLE_DEVICES_STR
65 +DCGM_FI_DEV_DEC_UTIL
66 +DCGM_FI_DEV_DIAG_CPU_EUD_RESULT
67 +DCGM_FI_DEV_DIAG_DIAGNOSTIC_RESULT
68 +DCGM_FI_DEV_DIAG_EUD_RESULT
69 +DCGM_FI_DEV_DIAG_MEMORY_BANDWIDTH_RESULT
70 +DCGM_FI_DEV_DIAG_MEMORY_RESULT
71 +DCGM_FI_DEV_DIAG_MEMTEST_RESULT
72 +DCGM_FI_DEV_DIAG_NCCL_TESTS_RESULT
73 +DCGM_FI_DEV_DIAG_NVBANDWIDTH_RESULT
74 +DCGM_FI_DEV_DIAG_PCIE_RESULT
75 +DCGM_FI_DEV_DIAG_PULSE_TEST_RESULT
76 +DCGM_FI_DEV_DIAG_SOFTWARE_RESULT
77 +DCGM_FI_DEV_DIAG_STATUS
78 +DCGM_FI_DEV_DIAG_TARGETED_POWER_RESULT
79 +DCGM_FI_DEV_DIAG_TARGETED_STRESS_RESULT
80 +DCGM_FI_DEV_ECC_CURRENT
81 +DCGM_FI_DEV_ECC_DBE_AGG_CBU
82 +DCGM_FI_DEV_ECC_DBE_AGG_DEV
83 +DCGM_FI_DEV_ECC_DBE_AGG_L1
84 +DCGM_FI_DEV_ECC_DBE_AGG_L2
85 +DCGM_FI_DEV_ECC_DBE_AGG_REG
86 +DCGM_FI_DEV_ECC_DBE_AGG_SHM
87 +DCGM_FI_DEV_ECC_DBE_AGG_SRM
88 +DCGM_FI_DEV_ECC_DBE_AGG_TEX
89 +DCGM_FI_DEV_ECC_DBE_AGG_TOTAL
90 +DCGM_FI_DEV_ECC_DBE_VOL_CBU
91 +DCGM_FI_DEV_ECC_DBE_VOL_DEV
92 +DCGM_FI_DEV_ECC_DBE_VOL_L1
93 +DCGM_FI_DEV_ECC_DBE_VOL_L2
94 +DCGM_FI_DEV_ECC_DBE_VOL_REG
95 +DCGM_FI_DEV_ECC_DBE_VOL_SHM
96 +DCGM_FI_DEV_ECC_DBE_VOL_SRM
97 +DCGM_FI_DEV_ECC_DBE_VOL_TEX
98 +DCGM_FI_DEV_ECC_DBE_VOL_TOTAL
99 +DCGM_FI_DEV_ECC_INFOROM_VER
100 +DCGM_FI_DEV_ECC_PENDING
101 +DCGM_FI_DEV_ECC_SBE_AGG_CBU
102 +DCGM_FI_DEV_ECC_SBE_AGG_DEV
103 +DCGM_FI_DEV_ECC_SBE_AGG_L1
104 +DCGM_FI_DEV_ECC_SBE_AGG_L2
105 +DCGM_FI_DEV_ECC_SBE_AGG_REG
106 +DCGM_FI_DEV_ECC_SBE_AGG_SHM
107 +DCGM_FI_DEV_ECC_SBE_AGG_SRM
108 +DCGM_FI_DEV_ECC_SBE_AGG_TEX
109 +DCGM_FI_DEV_ECC_SBE_AGG_TOTAL
110 +DCGM_FI_DEV_ECC_SBE_VOL_CBU
111 +DCGM_FI_DEV_ECC_SBE_VOL_DEV
112 +DCGM_FI_DEV_ECC_SBE_VOL_L1
113 +DCGM_FI_DEV_ECC_SBE_VOL_L2
114 +DCGM_FI_DEV_ECC_SBE_VOL_REG
115 +DCGM_FI_DEV_ECC_SBE_VOL_SHM
116 +DCGM_FI_DEV_ECC_SBE_VOL_SRM
117 +DCGM_FI_DEV_ECC_SBE_VOL_TEX
118 +DCGM_FI_DEV_ECC_SBE_VOL_TOTAL
119 +DCGM_FI_DEV_ENC_STATS
120 +DCGM_FI_DEV_ENC_UTIL
121 +DCGM_FI_DEV_ENFORCED_POWER_LIMIT
122 +DCGM_FI_DEV_ENFORCED_POWER_PROFILE_MASK
123 +DCGM_FI_DEV_FABRIC_CLIQUE_ID
124 +DCGM_FI_DEV_FABRIC_CLUSTER_UUID
125 +DCGM_FI_DEV_FABRIC_HEALTH_MASK
126 +DCGM_FI_DEV_FABRIC_MANAGER_ERROR_CODE
127 +DCGM_FI_DEV_FABRIC_MANAGER_STATUS
128 +DCGM_FI_DEV_FAN_SPEED
129 +DCGM_FI_DEV_FBC_SESSIONS_INFO
130 +DCGM_FI_DEV_FBC_STATS
131 +DCGM_FI_DEV_FB_FREE
132 +DCGM_FI_DEV_FB_RESERVED
133 +DCGM_FI_DEV_FB_TOTAL
134 +DCGM_FI_DEV_FB_USED
135 +DCGM_FI_DEV_FB_USED_PERCENT
136 +DCGM_FI_DEV_FIRST_CONNECTX_FIELD_ID
137 +DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION
138 +DCGM_FI_DEV_GPM_SUPPORT
139 +DCGM_FI_DEV_GPU_MAX_OP_TEMP
140 +DCGM_FI_DEV_GPU_NVLINK_ERRORS
141 +DCGM_FI_DEV_GPU_TEMP
142 +DCGM_FI_DEV_GPU_TEMP_LIMIT
143 +DCGM_FI_DEV_GPU_UTIL
144 +DCGM_FI_DEV_INFOROM_CONFIG_CHECK
145 +DCGM_FI_DEV_INFOROM_CONFIG_VALID
146 +DCGM_FI_DEV_INFOROM_IMAGE_VER
147 +DCGM_FI_DEV_LAST_CONNECTX_FIELD_ID
148 +DCGM_FI_DEV_LOW_UTIL_VIOLATION
149 +DCGM_FI_DEV_MAX_MEM_CLOCK
150 +DCGM_FI_DEV_MAX_SM_CLOCK
151 +DCGM_FI_DEV_MAX_VIDEO_CLOCK
152 +DCGM_FI_DEV_MEMORY_TEMP
153 +DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG
154 +DCGM_FI_DEV_MEM_AFFINITY_0
155 +DCGM_FI_DEV_MEM_AFFINITY_1
156 +DCGM_FI_DEV_MEM_AFFINITY_2
157 +DCGM_FI_DEV_MEM_AFFINITY_3
158 +DCGM_FI_DEV_MEM_CLOCK
159 +DCGM_FI_DEV_MEM_COPY_UTIL
160 +DCGM_FI_DEV_MEM_MAX_OP_TEMP
161 +DCGM_FI_DEV_MIG_ATTRIBUTES
162 +DCGM_FI_DEV_MIG_CI_INFO
163 +DCGM_FI_DEV_MIG_GI_INFO
164 +DCGM_FI_DEV_MIG_MAX_SLICES
165 +DCGM_FI_DEV_MIG_MODE
166 +DCGM_FI_DEV_MINOR_NUMBER
167 +DCGM_FI_DEV_MODULE_POWER_UTIL_CURRENT
168 +DCGM_FI_DEV_NAME
169 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L0
170 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L1
171 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L10
172 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L11
173 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L12
174 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L13
175 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L14
176 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L15
177 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L16
178 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L17
179 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L2
180 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L3
181 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L4
182 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L5
183 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L6
184 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L7
185 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L8
186 +DCGM_FI_DEV_NVLINK_BANDWIDTH_L9
187 +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL
188 +DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER
189 +DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER_FLOAT
190 +DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS
191 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_0
192 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_1
193 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_10
194 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_11
195 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_12
196 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_13
197 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_14
198 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_15
199 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_2
200 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_3
201 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_4
202 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_5
203 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_6
204 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_7
205 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_8
206 +DCGM_FI_DEV_NVLINK_COUNT_FEC_HISTORY_9
207 +DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS
208 +DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS
209 +DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_SUCCESSFUL_EVENTS
210 +DCGM_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS
211 +DCGM_FI_DEV_NVLINK_COUNT_RX_BUFFER_OVERRUN_ERRORS
212 +DCGM_FI_DEV_NVLINK_COUNT_RX_BYTES
213 +DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS
214 +DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS
215 +DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS
216 +DCGM_FI_DEV_NVLINK_COUNT_RX_PACKETS
217 +DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS
218 +DCGM_FI_DEV_NVLINK_COUNT_RX_SYMBOL_ERRORS
219 +DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER
220 +DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER_FLOAT
221 +DCGM_FI_DEV_NVLINK_COUNT_TX_BYTES
222 +DCGM_FI_DEV_NVLINK_COUNT_TX_DISCARDS
223 +DCGM_FI_DEV_NVLINK_COUNT_TX_PACKETS
224 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0
225 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1
226 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10
227 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11
228 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L12
229 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L13
230 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L14
231 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L15
232 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L16
233 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L17
234 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2
235 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3
236 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4
237 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5
238 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6
239 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7
240 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8
241 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9
242 +DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL
243 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0
244 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1
245 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10
246 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11
247 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L12
248 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L13
249 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L14
250 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L15
251 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L16
252 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L17
253 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2
254 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3
255 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4
256 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5
257 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6
258 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7
259 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8
260 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9
261 +DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL
262 +DCGM_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL
263 +DCGM_FI_DEV_NVLINK_ERROR_DL_CRC
264 +DCGM_FI_DEV_NVLINK_ERROR_DL_RECOVERY
265 +DCGM_FI_DEV_NVLINK_ERROR_DL_REPLAY
266 +DCGM_FI_DEV_NVLINK_GET_STATE
267 +DCGM_FI_DEV_NVLINK_PPCNT_IBPC_PORT_XMIT_WAIT
268 +DCGM_FI_DEV_NVLINK_PPCNT_PHYSICAL_LINK_DOWN_COUNTER
269 +DCGM_FI_DEV_NVLINK_PPCNT_PHYSICAL_SUCCESSFUL_RECOVERY_EVENTS
270 +DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_CODES
271 +DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_CODE_ERR
272 +DCGM_FI_DEV_NVLINK_PPCNT_PLR_RCV_UNCORRECTABLE_CODE
273 +DCGM_FI_DEV_NVLINK_PPCNT_PLR_SYNC_EVENTS
274 +DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_CODES
275 +DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_RETRY_CODES
276 +DCGM_FI_DEV_NVLINK_PPCNT_PLR_XMIT_RETRY_EVENTS
277 +DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TIME_BETWEEN_LAST_TWO
278 +DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TIME_SINCE_LAST
279 +DCGM_FI_DEV_NVLINK_PPCNT_RECOVERY_TOTAL_SUCCESSFUL_EVENTS
280 +DCGM_FI_DEV_NVLINK_PPRM_OPER_RECOVERY
281 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0
282 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1
283 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10
284 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11
285 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L12
286 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L13
287 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L14
288 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L15
289 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L16
290 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L17
291 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2
292 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3
293 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4
294 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5
295 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6
296 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7
297 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8
298 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9
299 +DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL
300 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0
301 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1
302 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10
303 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11
304 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L12
305 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L13
306 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L14
307 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L15
308 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L16
309 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L17
310 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2
311 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3
312 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4
313 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5
314 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6
315 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7
316 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8
317 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9
318 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL
319 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L0
320 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L1
321 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L10
322 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L11
323 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L12
324 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L13
325 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L14
326 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L15
327 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L16
328 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L17
329 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L2
330 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L3
331 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L4
332 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L5
333 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L6
334 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L7
335 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L8
336 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_L9
337 +DCGM_FI_DEV_NVLINK_RX_BANDWIDTH_TOTAL
338 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L0
339 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L1
340 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L10
341 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L11
342 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L12
343 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L13
344 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L14
345 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L15
346 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L16
347 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L17
348 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L2
349 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L3
350 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L4
351 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L5
352 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L6
353 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L7
354 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L8
355 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_L9
356 +DCGM_FI_DEV_NVLINK_TX_BANDWIDTH_TOTAL
357 +DCGM_FI_DEV_NVML_INDEX
358 +DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ
359 +DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ_DVDD
360 +DCGM_FI_DEV_NVSWITCH_CURRENT_IDDQ_REV
361 +DCGM_FI_DEV_NVSWITCH_DEVICE_UUID
362 +DCGM_FI_DEV_NVSWITCH_FATAL_ERRORS
363 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS
364 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE0
365 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE1
366 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE2
367 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE3
368 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE4
369 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE5
370 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE6
371 +DCGM_FI_DEV_NVSWITCH_LINK_CRC_ERRORS_LANE7
372 +DCGM_FI_DEV_NVSWITCH_LINK_DEVICE_LINK_ID
373 +DCGM_FI_DEV_NVSWITCH_LINK_DEVICE_LINK_SID
374 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS
375 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE0
376 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE1
377 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE2
378 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE3
379 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE4
380 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE5
381 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE6
382 +DCGM_FI_DEV_NVSWITCH_LINK_ECC_ERRORS_LANE7
383 +DCGM_FI_DEV_NVSWITCH_LINK_FATAL_ERRORS
384 +DCGM_FI_DEV_NVSWITCH_LINK_FLIT_ERRORS
385 +DCGM_FI_DEV_NVSWITCH_LINK_ID
386 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC0
387 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC1
388 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC2
389 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_COUNT_VC3
390 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC0
391 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC1
392 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC2
393 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_HIGH_VC3
394 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC0
395 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC1
396 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC2
397 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_LOW_VC3
398 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC0
399 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC1
400 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC2
401 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_MEDIUM_VC3
402 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC0
403 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC1
404 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC2
405 +DCGM_FI_DEV_NVSWITCH_LINK_LATENCY_PANIC_VC3
406 +DCGM_FI_DEV_NVSWITCH_LINK_NON_FATAL_ERRORS
407 +DCGM_FI_DEV_NVSWITCH_LINK_RECOVERY_ERRORS
408 +DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_BUS
409 +DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_DEVICE
410 +DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_DOMAIN
411 +DCGM_FI_DEV_NVSWITCH_LINK_REMOTE_PCIE_FUNCTION
412 +DCGM_FI_DEV_NVSWITCH_LINK_REPLAY_ERRORS
413 +DCGM_FI_DEV_NVSWITCH_LINK_STATUS
414 +DCGM_FI_DEV_NVSWITCH_LINK_THROUGHPUT_RX
415 +DCGM_FI_DEV_NVSWITCH_LINK_THROUGHPUT_TX
416 +DCGM_FI_DEV_NVSWITCH_LINK_TYPE
417 +DCGM_FI_DEV_NVSWITCH_NON_FATAL_ERRORS
418 +DCGM_FI_DEV_NVSWITCH_PCIE_BUS
419 +DCGM_FI_DEV_NVSWITCH_PCIE_DEVICE
420 +DCGM_FI_DEV_NVSWITCH_PCIE_DOMAIN
421 +DCGM_FI_DEV_NVSWITCH_PCIE_FUNCTION
422 +DCGM_FI_DEV_NVSWITCH_PHYS_ID
423 +DCGM_FI_DEV_NVSWITCH_POWER_DVDD
424 +DCGM_FI_DEV_NVSWITCH_POWER_HVDD
425 +DCGM_FI_DEV_NVSWITCH_POWER_VDD
426 +DCGM_FI_DEV_NVSWITCH_RESET_REQUIRED
427 +DCGM_FI_DEV_NVSWITCH_TEMPERATURE_CURRENT
428 +DCGM_FI_DEV_NVSWITCH_TEMPERATURE_LIMIT_SHUTDOWN
429 +DCGM_FI_DEV_NVSWITCH_TEMPERATURE_LIMIT_SLOWDOWN
430 +DCGM_FI_DEV_NVSWITCH_THROUGHPUT_RX
431 +DCGM_FI_DEV_NVSWITCH_THROUGHPUT_TX
432 +DCGM_FI_DEV_NVSWITCH_VOLTAGE_MVOLT
433 +DCGM_FI_DEV_OEM_INFOROM_VER
434 +DCGM_FI_DEV_P2P_NVLINK_STATUS
435 +DCGM_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS
436 +DCGM_FI_DEV_PCIE_LINK_GEN
437 +DCGM_FI_DEV_PCIE_LINK_WIDTH
438 +DCGM_FI_DEV_PCIE_MAX_LINK_GEN
439 +DCGM_FI_DEV_PCIE_MAX_LINK_WIDTH
440 +DCGM_FI_DEV_PCIE_REPLAY_COUNTER
441 +DCGM_FI_DEV_PCIE_RX_THROUGHPUT
442 +DCGM_FI_DEV_PCIE_TX_THROUGHPUT
443 +DCGM_FI_DEV_PCI_BUSID
444 +DCGM_FI_DEV_PCI_COMBINED_ID
445 +DCGM_FI_DEV_PCI_SUBSYS_ID
446 +DCGM_FI_DEV_PERSISTENCE_MODE
447 +DCGM_FI_DEV_PLATFORM_CHASSIS_SERIAL_NUMBER
448 +DCGM_FI_DEV_PLATFORM_CHASSIS_SLOT_NUMBER
449 +DCGM_FI_DEV_PLATFORM_HOST_ID
450 +DCGM_FI_DEV_PLATFORM_INFINIBAND_GUID
451 +DCGM_FI_DEV_PLATFORM_MODULE_ID
452 +DCGM_FI_DEV_PLATFORM_PEER_TYPE
453 +DCGM_FI_DEV_PLATFORM_TRAY_INDEX
454 +DCGM_FI_DEV_POWER_INFOROM_VER
455 +DCGM_FI_DEV_POWER_MGMT_LIMIT
456 +DCGM_FI_DEV_POWER_MGMT_LIMIT_DEF
457 +DCGM_FI_DEV_POWER_MGMT_LIMIT_MAX
458 +DCGM_FI_DEV_POWER_MGMT_LIMIT_MIN
459 +DCGM_FI_DEV_POWER_USAGE
460 +DCGM_FI_DEV_POWER_USAGE_INSTANT
461 +DCGM_FI_DEV_POWER_VIOLATION
462 +DCGM_FI_DEV_PSTATE
463 +DCGM_FI_DEV_PWR_SMOOTHING_ACTIVE_PRESET_PROFILE
464 +DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_PERCENT_TMP_FLOOR
465 +DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_HYST_VAL
466 +DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_RATE
467 +DCGM_FI_DEV_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_UP_RATE
468 +DCGM_FI_DEV_PWR_SMOOTHING_APPLIED_TMP_CEIL
469 +DCGM_FI_DEV_PWR_SMOOTHING_APPLIED_TMP_FLOOR
470 +DCGM_FI_DEV_PWR_SMOOTHING_ENABLED
471 +DCGM_FI_DEV_PWR_SMOOTHING_HW_CIRCUITRY_PERCENT_LIFETIME_REMAINING
472 +DCGM_FI_DEV_PWR_SMOOTHING_IMM_RAMP_DOWN_ENABLED
473 +DCGM_FI_DEV_PWR_SMOOTHING_MAX_NUM_PRESET_PROFILES
474 +DCGM_FI_DEV_PWR_SMOOTHING_MAX_PERCENT_TMP_FLOOR_SETTING
475 +DCGM_FI_DEV_PWR_SMOOTHING_MIN_PERCENT_TMP_FLOOR_SETTING
476 +DCGM_FI_DEV_PWR_SMOOTHING_PRIV_LVL
477 +DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_PERCENT_TMP_FLOOR
478 +DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_DOWN_HYST_VAL
479 +DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_DOWN_RATE
480 +DCGM_FI_DEV_PWR_SMOOTHING_PROFILE_RAMP_UP_RATE
481 +DCGM_FI_DEV_RELIABILITY_VIOLATION
482 +DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK
483 +DCGM_FI_DEV_RETIRED_DBE
484 +DCGM_FI_DEV_RETIRED_PENDING
485 +DCGM_FI_DEV_RETIRED_SBE
486 +DCGM_FI_DEV_ROW_REMAP_FAILURE
487 +DCGM_FI_DEV_ROW_REMAP_PENDING
488 +DCGM_FI_DEV_SERIAL
489 +DCGM_FI_DEV_SHUTDOWN_TEMP
490 +DCGM_FI_DEV_SLOWDOWN_TEMP
491 +DCGM_FI_DEV_SM_CLOCK
492 +DCGM_FI_DEV_SUPPORTED_CLOCKS
493 +DCGM_FI_DEV_SUPPORTED_TYPE_INFO
494 +DCGM_FI_DEV_SUPPORTED_VGPU_TYPE_IDS
495 +DCGM_FI_DEV_SYNC_BOOST_VIOLATION
496 +DCGM_FI_DEV_SYSIO_POWER_UTIL_CURRENT
497 +DCGM_FI_DEV_THERMAL_VIOLATION
498 +DCGM_FI_DEV_THRESHOLD_SRM
499 +DCGM_FI_DEV_TOTAL_APP_CLOCKS_VIOLATION
500 +DCGM_FI_DEV_TOTAL_BASE_CLOCKS_VIOLATION
501 +DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION
502 +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS
503 +DCGM_FI_DEV_UUID
504 +DCGM_FI_DEV_VALID_POWER_PROFILE_MASK
505 +DCGM_FI_DEV_VBIOS_VERSION
506 +DCGM_FI_DEV_VGPU_DRIVER_VERSION
507 +DCGM_FI_DEV_VGPU_ENC_SESSIONS_INFO
508 +DCGM_FI_DEV_VGPU_ENC_STATS
509 +DCGM_FI_DEV_VGPU_FBC_SESSIONS_INFO
510 +DCGM_FI_DEV_VGPU_FBC_STATS
511 +DCGM_FI_DEV_VGPU_FRAME_RATE_LIMIT
512 +DCGM_FI_DEV_VGPU_INSTANCE_IDS
513 +DCGM_FI_DEV_VGPU_INSTANCE_LICENSE_STATE
514 +DCGM_FI_DEV_VGPU_LICENSE_STATUS
515 +DCGM_FI_DEV_VGPU_MEMORY_USAGE
516 +DCGM_FI_DEV_VGPU_PCI_ID
517 +DCGM_FI_DEV_VGPU_PER_PROCESS_UTILIZATION
518 +DCGM_FI_DEV_VGPU_TYPE
519 +DCGM_FI_DEV_VGPU_TYPE_CLASS
520 +DCGM_FI_DEV_VGPU_TYPE_INFO
521 +DCGM_FI_DEV_VGPU_TYPE_LICENSE
522 +DCGM_FI_DEV_VGPU_TYPE_NAME
523 +DCGM_FI_DEV_VGPU_UTILIZATIONS
524 +DCGM_FI_DEV_VGPU_UUID
525 +DCGM_FI_DEV_VGPU_VM_GPU_INSTANCE_ID
526 +DCGM_FI_DEV_VGPU_VM_ID
527 +DCGM_FI_DEV_VGPU_VM_NAME
528 +DCGM_FI_DEV_VIDEO_CLOCK
529 +DCGM_FI_DEV_VIRTUAL_MODE
530 +DCGM_FI_DEV_XID_ERRORS
531 +DCGM_FI_DRIVER_VERSION
532 +DCGM_FI_FIRST_NVSWITCH_FIELD_ID
533 +DCGM_FI_FIRST_VGPU_FIELD_ID
534 +DCGM_FI_GPU_TOPOLOGY_AFFINITY
535 +DCGM_FI_GPU_TOPOLOGY_NVLINK
536 +DCGM_FI_GPU_TOPOLOGY_PCI
537 +DCGM_FI_IMEX_DAEMON_STATUS
538 +DCGM_FI_IMEX_DOMAIN_STATUS
539 +DCGM_FI_INTERNAL_FIELDS_0_END
540 +DCGM_FI_INTERNAL_FIELDS_0_START
541 +DCGM_FI_LAST_NVSWITCH_FIELD_ID
542 +DCGM_FI_LAST_VGPU_FIELD_ID
543 +DCGM_FI_NVML_VERSION
544 +DCGM_FI_PROCESS_NAME
545 +DCGM_FI_PROF_C2C_RX_ALL_BYTES
546 +DCGM_FI_PROF_C2C_RX_DATA_BYTES
547 +DCGM_FI_PROF_C2C_TX_ALL_BYTES
548 +DCGM_FI_PROF_C2C_TX_DATA_BYTES
549 +DCGM_FI_PROF_DRAM_ACTIVE
550 +DCGM_FI_PROF_GR_ENGINE_ACTIVE
551 +DCGM_FI_PROF_HOSTMEM_CACHE_HIT
552 +DCGM_FI_PROF_HOSTMEM_CACHE_MISS
553 +DCGM_FI_PROF_NVDEC0_ACTIVE
554 +DCGM_FI_PROF_NVDEC1_ACTIVE
555 +DCGM_FI_PROF_NVDEC2_ACTIVE
556 +DCGM_FI_PROF_NVDEC3_ACTIVE
557 +DCGM_FI_PROF_NVDEC4_ACTIVE
558 +DCGM_FI_PROF_NVDEC5_ACTIVE
559 +DCGM_FI_PROF_NVDEC6_ACTIVE
560 +DCGM_FI_PROF_NVDEC7_ACTIVE
561 +DCGM_FI_PROF_NVJPG0_ACTIVE
562 +DCGM_FI_PROF_NVJPG1_ACTIVE
563 +DCGM_FI_PROF_NVJPG2_ACTIVE
564 +DCGM_FI_PROF_NVJPG3_ACTIVE
565 +DCGM_FI_PROF_NVJPG4_ACTIVE
566 +DCGM_FI_PROF_NVJPG5_ACTIVE
567 +DCGM_FI_PROF_NVJPG6_ACTIVE
568 +DCGM_FI_PROF_NVJPG7_ACTIVE
569 +DCGM_FI_PROF_NVLINK_L0_RX_BYTES
570 +DCGM_FI_PROF_NVLINK_L0_TX_BYTES
571 +DCGM_FI_PROF_NVLINK_L10_RX_BYTES
572 +DCGM_FI_PROF_NVLINK_L10_TX_BYTES
573 +DCGM_FI_PROF_NVLINK_L11_RX_BYTES
574 +DCGM_FI_PROF_NVLINK_L11_TX_BYTES
575 +DCGM_FI_PROF_NVLINK_L12_RX_BYTES
576 +DCGM_FI_PROF_NVLINK_L12_TX_BYTES
577 +DCGM_FI_PROF_NVLINK_L13_RX_BYTES
578 +DCGM_FI_PROF_NVLINK_L13_TX_BYTES
579 +DCGM_FI_PROF_NVLINK_L14_RX_BYTES
580 +DCGM_FI_PROF_NVLINK_L14_TX_BYTES
581 +DCGM_FI_PROF_NVLINK_L15_RX_BYTES
582 +DCGM_FI_PROF_NVLINK_L15_TX_BYTES
583 +DCGM_FI_PROF_NVLINK_L16_RX_BYTES
584 +DCGM_FI_PROF_NVLINK_L16_TX_BYTES
585 +DCGM_FI_PROF_NVLINK_L17_RX_BYTES
586 +DCGM_FI_PROF_NVLINK_L17_TX_BYTES
587 +DCGM_FI_PROF_NVLINK_L1_RX_BYTES
588 +DCGM_FI_PROF_NVLINK_L1_TX_BYTES
589 +DCGM_FI_PROF_NVLINK_L2_RX_BYTES
590 +DCGM_FI_PROF_NVLINK_L2_TX_BYTES
591 +DCGM_FI_PROF_NVLINK_L3_RX_BYTES
592 +DCGM_FI_PROF_NVLINK_L3_TX_BYTES
593 +DCGM_FI_PROF_NVLINK_L4_RX_BYTES
594 +DCGM_FI_PROF_NVLINK_L4_TX_BYTES
595 +DCGM_FI_PROF_NVLINK_L5_RX_BYTES
596 +DCGM_FI_PROF_NVLINK_L5_TX_BYTES
597 +DCGM_FI_PROF_NVLINK_L6_RX_BYTES
598 +DCGM_FI_PROF_NVLINK_L6_TX_BYTES
599 +DCGM_FI_PROF_NVLINK_L7_RX_BYTES
600 +DCGM_FI_PROF_NVLINK_L7_TX_BYTES
601 +DCGM_FI_PROF_NVLINK_L8_RX_BYTES
602 +DCGM_FI_PROF_NVLINK_L8_TX_BYTES
603 +DCGM_FI_PROF_NVLINK_L9_RX_BYTES
604 +DCGM_FI_PROF_NVLINK_L9_TX_BYTES
605 +DCGM_FI_PROF_NVLINK_RX_BYTES
606 +DCGM_FI_PROF_NVLINK_TX_BYTES
607 +DCGM_FI_PROF_NVOFA0_ACTIVE
608 +DCGM_FI_PROF_NVOFA1_ACTIVE
609 +DCGM_FI_PROF_PCIE_RX_BYTES
610 +DCGM_FI_PROF_PCIE_TX_BYTES
611 +DCGM_FI_PROF_PEERMEM_CACHE_HIT
612 +DCGM_FI_PROF_PEERMEM_CACHE_MISS
613 +DCGM_FI_PROF_PIPE_FP16_ACTIVE
614 +DCGM_FI_PROF_PIPE_FP32_ACTIVE
615 +DCGM_FI_PROF_PIPE_FP64_ACTIVE
616 +DCGM_FI_PROF_PIPE_INT_ACTIVE
617 +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE
618 +DCGM_FI_PROF_PIPE_TENSOR_DFMA_ACTIVE
619 +DCGM_FI_PROF_PIPE_TENSOR_HMMA_ACTIVE
620 +DCGM_FI_PROF_PIPE_TENSOR_IMMA_ACTIVE
621 +DCGM_FI_PROF_SM_ACTIVE
622 +DCGM_FI_PROF_SM_OCCUPANCY
623 +DCGM_FI_SYNC_BOOST
src/go/plugin/go.d/collector/dcgm/testdata/config.json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "update_every": 123,
3 + "autodetection_retry": 123,
4 + "vnode": "vnode",
5 + "url": "http://127.0.0.1:9400/metrics",
6 + "timeout": 123.123,
7 + "max_time_series": 3210,
8 + "max_time_series_per_metric": 321,
9 + "username": "username",
10 + "password": "password",
11 + "bearer_token_file": "/tmp/token",
12 + "proxy_url": "http://127.0.0.1:3128",
13 + "proxy_username": "proxy-user",
14 + "proxy_password": "proxy-pass",
15 + "headers": {
16 + "X-Test": "1"
17 + },
18 + "tls_skip_verify": true,
19 + "tls_ca": "/tmp/ca.crt",
20 + "tls_cert": "/tmp/client.crt",
21 + "tls_key": "/tmp/client.key",
22 + "body": "hello",
23 + "method": "POST",
24 + "not_follow_redirects": true,
25 + "force_http2": true
26 +}
src/go/plugin/go.d/collector/dcgm/testdata/config.yaml new
+23
@@ -0,0 +1,23 @@
1 +update_every: 123
2 +autodetection_retry: 123
3 +vnode: vnode
4 +url: http://127.0.0.1:9400/metrics
5 +timeout: 123.123
6 +max_time_series: 3210
7 +max_time_series_per_metric: 321
8 +username: username
9 +password: password
10 +bearer_token_file: /tmp/token
11 +proxy_url: http://127.0.0.1:3128
12 +proxy_username: proxy-user
13 +proxy_password: proxy-pass
14 +headers:
15 + X-Test: "1"
16 +tls_skip_verify: true
17 +tls_ca: /tmp/ca.crt
18 +tls_cert: /tmp/client.crt
19 +tls_key: /tmp/client.key
20 +body: hello
21 +method: POST
22 +not_follow_redirects: true
23 +force_http2: true
src/go/plugin/go.d/collector/dcgm/testdata/metrics_non_dcgm.prom new
+3
@@ -0,0 +1,3 @@
1 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
2 +# TYPE process_cpu_seconds_total counter
3 +process_cpu_seconds_total 12
src/go/plugin/go.d/collector/dcgm/testdata/metrics_valid.prom new
+46
@@ -0,0 +1,46 @@
1 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
2 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
3 +DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa",Hostname="host1"} 80
4 +
5 +# HELP DCGM_FI_DEV_FB_USED Framebuffer memory used (in MiB).
6 +# TYPE DCGM_FI_DEV_FB_USED gauge
7 +DCGM_FI_DEV_FB_USED{gpu="0",UUID="GPU-aaa",Hostname="host1"} 1024
8 +
9 +# HELP DCGM_FI_DEV_XID_ERRORS Value of the last XID error encountered.
10 +# TYPE DCGM_FI_DEV_XID_ERRORS gauge
11 +DCGM_FI_DEV_XID_ERRORS{gpu="0",UUID="GPU-aaa"} 31
12 +
13 +# HELP DCGM_FI_DEV_ROW_REMAP_FAILURE Whether remapping of rows has failed.
14 +# TYPE DCGM_FI_DEV_ROW_REMAP_FAILURE gauge
15 +DCGM_FI_DEV_ROW_REMAP_FAILURE{gpu="0",UUID="GPU-aaa"} 1
16 +
17 +# HELP DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS Number of remapped rows for uncorrectable errors.
18 +# TYPE DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS counter
19 +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS{gpu="0",UUID="GPU-aaa"} 7
20 +
21 +# HELP DCGM_FI_DEV_POWER_VIOLATION Throttling duration due to power constraints (in ns).
22 +# TYPE DCGM_FI_DEV_POWER_VIOLATION counter
23 +DCGM_FI_DEV_POWER_VIOLATION{gpu="0",UUID="GPU-aaa"} 2000000
24 +
25 +# HELP DCGM_FI_DEV_THERMAL_VIOLATION Throttling duration due to thermal constraints (in ns).
26 +# TYPE DCGM_FI_DEV_THERMAL_VIOLATION counter
27 +DCGM_FI_DEV_THERMAL_VIOLATION{gpu="0",UUID="GPU-aaa"} 5000000
28 +
29 +# HELP DCGM_FI_DEV_MEMORY_TEMP Memory temperature (in C).
30 +# TYPE DCGM_FI_DEV_MEMORY_TEMP gauge
31 +DCGM_FI_DEV_MEMORY_TEMP{gpu="0",UUID="GPU-aaa"} 9223372036854775794
32 +
33 +# HELP DCGM_FI_PROF_PCIE_TX_BYTES PCIe TX bytes.
34 +# TYPE DCGM_FI_PROF_PCIE_TX_BYTES gauge
35 +DCGM_FI_PROF_PCIE_TX_BYTES{gpu="0",UUID="GPU-aaa"} 123456
36 +
37 +# HELP DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL Total number of NVLink retries.
38 +# TYPE DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL counter
39 +DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL{gpu="0",gpu_uuid="GPU-aaa",nvlink="1"} 4
40 +
41 +# MIG sample for the same DCGM_FI_DEV_GPU_UTIL metric family.
42 +DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-aaa",GPU_I_ID="2",GPU_I_PROFILE="1g.10gb"} 60
43 +
44 +# HELP go_gc_duration_seconds A Go runtime metric that should be ignored by the collector.
45 +# TYPE go_gc_duration_seconds summary
46 +go_gc_duration_seconds{quantile="0.5"} 0.001
src/go/plugin/go.d/collector/init.go
+1
@@ -21,6 +21,7 @@ import (
21 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/coredns"
22 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/couchbase"
23 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/couchdb"
24 + _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/dcgm"
25 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/dmcache"
26 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/dnsdist"
27 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/dnsmasq"
src/go/plugin/go.d/config/go.d.conf
+1
@@ -31,6 +31,7 @@ modules:
31 # coredns: yes
32 # couchbase: yes
33 # couchdb: yes
34 +# dcgm: yes
35 # dmcache: yes
36 # dnsdist: yes
37 # dnsmasq: yes
src/go/plugin/go.d/config/go.d/dcgm.conf new
+9
@@ -0,0 +1,9 @@
1 +## All available configuration options, their descriptions and default values:
2 +## https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/dcgm#readme
3 +
4 +jobs:
5 + - name: local
6 + url: http://127.0.0.1:9400/metrics
7 + update_every: 30
8 + # Keep update_every aligned with dcgm-exporter collection interval.
9 + # If you change one side, update the other side too.
src/health/health.d/dcgm.conf new
+73
@@ -0,0 +1,73 @@
1 +# DCGM GPU reliability alerts.
2 +
3 + template: dcgm_gpu_xid_errors
4 + on: dcgm.gpu.reliability.xid
5 + class: Errors
6 + type: GPU
7 +component: NVIDIA
8 + lookup: max -1m unaligned absolute of xid
9 + units: code
10 + every: 30s
11 + warn: $this > 0
12 + delay: up 30s down 5m multiplier 1.5 max 1h
13 + summary: DCGM reported XID error on GPU ${label:gpu}
14 + info: NVIDIA driver reported a GPU XID error (metric ${label:chart_context}).
15 + to: sysadmin
16 +
17 + template: dcgm_gpu_row_remap_failure
18 + on: dcgm.gpu.reliability.row_remap_status
19 + class: Errors
20 + type: GPU
21 +component: NVIDIA
22 + lookup: max -1m unaligned absolute of row_remap_failure
23 + units: state
24 + every: 30s
25 + warn: $this > 0
26 + delay: up 30s down 5m multiplier 1.5 max 1h
27 + summary: DCGM row remap failure on GPU ${label:gpu}
28 + info: Row remapping has failed, indicating a persistent memory reliability problem.
29 + to: sysadmin
30 +
31 + template: dcgm_gpu_uncorrectable_remapped_rows
32 + on: dcgm.gpu.reliability.row_remap_events
33 + class: Errors
34 + type: GPU
35 +component: NVIDIA
36 + lookup: sum -5m unaligned absolute of uncorrectable_remapped_rows
37 + units: rows
38 + every: 30s
39 + warn: $this > 0
40 + delay: up 30s down 10m multiplier 1.5 max 1h
41 + summary: DCGM uncorrectable remapped rows on GPU ${label:gpu}
42 + info: New uncorrectable row remap events were detected in the last 5 minutes.
43 + to: sysadmin
44 +
45 +# DCGM throttle violation alerts.
46 +
47 + template: dcgm_gpu_power_violation
48 + on: dcgm.gpu.throttle.violations
49 + class: Workload
50 + type: GPU
51 +component: NVIDIA
52 + lookup: sum -5m unaligned absolute of power_violation
53 + units: milliseconds
54 + every: 30s
55 + warn: $this > 0
56 + delay: up 1m down 10m multiplier 1.5 max 1h
57 + summary: DCGM power throttling detected on GPU ${label:gpu}
58 + info: The GPU was power-throttled during the last 5 minutes.
59 + to: sysadmin
60 +
61 + template: dcgm_gpu_thermal_violation
62 + on: dcgm.gpu.throttle.violations
63 + class: Workload
64 + type: GPU
65 +component: NVIDIA
66 + lookup: sum -5m unaligned absolute of thermal_violation
67 + units: milliseconds
68 + every: 30s
69 + warn: $this > 0
70 + delay: up 1m down 10m multiplier 1.5 max 1h
71 + summary: DCGM thermal throttling detected on GPU ${label:gpu}
72 + info: The GPU was thermally throttled during the last 5 minutes.
73 + to: sysadmin