master
yaml 2,225 lines 92.4 KB
Raw
1 plugin_name: go.d.plugin
2 modules:
3 - meta:
4 id: collector-go.d.plugin-dcgm
5 plugin_name: go.d.plugin
6 module_name: dcgm
7 monitored_instance:
8 name: Nvidia Data Center GPU Manager (DCGM)
9 link: https://github.com/NVIDIA/dcgm-exporter
10 icon_filename: nvidia.svg
11 categories:
12 - data-collection.hardware-and-sensors
13 keywords:
14 - nvidia
15 - gpu
16 - dcgm
17 - dcgm-exporter
18 related_resources:
19 integrations:
20 list:
21 - plugin_name: go.d.plugin
22 module_name: nvidia_smi
23 info_provided_to_referring_integrations:
24 description: ""
25 overview:
26 data_collection:
27 metrics_description: |
28 This collector gathers NVIDIA GPU telemetry from a `dcgm-exporter` endpoint.
29 It supports all numeric fields exposed by the exporter and maps them into Netdata-native contexts.
30 method_description: |
31 It collects metrics by periodically scraping the exporter Prometheus endpoint over HTTP.
32 supported_platforms:
33 include: []
34 exclude: []
35 multi_instance: true
36 additional_permissions:
37 description: ""
38 default_behavior:
39 auto_detection:
40 description: This integration does not support auto-detection in v1.
41 limits:
42 description: |
43 The collector applies global and per-metric time series limits to prevent excessive cardinality.
44 performance_impact:
45 description: The impact depends on dcgm-exporter field selection and resulting series cardinality.
46 setup:
47 prerequisites:
48 list:
49 - title: Run dcgm-exporter
50 description: Install DCGM and run `dcgm-exporter` so that a Prometheus endpoint is available (default `:9400/metrics`).
51 - title: Configure exporter field list
52 description: |
53 The default exporter profile exposes a small subset of fields.
54 Use the Netdata recommended profile:
55 [`dcgm-exporter-netdata.csv`](https://github.com/netdata/netdata/blob/master/src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv)
56 (raw download: `https://raw.githubusercontent.com/netdata/netdata/master/src/go/plugin/go.d/collector/dcgm/dcgm-exporter-netdata.csv`).
57
58 The Netdata profile enables 127 fields by default and documents all remaining known DCGM fields as commented entries.
59 To customize beyond the baseline, uncomment the field you need and comment one currently enabled field.
60
61 Runtime validation artifact:
62 `src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.md`
63 and
64 `src/go/plugin/go.d/collector/dcgm/runtime-validation-driver-590.48.01-dcgm-exporter-4.4.1-4.5.2.json`
65
66 Validation is primarily version-scoped (NVIDIA driver + DCGM/DCGM-exporter versions), so treat it as a strong baseline rather than universal compatibility.
67
68 Example:
69 `dcgm-exporter -f /path/to/dcgm-exporter-netdata.csv`
70 - title: Keep collection intervals aligned
71 description: |
72 Set Netdata `update_every` to the same value as dcgm-exporter collection interval (default 30 seconds).
73 Example exporter interval: `dcgm-exporter -c 30000` and Netdata `update_every: 30`.
74 - title: Enable profiling capabilities (optional)
75 description: Profiling fields may require additional privileges/capabilities in your runtime environment.
76 configuration:
77 file:
78 name: go.d/dcgm.conf
79 options:
80 description: |
81 The following options can be defined globally: update_every, autodetection_retry.
82 folding:
83 title: Config options
84 enabled: true
85 list:
86 - name: update_every
87 description: Data collection interval (seconds). Keep this aligned with dcgm-exporter collection interval.
88 default_value: 30
89 required: false
90 group: Collection
91 - name: autodetection_retry
92 description: Autodetection retry interval (seconds). Set 0 to disable.
93 default_value: 0
94 required: false
95 group: Collection
96 - name: url
97 description: DCGM exporter metrics endpoint URL.
98 default_value: http://127.0.0.1:9400/metrics
99 required: true
100 group: Target
101 - name: timeout
102 description: HTTP request timeout (seconds).
103 default_value: 10
104 required: false
105 group: Target
106 - name: max_time_series
107 description: Global time series limit. If exceeded, collection is skipped for this cycle.
108 default_value: 2000
109 required: false
110 group: Limits
111 - name: max_time_series_per_metric
112 description: Per-metric time series limit. Metrics above this limit are skipped.
113 default_value: 200
114 required: false
115 group: Limits
116 - name: username
117 description: Username for Basic HTTP authentication.
118 default_value: ""
119 required: false
120 group: HTTP Auth
121 - name: password
122 description: Password for Basic HTTP authentication.
123 default_value: ""
124 required: false
125 group: HTTP Auth
126 - name: bearer_token_file
127 description: Path to a file containing a bearer token.
128 default_value: ""
129 required: false
130 group: HTTP Auth
131 - name: tls_skip_verify
132 description: Skip TLS certificate and hostname verification (insecure).
133 default_value: no
134 required: false
135 group: TLS
136 - name: tls_ca
137 description: Path to CA bundle used to validate the server certificate.
138 default_value: ""
139 required: false
140 group: TLS
141 - name: tls_cert
142 description: Path to client TLS certificate (for mTLS).
143 default_value: ""
144 required: false
145 group: TLS
146 - name: tls_key
147 description: Path to client TLS private key (for mTLS).
148 default_value: ""
149 required: false
150 group: TLS
151 - name: proxy_url
152 description: HTTP proxy URL.
153 default_value: ""
154 required: false
155 group: Proxy
156 - name: proxy_username
157 description: Username for proxy authentication.
158 default_value: ""
159 required: false
160 group: Proxy
161 - name: proxy_password
162 description: Password for proxy authentication.
163 default_value: ""
164 required: false
165 group: Proxy
166 - name: headers
167 description: Additional HTTP headers to include in the request.
168 default_value: ""
169 required: false
170 group: Request
171 - name: method
172 description: HTTP method.
173 default_value: GET
174 required: false
175 group: Request
176 - name: body
177 description: HTTP request body.
178 default_value: ""
179 required: false
180 group: Request
181 - name: not_follow_redirects
182 description: Do not follow HTTP redirects.
183 default_value: no
184 required: false
185 group: Request
186 - name: force_http2
187 description: Force HTTP/2 (including h2c over TCP).
188 default_value: no
189 required: false
190 group: Request
191 - name: vnode
192 description: Associate this job with a Virtual Node.
193 default_value: ""
194 required: false
195 group: Virtual Node
196 examples:
197 folding:
198 title: Config
199 enabled: true
200 list:
201 - name: Local exporter
202 description: Collect metrics from a local dcgm-exporter endpoint.
203 config: |
204 jobs:
205 - name: local
206 url: http://127.0.0.1:9400/metrics
207 update_every: 30
208 - name: TLS endpoint
209 description: Collect metrics over HTTPS with custom CA certificate.
210 config: |
211 jobs:
212 - name: secure
213 url: https://dcgm-exporter.example.com:9400/metrics
214 update_every: 30
215 tls_ca: /etc/netdata/certs/dcgm-ca.crt
216 - name: Increased cardinality limits
217 description: Increase limits when collecting large field sets and multiple entities.
218 config: |
219 jobs:
220 - name: dcgm_large
221 url: http://127.0.0.1:9400/metrics
222 update_every: 30
223 max_time_series: 10000
224 max_time_series_per_metric: 2000
225 troubleshooting:
226 problems:
227 list: []
228 alerts:
229 - name: dcgm_gpu_xid_errors
230 metric: dcgm.gpu.reliability.xid
231 info: NVIDIA driver reported GPU XID error on GPU ${label:gpu}
232 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
233 - name: dcgm_gpu_row_remap_failure
234 metric: dcgm.gpu.reliability.row_remap_status
235 info: GPU row remapping failed on GPU ${label:gpu}
236 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
237 - name: dcgm_gpu_uncorrectable_remapped_rows
238 metric: dcgm.gpu.reliability.row_remap_events
239 info: Uncorrectable remapped rows increased on GPU ${label:gpu}
240 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
241 - name: dcgm_gpu_power_violation
242 metric: dcgm.gpu.throttle.violations
243 info: Power throttling detected on GPU ${label:gpu}
244 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
245 - name: dcgm_gpu_thermal_violation
246 metric: dcgm.gpu.throttle.violations
247 info: Thermal throttling detected on GPU ${label:gpu}
248 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/dcgm.conf
249 metrics:
250 folding:
251 title: Metrics
252 enabled: false
253 description: |
254 Metrics are grouped into static Netdata contexts. Contexts are created only when matching DCGM fields are present in the exporter output.
255 availability: []
256 scopes:
257 - name: gpu
258 description: These metrics refer to GPU device instances.
259 labels:
260 - name: gpu
261 description: gpu label from exporter metrics.
262 - name: uuid
263 description: uuid label from exporter metrics.
264 metrics:
265 - name: dcgm.gpu.capability.support
266 description: GPU Capability Support metrics.
267 unit: state
268 chart_type: line
269 dimensions:
270 - name: cc_mode
271 - name: cuda_compute_capability
272 - name: gpm_support
273 - name: mig_attributes
274 - name: mig_ci_info
275 - name: mig_gi_info
276 - name: mig_max_slices
277 - name: supported_clocks
278 - name: supported_type_info
279 - name: dcgm.gpu.clock.frequency
280 description: GPU Clock Frequency metrics.
281 unit: MHz
282 chart_type: line
283 dimensions:
284 - name: app_mem_clock
285 visibility: hidden
286 - name: app_sm_clock
287 visibility: hidden
288 - name: max_mem_clock
289 visibility: hidden
290 - name: max_sm_clock
291 visibility: hidden
292 - name: max_video_clock
293 visibility: hidden
294 - name: memory
295 - name: sm
296 - name: video_clock
297 - name: dcgm.gpu.compute.activity
298 description: GPU Compute Pipeline Activity metrics.
299 unit: '%'
300 chart_type: line
301 dimensions:
302 - name: dram
303 - name: fp16
304 - name: fp32
305 - name: fp64
306 - name: graphics_engine_active
307 - name: integer
308 - name: sm_active
309 - name: sm_occupancy
310 - name: tensor
311 - name: dcgm.gpu.compute.tensor.activity
312 description: GPU Tensor Core Activity by precision type.
313 unit: '%'
314 chart_type: line
315 dimensions:
316 - name: tensor_dfma
317 - name: tensor_hmma
318 - name: tensor_imma
319 - name: dcgm.gpu.compute.media.activity
320 description: GPU Media Engine Activity metrics.
321 unit: '%'
322 chart_type: line
323 dimensions:
324 - name: nvdec0_active
325 - name: nvdec1_active
326 - name: nvdec2_active
327 - name: nvdec3_active
328 - name: nvdec4_active
329 - name: nvdec5_active
330 - name: nvdec6_active
331 - name: nvdec7_active
332 - name: nvjpg0_active
333 - name: nvjpg1_active
334 - name: nvjpg2_active
335 - name: nvjpg3_active
336 - name: nvjpg4_active
337 - name: nvjpg5_active
338 - name: nvjpg6_active
339 - name: nvjpg7_active
340 - name: nvofa0_active
341 - name: nvofa1_active
342 - name: dcgm.gpu.compute.cache.activity
343 description: GPU Memory Cache Hit/Miss metrics.
344 unit: 'events/s'
345 chart_type: line
346 dimensions:
347 - name: hostmem_cache_hit
348 - name: hostmem_cache_miss
349 - name: peermem_cache_hit
350 - name: peermem_cache_miss
351 - name: dcgm.gpu.compute.utilization
352 description: GPU Compute Utilization metrics.
353 unit: '%'
354 chart_type: line
355 dimensions:
356 - name: decoder
357 - name: encoder
358 - name: gpu
359 - name: memory_copy
360 - name: dcgm.gpu.cpu.power
361 description: GPU CPU Power metrics.
362 unit: Watts
363 chart_type: line
364 dimensions:
365 - name: module_power_util_current
366 - name: sysio_power_util_current
367 - name: dcgm.gpu.cpu.info
368 description: GPU CPU Information metrics.
369 unit: value
370 chart_type: line
371 dimensions:
372 - name: cpu_model
373 - name: cpu_vendor
374 - name: dcgm.gpu.diagnostics.results
375 description: GPU Diagnostics Results metrics.
376 unit: state
377 chart_type: line
378 dimensions:
379 - name: diag_diagnostic_result
380 - name: diag_eud_result
381 - name: diag_memory_bandwidth_result
382 - name: diag_memory_result
383 - name: diag_memtest_result
384 - name: diag_nccl_tests_result
385 - name: diag_nvbandwidth_result
386 - name: diag_pulse_test_result
387 - name: diag_software_result
388 - name: diag_targeted_power_result
389 - name: diag_targeted_stress_result
390 - name: dcgm.gpu.diagnostics.status
391 description: GPU Diagnostics Status metrics.
392 unit: state
393 chart_type: line
394 dimensions:
395 - name: diag_status
396 - name: dcgm.gpu.health.status
397 description: GPU Health Status metrics.
398 unit: state
399 chart_type: line
400 dimensions:
401 - name: imex_daemon_status
402 - name: imex_domain_status
403 - name: dcgm.gpu.interconnect.connectx.error_status
404 description: GPU ConnectX Error Status metrics.
405 unit: state
406 chart_type: line
407 dimensions:
408 - name: connectx_correctable_err_mask
409 - name: connectx_correctable_err_status
410 - name: connectx_uncorrectable_err_mask
411 - name: connectx_uncorrectable_err_severity
412 - name: connectx_uncorrectable_err_status
413 - name: dcgm.gpu.interconnect.connectx.errors
414 description: GPU ConnectX Errors metrics.
415 unit: errors/s
416 chart_type: line
417 dimensions:
418 - name: connectx_correctable_err_mask
419 - name: connectx_correctable_err_status
420 - name: connectx_uncorrectable_err_mask
421 - name: connectx_uncorrectable_err_severity
422 - name: connectx_uncorrectable_err_status
423 - name: dcgm.gpu.interconnect.connectx.link
424 description: GPU ConnectX Link metrics.
425 unit: value
426 chart_type: line
427 dimensions:
428 - name: connectx_active_pcie_link_speed
429 - name: connectx_expect_pcie_link_speed
430 - name: dcgm.gpu.interconnect.connectx.status
431 description: GPU ConnectX Status metrics.
432 unit: state
433 chart_type: line
434 dimensions:
435 - name: connectx_health
436 - name: dcgm.gpu.interconnect.error_rate
437 description: GPU Interconnect Error Rate metrics.
438 unit: errors/s
439 chart_type: line
440 dimensions:
441 - name: c2c_link_error_intr
442 - name: c2c_link_error_replay
443 - name: c2c_link_error_replay_b2b
444 - name: dcgm.gpu.interconnect.fabric
445 description: GPU Fabric State metrics.
446 unit: state
447 chart_type: line
448 dimensions:
449 - name: fabric_clique_id
450 - name: fabric_cluster_uuid
451 - name: fabric_health_mask
452 - name: fabric_manager_error_code
453 - name: fabric_manager_status
454 - name: dcgm.gpu.interconnect.nvlink.error_rate
455 description: GPU NVLink Error Rate metrics.
456 unit: errors/s
457 chart_type: line
458 dimensions:
459 - name: gpu_nvlink_errors
460 - name: dcgm.gpu.interconnect.pcie.error_rate
461 description: GPU PCIe Error Rate metrics.
462 unit: errors/s
463 chart_type: line
464 dimensions:
465 - name: pcie_count_correctable_errors
466 - name: pcie_replay
467 - name: dcgm.gpu.interconnect.pcie.link.generation
468 description: GPU PCIe Link Generation metrics.
469 unit: generation
470 chart_type: line
471 dimensions:
472 - name: link_gen
473 - name: max_link_gen
474 visibility: hidden
475 - name: dcgm.gpu.interconnect.pcie.link.width
476 description: GPU PCIe Link Width metrics.
477 unit: lanes
478 chart_type: line
479 dimensions:
480 - name: connectx_active_pcie_link_width
481 - name: connectx_expect_pcie_link_width
482 - name: link_width
483 - name: max_link_width
484 visibility: hidden
485 - name: dcgm.gpu.interconnect.state
486 description: GPU Interconnect State metrics.
487 unit: state
488 chart_type: line
489 dimensions:
490 - name: c2c_link
491 - name: c2c_link_power_state
492 - name: c2c_link_status
493 - name: dcgm.gpu.interconnect.pcie.state
494 description: GPU PCIe State metrics.
495 unit: state
496 chart_type: line
497 dimensions:
498 - name: diag_pcie_result
499 - name: dcgm.gpu.interconnect.throughput
500 description: GPU Interconnect Throughput metrics.
501 unit: B/s
502 chart_type: area
503 dimensions:
504 - name: c2c_max_bandwidth
505 - name: c2c_rx_all_bytes
506 - name: c2c_rx_data_bytes
507 - name: c2c_tx_all_bytes
508 - name: c2c_tx_data_bytes
509 - name: dcgm.gpu.interconnect.pcie.throughput
510 description: GPU PCIe Throughput metrics.
511 unit: B/s
512 chart_type: area
513 dimensions:
514 - name: pcie_rx
515 - name: pcie_rx_throughput
516 - name: pcie_tx
517 - name: pcie_tx_throughput
518 - name: dcgm.gpu.interconnect.nvlink.throughput
519 description: GPU NVLink Throughput metrics.
520 unit: B/s
521 chart_type: area
522 dimensions:
523 - name: nvlink_rx
524 - name: nvlink_tx
525 - name: dcgm.gpu.interconnect.total.throughput
526 description: GPU Interconnect Total Throughput metrics.
527 unit: B/s
528 chart_type: area
529 dimensions:
530 - name: pcie
531 - name: nvlink
532 - name: dcgm.gpu.internal.boundary
533 description: GPU Internal Boundary Fields metrics.
534 unit: state
535 chart_type: line
536 dimensions:
537 - name: first_connectx_field_id
538 - name: first_vgpu_field_id
539 - name: internal_fields_0_end
540 - name: internal_fields_0_start
541 - name: last_connectx_field_id
542 - name: last_vgpu_field_id
543 - name: dcgm.gpu.inventory.identity
544 description: GPU Inventory Identity metrics.
545 unit: value
546 chart_type: line
547 dimensions:
548 - name: brand
549 - name: count
550 - name: cuda_visible_devices_str
551 - name: minor_number
552 - name: name
553 - name: nvml_index
554 - name: serial
555 - name: uuid
556 - name: dcgm.gpu.inventory.platform
557 description: GPU Platform Inventory metrics.
558 unit: value
559 chart_type: line
560 dimensions:
561 - name: platform_chassis_serial_number
562 - name: platform_chassis_slot_number
563 - name: platform_host_id
564 - name: platform_infiniband_guid
565 - name: platform_module_id
566 - name: platform_peer_type
567 - name: platform_tray_index
568 - name: dcgm.gpu.inventory.software
569 description: GPU Software and Firmware metrics.
570 unit: value
571 chart_type: line
572 dimensions:
573 - name: inforom_config_check
574 - name: inforom_config_valid
575 - name: inforom_image_ver
576 - name: oem_inforom_ver
577 - name: power_inforom_ver
578 - name: process_name
579 - name: vbios_version
580 - name: dcgm.gpu.memory.bar1_usage
581 description: GPU BAR1 Memory Usage metrics.
582 unit: B
583 chart_type: stacked
584 dimensions:
585 - name: free
586 - name: used
587 - name: dcgm.gpu.memory.bar1_capacity
588 description: GPU BAR1 Memory Capacity metrics.
589 unit: B
590 chart_type: line
591 dimensions:
592 - name: total
593 - name: dcgm.gpu.memory.ecc_error_rate
594 description: GPU ECC Error Rate metrics.
595 unit: errors/s
596 chart_type: line
597 dimensions:
598 - name: ecc_current
599 - name: ecc_dbe_agg
600 - name: ecc_dbe_agg_cbu
601 - name: ecc_dbe_agg_dev
602 - name: ecc_dbe_agg_l1
603 - name: ecc_dbe_agg_l2
604 - name: ecc_dbe_agg_reg
605 - name: ecc_dbe_agg_shm
606 - name: ecc_dbe_agg_srm
607 - name: ecc_dbe_agg_tex
608 - name: ecc_dbe_vol
609 - name: ecc_dbe_vol_cbu
610 - name: ecc_dbe_vol_dev
611 - name: ecc_dbe_vol_l1
612 - name: ecc_dbe_vol_l2
613 - name: ecc_dbe_vol_reg
614 - name: ecc_dbe_vol_shm
615 - name: ecc_dbe_vol_srm
616 - name: ecc_dbe_vol_tex
617 - name: ecc_pending
618 - name: ecc_sbe_agg
619 - name: ecc_sbe_agg_cbu
620 - name: ecc_sbe_agg_dev
621 - name: ecc_sbe_agg_l1
622 - name: ecc_sbe_agg_l2
623 - name: ecc_sbe_agg_reg
624 - name: ecc_sbe_agg_shm
625 - name: ecc_sbe_agg_srm
626 - name: ecc_sbe_agg_tex
627 - name: ecc_sbe_vol
628 - name: ecc_sbe_vol_cbu
629 - name: ecc_sbe_vol_dev
630 - name: ecc_sbe_vol_l1
631 - name: ecc_sbe_vol_l2
632 - name: ecc_sbe_vol_reg
633 - name: ecc_sbe_vol_shm
634 - name: ecc_sbe_vol_srm
635 - name: ecc_sbe_vol_tex
636 - name: dcgm.gpu.memory.ecc_errors
637 description: GPU ECC Errors metrics.
638 unit: errors
639 chart_type: line
640 dimensions:
641 - name: ecc_current
642 - name: ecc_dbe_agg_cbu
643 - name: ecc_dbe_agg_dev
644 - name: ecc_dbe_agg_l1
645 - name: ecc_dbe_agg_l2
646 - name: ecc_dbe_agg_reg
647 - name: ecc_dbe_agg_shm
648 - name: ecc_dbe_agg_srm
649 - name: ecc_dbe_agg_tex
650 - name: ecc_dbe_vol_cbu
651 - name: ecc_dbe_vol_dev
652 - name: ecc_dbe_vol_l1
653 - name: ecc_dbe_vol_l2
654 - name: ecc_dbe_vol_reg
655 - name: ecc_dbe_vol_shm
656 - name: ecc_dbe_vol_srm
657 - name: ecc_dbe_vol_tex
658 - name: ecc_inforom_ver
659 - name: ecc_pending
660 - name: ecc_sbe_agg_cbu
661 - name: ecc_sbe_agg_dev
662 - name: ecc_sbe_agg_l1
663 - name: ecc_sbe_agg_l2
664 - name: ecc_sbe_agg_reg
665 - name: ecc_sbe_agg_shm
666 - name: ecc_sbe_agg_srm
667 - name: ecc_sbe_agg_tex
668 - name: ecc_sbe_vol_cbu
669 - name: ecc_sbe_vol_dev
670 - name: ecc_sbe_vol_l1
671 - name: ecc_sbe_vol_l2
672 - name: ecc_sbe_vol_reg
673 - name: ecc_sbe_vol_shm
674 - name: ecc_sbe_vol_srm
675 - name: ecc_sbe_vol_tex
676 - name: dcgm.gpu.memory.page_retirements
677 description: GPU Retired Memory Pages metrics.
678 unit: pages/s
679 chart_type: line
680 dimensions:
681 - name: retired_dbe
682 - name: retired_pending
683 - name: retired_sbe
684 - name: dcgm.gpu.memory.usage
685 description: GPU Memory Usage metrics.
686 unit: B
687 chart_type: stacked
688 dimensions:
689 - name: free
690 - name: reserved
691 - name: used
692 - name: dcgm.gpu.memory.capacity
693 description: GPU Memory Capacity metrics.
694 unit: B
695 chart_type: line
696 dimensions:
697 - name: total
698 - name: dcgm.gpu.memory.utilization
699 description: GPU Memory Utilization metrics.
700 unit: '%'
701 chart_type: line
702 dimensions:
703 - name: used_percent
704 - name: dcgm.gpu.power.energy
705 description: GPU Energy Consumption Rate metrics.
706 unit: mJ/s
707 chart_type: line
708 dimensions:
709 - name: total
710 - name: dcgm.gpu.power.profiles
711 description: GPU Power Profiles metrics.
712 unit: state
713 chart_type: line
714 dimensions:
715 - name: enforced_power_profile_mask
716 - name: requested_power_profile_mask
717 - name: valid_power_profile_mask
718 - name: dcgm.gpu.power.smoothing
719 description: GPU Power Smoothing metrics.
720 unit: value
721 chart_type: line
722 dimensions:
723 - name: pwr_smoothing_active_preset_profile
724 - name: pwr_smoothing_admin_override_percent_tmp_floor
725 - name: pwr_smoothing_admin_override_ramp_down_hyst_val
726 - name: pwr_smoothing_admin_override_ramp_down_rate
727 - name: pwr_smoothing_admin_override_ramp_up_rate
728 - name: pwr_smoothing_applied_tmp_ceil
729 - name: pwr_smoothing_applied_tmp_floor
730 - name: pwr_smoothing_enabled
731 - name: pwr_smoothing_hw_circuitry_percent_lifetime_remaining
732 - name: pwr_smoothing_imm_ramp_down_enabled
733 - name: pwr_smoothing_max_num_preset_profiles
734 - name: pwr_smoothing_max_percent_tmp_floor_setting
735 - name: pwr_smoothing_min_percent_tmp_floor_setting
736 - name: pwr_smoothing_priv_lvl
737 - name: pwr_smoothing_profile_percent_tmp_floor
738 - name: pwr_smoothing_profile_ramp_down_hyst_val
739 - name: pwr_smoothing_profile_ramp_down_rate
740 - name: pwr_smoothing_profile_ramp_up_rate
741 - name: dcgm.gpu.power.usage
742 description: GPU Power Usage metrics.
743 unit: Watts
744 chart_type: line
745 dimensions:
746 - name: draw
747 - name: enforced_limit
748 visibility: hidden
749 - name: power_mgmt_limit
750 visibility: hidden
751 - name: power_mgmt_limit_def
752 visibility: hidden
753 - name: power_mgmt_limit_max
754 visibility: hidden
755 - name: power_mgmt_limit_min
756 visibility: hidden
757 - name: power_usage_instant
758 - name: dcgm.gpu.reliability.memory_health
759 description: GPU Memory Health metrics.
760 unit: state
761 chart_type: line
762 dimensions:
763 - name: banks_remap_rows_avail_high
764 - name: banks_remap_rows_avail_low
765 - name: banks_remap_rows_avail_max
766 - name: banks_remap_rows_avail_none
767 - name: banks_remap_rows_avail_partial
768 - name: memory_unrepairable_flag
769 - name: threshold_srm
770 - name: dcgm.gpu.reliability.recovery_action
771 description: GPU Recovery Action metrics.
772 unit: state
773 chart_type: line
774 dimensions:
775 - name: get_gpu_recovery_action
776 - name: dcgm.gpu.reliability.row_remap_events
777 description: GPU Row Remap Events metrics.
778 unit: rows/s
779 chart_type: line
780 dimensions:
781 - name: correctable_remapped_rows
782 - name: uncorrectable_remapped_rows
783 - name: dcgm.gpu.reliability.row_remap_status
784 description: GPU Row Remap Status metrics.
785 unit: state
786 chart_type: line
787 dimensions:
788 - name: row_remap_failure
789 - name: row_remap_pending
790 - name: dcgm.gpu.reliability.xid
791 description: GPU XID Errors metrics.
792 unit: code
793 chart_type: line
794 dimensions:
795 - name: xid
796 - name: dcgm.gpu.state.configuration
797 description: GPU Configuration State metrics.
798 unit: state
799 chart_type: line
800 dimensions:
801 - name: autoboost
802 - name: compute_mode
803 - name: persistence_mode
804 - name: sync_boost
805 - name: sync_boost_violation
806 - name: dcgm.gpu.state.performance
807 description: GPU Performance State metrics.
808 unit: state
809 chart_type: line
810 dimensions:
811 - name: pstate
812 - name: dcgm.gpu.state.virtualization
813 description: GPU Virtualization State metrics.
814 unit: state
815 chart_type: line
816 dimensions:
817 - name: mig_mode
818 - name: virtual_mode
819 - name: dcgm.gpu.thermal.fan_speed
820 description: GPU Fan Speed metrics.
821 unit: '%'
822 chart_type: line
823 dimensions:
824 - name: fan_speed
825 - name: dcgm.gpu.thermal.temperature
826 description: GPU Temperature metrics.
827 unit: Celsius
828 chart_type: line
829 dimensions:
830 - name: connectx_device_temperature
831 - name: gpu
832 - name: gpu_max_op_temp
833 visibility: hidden
834 - name: gpu_temp_limit
835 visibility: hidden
836 - name: mem_max_op_temp
837 visibility: hidden
838 - name: memory
839 - name: shutdown_temp
840 visibility: hidden
841 - name: slowdown_temp
842 visibility: hidden
843 - name: dcgm.gpu.throttle.reasons
844 description: GPU Throttle Reasons metrics.
845 unit: bitmask
846 chart_type: line
847 dimensions:
848 - name: clocks_event_reasons
849 - name: dcgm.gpu.throttle.violations
850 description: GPU Throttle Violation Duration metrics.
851 unit: milliseconds/s
852 chart_type: line
853 dimensions:
854 - name: board_limit_violation
855 - name: hw_power_brake_slowdown
856 - name: hw_therm_slowdown
857 - name: low_utilization_violation
858 - name: power_violation
859 - name: reliability_violation
860 - name: sw_power_cap
861 - name: sw_therm_slowdown
862 - name: sync_boost
863 - name: thermal_violation
864 - name: total_app_clocks_violation
865 - name: total_base_clocks_violation
866 - name: dcgm.gpu.topology.affinity
867 description: GPU Topology and Affinity metrics.
868 unit: value
869 chart_type: line
870 dimensions:
871 - name: cpu_affinity_0
872 - name: cpu_affinity_1
873 - name: cpu_affinity_2
874 - name: cpu_affinity_3
875 - name: gpu_topology_affinity
876 - name: gpu_topology_pci
877 - name: mem_affinity_0
878 - name: mem_affinity_1
879 - name: mem_affinity_2
880 - name: mem_affinity_3
881 - name: pci_busid
882 - name: pci_combined_id
883 - name: pci_subsys_id
884 - name: dcgm.gpu.virtualization.vgpu.frame_rate
885 description: GPU vGPU Frame Rate metrics.
886 unit: fps
887 chart_type: line
888 dimensions:
889 - name: vgpu_frame_rate_limit
890 - name: dcgm.gpu.virtualization.vgpu.instance
891 description: GPU vGPU Instance metrics.
892 unit: value
893 chart_type: line
894 dimensions:
895 - name: vgpu_instance_ids
896 - name: vgpu_pci_id
897 - name: vgpu_uuid
898 - name: dcgm.gpu.virtualization.vgpu.license
899 description: GPU vGPU License metrics.
900 unit: state
901 chart_type: line
902 dimensions:
903 - name: vgpu_instance_license_state
904 - name: vgpu_license_status
905 - name: vgpu_type_license
906 - name: dcgm.gpu.virtualization.vgpu.memory
907 description: GPU vGPU Memory metrics.
908 unit: B
909 chart_type: line
910 dimensions:
911 - name: vgpu_memory_usage
912 - name: dcgm.gpu.virtualization.vgpu.sessions
913 description: GPU vGPU Sessions metrics.
914 unit: value
915 chart_type: line
916 dimensions:
917 - name: vgpu_enc_sessions_info
918 - name: vgpu_enc_stats
919 - name: vgpu_fbc_sessions_info
920 - name: vgpu_fbc_stats
921 - name: dcgm.gpu.virtualization.vgpu.software
922 description: GPU vGPU Software metrics.
923 unit: value
924 chart_type: line
925 dimensions:
926 - name: vgpu_driver_version
927 - name: dcgm.gpu.virtualization.vgpu.type
928 description: GPU vGPU Type metrics.
929 unit: value
930 chart_type: line
931 dimensions:
932 - name: creatable_vgpu_type_ids
933 - name: supported_vgpu_type_ids
934 - name: vgpu_type
935 - name: vgpu_type_class
936 - name: vgpu_type_info
937 - name: vgpu_type_name
938 - name: dcgm.gpu.virtualization.vgpu.utilization
939 description: GPU vGPU Utilization metrics.
940 unit: '%'
941 chart_type: line
942 dimensions:
943 - name: vgpu_per_process_utilization
944 - name: dcgm.gpu.virtualization.vgpu.vm
945 description: GPU vGPU VM metrics.
946 unit: value
947 chart_type: line
948 dimensions:
949 - name: vgpu_vm_gpu_instance_id
950 - name: vgpu_vm_id
951 - name: vgpu_vm_name
952 - name: dcgm.gpu.workload.sessions
953 description: GPU Workload Sessions metrics.
954 unit: value
955 chart_type: line
956 dimensions:
957 - name: accounting_data
958 - name: enc_stats
959 - name: fbc_sessions_info
960 - name: fbc_stats
961 - name: mig
962 description: These metrics refer to MIG instances.
963 labels:
964 - name: gpu
965 description: gpu label from exporter metrics.
966 - name: gpu_i_id
967 description: gpu_i_id label from exporter metrics.
968 - name: gpu_i_profile
969 description: gpu_i_profile label from exporter metrics.
970 metrics:
971 - name: dcgm.mig.clock.frequency
972 description: MIG Clock Frequency metrics.
973 unit: MHz
974 chart_type: line
975 dimensions:
976 - name: app_mem_clock
977 visibility: hidden
978 - name: app_sm_clock
979 visibility: hidden
980 - name: max_mem_clock
981 visibility: hidden
982 - name: max_sm_clock
983 visibility: hidden
984 - name: max_video_clock
985 visibility: hidden
986 - name: memory
987 - name: sm
988 - name: video_clock
989 - name: dcgm.mig.compute.activity
990 description: MIG Compute Pipeline Activity metrics.
991 unit: '%'
992 chart_type: line
993 dimensions:
994 - name: dram
995 - name: fp16
996 - name: fp32
997 - name: fp64
998 - name: graphics_engine_active
999 - name: integer
1000 - name: sm_active
1001 - name: sm_occupancy
1002 - name: tensor
1003 - name: dcgm.mig.compute.tensor.activity
1004 description: MIG Tensor Core Activity by precision type.
1005 unit: '%'
1006 chart_type: line
1007 dimensions:
1008 - name: tensor_dfma
1009 - name: tensor_hmma
1010 - name: tensor_imma
1011 - name: dcgm.mig.compute.media.activity
1012 description: MIG Media Engine Activity metrics.
1013 unit: '%'
1014 chart_type: line
1015 dimensions:
1016 - name: nvdec0_active
1017 - name: nvdec1_active
1018 - name: nvdec2_active
1019 - name: nvdec3_active
1020 - name: nvdec4_active
1021 - name: nvdec5_active
1022 - name: nvdec6_active
1023 - name: nvdec7_active
1024 - name: nvjpg0_active
1025 - name: nvjpg1_active
1026 - name: nvjpg2_active
1027 - name: nvjpg3_active
1028 - name: nvjpg4_active
1029 - name: nvjpg5_active
1030 - name: nvjpg6_active
1031 - name: nvjpg7_active
1032 - name: nvofa0_active
1033 - name: nvofa1_active
1034 - name: dcgm.mig.compute.cache.activity
1035 description: MIG Memory Cache Hit/Miss metrics.
1036 unit: 'events/s'
1037 chart_type: line
1038 dimensions:
1039 - name: hostmem_cache_hit
1040 - name: hostmem_cache_miss
1041 - name: peermem_cache_hit
1042 - name: peermem_cache_miss
1043 - name: dcgm.mig.compute.utilization
1044 description: MIG Compute Utilization metrics.
1045 unit: '%'
1046 chart_type: line
1047 dimensions:
1048 - name: decoder
1049 - name: encoder
1050 - name: gpu
1051 - name: memory_copy
1052 - name: dcgm.mig.interconnect.nvlink.ber
1053 description: MIG NVLink Bit Error Rate metrics.
1054 unit: ratio
1055 chart_type: line
1056 dimensions:
1057 - name: nvlink_count_effective_ber
1058 - name: nvlink_count_effective_ber_float
1059 - name: nvlink_count_symbol_ber
1060 - name: nvlink_count_symbol_ber_float
1061 - name: dcgm.mig.interconnect.nvlink.congestion
1062 description: MIG NVLink Congestion metrics.
1063 unit: events/s
1064 chart_type: line
1065 dimensions:
1066 - name: nvlink_ppcnt_ibpc_port_xmit_wait
1067 - name: dcgm.mig.interconnect.error_rate
1068 description: MIG Interconnect Error Rate metrics.
1069 unit: errors/s
1070 chart_type: line
1071 dimensions:
1072 - name: c2c_link_error_intr
1073 - name: c2c_link_error_replay
1074 - name: c2c_link_error_replay_b2b
1075 - name: dcgm.mig.interconnect.nvlink.error_rate
1076 description: MIG NVLink Error Rate metrics.
1077 unit: errors/s
1078 chart_type: line
1079 dimensions:
1080 - name: gpu_nvlink_errors
1081 - name: nvlink_count_effective_errors
1082 - name: nvlink_count_fec_history_0
1083 - name: nvlink_count_fec_history_1
1084 - name: nvlink_count_fec_history_10
1085 - name: nvlink_count_fec_history_11
1086 - name: nvlink_count_fec_history_12
1087 - name: nvlink_count_fec_history_13
1088 - name: nvlink_count_fec_history_14
1089 - name: nvlink_count_fec_history_15
1090 - name: nvlink_count_fec_history_2
1091 - name: nvlink_count_fec_history_3
1092 - name: nvlink_count_fec_history_4
1093 - name: nvlink_count_fec_history_5
1094 - name: nvlink_count_fec_history_6
1095 - name: nvlink_count_fec_history_7
1096 - name: nvlink_count_fec_history_8
1097 - name: nvlink_count_fec_history_9
1098 - name: nvlink_count_link_recovery_events
1099 - name: nvlink_count_link_recovery_failed_events
1100 - name: nvlink_count_link_recovery_successful_events
1101 - name: nvlink_count_local_link_integrity_errors
1102 - name: nvlink_count_rx_buffer_overrun_errors
1103 - name: nvlink_count_rx_errors
1104 - name: nvlink_count_rx_general_errors
1105 - name: nvlink_count_rx_malformed_packet_errors
1106 - name: nvlink_count_rx_remote_errors
1107 - name: nvlink_count_rx_symbol_errors
1108 - name: nvlink_count_tx_discards
1109 - name: nvlink_crc_data_error
1110 - name: nvlink_crc_data_error_count_l0
1111 - name: nvlink_crc_data_error_count_l1
1112 - name: nvlink_crc_data_error_count_l10
1113 - name: nvlink_crc_data_error_count_l11
1114 - name: nvlink_crc_data_error_count_l12
1115 - name: nvlink_crc_data_error_count_l13
1116 - name: nvlink_crc_data_error_count_l14
1117 - name: nvlink_crc_data_error_count_l15
1118 - name: nvlink_crc_data_error_count_l16
1119 - name: nvlink_crc_data_error_count_l17
1120 - name: nvlink_crc_data_error_count_l2
1121 - name: nvlink_crc_data_error_count_l3
1122 - name: nvlink_crc_data_error_count_l4
1123 - name: nvlink_crc_data_error_count_l5
1124 - name: nvlink_crc_data_error_count_l6
1125 - name: nvlink_crc_data_error_count_l7
1126 - name: nvlink_crc_data_error_count_l8
1127 - name: nvlink_crc_data_error_count_l9
1128 - name: nvlink_crc_flit_error
1129 - name: nvlink_crc_flit_error_count_l0
1130 - name: nvlink_crc_flit_error_count_l1
1131 - name: nvlink_crc_flit_error_count_l10
1132 - name: nvlink_crc_flit_error_count_l11
1133 - name: nvlink_crc_flit_error_count_l12
1134 - name: nvlink_crc_flit_error_count_l13
1135 - name: nvlink_crc_flit_error_count_l14
1136 - name: nvlink_crc_flit_error_count_l15
1137 - name: nvlink_crc_flit_error_count_l16
1138 - name: nvlink_crc_flit_error_count_l17
1139 - name: nvlink_crc_flit_error_count_l2
1140 - name: nvlink_crc_flit_error_count_l3
1141 - name: nvlink_crc_flit_error_count_l4
1142 - name: nvlink_crc_flit_error_count_l5
1143 - name: nvlink_crc_flit_error_count_l6
1144 - name: nvlink_crc_flit_error_count_l7
1145 - name: nvlink_crc_flit_error_count_l8
1146 - name: nvlink_crc_flit_error_count_l9
1147 - name: nvlink_error_dl_crc
1148 - name: nvlink_error_dl_recovery
1149 - name: nvlink_error_dl_replay
1150 - name: nvlink_ppcnt_physical_successful_recovery_events
1151 - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1152 - name: nvlink_ppcnt_recovery_time_since_last
1153 - name: nvlink_ppcnt_recovery_total_successful_events
1154 - name: nvlink_pprm_oper_recovery
1155 - name: nvlink_recovery_error
1156 - name: nvlink_recovery_error_count_l0
1157 - name: nvlink_recovery_error_count_l1
1158 - name: nvlink_recovery_error_count_l10
1159 - name: nvlink_recovery_error_count_l11
1160 - name: nvlink_recovery_error_count_l12
1161 - name: nvlink_recovery_error_count_l13
1162 - name: nvlink_recovery_error_count_l14
1163 - name: nvlink_recovery_error_count_l15
1164 - name: nvlink_recovery_error_count_l16
1165 - name: nvlink_recovery_error_count_l17
1166 - name: nvlink_recovery_error_count_l2
1167 - name: nvlink_recovery_error_count_l3
1168 - name: nvlink_recovery_error_count_l4
1169 - name: nvlink_recovery_error_count_l5
1170 - name: nvlink_recovery_error_count_l6
1171 - name: nvlink_recovery_error_count_l7
1172 - name: nvlink_recovery_error_count_l8
1173 - name: nvlink_recovery_error_count_l9
1174 - name: nvlink_replay_error
1175 - name: nvlink_replay_error_count_l0
1176 - name: nvlink_replay_error_count_l1
1177 - name: nvlink_replay_error_count_l10
1178 - name: nvlink_replay_error_count_l11
1179 - name: nvlink_replay_error_count_l12
1180 - name: nvlink_replay_error_count_l13
1181 - name: nvlink_replay_error_count_l14
1182 - name: nvlink_replay_error_count_l15
1183 - name: nvlink_replay_error_count_l16
1184 - name: nvlink_replay_error_count_l17
1185 - name: nvlink_replay_error_count_l2
1186 - name: nvlink_replay_error_count_l3
1187 - name: nvlink_replay_error_count_l4
1188 - name: nvlink_replay_error_count_l5
1189 - name: nvlink_replay_error_count_l6
1190 - name: nvlink_replay_error_count_l7
1191 - name: nvlink_replay_error_count_l8
1192 - name: nvlink_replay_error_count_l9
1193 - name: dcgm.mig.interconnect.pcie.error_rate
1194 description: MIG PCIe Error Rate metrics.
1195 unit: errors/s
1196 chart_type: line
1197 dimensions:
1198 - name: pcie_count_correctable_errors
1199 - name: pcie_replay
1200 - name: dcgm.mig.interconnect.nvlink.errors
1201 description: MIG NVLink Errors metrics.
1202 unit: errors
1203 chart_type: line
1204 dimensions:
1205 - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1206 - name: dcgm.mig.interconnect.fabric
1207 description: MIG Fabric State metrics.
1208 unit: state
1209 chart_type: line
1210 dimensions:
1211 - name: fabric_clique_id
1212 - name: fabric_cluster_uuid
1213 - name: fabric_health_mask
1214 - name: fabric_manager_error_code
1215 - name: fabric_manager_status
1216 - name: dcgm.mig.interconnect.pcie.link.generation
1217 description: MIG PCIe Link Generation metrics.
1218 unit: generation
1219 chart_type: line
1220 dimensions:
1221 - name: link_gen
1222 - name: max_link_gen
1223 visibility: hidden
1224 - name: dcgm.mig.interconnect.pcie.link.width
1225 description: MIG PCIe Link Width metrics.
1226 unit: lanes
1227 chart_type: line
1228 dimensions:
1229 - name: link_width
1230 - name: max_link_width
1231 visibility: hidden
1232 - name: dcgm.mig.interconnect.state
1233 description: MIG Interconnect State metrics.
1234 unit: state
1235 chart_type: line
1236 dimensions:
1237 - name: c2c_link
1238 - name: c2c_link_power_state
1239 - name: c2c_link_status
1240 - name: dcgm.mig.interconnect.pcie.state
1241 description: MIG PCIe State metrics.
1242 unit: state
1243 chart_type: line
1244 dimensions:
1245 - name: diag_pcie_result
1246 - name: dcgm.mig.interconnect.nvlink.state
1247 description: MIG NVLink State metrics.
1248 unit: state
1249 chart_type: line
1250 dimensions:
1251 - name: gpu_topology_nvlink
1252 - name: nvlink_get_state
1253 - name: nvlink_ppcnt_physical_link_down_counter
1254 - name: nvlink_ppcnt_plr_rcv_code_err
1255 - name: nvlink_ppcnt_plr_sync_events
1256 - name: nvlink_ppcnt_plr_xmit_retry_events
1257 - name: p2p_nvlink_status
1258 - name: dcgm.mig.interconnect.throughput
1259 description: MIG Interconnect Throughput metrics.
1260 unit: B/s
1261 chart_type: area
1262 dimensions:
1263 - name: c2c_max_bandwidth
1264 - name: c2c_rx_all_bytes
1265 - name: c2c_rx_data_bytes
1266 - name: c2c_tx_all_bytes
1267 - name: c2c_tx_data_bytes
1268 - name: dcgm.mig.interconnect.nvlink.throughput
1269 description: MIG NVLink Throughput metrics.
1270 unit: B/s
1271 chart_type: area
1272 dimensions:
1273 - name: nvlink_bandwidth_l0
1274 - name: nvlink_bandwidth_l1
1275 - name: nvlink_bandwidth_l10
1276 - name: nvlink_bandwidth_l11
1277 - name: nvlink_bandwidth_l12
1278 - name: nvlink_bandwidth_l13
1279 - name: nvlink_bandwidth_l14
1280 - name: nvlink_bandwidth_l15
1281 - name: nvlink_bandwidth_l16
1282 - name: nvlink_bandwidth_l17
1283 - name: nvlink_bandwidth_l2
1284 - name: nvlink_bandwidth_l3
1285 - name: nvlink_bandwidth_l4
1286 - name: nvlink_bandwidth_l5
1287 - name: nvlink_bandwidth_l6
1288 - name: nvlink_bandwidth_l7
1289 - name: nvlink_bandwidth_l8
1290 - name: nvlink_bandwidth_l9
1291 - name: nvlink_count_rx
1292 - name: nvlink_count_tx
1293 - name: nvlink_l0_rx
1294 - name: nvlink_l0_tx
1295 - name: nvlink_l10_rx
1296 - name: nvlink_l10_tx
1297 - name: nvlink_l11_rx
1298 - name: nvlink_l11_tx
1299 - name: nvlink_l12_rx
1300 - name: nvlink_l12_tx
1301 - name: nvlink_l13_rx
1302 - name: nvlink_l13_tx
1303 - name: nvlink_l14_rx
1304 - name: nvlink_l14_tx
1305 - name: nvlink_l15_rx
1306 - name: nvlink_l15_tx
1307 - name: nvlink_l16_rx
1308 - name: nvlink_l16_tx
1309 - name: nvlink_l17_rx
1310 - name: nvlink_l17_tx
1311 - name: nvlink_l1_rx
1312 - name: nvlink_l1_tx
1313 - name: nvlink_l2_rx
1314 - name: nvlink_l2_tx
1315 - name: nvlink_l3_rx
1316 - name: nvlink_l3_tx
1317 - name: nvlink_l4_rx
1318 - name: nvlink_l4_tx
1319 - name: nvlink_l5_rx
1320 - name: nvlink_l5_tx
1321 - name: nvlink_l6_rx
1322 - name: nvlink_l6_tx
1323 - name: nvlink_l7_rx
1324 - name: nvlink_l7_tx
1325 - name: nvlink_l8_rx
1326 - name: nvlink_l8_tx
1327 - name: nvlink_l9_rx
1328 - name: nvlink_l9_tx
1329 - name: nvlink_rx_bandwidth
1330 - name: nvlink_rx_bandwidth_l0
1331 - name: nvlink_rx_bandwidth_l1
1332 - name: nvlink_rx_bandwidth_l10
1333 - name: nvlink_rx_bandwidth_l11
1334 - name: nvlink_rx_bandwidth_l12
1335 - name: nvlink_rx_bandwidth_l13
1336 - name: nvlink_rx_bandwidth_l14
1337 - name: nvlink_rx_bandwidth_l15
1338 - name: nvlink_rx_bandwidth_l16
1339 - name: nvlink_rx_bandwidth_l17
1340 - name: nvlink_rx_bandwidth_l2
1341 - name: nvlink_rx_bandwidth_l3
1342 - name: nvlink_rx_bandwidth_l4
1343 - name: nvlink_rx_bandwidth_l5
1344 - name: nvlink_rx_bandwidth_l6
1345 - name: nvlink_rx_bandwidth_l7
1346 - name: nvlink_rx_bandwidth_l8
1347 - name: nvlink_rx_bandwidth_l9
1348 - name: nvlink_rx
1349 - name: nvlink_tx_bandwidth
1350 - name: nvlink_tx_bandwidth_l0
1351 - name: nvlink_tx_bandwidth_l1
1352 - name: nvlink_tx_bandwidth_l10
1353 - name: nvlink_tx_bandwidth_l11
1354 - name: nvlink_tx_bandwidth_l12
1355 - name: nvlink_tx_bandwidth_l13
1356 - name: nvlink_tx_bandwidth_l14
1357 - name: nvlink_tx_bandwidth_l15
1358 - name: nvlink_tx_bandwidth_l16
1359 - name: nvlink_tx_bandwidth_l17
1360 - name: nvlink_tx_bandwidth_l2
1361 - name: nvlink_tx_bandwidth_l3
1362 - name: nvlink_tx_bandwidth_l4
1363 - name: nvlink_tx_bandwidth_l5
1364 - name: nvlink_tx_bandwidth_l6
1365 - name: nvlink_tx_bandwidth_l7
1366 - name: nvlink_tx_bandwidth_l8
1367 - name: nvlink_tx_bandwidth_l9
1368 - name: nvlink_tx
1369 - name: dcgm.mig.interconnect.pcie.throughput
1370 description: MIG PCIe Throughput metrics.
1371 unit: B/s
1372 chart_type: area
1373 dimensions:
1374 - name: pcie_rx
1375 - name: pcie_rx_throughput
1376 - name: pcie_tx
1377 - name: pcie_tx_throughput
1378 - name: dcgm.mig.interconnect.total.throughput
1379 description: MIG Interconnect Total Throughput metrics.
1380 unit: B/s
1381 chart_type: area
1382 dimensions:
1383 - name: pcie
1384 - name: nvlink
1385 - name: dcgm.mig.interconnect.nvlink.traffic
1386 description: MIG NVLink Traffic metrics.
1387 unit: events/s
1388 chart_type: line
1389 dimensions:
1390 - name: nvlink_count_rx_packets
1391 - name: nvlink_count_tx_packets
1392 - name: nvlink_ppcnt_plr_rcv_codes
1393 - name: nvlink_ppcnt_plr_xmit_codes
1394 - name: nvlink_ppcnt_plr_xmit_retry_codes
1395 - name: dcgm.mig.memory.bar1_usage
1396 description: MIG BAR1 Memory Usage metrics.
1397 unit: B
1398 chart_type: stacked
1399 dimensions:
1400 - name: free
1401 - name: used
1402 - name: dcgm.mig.memory.bar1_capacity
1403 description: MIG BAR1 Memory Capacity metrics.
1404 unit: B
1405 chart_type: line
1406 dimensions:
1407 - name: total
1408 - name: dcgm.mig.memory.ecc_error_rate
1409 description: MIG ECC Error Rate metrics.
1410 unit: errors/s
1411 chart_type: line
1412 dimensions:
1413 - name: ecc_current
1414 - name: ecc_dbe_agg
1415 - name: ecc_dbe_agg_cbu
1416 - name: ecc_dbe_agg_dev
1417 - name: ecc_dbe_agg_l1
1418 - name: ecc_dbe_agg_l2
1419 - name: ecc_dbe_agg_reg
1420 - name: ecc_dbe_agg_shm
1421 - name: ecc_dbe_agg_srm
1422 - name: ecc_dbe_agg_tex
1423 - name: ecc_dbe_vol
1424 - name: ecc_dbe_vol_cbu
1425 - name: ecc_dbe_vol_dev
1426 - name: ecc_dbe_vol_l1
1427 - name: ecc_dbe_vol_l2
1428 - name: ecc_dbe_vol_reg
1429 - name: ecc_dbe_vol_shm
1430 - name: ecc_dbe_vol_srm
1431 - name: ecc_dbe_vol_tex
1432 - name: ecc_pending
1433 - name: ecc_sbe_agg
1434 - name: ecc_sbe_agg_cbu
1435 - name: ecc_sbe_agg_dev
1436 - name: ecc_sbe_agg_l1
1437 - name: ecc_sbe_agg_l2
1438 - name: ecc_sbe_agg_reg
1439 - name: ecc_sbe_agg_shm
1440 - name: ecc_sbe_agg_srm
1441 - name: ecc_sbe_agg_tex
1442 - name: ecc_sbe_vol
1443 - name: ecc_sbe_vol_cbu
1444 - name: ecc_sbe_vol_dev
1445 - name: ecc_sbe_vol_l1
1446 - name: ecc_sbe_vol_l2
1447 - name: ecc_sbe_vol_reg
1448 - name: ecc_sbe_vol_shm
1449 - name: ecc_sbe_vol_srm
1450 - name: ecc_sbe_vol_tex
1451 - name: nvlink_ecc_data_error
1452 - name: dcgm.mig.memory.ecc_errors
1453 description: MIG ECC Errors metrics.
1454 unit: errors
1455 chart_type: line
1456 dimensions:
1457 - name: ecc_current
1458 - name: ecc_dbe_agg_cbu
1459 - name: ecc_dbe_agg_dev
1460 - name: ecc_dbe_agg_l1
1461 - name: ecc_dbe_agg_l2
1462 - name: ecc_dbe_agg_reg
1463 - name: ecc_dbe_agg_shm
1464 - name: ecc_dbe_agg_srm
1465 - name: ecc_dbe_agg_tex
1466 - name: ecc_dbe_vol_cbu
1467 - name: ecc_dbe_vol_dev
1468 - name: ecc_dbe_vol_l1
1469 - name: ecc_dbe_vol_l2
1470 - name: ecc_dbe_vol_reg
1471 - name: ecc_dbe_vol_shm
1472 - name: ecc_dbe_vol_srm
1473 - name: ecc_dbe_vol_tex
1474 - name: ecc_inforom_ver
1475 - name: ecc_pending
1476 - name: ecc_sbe_agg_cbu
1477 - name: ecc_sbe_agg_dev
1478 - name: ecc_sbe_agg_l1
1479 - name: ecc_sbe_agg_l2
1480 - name: ecc_sbe_agg_reg
1481 - name: ecc_sbe_agg_shm
1482 - name: ecc_sbe_agg_srm
1483 - name: ecc_sbe_agg_tex
1484 - name: ecc_sbe_vol_cbu
1485 - name: ecc_sbe_vol_dev
1486 - name: ecc_sbe_vol_l1
1487 - name: ecc_sbe_vol_l2
1488 - name: ecc_sbe_vol_reg
1489 - name: ecc_sbe_vol_shm
1490 - name: ecc_sbe_vol_srm
1491 - name: ecc_sbe_vol_tex
1492 - name: dcgm.mig.memory.page_retirements
1493 description: MIG Retired Memory Pages metrics.
1494 unit: pages/s
1495 chart_type: line
1496 dimensions:
1497 - name: retired_dbe
1498 - name: retired_pending
1499 - name: retired_sbe
1500 - name: dcgm.mig.memory.usage
1501 description: MIG Memory Usage metrics.
1502 unit: B
1503 chart_type: stacked
1504 dimensions:
1505 - name: free
1506 - name: reserved
1507 - name: used
1508 - name: dcgm.mig.memory.capacity
1509 description: MIG Memory Capacity metrics.
1510 unit: B
1511 chart_type: line
1512 dimensions:
1513 - name: total
1514 - name: dcgm.mig.memory.utilization
1515 description: MIG Memory Utilization metrics.
1516 unit: '%'
1517 chart_type: line
1518 dimensions:
1519 - name: used_percent
1520 - name: dcgm.mig.power.energy
1521 description: MIG Energy Consumption Rate metrics.
1522 unit: mJ/s
1523 chart_type: line
1524 dimensions:
1525 - name: total
1526 - name: dcgm.mig.power.profiles
1527 description: MIG Power Profiles metrics.
1528 unit: state
1529 chart_type: line
1530 dimensions:
1531 - name: enforced_power_profile_mask
1532 - name: requested_power_profile_mask
1533 - name: valid_power_profile_mask
1534 - name: dcgm.mig.power.smoothing
1535 description: MIG Power Smoothing metrics.
1536 unit: value
1537 chart_type: line
1538 dimensions:
1539 - name: pwr_smoothing_active_preset_profile
1540 - name: pwr_smoothing_admin_override_percent_tmp_floor
1541 - name: pwr_smoothing_admin_override_ramp_down_hyst_val
1542 - name: pwr_smoothing_admin_override_ramp_down_rate
1543 - name: pwr_smoothing_admin_override_ramp_up_rate
1544 - name: pwr_smoothing_applied_tmp_ceil
1545 - name: pwr_smoothing_applied_tmp_floor
1546 - name: pwr_smoothing_enabled
1547 - name: pwr_smoothing_hw_circuitry_percent_lifetime_remaining
1548 - name: pwr_smoothing_imm_ramp_down_enabled
1549 - name: pwr_smoothing_max_num_preset_profiles
1550 - name: pwr_smoothing_max_percent_tmp_floor_setting
1551 - name: pwr_smoothing_min_percent_tmp_floor_setting
1552 - name: pwr_smoothing_priv_lvl
1553 - name: pwr_smoothing_profile_percent_tmp_floor
1554 - name: pwr_smoothing_profile_ramp_down_hyst_val
1555 - name: pwr_smoothing_profile_ramp_down_rate
1556 - name: pwr_smoothing_profile_ramp_up_rate
1557 - name: dcgm.mig.power.usage
1558 description: MIG Power Usage metrics.
1559 unit: Watts
1560 chart_type: line
1561 dimensions:
1562 - name: draw
1563 - name: enforced_limit
1564 visibility: hidden
1565 - name: power_mgmt_limit
1566 visibility: hidden
1567 - name: power_mgmt_limit_def
1568 visibility: hidden
1569 - name: power_mgmt_limit_max
1570 visibility: hidden
1571 - name: power_mgmt_limit_min
1572 visibility: hidden
1573 - name: power_usage_instant
1574 - name: dcgm.mig.reliability.memory_health
1575 description: MIG Memory Health metrics.
1576 unit: state
1577 chart_type: line
1578 dimensions:
1579 - name: banks_remap_rows_avail_high
1580 - name: banks_remap_rows_avail_low
1581 - name: banks_remap_rows_avail_max
1582 - name: banks_remap_rows_avail_none
1583 - name: banks_remap_rows_avail_partial
1584 - name: memory_unrepairable_flag
1585 - name: threshold_srm
1586 - name: dcgm.mig.reliability.recovery_action
1587 description: MIG Recovery Action metrics.
1588 unit: state
1589 chart_type: line
1590 dimensions:
1591 - name: get_gpu_recovery_action
1592 - name: dcgm.mig.reliability.row_remap_events
1593 description: MIG Row Remap Events metrics.
1594 unit: rows/s
1595 chart_type: line
1596 dimensions:
1597 - name: correctable_remapped_rows
1598 - name: uncorrectable_remapped_rows
1599 - name: dcgm.mig.reliability.row_remap_status
1600 description: MIG Row Remap Status metrics.
1601 unit: state
1602 chart_type: line
1603 dimensions:
1604 - name: row_remap_failure
1605 - name: row_remap_pending
1606 - name: dcgm.mig.reliability.xid
1607 description: MIG XID Errors metrics.
1608 unit: code
1609 chart_type: line
1610 dimensions:
1611 - name: xid
1612 - name: dcgm.mig.state.configuration
1613 description: MIG Configuration State metrics.
1614 unit: state
1615 chart_type: line
1616 dimensions:
1617 - name: autoboost
1618 - name: compute_mode
1619 - name: persistence_mode
1620 - name: sync_boost
1621 - name: sync_boost_violation
1622 - name: dcgm.mig.state.performance
1623 description: MIG Performance State metrics.
1624 unit: state
1625 chart_type: line
1626 dimensions:
1627 - name: pstate
1628 - name: dcgm.mig.state.virtualization
1629 description: MIG Virtualization State metrics.
1630 unit: state
1631 chart_type: line
1632 dimensions:
1633 - name: mig_mode
1634 - name: virtual_mode
1635 - name: dcgm.mig.thermal.fan_speed
1636 description: MIG Fan Speed metrics.
1637 unit: '%'
1638 chart_type: line
1639 dimensions:
1640 - name: fan_speed
1641 - name: dcgm.mig.thermal.temperature
1642 description: MIG Temperature metrics.
1643 unit: Celsius
1644 chart_type: line
1645 dimensions:
1646 - name: gpu
1647 - name: gpu_max_op_temp
1648 visibility: hidden
1649 - name: gpu_temp_limit
1650 visibility: hidden
1651 - name: mem_max_op_temp
1652 visibility: hidden
1653 - name: memory
1654 - name: shutdown_temp
1655 visibility: hidden
1656 - name: slowdown_temp
1657 visibility: hidden
1658 - name: dcgm.mig.throttle.reasons
1659 description: MIG Throttle Reasons metrics.
1660 unit: bitmask
1661 chart_type: line
1662 dimensions:
1663 - name: clocks_event_reasons
1664 - name: dcgm.mig.throttle.violations
1665 description: MIG Throttle Violation Duration metrics.
1666 unit: milliseconds/s
1667 chart_type: line
1668 dimensions:
1669 - name: board_limit_violation
1670 - name: hw_power_brake_slowdown
1671 - name: hw_therm_slowdown
1672 - name: low_utilization_violation
1673 - name: power_violation
1674 - name: reliability_violation
1675 - name: sw_power_cap
1676 - name: sw_therm_slowdown
1677 - name: sync_boost
1678 - name: thermal_violation
1679 - name: total_app_clocks_violation
1680 - name: total_base_clocks_violation
1681 - name: nvlink
1682 description: These metrics refer to NVLink link instances.
1683 labels:
1684 - name: gpu
1685 description: gpu label from exporter metrics.
1686 - name: gpu_uuid
1687 description: gpu_uuid label from exporter metrics.
1688 - name: nvlink
1689 description: nvlink label from exporter metrics.
1690 metrics:
1691 - name: dcgm.nvlink.interconnect.ber
1692 description: NVLink Interconnect Bit Error Rate metrics.
1693 unit: ratio
1694 chart_type: line
1695 dimensions:
1696 - name: nvlink_count_effective_ber
1697 - name: nvlink_count_effective_ber_float
1698 - name: nvlink_count_symbol_ber
1699 - name: nvlink_count_symbol_ber_float
1700 - name: dcgm.nvlink.interconnect.congestion
1701 description: NVLink Interconnect Congestion metrics.
1702 unit: events/s
1703 chart_type: line
1704 dimensions:
1705 - name: nvlink_ppcnt_ibpc_port_xmit_wait
1706 - name: dcgm.nvlink.interconnect.error_rate
1707 description: NVLink Interconnect Error Rate metrics.
1708 unit: errors/s
1709 chart_type: line
1710 dimensions:
1711 - name: gpu_nvlink_errors
1712 - name: nvlink_count_effective_errors
1713 - name: nvlink_count_fec_history_0
1714 - name: nvlink_count_fec_history_1
1715 - name: nvlink_count_fec_history_10
1716 - name: nvlink_count_fec_history_11
1717 - name: nvlink_count_fec_history_12
1718 - name: nvlink_count_fec_history_13
1719 - name: nvlink_count_fec_history_14
1720 - name: nvlink_count_fec_history_15
1721 - name: nvlink_count_fec_history_2
1722 - name: nvlink_count_fec_history_3
1723 - name: nvlink_count_fec_history_4
1724 - name: nvlink_count_fec_history_5
1725 - name: nvlink_count_fec_history_6
1726 - name: nvlink_count_fec_history_7
1727 - name: nvlink_count_fec_history_8
1728 - name: nvlink_count_fec_history_9
1729 - name: nvlink_count_link_recovery_events
1730 - name: nvlink_count_link_recovery_failed_events
1731 - name: nvlink_count_link_recovery_successful_events
1732 - name: nvlink_count_local_link_integrity_errors
1733 - name: nvlink_count_rx_buffer_overrun_errors
1734 - name: nvlink_count_rx_errors
1735 - name: nvlink_count_rx_general_errors
1736 - name: nvlink_count_rx_malformed_packet_errors
1737 - name: nvlink_count_rx_remote_errors
1738 - name: nvlink_count_rx_symbol_errors
1739 - name: nvlink_count_tx_discards
1740 - name: nvlink_crc_data_error
1741 - name: nvlink_crc_data_error_count_l0
1742 - name: nvlink_crc_data_error_count_l1
1743 - name: nvlink_crc_data_error_count_l10
1744 - name: nvlink_crc_data_error_count_l11
1745 - name: nvlink_crc_data_error_count_l12
1746 - name: nvlink_crc_data_error_count_l13
1747 - name: nvlink_crc_data_error_count_l14
1748 - name: nvlink_crc_data_error_count_l15
1749 - name: nvlink_crc_data_error_count_l16
1750 - name: nvlink_crc_data_error_count_l17
1751 - name: nvlink_crc_data_error_count_l2
1752 - name: nvlink_crc_data_error_count_l3
1753 - name: nvlink_crc_data_error_count_l4
1754 - name: nvlink_crc_data_error_count_l5
1755 - name: nvlink_crc_data_error_count_l6
1756 - name: nvlink_crc_data_error_count_l7
1757 - name: nvlink_crc_data_error_count_l8
1758 - name: nvlink_crc_data_error_count_l9
1759 - name: nvlink_crc_flit_error
1760 - name: nvlink_crc_flit_error_count_l0
1761 - name: nvlink_crc_flit_error_count_l1
1762 - name: nvlink_crc_flit_error_count_l10
1763 - name: nvlink_crc_flit_error_count_l11
1764 - name: nvlink_crc_flit_error_count_l12
1765 - name: nvlink_crc_flit_error_count_l13
1766 - name: nvlink_crc_flit_error_count_l14
1767 - name: nvlink_crc_flit_error_count_l15
1768 - name: nvlink_crc_flit_error_count_l16
1769 - name: nvlink_crc_flit_error_count_l17
1770 - name: nvlink_crc_flit_error_count_l2
1771 - name: nvlink_crc_flit_error_count_l3
1772 - name: nvlink_crc_flit_error_count_l4
1773 - name: nvlink_crc_flit_error_count_l5
1774 - name: nvlink_crc_flit_error_count_l6
1775 - name: nvlink_crc_flit_error_count_l7
1776 - name: nvlink_crc_flit_error_count_l8
1777 - name: nvlink_crc_flit_error_count_l9
1778 - name: nvlink_error_dl_crc
1779 - name: nvlink_error_dl_recovery
1780 - name: nvlink_error_dl_replay
1781 - name: nvlink_ppcnt_physical_successful_recovery_events
1782 - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1783 - name: nvlink_ppcnt_recovery_time_since_last
1784 - name: nvlink_ppcnt_recovery_total_successful_events
1785 - name: nvlink_pprm_oper_recovery
1786 - name: nvlink_recovery_error
1787 - name: nvlink_recovery_error_count_l0
1788 - name: nvlink_recovery_error_count_l1
1789 - name: nvlink_recovery_error_count_l10
1790 - name: nvlink_recovery_error_count_l11
1791 - name: nvlink_recovery_error_count_l12
1792 - name: nvlink_recovery_error_count_l13
1793 - name: nvlink_recovery_error_count_l14
1794 - name: nvlink_recovery_error_count_l15
1795 - name: nvlink_recovery_error_count_l16
1796 - name: nvlink_recovery_error_count_l17
1797 - name: nvlink_recovery_error_count_l2
1798 - name: nvlink_recovery_error_count_l3
1799 - name: nvlink_recovery_error_count_l4
1800 - name: nvlink_recovery_error_count_l5
1801 - name: nvlink_recovery_error_count_l6
1802 - name: nvlink_recovery_error_count_l7
1803 - name: nvlink_recovery_error_count_l8
1804 - name: nvlink_recovery_error_count_l9
1805 - name: nvlink_replay_error
1806 - name: nvlink_replay_error_count_l0
1807 - name: nvlink_replay_error_count_l1
1808 - name: nvlink_replay_error_count_l10
1809 - name: nvlink_replay_error_count_l11
1810 - name: nvlink_replay_error_count_l12
1811 - name: nvlink_replay_error_count_l13
1812 - name: nvlink_replay_error_count_l14
1813 - name: nvlink_replay_error_count_l15
1814 - name: nvlink_replay_error_count_l16
1815 - name: nvlink_replay_error_count_l17
1816 - name: nvlink_replay_error_count_l2
1817 - name: nvlink_replay_error_count_l3
1818 - name: nvlink_replay_error_count_l4
1819 - name: nvlink_replay_error_count_l5
1820 - name: nvlink_replay_error_count_l6
1821 - name: nvlink_replay_error_count_l7
1822 - name: nvlink_replay_error_count_l8
1823 - name: nvlink_replay_error_count_l9
1824 - name: dcgm.nvlink.interconnect.errors
1825 description: NVLink Interconnect Errors metrics.
1826 unit: errors
1827 chart_type: line
1828 dimensions:
1829 - name: nvlink_ppcnt_plr_rcv_uncorrectable_code
1830 - name: dcgm.nvlink.interconnect.state
1831 description: NVLink Interconnect State metrics.
1832 unit: state
1833 chart_type: line
1834 dimensions:
1835 - name: gpu_topology_nvlink
1836 - name: nvlink_get_state
1837 - name: nvlink_ppcnt_physical_link_down_counter
1838 - name: nvlink_ppcnt_plr_rcv_code_err
1839 - name: nvlink_ppcnt_plr_sync_events
1840 - name: nvlink_ppcnt_plr_xmit_retry_events
1841 - name: p2p_nvlink_status
1842 - name: dcgm.nvlink.interconnect.throughput
1843 description: NVLink Interconnect Throughput metrics.
1844 unit: B/s
1845 chart_type: area
1846 dimensions:
1847 - name: nvlink_bandwidth
1848 - name: nvlink_bandwidth_l0
1849 - name: nvlink_bandwidth_l1
1850 - name: nvlink_bandwidth_l10
1851 - name: nvlink_bandwidth_l11
1852 - name: nvlink_bandwidth_l12
1853 - name: nvlink_bandwidth_l13
1854 - name: nvlink_bandwidth_l14
1855 - name: nvlink_bandwidth_l15
1856 - name: nvlink_bandwidth_l16
1857 - name: nvlink_bandwidth_l17
1858 - name: nvlink_bandwidth_l2
1859 - name: nvlink_bandwidth_l3
1860 - name: nvlink_bandwidth_l4
1861 - name: nvlink_bandwidth_l5
1862 - name: nvlink_bandwidth_l6
1863 - name: nvlink_bandwidth_l7
1864 - name: nvlink_bandwidth_l8
1865 - name: nvlink_bandwidth_l9
1866 - name: nvlink_count_rx
1867 - name: nvlink_count_tx
1868 - name: nvlink_l0_rx
1869 - name: nvlink_l0_tx
1870 - name: nvlink_l10_rx
1871 - name: nvlink_l10_tx
1872 - name: nvlink_l11_rx
1873 - name: nvlink_l11_tx
1874 - name: nvlink_l12_rx
1875 - name: nvlink_l12_tx
1876 - name: nvlink_l13_rx
1877 - name: nvlink_l13_tx
1878 - name: nvlink_l14_rx
1879 - name: nvlink_l14_tx
1880 - name: nvlink_l15_rx
1881 - name: nvlink_l15_tx
1882 - name: nvlink_l16_rx
1883 - name: nvlink_l16_tx
1884 - name: nvlink_l17_rx
1885 - name: nvlink_l17_tx
1886 - name: nvlink_l1_rx
1887 - name: nvlink_l1_tx
1888 - name: nvlink_l2_rx
1889 - name: nvlink_l2_tx
1890 - name: nvlink_l3_rx
1891 - name: nvlink_l3_tx
1892 - name: nvlink_l4_rx
1893 - name: nvlink_l4_tx
1894 - name: nvlink_l5_rx
1895 - name: nvlink_l5_tx
1896 - name: nvlink_l6_rx
1897 - name: nvlink_l6_tx
1898 - name: nvlink_l7_rx
1899 - name: nvlink_l7_tx
1900 - name: nvlink_l8_rx
1901 - name: nvlink_l8_tx
1902 - name: nvlink_l9_rx
1903 - name: nvlink_l9_tx
1904 - name: nvlink_rx_bandwidth
1905 - name: nvlink_rx_bandwidth_l0
1906 - name: nvlink_rx_bandwidth_l1
1907 - name: nvlink_rx_bandwidth_l10
1908 - name: nvlink_rx_bandwidth_l11
1909 - name: nvlink_rx_bandwidth_l12
1910 - name: nvlink_rx_bandwidth_l13
1911 - name: nvlink_rx_bandwidth_l14
1912 - name: nvlink_rx_bandwidth_l15
1913 - name: nvlink_rx_bandwidth_l16
1914 - name: nvlink_rx_bandwidth_l17
1915 - name: nvlink_rx_bandwidth_l2
1916 - name: nvlink_rx_bandwidth_l3
1917 - name: nvlink_rx_bandwidth_l4
1918 - name: nvlink_rx_bandwidth_l5
1919 - name: nvlink_rx_bandwidth_l6
1920 - name: nvlink_rx_bandwidth_l7
1921 - name: nvlink_rx_bandwidth_l8
1922 - name: nvlink_rx_bandwidth_l9
1923 - name: nvlink_rx
1924 - name: nvlink_tx_bandwidth
1925 - name: nvlink_tx_bandwidth_l0
1926 - name: nvlink_tx_bandwidth_l1
1927 - name: nvlink_tx_bandwidth_l10
1928 - name: nvlink_tx_bandwidth_l11
1929 - name: nvlink_tx_bandwidth_l12
1930 - name: nvlink_tx_bandwidth_l13
1931 - name: nvlink_tx_bandwidth_l14
1932 - name: nvlink_tx_bandwidth_l15
1933 - name: nvlink_tx_bandwidth_l16
1934 - name: nvlink_tx_bandwidth_l17
1935 - name: nvlink_tx_bandwidth_l2
1936 - name: nvlink_tx_bandwidth_l3
1937 - name: nvlink_tx_bandwidth_l4
1938 - name: nvlink_tx_bandwidth_l5
1939 - name: nvlink_tx_bandwidth_l6
1940 - name: nvlink_tx_bandwidth_l7
1941 - name: nvlink_tx_bandwidth_l8
1942 - name: nvlink_tx_bandwidth_l9
1943 - name: nvlink_tx
1944 - name: dcgm.nvlink.interconnect.traffic
1945 description: NVLink Interconnect Traffic metrics.
1946 unit: events/s
1947 chart_type: line
1948 dimensions:
1949 - name: nvlink_count_rx_packets
1950 - name: nvlink_count_tx_packets
1951 - name: nvlink_ppcnt_plr_rcv_codes
1952 - name: nvlink_ppcnt_plr_xmit_codes
1953 - name: nvlink_ppcnt_plr_xmit_retry_codes
1954 - name: dcgm.nvlink.internal.boundary
1955 description: NVLink Internal Boundary Fields metrics.
1956 unit: state
1957 chart_type: line
1958 dimensions:
1959 - name: nvlink_ppcnt_recovery_time_between_last_two
1960 - name: dcgm.nvlink.memory.ecc_error_rate
1961 description: NVLink ECC Error Rate metrics.
1962 unit: errors/s
1963 chart_type: line
1964 dimensions:
1965 - name: nvlink_ecc_data_error
1966 - name: nvswitch
1967 description: These metrics refer to NVSwitch instances.
1968 labels:
1969 - name: nvswitch
1970 description: nvswitch label from exporter metrics.
1971 metrics:
1972 - name: dcgm.nvswitch.interconnect.nvswitch.current
1973 description: NVSwitch Current metrics.
1974 unit: value
1975 chart_type: line
1976 dimensions:
1977 - name: nvswitch_current_iddq
1978 - name: nvswitch_current_iddq_dvdd
1979 - name: nvswitch_current_iddq_rev
1980 - name: dcgm.nvswitch.interconnect.nvswitch.errors
1981 description: NVSwitch NVSwitch Errors metrics.
1982 unit: errors/s
1983 chart_type: line
1984 dimensions:
1985 - name: nvswitch_fatal_errors
1986 - name: nvswitch_link_crc_errors
1987 - name: nvswitch_link_crc_errors_lane0
1988 - name: nvswitch_link_crc_errors_lane1
1989 - name: nvswitch_link_crc_errors_lane2
1990 - name: nvswitch_link_crc_errors_lane3
1991 - name: nvswitch_link_crc_errors_lane4
1992 - name: nvswitch_link_crc_errors_lane5
1993 - name: nvswitch_link_crc_errors_lane6
1994 - name: nvswitch_link_crc_errors_lane7
1995 - name: nvswitch_link_fatal_errors
1996 - name: nvswitch_link_flit_errors
1997 - name: nvswitch_link_non_fatal_errors
1998 - name: nvswitch_link_recovery_errors
1999 - name: nvswitch_link_replay_errors
2000 - name: nvswitch_non_fatal_errors
2001 - name: dcgm.nvswitch.interconnect.nvswitch.latency
2002 description: NVSwitch NVSwitch Link Latency metrics.
2003 unit: events/s
2004 chart_type: line
2005 dimensions:
2006 - name: nvswitch_link_latency_count_vc0
2007 - name: nvswitch_link_latency_count_vc1
2008 - name: nvswitch_link_latency_count_vc2
2009 - name: nvswitch_link_latency_count_vc3
2010 - name: nvswitch_link_latency_high_vc0
2011 - name: nvswitch_link_latency_high_vc1
2012 - name: nvswitch_link_latency_high_vc2
2013 - name: nvswitch_link_latency_high_vc3
2014 - name: nvswitch_link_latency_low_vc0
2015 - name: nvswitch_link_latency_low_vc1
2016 - name: nvswitch_link_latency_low_vc2
2017 - name: nvswitch_link_latency_low_vc3
2018 - name: nvswitch_link_latency_medium_vc0
2019 - name: nvswitch_link_latency_medium_vc1
2020 - name: nvswitch_link_latency_medium_vc2
2021 - name: nvswitch_link_latency_medium_vc3
2022 - name: nvswitch_link_latency_panic_vc0
2023 - name: nvswitch_link_latency_panic_vc1
2024 - name: nvswitch_link_latency_panic_vc2
2025 - name: nvswitch_link_latency_panic_vc3
2026 - name: dcgm.nvswitch.interconnect.nvswitch.power
2027 description: NVSwitch NVSwitch Power metrics.
2028 unit: Watts
2029 chart_type: line
2030 dimensions:
2031 - name: nvswitch_power_dvdd
2032 - name: nvswitch_power_hvdd
2033 - name: nvswitch_power_vdd
2034 - name: dcgm.nvswitch.interconnect.nvswitch.status
2035 description: NVSwitch NVSwitch Status metrics.
2036 unit: state
2037 chart_type: line
2038 dimensions:
2039 - name: nvswitch_link_status
2040 - name: nvswitch_link_type
2041 - name: nvswitch_reset_required
2042 - name: dcgm.nvswitch.interconnect.nvswitch.throughput
2043 description: NVSwitch NVSwitch Throughput metrics.
2044 unit: B/s
2045 chart_type: area
2046 dimensions:
2047 - name: nvswitch_link_throughput_rx
2048 - name: nvswitch_link_throughput_tx
2049 - name: nvswitch_throughput_rx
2050 - name: nvswitch_throughput_tx
2051 - name: dcgm.nvswitch.interconnect.nvswitch.topology
2052 description: NVSwitch NVSwitch Topology metrics.
2053 unit: value
2054 chart_type: line
2055 dimensions:
2056 - name: nvswitch_device_uuid
2057 - name: nvswitch_link_device_link_id
2058 - name: nvswitch_link_device_link_sid
2059 - name: nvswitch_link_id
2060 - name: nvswitch_link_remote_pcie_bus
2061 - name: nvswitch_link_remote_pcie_device
2062 - name: nvswitch_link_remote_pcie_domain
2063 - name: nvswitch_link_remote_pcie_function
2064 - name: nvswitch_pcie_bus
2065 - name: nvswitch_pcie_device
2066 - name: nvswitch_pcie_domain
2067 - name: nvswitch_pcie_function
2068 - name: nvswitch_phys_id
2069 - name: dcgm.nvswitch.interconnect.nvswitch.voltage
2070 description: NVSwitch NVSwitch Voltage metrics.
2071 unit: mV
2072 chart_type: line
2073 dimensions:
2074 - name: nvswitch_voltage_mvolt
2075 - name: dcgm.nvswitch.internal.boundary
2076 description: NVSwitch Internal Boundary Fields metrics.
2077 unit: state
2078 chart_type: line
2079 dimensions:
2080 - name: first_nvswitch_field_id
2081 - name: last_nvswitch_field_id
2082 - name: dcgm.nvswitch.memory.ecc_error_rate
2083 description: NVSwitch ECC Error Rate metrics.
2084 unit: errors/s
2085 chart_type: line
2086 dimensions:
2087 - name: nvswitch_link_ecc_errors
2088 - name: nvswitch_link_ecc_errors_lane0
2089 - name: nvswitch_link_ecc_errors_lane1
2090 - name: nvswitch_link_ecc_errors_lane2
2091 - name: nvswitch_link_ecc_errors_lane3
2092 - name: nvswitch_link_ecc_errors_lane4
2093 - name: nvswitch_link_ecc_errors_lane5
2094 - name: nvswitch_link_ecc_errors_lane6
2095 - name: nvswitch_link_ecc_errors_lane7
2096 - name: dcgm.nvswitch.thermal.temperature
2097 description: NVSwitch Temperature metrics.
2098 unit: Celsius
2099 chart_type: line
2100 dimensions:
2101 - name: nvswitch_temperature_current
2102 - name: nvswitch_temperature_limit_shutdown
2103 - name: nvswitch_temperature_limit_slowdown
2104 - name: cpu
2105 description: These metrics refer to host CPU instances.
2106 labels:
2107 - name: cpu
2108 description: cpu label from exporter metrics.
2109 metrics:
2110 - name: dcgm.cpu.clock.frequency
2111 description: CPU Clock Frequency metrics.
2112 unit: MHz
2113 chart_type: line
2114 dimensions:
2115 - name: cpu_clock_current
2116 - name: dcgm.cpu.cpu.info
2117 description: CPU CPU Information metrics.
2118 unit: value
2119 chart_type: line
2120 dimensions:
2121 - name: cpu_model
2122 - name: cpu_vendor
2123 - name: dcgm.cpu.cpu.power
2124 description: CPU CPU Power metrics.
2125 unit: Watts
2126 chart_type: line
2127 dimensions:
2128 - name: cpu_power_limit
2129 - name: cpu_power_util_current
2130 - name: dcgm.cpu.cpu.temperature
2131 description: CPU CPU Temperature metrics.
2132 unit: Celsius
2133 chart_type: line
2134 dimensions:
2135 - name: cpu_temp_critical
2136 - name: cpu_temp_current
2137 - name: cpu_temp_warning
2138 - name: dcgm.cpu.cpu.utilization
2139 description: CPU CPU Utilization metrics.
2140 unit: '%'
2141 chart_type: line
2142 dimensions:
2143 - name: cpu_util
2144 - name: cpu_util_irq
2145 - name: cpu_util_nice
2146 - name: cpu_util_sys
2147 - name: cpu_util_user
2148 - name: dcgm.cpu.diagnostics.results
2149 description: CPU Diagnostics Results metrics.
2150 unit: state
2151 chart_type: line
2152 dimensions:
2153 - name: diag_cpu_eud_result
2154 - name: cpu_core
2155 description: These metrics refer to host CPU core instances.
2156 labels:
2157 - name: cpu
2158 description: cpu label from exporter metrics.
2159 - name: cpucore
2160 description: cpucore label from exporter metrics.
2161 metrics:
2162 - name: dcgm.cpu_core.clock.frequency
2163 description: CPU Core Clock Frequency metrics.
2164 unit: MHz
2165 chart_type: line
2166 dimensions:
2167 - name: cpu_clock_current
2168 - name: dcgm.cpu_core.cpu.info
2169 description: CPU Core CPU Information metrics.
2170 unit: value
2171 chart_type: line
2172 dimensions:
2173 - name: cpu_model
2174 - name: cpu_vendor
2175 - name: dcgm.cpu_core.cpu.power
2176 description: CPU Core CPU Power metrics.
2177 unit: Watts
2178 chart_type: line
2179 dimensions:
2180 - name: cpu_power_limit
2181 - name: cpu_power_util_current
2182 - name: dcgm.cpu_core.cpu.temperature
2183 description: CPU Core CPU Temperature metrics.
2184 unit: Celsius
2185 chart_type: line
2186 dimensions:
2187 - name: cpu_temp_critical
2188 - name: cpu_temp_current
2189 - name: cpu_temp_warning
2190 - name: dcgm.cpu_core.cpu.utilization
2191 description: CPU Core CPU Utilization metrics.
2192 unit: '%'
2193 chart_type: line
2194 dimensions:
2195 - name: cpu_util
2196 - name: cpu_util_irq
2197 - name: cpu_util_nice
2198 - name: cpu_util_sys
2199 - name: cpu_util_user
2200 - name: dcgm.cpu_core.diagnostics.results
2201 description: CPU Core Diagnostics Results metrics.
2202 unit: state
2203 chart_type: line
2204 dimensions:
2205 - name: diag_cpu_eud_result
2206 - name: exporter
2207 description: These metrics refer to exporter/global instances.
2208 labels:
2209 - name: job
2210 description: job label from exporter metrics.
2211 metrics:
2212 - name: dcgm.exporter.health.status
2213 description: Exporter Health Status metrics.
2214 unit: state
2215 chart_type: line
2216 dimensions:
2217 - name: bind_unbind_event
2218 - name: dcgm.exporter.inventory.software
2219 description: Exporter Software and Firmware metrics.
2220 unit: value
2221 chart_type: line
2222 dimensions:
2223 - name: cuda_driver_version
2224 - name: driver_version
2225 - name: nvml_version