| 1 | # DCGM GPU reliability alerts. |
| 2 | |
| 3 | template: dcgm_gpu_xid_errors |
| 4 | on: dcgm.gpu.reliability.xid |
| 5 | class: Errors |
| 6 | type: GPU |
| 7 | component: NVIDIA |
| 8 | lookup: max -1m unaligned absolute of xid |
| 9 | units: code |
| 10 | every: 30s |
| 11 | warn: $this > 0 |
| 12 | delay: up 30s down 5m multiplier 1.5 max 1h |
| 13 | summary: DCGM reported XID error on GPU ${label:gpu} |
| 14 | info: NVIDIA driver reported a GPU XID error (metric ${label:chart_context}). |
| 15 | to: sysadmin |
| 16 | |
| 17 | template: dcgm_gpu_row_remap_failure |
| 18 | on: dcgm.gpu.reliability.row_remap_status |
| 19 | class: Errors |
| 20 | type: GPU |
| 21 | component: NVIDIA |
| 22 | lookup: max -1m unaligned absolute of row_remap_failure |
| 23 | units: state |
| 24 | every: 30s |
| 25 | warn: $this > 0 |
| 26 | delay: up 30s down 5m multiplier 1.5 max 1h |
| 27 | summary: DCGM row remap failure on GPU ${label:gpu} |
| 28 | info: Row remapping has failed, indicating a persistent memory reliability problem. |
| 29 | to: sysadmin |
| 30 | |
| 31 | template: dcgm_gpu_uncorrectable_remapped_rows |
| 32 | on: dcgm.gpu.reliability.row_remap_events |
| 33 | class: Errors |
| 34 | type: GPU |
| 35 | component: NVIDIA |
| 36 | lookup: sum -5m unaligned absolute of uncorrectable_remapped_rows |
| 37 | units: rows |
| 38 | every: 30s |
| 39 | warn: $this > 0 |
| 40 | delay: up 30s down 10m multiplier 1.5 max 1h |
| 41 | summary: DCGM uncorrectable remapped rows on GPU ${label:gpu} |
| 42 | info: New uncorrectable row remap events were detected in the last 5 minutes. |
| 43 | to: sysadmin |
| 44 | |
| 45 | # DCGM throttle violation alerts. |
| 46 | |
| 47 | template: dcgm_gpu_power_violation |
| 48 | on: dcgm.gpu.throttle.violations |
| 49 | class: Workload |
| 50 | type: GPU |
| 51 | component: NVIDIA |
| 52 | lookup: sum -5m unaligned absolute of power_violation |
| 53 | units: milliseconds |
| 54 | every: 30s |
| 55 | warn: $this > 0 |
| 56 | delay: up 1m down 10m multiplier 1.5 max 1h |
| 57 | summary: DCGM power throttling detected on GPU ${label:gpu} |
| 58 | info: The GPU was power-throttled during the last 5 minutes. |
| 59 | to: sysadmin |
| 60 | |
| 61 | template: dcgm_gpu_thermal_violation |
| 62 | on: dcgm.gpu.throttle.violations |
| 63 | class: Workload |
| 64 | type: GPU |
| 65 | component: NVIDIA |
| 66 | lookup: sum -5m unaligned absolute of thermal_violation |
| 67 | units: milliseconds |
| 68 | every: 30s |
| 69 | warn: $this > 0 |
| 70 | delay: up 1m down 10m multiplier 1.5 max 1h |
| 71 | summary: DCGM thermal throttling detected on GPU ${label:gpu} |
| 72 | info: The GPU was thermally throttled during the last 5 minutes. |
| 73 | to: sysadmin |