Add nvidia smi pci bandwidth percent collector (#14315)
Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>
Van Phan Quang committed
Feb 1, 2023 at 16:37 UTC
f444be6a847cc255b53483fe1a01d79fe8e6dd6c
1 file changed
+52
-1
collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py
+52
-1
@@ -22,6 +22,7 @@ EMPTY_ROW_LIMIT = 500
22
POLLER_BREAK_ROW = '</nvidia_smi_log>'
23
24
PCI_BANDWIDTH = 'pci_bandwidth'
25
+PCI_BANDWIDTH_PERCENT = 'pci_bandwidth_percent'
26
FAN_SPEED = 'fan_speed'
27
GPU_UTIL = 'gpu_utilization'
28
MEM_UTIL = 'mem_utilization'
@@ -38,6 +39,7 @@ USER_NUM = 'user_num'
39
40
ORDER = [
41
PCI_BANDWIDTH,
42
+ PCI_BANDWIDTH_PERCENT,
43
FAN_SPEED,
44
GPU_UTIL,
45
MEM_UTIL,
@@ -56,7 +58,22 @@ ORDER = [
58
# https://docs.nvidia.com/gameworks/content/gameworkslibrary/coresdk/nvapi/group__gpupstate.html
59
POWER_STATES = ['P' + str(i) for i in range(0, 16)]
60
59
-
61
+# PCI Transfer data rate in gigabits per second (Gb/s) per generation
62
+PCI_SPEED = {
63
+ "1": 2.5,
64
+ "2": 5,
65
+ "3": 8,
66
+ "4": 16,
67
+ "5": 32
68
+}
69
+# PCI encoding per generation
70
+PCI_ENCODING = {
71
+ "1": 2/10,
72
+ "2": 2/10,
73
+ "3": 2/130,
74
+ "4": 2/130,
75
+ "5": 2/130
76
+}
77
def gpu_charts(gpu):
78
fam = gpu.full_name()
79
@@ -68,6 +85,13 @@ def gpu_charts(gpu):
85
['tx_util', 'tx', 'absolute', 1, -1],
86
]
87
},
88
+ PCI_BANDWIDTH_PERCENT: {
89
+ 'options': [None, 'PCI Express Bandwidth Percent', 'percentage', fam, 'nvidia_smi.pci_bandwidth_percent', 'area'],
90
+ 'lines': [
91
+ ['rx_util_percent', 'rx_percent'],
92
+ ['tx_util_percent', 'tx_percent'],
93
+ ]
94
+ },
95
FAN_SPEED: {
96
'options': [None, 'Fan Speed', 'percentage', fam, 'nvidia_smi.fan_speed', 'line'],
97
'lines': [
@@ -326,6 +350,24 @@ class GPU:
350
def full_name(self):
351
return 'gpu{0} {1}'.format(self.num, self.name())
352
353
+ @handle_attr_error
354
+ def pci_link_gen(self):
355
+ return self.root.find('pci').find('pci_gpu_link_info').find('pcie_gen').find('max_link_gen').text
356
+
357
+ @handle_attr_error
358
+ def pci_link_width(self):
359
+ return self.root.find('pci').find('pci_gpu_link_info').find('link_widths').find('max_link_width').text.split('x')[0]
360
+
361
+ def pci_bw_max(self):
362
+ link_gen = self.pci_link_gen()
363
+ link_width = int(self.pci_link_width())
364
+ if link_gen not in PCI_SPEED or link_gen not in PCI_ENCODING or not link_width:
365
+ return None
366
+ # Maximum PCIe Bandwidth = SPEED * WIDTH * (1 - ENCODING) - 1Gb/s.
367
+ # see details https://enterprise-support.nvidia.com/s/article/understanding-pcie-configuration-for-maximum-performance
368
+ # return max bandwidth in kilobytes per second (kB/s)
369
+ return (PCI_SPEED[link_gen] * link_width * (1- PCI_ENCODING[link_gen]) - 1) * 1000 * 1000 / 8
370
+
371
@handle_attr_error
372
def rx_util(self):
373
return self.root.find('pci').find('rx_util').text.split()[0]
@@ -439,6 +481,15 @@ class GPU:
481
'power_draw': self.power_draw(),
482
}
483
484
+ pci_bw_max = self.pci_bw_max()
485
+ if not pci_bw_max:
486
+ data['rx_util_percent'] = 0
487
+ data['tx_util_percent'] = 0
488
+ else :
489
+ data['rx_util_percent'] = str(int(int(self.rx_util())*100/self.pci_bw_max()))
490
+ data['tx_util_percent'] = str(int(int(self.tx_util())*100/self.pci_bw_max()))
491
+
492
+
493
for v in POWER_STATES:
494
data['power_state_' + v.lower()] = 0
495
p_state = self.power_state()