nvidia_smi: charts for memory used by each user and number of distinct users (#9372)
Co-authored-by: ilyam8 <ilya@netdata.cloud>
Guido committed
Aug 4, 2020 at 18:09 UTC
211137df801d1b3e4d4d484a10c4ec42d8640040
2 files changed
+123
-29
collectors/python.d.plugin/nvidia_smi/README.md
+17
-20
@@ -9,36 +9,33 @@ sidebar_label: "Nvidia GPUs"
9
Monitors performance metrics (memory usage, fan speed, pcie bandwidth utilization, temperature, etc.) using `nvidia-smi` cli tool.
10
11
12
-**Requirements and Notes:**
12
+## Requirements and Notes
13
14
- You must have the `nvidia-smi` tool installed and your NVIDIA GPU(s) must support the tool. Mostly the newer high end models used for AI / ML and Crypto or Pro range, read more about [nvidia_smi](https://developer.nvidia.com/nvidia-system-management-interface).
15
-
15
- You must enable this plugin as its disabled by default due to minor performance issues.
17
-
16
- On some systems when the GPU is idle the `nvidia-smi` tool unloads and there is added latency again when it is next queried. If you are running GPUs under constant workload this isn't likely to be an issue.
19
-
17
- Currently the `nvidia-smi` tool is being queried via cli. Updating the plugin to use the nvidia c/c++ API directly should resolve this issue. See discussion here: <https://github.com/netdata/netdata/pull/4357>
21
-
18
- Contributions are welcome.
23
-
19
- Make sure `netdata` user can execute `/usr/bin/nvidia-smi` or wherever your binary is.
25
-
20
- If `nvidia-smi` process [is not killed after netdata restart](https://github.com/netdata/netdata/issues/7143) you need to off `loop_mode`.
27
-
21
- `poll_seconds` is how often in seconds the tool is polled for as an integer.
22
30
-It produces:
31
-
32
-1. Per GPU
33
-
34
- - GPU utilization
35
- - memory allocation
36
- - memory utilization
37
- - fan speed
38
- - power usage
39
- - temperature
40
- - clock speed
41
- - PCI bandwidth
23
+## Charts
24
+
25
+It produces the following charts:
26
+
27
+- PCI Express Bandwidth Utilization in `KiB/s`
28
+- Fan Speed in `percentage`
29
+- GPU Utilization in `percentage`
30
+- Memory Bandwidth Utilization in `percentage`
31
+- Encoder/Decoder Utilization in `percentage`
32
+- Memory Usage in `MiB`
33
+- Temperature in `celsius`
34
+- Clock Frequencies in `MHz`
35
+- Power Utilization in `Watts`
36
+- Memory Used by Each Process in `MiB`
37
+- Memory Used by Each User in `MiB`
38
+- Number of User on GPU in `num`
39
40
## Configuration
41
collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py
+106
-9
@@ -2,9 +2,12 @@
2
# Description: nvidia-smi netdata python.d module
3
# Original Author: Steven Noonan (tycho)
4
# Author: Ilya Mashchenko (ilyam8)
5
+# User Memory Stat Author: Guido Scatena (scatenag)
6
7
import subprocess
8
import threading
9
+import os
10
+
11
import xml.etree.ElementTree as et
12
13
from bases.FrameworkServices.SimpleService import SimpleService
@@ -30,6 +33,8 @@ TEMPERATURE = 'temperature'
33
CLOCKS = 'clocks'
34
POWER = 'power'
35
PROCESSES_MEM = 'processes_mem'
36
+USER_MEM = 'user_mem'
37
+USER_NUM = 'user_num'
38
39
ORDER = [
40
PCI_BANDWIDTH,
@@ -42,6 +47,8 @@ ORDER = [
47
CLOCKS,
48
POWER,
49
PROCESSES_MEM,
50
+ USER_MEM,
51
+ USER_NUM,
52
]
53
54
@@ -114,6 +121,16 @@ def gpu_charts(gpu):
121
'options': [None, 'Memory Used by Each Process', 'MiB', fam, 'nvidia_smi.processes_mem', 'stacked'],
122
'lines': []
123
},
124
+ USER_MEM: {
125
+ 'options': [None, 'Memory Used by Each User', 'MiB', fam, 'nvidia_smi.user_mem', 'stacked'],
126
+ 'lines': []
127
+ },
128
+ USER_NUM: {
129
+ 'options': [None, 'Number of User on GPU', 'num', fam, 'nvidia_smi.user_num', 'line'],
130
+ 'lines': [
131
+ ['user_num', 'users'],
132
+ ]
133
+ },
134
}
135
136
idx = gpu.num
@@ -226,6 +243,50 @@ def handle_value_error(method):
243
return on_call
244
245
246
+HOST_PREFIX = os.getenv('NETDATA_HOST_PREFIX')
247
+ETC_PASSWD_PATH = '/etc/passwd'
248
+PROC_PATH = '/proc'
249
+
250
+if HOST_PREFIX:
251
+ ETC_PASSWD_PATH = os.path.join(HOST_PREFIX, ETC_PASSWD_PATH[1:])
252
+ PROC_PATH = os.path.join(HOST_PREFIX, PROC_PATH[1:])
253
+
254
+
255
+def read_passwd_file():
256
+ data = dict()
257
+ with open(ETC_PASSWD_PATH, 'r') as f:
258
+ for line in f:
259
+ line = line.strip()
260
+ if line.startswith("#"):
261
+ continue
262
+ fields = line.split(":")
263
+ # name, passwd, uid, gid, comment, home_dir, shell
264
+ if len(fields) != 7:
265
+ continue
266
+ # uid, guid
267
+ fields[2], fields[3] = int(fields[2]), int(fields[3])
268
+ data[fields[2]] = fields
269
+ return data
270
+
271
+
272
+def read_passwd_file_safe():
273
+ try:
274
+ return read_passwd_file()
275
+ except (OSError, IOError):
276
+ return dict()
277
+
278
+
279
+def get_username_by_pid_safe(pid, passwd_file):
280
+ if not passwd_file:
281
+ return ''
282
+ path = os.path.join(PROC_PATH, pid)
283
+ try:
284
+ uid = os.stat(path).st_uid
285
+ return passwd_file[uid][0]
286
+ except (OSError, IOError, KeyError):
287
+ return ''
288
+
289
+
290
class GPU:
291
def __init__(self, num, root):
292
self.num = num
@@ -303,15 +364,22 @@ class GPU:
364
365
@handle_attr_error
366
def processes(self):
306
- p_nodes = self.root.find('processes').findall('process_info')
307
- ps = []
308
- for p in p_nodes:
309
- ps.append({
310
- 'pid': p.find('pid').text,
311
- 'process_name': p.find('process_name').text,
312
- 'used_memory': int(p.find('used_memory').text.split()[0]),
367
+ processes_info = self.root.find('processes').findall('process_info')
368
+ if not processes_info:
369
+ return list()
370
+
371
+ passwd_file = read_passwd_file_safe()
372
+ processes = list()
373
+
374
+ for info in processes_info:
375
+ pid = info.find('pid').text
376
+ processes.append({
377
+ 'pid': int(pid),
378
+ 'process_name': info.find('process_name').text,
379
+ 'used_memory': int(info.find('used_memory').text.split()[0]),
380
+ 'username': get_username_by_pid_safe(pid, passwd_file),
381
})
314
- return ps
382
+ return processes
383
384
def data(self):
385
data = {
@@ -332,7 +400,17 @@ class GPU:
400
'power_draw': self.power_draw(),
401
}
402
processes = self.processes() or []
335
- data.update({'process_mem_{0}'.format(p['pid']): p['used_memory'] for p in processes})
403
+ users = set()
404
+ for p in processes:
405
+ data['process_mem_{0}'.format(p['pid'])] = p['used_memory']
406
+ if p['username']:
407
+ users.add(p['username'])
408
+ key = 'user_mem_{0}'.format(p['username'])
409
+ if key in data:
410
+ data[key] += p['used_memory']
411
+ else:
412
+ data[key] = p['used_memory']
413
+ data['user_num'] = len(users)
414
415
return dict(
416
('gpu{0}_{1}'.format(self.num, k), v) for k, v in data.items() if v is not None and v != BAD_VALUE
@@ -379,6 +457,7 @@ class Service(SimpleService):
457
gpu = GPU(idx, root)
458
data.update(gpu.data())
459
self.update_processes_mem_chart(gpu)
460
+ self.update_processes_user_mem_chart(gpu)
461
462
return data or None
463
@@ -397,6 +476,24 @@ class Service(SimpleService):
476
if dim.id not in active_dim_ids:
477
chart.del_dimension(dim.id, hide=False)
478
479
+ def update_processes_user_mem_chart(self, gpu):
480
+ ps = gpu.processes()
481
+ if not ps:
482
+ return
483
+ chart = self.charts['gpu{0}_{1}'.format(gpu.num, USER_MEM)]
484
+ active_dim_ids = []
485
+ for p in ps:
486
+ if not p.get('username'):
487
+ continue
488
+ dim_id = 'gpu{0}_user_mem_{1}'.format(gpu.num, p['username'])
489
+ active_dim_ids.append(dim_id)
490
+ if dim_id not in chart:
491
+ chart.add_dimension([dim_id, '{0}'.format(p['username'])])
492
+
493
+ for dim in chart:
494
+ if dim.id not in active_dim_ids:
495
+ chart.del_dimension(dim.id, hide=False)
496
+
497
def check(self):
498
if not self.poller.has_smi():
499
self.error("couldn't find '{0}' binary".format(NVIDIA_SMI))