Add memory free and per process memory usage to nvidia_smi (#5796)
<!-- Describe the change in summary section, including rationale and degin decisions. Include "Fixes #nnn" if you are fixing an existing issue. In "Component Name" section write which component is changed in this PR. This will help us review your PR quicker. If you have more information you want to add, write them in "Additional Information" section. This is usually used to help others understand your motivation behind this change. A step-by-step reproduction of the problem is helpful if there is no related issue. --> ##### Summary Add memory free Add per process memory usage ##### Component Name nvidia_smi ##### Additional Information
胡玮文 committed
Apr 5, 2019 at 15:59 UTC
01b91117caecce0d50020a51673b9d3b3b0e01f0
2 files changed
+60
-13
collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py
+57
-11
@@ -26,10 +26,11 @@ FAN_SPEED = 'fan_speed'
26
GPU_UTIL = 'gpu_utilization'
27
MEM_UTIL = 'mem_utilization'
28
ENCODER_UTIL = 'encoder_utilization'
29
-MEM_ALLOCATED = 'mem_allocated'
29
+MEM_USAGE = 'mem_usage'
30
TEMPERATURE = 'temperature'
31
CLOCKS = 'clocks'
32
POWER = 'power'
33
+PROCESSES_MEM = 'processes_mem'
34
35
ORDER = [
36
PCI_BANDWIDTH,
@@ -37,10 +38,11 @@ ORDER = [
38
GPU_UTIL,
39
MEM_UTIL,
40
ENCODER_UTIL,
40
- MEM_ALLOCATED,
41
+ MEM_USAGE,
42
TEMPERATURE,
43
CLOCKS,
44
POWER,
45
+ PROCESSES_MEM,
46
]
47
48
@@ -80,10 +82,11 @@ def gpu_charts(gpu):
82
['decoder_util', 'decoder'],
83
]
84
},
83
- MEM_ALLOCATED: {
84
- 'options': [None, 'Memory Allocated', 'MiB', fam, 'nvidia_smi.memory_allocated', 'line'],
85
+ MEM_USAGE: {
86
+ 'options': [None, 'Memory Usage', 'MiB', fam, 'nvidia_smi.memory_allocated', 'stacked'],
87
'lines': [
86
- ['fb_memory_usage', 'used'],
88
+ ['fb_memory_free', 'free'],
89
+ ['fb_memory_used', 'used'],
90
]
91
},
92
TEMPERATURE: {
@@ -107,6 +110,10 @@ def gpu_charts(gpu):
110
['power_draw', 'power', 1, 100],
111
]
112
},
113
+ PROCESSES_MEM: {
114
+ 'options': [None, 'Memory Used by Each Process', 'MiB', fam, 'nvidia_smi.processes_mem', 'stacked'],
115
+ 'lines': []
116
+ },
117
}
118
119
idx = gpu.num
@@ -260,9 +267,13 @@ class GPU:
267
return self.root.find('utilization').find('decoder_util').text.split()[0]
268
269
@handle_attr_error
263
- def fb_memory_usage(self):
270
+ def fb_memory_used(self):
271
return self.root.find('fb_memory_usage').find('used').text.split()[0]
272
273
+ @handle_attr_error
274
+ def fb_memory_free(self):
275
+ return self.root.find('fb_memory_usage').find('free').text.split()[0]
276
+
277
@handle_attr_error
278
def temperature(self):
279
return self.root.find('temperature').find('gpu_temp').text.split()[0]
@@ -288,6 +299,18 @@ class GPU:
299
def power_draw(self):
300
return float(self.root.find('power_readings').find('power_draw').text.split()[0]) * 100
301
302
+ @handle_attr_error
303
+ def processes(self):
304
+ p_nodes = self.root.find('processes').findall('process_info')
305
+ ps = []
306
+ for p in p_nodes:
307
+ ps.append({
308
+ 'pid': p.find('pid').text,
309
+ 'process_name': p.find('process_name').text,
310
+ 'used_memory': int(p.find('used_memory').text.split()[0]),
311
+ })
312
+ return ps
313
+
314
def data(self):
315
data = {
316
'rx_util': self.rx_util(),
@@ -297,7 +320,8 @@ class GPU:
320
'memory_util': self.memory_util(),
321
'encoder_util': self.encoder_util(),
322
'decoder_util': self.decoder_util(),
300
- 'fb_memory_usage': self.fb_memory_usage(),
323
+ 'fb_memory_used': self.fb_memory_used(),
324
+ 'fb_memory_free': self.fb_memory_free(),
325
'gpu_temp': self.temperature(),
326
'graphics_clock': self.graphics_clock(),
327
'video_clock': self.video_clock(),
@@ -305,12 +329,13 @@ class GPU:
329
'mem_clock': self.mem_clock(),
330
'power_draw': self.power_draw(),
331
}
332
+ processes = self.processes() or []
333
+ data.update({'process_mem_{0}'.format(p['pid']): p['used_memory'] for p in processes})
334
335
return dict(
336
('gpu{0}_{1}'.format(self.num, k), v) for k, v in data.items() if v is not None and v != BAD_VALUE
337
)
338
313
-
339
class Service(SimpleService):
340
def __init__(self, configuration=None, name=None):
341
super(Service, self).__init__(configuration=configuration, name=name)
@@ -320,11 +345,16 @@ class Service(SimpleService):
345
self.poller = NvidiaSMIPoller(poll)
346
347
def get_data(self):
348
+ if not self.poller.is_started():
349
+ self.poller.start()
350
+
351
if not self.poller.is_alive():
352
self.debug('poller is off')
353
return None
354
355
last_data = self.poller.data()
356
+ if not last_data:
357
+ return None
358
359
parsed = self.parse_xml(last_data)
360
if parsed is None:
@@ -332,10 +362,27 @@ class Service(SimpleService):
362
363
data = dict()
364
for idx, root in enumerate(parsed.findall('gpu')):
335
- data.update(GPU(idx, root).data())
365
+ gpu = GPU(idx, root)
366
+ data.update(gpu.data())
367
+ self.update_processes_mem_chart(gpu)
368
369
return data or None
370
371
+ def update_processes_mem_chart(self, gpu):
372
+ ps = gpu.processes()
373
+ if not ps:
374
+ return
375
+ chart = self.charts['gpu{0}_{1}'.format(gpu.num, PROCESSES_MEM)]
376
+ active_dim_ids = []
377
+ for p in ps:
378
+ dim_id = 'gpu{0}_process_mem_{1}'.format(gpu.num, p['pid'])
379
+ active_dim_ids.append(dim_id)
380
+ if dim_id not in chart:
381
+ chart.add_dimension([dim_id, '{0} {1}'.format(p['pid'], p['process_name'])])
382
+ for dim in chart:
383
+ if dim.id not in active_dim_ids:
384
+ chart.del_dimension(dim.id, hide=False)
385
+
386
def check(self):
387
if not self.poller.has_smi():
388
self.error("couldn't find '{0}' binary".format(NVIDIA_SMI))
@@ -355,7 +402,6 @@ class Service(SimpleService):
402
return False
403
404
self.create_charts(gpus)
358
- self.poller.start()
405
406
return True
407
@@ -363,7 +409,7 @@ class Service(SimpleService):
409
try:
410
return et.fromstring(data)
411
except et.ParseError as error:
366
- self.error(error)
412
+ self.error('xml parse failed: "{0}", error: {1}'.format(data, error))
413
414
return None
415
collectors/python.d.plugin/python_modules/bases/charts.py
+3
-2
@@ -200,12 +200,13 @@ class Chart:
200
self.dimensions.append(dim)
201
return dim
202
203
- def del_dimension(self, dimension_id):
203
+ def del_dimension(self, dimension_id, hide=True):
204
if dimension_id not in self:
205
return
206
idx = self.dimensions.index(dimension_id)
207
dimension = self.dimensions[idx]
208
- dimension.params['hidden'] = 'hidden'
208
+ if hide:
209
+ dimension.params['hidden'] = 'hidden'
210
dimension.params['obsolete'] = 'obsolete'
211
self.create()
212
self.dimensions.remove(dimension)