@cryptotaxi247 / netdata-1 / commits / a36af8a34

remove python.d/nvidia_smi (#18316)

* remove python.d/nvidia_smi * update python.d.conf

Ilya Mashchenko committed Aug 12, 2024 at 15:07 UTC a36af8a3457849b238e5e0e833f92b6090448fd3

6 files changed +1 -969

CMakeLists.txt

-2

@@ -2785,7 +2785,6 @@ install(FILES

2785 src/collectors/python.d.plugin/go_expvar/go_expvar.conf

2786 src/collectors/python.d.plugin/haproxy/haproxy.conf

2787 src/collectors/python.d.plugin/monit/monit.conf

2788 - src/collectors/python.d.plugin/nvidia_smi/nvidia_smi.conf

2788 src/collectors/python.d.plugin/openldap/openldap.conf

2789 src/collectors/python.d.plugin/oracledb/oracledb.conf

2790 src/collectors/python.d.plugin/pandas/pandas.conf

@@ -2813,7 +2812,6 @@ install(FILES

2812 src/collectors/python.d.plugin/go_expvar/go_expvar.chart.py

2813 src/collectors/python.d.plugin/haproxy/haproxy.chart.py

2814 src/collectors/python.d.plugin/monit/monit.chart.py

2816 - src/collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py

2815 src/collectors/python.d.plugin/openldap/openldap.chart.py

2816 src/collectors/python.d.plugin/oracledb/oracledb.chart.py

2817 src/collectors/python.d.plugin/pandas/pandas.chart.py

src/collectors/python.d.plugin/nvidia_smi/README.md deleted

-81

@@ -1,81 +0,0 @@

1 -<!--

2 -title: "Nvidia GPU monitoring with Netdata"

3 -custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/collectors/python.d.plugin/nvidia_smi/README.md"

4 -sidebar_label: "nvidia_smi-python.d.plugin"

5 -learn_status: "Published"

6 -learn_topic_type: "References"

7 -learn_rel_path: "Integrations/Monitor/Devices"

8 --->

9 -

10 -# Nvidia GPU collector

11 -

12 -Monitors performance metrics (memory usage, fan speed, pcie bandwidth utilization, temperature, etc.) using `nvidia-smi` cli tool.

13 -

14 -## Requirements

15 -

16 -- The `nvidia-smi` tool installed and your NVIDIA GPU(s) must support the tool. Mostly the newer high end models used for AI / ML and Crypto or Pro range, read more about [nvidia_smi](https://developer.nvidia.com/nvidia-system-management-interface).

17 -- Enable this plugin, as it's disabled by default due to minor performance issues:

18 - ```bash

19 - cd /etc/netdata # Replace this path with your Netdata config directory, if different

20 - sudo ./edit-config python.d.conf

21 - ```

22 - Remove the '#' before nvidia_smi so it reads: `nvidia_smi: yes`.

23 -- On some systems when the GPU is idle the `nvidia-smi` tool unloads and there is added latency again when it is next queried. If you are running GPUs under constant workload this isn't likely to be an issue.

24 -

25 -If using Docker, see [Netdata Docker container with NVIDIA GPUs monitoring](https://github.com/netdata/netdata/tree/master/packaging/docker#with-nvidia-gpus-monitoring).

26 -

27 -## Charts

28 -

29 -It produces the following charts:

30 -

31 -- PCI Express Bandwidth Utilization in `KiB/s`

32 -- Fan Speed in `percentage`

33 -- GPU Utilization in `percentage`

34 -- Memory Bandwidth Utilization in `percentage`

35 -- Encoder/Decoder Utilization in `percentage`

36 -- Memory Usage in `MiB`

37 -- Temperature in `celsius`

38 -- Clock Frequencies in `MHz`

39 -- Power Utilization in `Watts`

40 -- Memory Used by Each Process in `MiB`

41 -- Memory Used by Each User in `MiB`

42 -- Number of User on GPU in `num`

43 -

44 -## Configuration

45 -

46 -Edit the `python.d/nvidia_smi.conf` configuration file using `edit-config` from the Netdata [config

47 -directory](/docs/netdata-agent/configuration/README.md), which is typically at `/etc/netdata`.

48 -

49 -```bash

50 -cd /etc/netdata # Replace this path with your Netdata config directory, if different

51 -sudo ./edit-config python.d/nvidia_smi.conf

52 -```

53 -

54 -Sample:

55 -

56 -```yaml

57 -loop_mode : yes

58 -poll_seconds : 1

59 -exclude_zero_memory_users : yes

60 -```

61 -

62 -

63 -### Troubleshooting

64 -

65 -To troubleshoot issues with the `nvidia_smi` module, run the `python.d.plugin` with the debug option enabled. The

66 -output will give you the output of the data collection job or error messages on why the collector isn't working.

67 -

68 -First, navigate to your plugins directory, usually they are located under `/usr/libexec/netdata/plugins.d/`. If that's

69 -not the case on your system, open `netdata.conf` and look for the setting `plugins directory`. Once you're in the

70 -plugin's directory, switch to the `netdata` user.

71 -

72 -```bash

73 -cd /usr/libexec/netdata/plugins.d/

74 -sudo su -s /bin/bash netdata

75 -```

76 -

77 -Now you can manually run the `nvidia_smi` module in debug mode:

78 -

79 -```bash

80 -./python.d.plugin nvidia_smi debug trace

81 -```

src/collectors/python.d.plugin/nvidia_smi/metadata.yaml deleted

-166

@@ -1,166 +0,0 @@

1 -# This collector will not appear in documentation, as the go version is preferred,

2 -# /src/go/plugin/go.d/modules/nvidia_smi/README.md

3 -#

4 -# meta:

5 -# plugin_name: python.d.plugin

6 -# module_name: nvidia_smi

7 -# monitored_instance:

8 -# name: python.d nvidia_smi

9 -# link: ''

10 -# categories: []

11 -# icon_filename: ''

12 -# related_resources:

13 -# integrations:

14 -# list: []

15 -# info_provided_to_referring_integrations:

16 -# description: ''

17 -# keywords: []

18 -# most_popular: false

19 -# overview:

20 -# data_collection:

21 -# metrics_description: ''

22 -# method_description: ''

23 -# supported_platforms:

24 -# include: []

25 -# exclude: []

26 -# multi_instance: true

27 -# additional_permissions:

28 -# description: ''

29 -# default_behavior:

30 -# auto_detection:

31 -# description: ''

32 -# limits:

33 -# description: ''

34 -# performance_impact:

35 -# description: ''

36 -# setup:

37 -# prerequisites:

38 -# list: []

39 -# configuration:

40 -# file:

41 -# name: ''

42 -# description: ''

43 -# options:

44 -# description: ''

45 -# folding:

46 -# title: ''

47 -# enabled: true

48 -# list: []

49 -# examples:

50 -# folding:

51 -# enabled: true

52 -# title: ''

53 -# list: []

54 -# troubleshooting:

55 -# problems:

56 -# list: []

57 -# alerts: []

58 -# metrics:

59 -# folding:

60 -# title: Metrics

61 -# enabled: false

62 -# description: ""

63 -# availability: []

64 -# scopes:

65 -# - name: GPU

66 -# description: ""

67 -# labels: []

68 -# metrics:

69 -# - name: nvidia_smi.pci_bandwidth

70 -# description: PCI Express Bandwidth Utilization

71 -# unit: "KiB/s"

72 -# chart_type: area

73 -# dimensions:

74 -# - name: rx

75 -# - name: tx

76 -# - name: nvidia_smi.pci_bandwidth_percent

77 -# description: PCI Express Bandwidth Percent

78 -# unit: "percentage"

79 -# chart_type: area

80 -# dimensions:

81 -# - name: rx_percent

82 -# - name: tx_percent

83 -# - name: nvidia_smi.fan_speed

84 -# description: Fan Speed

85 -# unit: "percentage"

86 -# chart_type: line

87 -# dimensions:

88 -# - name: speed

89 -# - name: nvidia_smi.gpu_utilization

90 -# description: GPU Utilization

91 -# unit: "percentage"

92 -# chart_type: line

93 -# dimensions:

94 -# - name: utilization

95 -# - name: nvidia_smi.mem_utilization

96 -# description: Memory Bandwidth Utilization

97 -# unit: "percentage"

98 -# chart_type: line

99 -# dimensions:

100 -# - name: utilization

101 -# - name: nvidia_smi.encoder_utilization

102 -# description: Encoder/Decoder Utilization

103 -# unit: "percentage"

104 -# chart_type: line

105 -# dimensions:

106 -# - name: encoder

107 -# - name: decoder

108 -# - name: nvidia_smi.memory_allocated

109 -# description: Memory Usage

110 -# unit: "MiB"

111 -# chart_type: stacked

112 -# dimensions:

113 -# - name: free

114 -# - name: used

115 -# - name: nvidia_smi.bar1_memory_usage

116 -# description: Bar1 Memory Usage

117 -# unit: "MiB"

118 -# chart_type: stacked

119 -# dimensions:

120 -# - name: free

121 -# - name: used

122 -# - name: nvidia_smi.temperature

123 -# description: Temperature

124 -# unit: "celsius"

125 -# chart_type: line

126 -# dimensions:

127 -# - name: temp

128 -# - name: nvidia_smi.clocks

129 -# description: Clock Frequencies

130 -# unit: "MHz"

131 -# chart_type: line

132 -# dimensions:

133 -# - name: graphics

134 -# - name: video

135 -# - name: sm

136 -# - name: mem

137 -# - name: nvidia_smi.power

138 -# description: Power Utilization

139 -# unit: "Watts"

140 -# chart_type: line

141 -# dimensions:

142 -# - name: power

143 -# - name: nvidia_smi.power_state

144 -# description: Power State

145 -# unit: "state"

146 -# chart_type: line

147 -# dimensions:

148 -# - name: a dimension per {power_state}

149 -# - name: nvidia_smi.processes_mem

150 -# description: Memory Used by Each Process

151 -# unit: "MiB"

152 -# chart_type: stacked

153 -# dimensions:

154 -# - name: a dimension per process

155 -# - name: nvidia_smi.user_mem

156 -# description: Memory Used by Each User

157 -# unit: "MiB"

158 -# chart_type: stacked

159 -# dimensions:

160 -# - name: a dimension per user

161 -# - name: nvidia_smi.user_num

162 -# description: Number of User on GPU

163 -# unit: "num"

164 -# chart_type: line

165 -# dimensions:

166 -# - name: users

src/collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py deleted

-651

@@ -1,651 +0,0 @@

1 -# -*- coding: utf-8 -*-

2 -# Description: nvidia-smi netdata python.d module

3 -# Original Author: Steven Noonan (tycho)

4 -# Author: Ilya Mashchenko (ilyam8)

5 -# User Memory Stat Author: Guido Scatena (scatenag)

6 -

7 -import os

8 -import pwd

9 -import subprocess

10 -import threading

11 -import xml.etree.ElementTree as et

12 -

13 -from bases.FrameworkServices.SimpleService import SimpleService

14 -from bases.collection import find_binary

15 -

16 -disabled_by_default = True

17 -

18 -NVIDIA_SMI = 'nvidia-smi'

19 -

20 -NOT_AVAILABLE = 'N/A'

21 -

22 -EMPTY_ROW = ''

23 -EMPTY_ROW_LIMIT = 500

24 -POLLER_BREAK_ROW = '</nvidia_smi_log>'

25 -

26 -PCI_BANDWIDTH = 'pci_bandwidth'

27 -PCI_BANDWIDTH_PERCENT = 'pci_bandwidth_percent'

28 -FAN_SPEED = 'fan_speed'

29 -GPU_UTIL = 'gpu_utilization'

30 -MEM_UTIL = 'mem_utilization'

31 -ENCODER_UTIL = 'encoder_utilization'

32 -MEM_USAGE = 'mem_usage'

33 -BAR_USAGE = 'bar1_mem_usage'

34 -TEMPERATURE = 'temperature'

35 -CLOCKS = 'clocks'

36 -POWER = 'power'

37 -POWER_STATE = 'power_state'

38 -PROCESSES_MEM = 'processes_mem'

39 -USER_MEM = 'user_mem'

40 -USER_NUM = 'user_num'

41 -

42 -ORDER = [

43 - PCI_BANDWIDTH,

44 - PCI_BANDWIDTH_PERCENT,

45 - FAN_SPEED,

46 - GPU_UTIL,

47 - MEM_UTIL,

48 - ENCODER_UTIL,

49 - MEM_USAGE,

50 - BAR_USAGE,

51 - TEMPERATURE,

52 - CLOCKS,

53 - POWER,

54 - POWER_STATE,

55 - PROCESSES_MEM,

56 - USER_MEM,

57 - USER_NUM,

58 -]

59 -

60 -# https://docs.nvidia.com/gameworks/content/gameworkslibrary/coresdk/nvapi/group__gpupstate.html

61 -POWER_STATES = ['P' + str(i) for i in range(0, 16)]

62 -

63 -# PCI Transfer data rate in gigabits per second (Gb/s) per generation

64 -PCI_SPEED = {

65 - "1": 2.5,

66 - "2": 5,

67 - "3": 8,

68 - "4": 16,

69 - "5": 32

70 -}

71 -# PCI encoding per generation

72 -PCI_ENCODING = {

73 - "1": 2 / 10,

74 - "2": 2 / 10,

75 - "3": 2 / 130,

76 - "4": 2 / 130,

77 - "5": 2 / 130

78 -}

79 -

80 -

81 -def gpu_charts(gpu):

82 - fam = gpu.full_name()

83 -

84 - charts = {

85 - PCI_BANDWIDTH: {

86 - 'options': [None, 'PCI Express Bandwidth Utilization', 'KiB/s', fam, 'nvidia_smi.pci_bandwidth', 'area'],

87 - 'lines': [

88 - ['rx_util', 'rx', 'absolute', 1, 1],

89 - ['tx_util', 'tx', 'absolute', 1, -1],

90 - ]

91 - },

92 - PCI_BANDWIDTH_PERCENT: {

93 - 'options': [None, 'PCI Express Bandwidth Percent', 'percentage', fam, 'nvidia_smi.pci_bandwidth_percent',

94 - 'area'],

95 - 'lines': [

96 - ['rx_util_percent', 'rx_percent'],

97 - ['tx_util_percent', 'tx_percent'],

98 - ]

99 - },

100 - FAN_SPEED: {

101 - 'options': [None, 'Fan Speed', 'percentage', fam, 'nvidia_smi.fan_speed', 'line'],

102 - 'lines': [

103 - ['fan_speed', 'speed'],

104 - ]

105 - },

106 - GPU_UTIL: {

107 - 'options': [None, 'GPU Utilization', 'percentage', fam, 'nvidia_smi.gpu_utilization', 'line'],

108 - 'lines': [

109 - ['gpu_util', 'utilization'],

110 - ]

111 - },

112 - MEM_UTIL: {

113 - 'options': [None, 'Memory Bandwidth Utilization', 'percentage', fam, 'nvidia_smi.mem_utilization', 'line'],

114 - 'lines': [

115 - ['memory_util', 'utilization'],

116 - ]

117 - },

118 - ENCODER_UTIL: {

119 - 'options': [None, 'Encoder/Decoder Utilization', 'percentage', fam, 'nvidia_smi.encoder_utilization',

120 - 'line'],

121 - 'lines': [

122 - ['encoder_util', 'encoder'],

123 - ['decoder_util', 'decoder'],

124 - ]

125 - },

126 - MEM_USAGE: {

127 - 'options': [None, 'Memory Usage', 'MiB', fam, 'nvidia_smi.memory_allocated', 'stacked'],

128 - 'lines': [

129 - ['fb_memory_free', 'free'],

130 - ['fb_memory_used', 'used'],

131 - ]

132 - },

133 - BAR_USAGE: {

134 - 'options': [None, 'Bar1 Memory Usage', 'MiB', fam, 'nvidia_smi.bar1_memory_usage', 'stacked'],

135 - 'lines': [

136 - ['bar1_memory_free', 'free'],

137 - ['bar1_memory_used', 'used'],

138 - ]

139 - },

140 - TEMPERATURE: {

141 - 'options': [None, 'Temperature', 'celsius', fam, 'nvidia_smi.temperature', 'line'],

142 - 'lines': [

143 - ['gpu_temp', 'temp'],

144 - ]

145 - },

146 - CLOCKS: {

147 - 'options': [None, 'Clock Frequencies', 'MHz', fam, 'nvidia_smi.clocks', 'line'],

148 - 'lines': [

149 - ['graphics_clock', 'graphics'],

150 - ['video_clock', 'video'],

151 - ['sm_clock', 'sm'],

152 - ['mem_clock', 'mem'],

153 - ]

154 - },

155 - POWER: {

156 - 'options': [None, 'Power Utilization', 'Watts', fam, 'nvidia_smi.power', 'line'],

157 - 'lines': [

158 - ['power_draw', 'power', 'absolute', 1, 100],

159 - ]

160 - },

161 - POWER_STATE: {

162 - 'options': [None, 'Power State', 'state', fam, 'nvidia_smi.power_state', 'line'],

163 - 'lines': [['power_state_' + v.lower(), v, 'absolute'] for v in POWER_STATES]

164 - },

165 - PROCESSES_MEM: {

166 - 'options': [None, 'Memory Used by Each Process', 'MiB', fam, 'nvidia_smi.processes_mem', 'stacked'],

167 - 'lines': []

168 - },

169 - USER_MEM: {

170 - 'options': [None, 'Memory Used by Each User', 'MiB', fam, 'nvidia_smi.user_mem', 'stacked'],

171 - 'lines': []

172 - },

173 - USER_NUM: {

174 - 'options': [None, 'Number of User on GPU', 'num', fam, 'nvidia_smi.user_num', 'line'],

175 - 'lines': [

176 - ['user_num', 'users'],

177 - ]

178 - },

179 - }

180 -

181 - idx = gpu.num

182 -

183 - order = ['gpu{0}_{1}'.format(idx, v) for v in ORDER]

184 - charts = dict(('gpu{0}_{1}'.format(idx, k), v) for k, v in charts.items())

185 -

186 - for chart in charts.values():

187 - for line in chart['lines']:

188 - line[0] = 'gpu{0}_{1}'.format(idx, line[0])

189 -

190 - return order, charts

191 -

192 -

193 -class NvidiaSMI:

194 - def __init__(self):

195 - self.command = find_binary(NVIDIA_SMI)

196 - self.active_proc = None

197 -

198 - def run_once(self):

199 - proc = subprocess.Popen([self.command, '-x', '-q'], stdout=subprocess.PIPE)

200 - stdout, _ = proc.communicate()

201 - return stdout

202 -

203 - def run_loop(self, interval):

204 - if self.active_proc:

205 - self.kill()

206 - proc = subprocess.Popen([self.command, '-x', '-q', '-l', str(interval)], stdout=subprocess.PIPE)

207 - self.active_proc = proc

208 - return proc.stdout

209 -

210 - def kill(self):

211 - if self.active_proc:

212 - self.active_proc.kill()

213 - self.active_proc = None

214 -

215 -

216 -class NvidiaSMIPoller(threading.Thread):

217 - def __init__(self, poll_interval):

218 - threading.Thread.__init__(self)

219 - self.daemon = True

220 -

221 - self.smi = NvidiaSMI()

222 - self.interval = poll_interval

223 -

224 - self.lock = threading.RLock()

225 - self.last_data = str()

226 - self.exit = False

227 - self.empty_rows = 0

228 - self.rows = list()

229 -

230 - def has_smi(self):

231 - return bool(self.smi.command)

232 -

233 - def run_once(self):

234 - return self.smi.run_once()

235 -

236 - def run(self):

237 - out = self.smi.run_loop(self.interval)

238 -

239 - for row in out:

240 - if self.exit or self.empty_rows > EMPTY_ROW_LIMIT:

241 - break

242 - self.process_row(row)

243 - self.smi.kill()

244 -

245 - def process_row(self, row):

246 - row = row.decode()

247 - self.empty_rows += (row == EMPTY_ROW)

248 - self.rows.append(row)

249 -

250 - if POLLER_BREAK_ROW in row:

251 - self.lock.acquire()

252 - self.last_data = '\n'.join(self.rows)

253 - self.lock.release()

254 -

255 - self.rows = list()

256 - self.empty_rows = 0

257 -

258 - def is_started(self):

259 - return self.ident is not None

260 -

261 - def shutdown(self):

262 - self.exit = True

263 -

264 - def data(self):

265 - self.lock.acquire()

266 - data = self.last_data

267 - self.lock.release()

268 - return data

269 -

270 -

271 -def handle_attr_error(method):

272 - def on_call(*args, **kwargs):

273 - try:

274 - return method(*args, **kwargs)

275 - except AttributeError:

276 - return None

277 -

278 - return on_call

279 -

280 -

281 -def handle_value_error(method):

282 - def on_call(*args, **kwargs):

283 - try:

284 - return method(*args, **kwargs)

285 - except ValueError:

286 - return None

287 -

288 - return on_call

289 -

290 -

291 -HOST_PREFIX = os.getenv('NETDATA_HOST_PREFIX')

292 -ETC_PASSWD_PATH = '/etc/passwd'

293 -PROC_PATH = '/proc'

294 -

295 -IS_INSIDE_DOCKER = False

296 -

297 -if HOST_PREFIX:

298 - ETC_PASSWD_PATH = os.path.join(HOST_PREFIX, ETC_PASSWD_PATH[1:])

299 - PROC_PATH = os.path.join(HOST_PREFIX, PROC_PATH[1:])

300 - IS_INSIDE_DOCKER = True

301 -

302 -

303 -def read_passwd_file():

304 - data = dict()

305 - with open(ETC_PASSWD_PATH, 'r') as f:

306 - for line in f:

307 - line = line.strip()

308 - if line.startswith("#"):

309 - continue

310 - fields = line.split(":")

311 - # name, passwd, uid, gid, comment, home_dir, shell

312 - if len(fields) != 7:

313 - continue

314 - # uid, guid

315 - fields[2], fields[3] = int(fields[2]), int(fields[3])

316 - data[fields[2]] = fields

317 - return data

318 -

319 -

320 -def read_passwd_file_safe():

321 - try:

322 - if IS_INSIDE_DOCKER:

323 - return read_passwd_file()

324 - return dict((k[2], k) for k in pwd.getpwall())

325 - except (OSError, IOError):

326 - return dict()

327 -

328 -

329 -def get_username_by_pid_safe(pid, passwd_file):

330 - path = os.path.join(PROC_PATH, pid)

331 - try:

332 - uid = os.stat(path).st_uid

333 - except (OSError, IOError):

334 - return ''

335 - try:

336 - if IS_INSIDE_DOCKER:

337 - return passwd_file[uid][0]

338 - return pwd.getpwuid(uid)[0]

339 - except KeyError:

340 - return str(uid)

341 -

342 -

343 -class GPU:

344 - def __init__(self, num, root, exclude_zero_memory_users=False):

345 - self.num = num

346 - self.root = root

347 - self.exclude_zero_memory_users = exclude_zero_memory_users

348 -

349 - def id(self):

350 - return self.root.get('id')

351 -

352 - def name(self):

353 - return self.root.find('product_name').text

354 -

355 - def full_name(self):

356 - return 'gpu{0} {1}'.format(self.num, self.name())

357 -

358 - @handle_attr_error

359 - def pci_link_gen(self):

360 - return self.root.find('pci').find('pci_gpu_link_info').find('pcie_gen').find('max_link_gen').text

361 -

362 - @handle_attr_error

363 - def pci_link_width(self):

364 - info = self.root.find('pci').find('pci_gpu_link_info')

365 - return info.find('link_widths').find('max_link_width').text.split('x')[0]

366 -

367 - def pci_bw_max(self):

368 - link_gen = self.pci_link_gen()

369 - link_width = int(self.pci_link_width())

370 - if link_gen not in PCI_SPEED or link_gen not in PCI_ENCODING or not link_width:

371 - return None

372 - # Maximum PCIe Bandwidth = SPEED * WIDTH * (1 - ENCODING) - 1Gb/s.

373 - # see details https://enterprise-support.nvidia.com/s/article/understanding-pcie-configuration-for-maximum-performance

374 - # return max bandwidth in kilobytes per second (kB/s)

375 - return (PCI_SPEED[link_gen] * link_width * (1 - PCI_ENCODING[link_gen]) - 1) * 1000 * 1000 / 8

376 -

377 - @handle_attr_error

378 - def rx_util(self):

379 - return self.root.find('pci').find('rx_util').text.split()[0]

380 -

381 - @handle_attr_error

382 - def tx_util(self):

383 - return self.root.find('pci').find('tx_util').text.split()[0]

384 -

385 - @handle_attr_error

386 - def fan_speed(self):

387 - return self.root.find('fan_speed').text.split()[0]

388 -

389 - @handle_attr_error

390 - def gpu_util(self):

391 - return self.root.find('utilization').find('gpu_util').text.split()[0]

392 -

393 - @handle_attr_error

394 - def memory_util(self):

395 - return self.root.find('utilization').find('memory_util').text.split()[0]

396 -

397 - @handle_attr_error

398 - def encoder_util(self):

399 - return self.root.find('utilization').find('encoder_util').text.split()[0]

400 -

401 - @handle_attr_error

402 - def decoder_util(self):

403 - return self.root.find('utilization').find('decoder_util').text.split()[0]

404 -

405 - @handle_attr_error

406 - def fb_memory_used(self):

407 - return self.root.find('fb_memory_usage').find('used').text.split()[0]

408 -

409 - @handle_attr_error

410 - def fb_memory_free(self):

411 - return self.root.find('fb_memory_usage').find('free').text.split()[0]

412 -

413 - @handle_attr_error

414 - def bar1_memory_used(self):

415 - return self.root.find('bar1_memory_usage').find('used').text.split()[0]

416 -

417 - @handle_attr_error

418 - def bar1_memory_free(self):

419 - return self.root.find('bar1_memory_usage').find('free').text.split()[0]

420 -

421 - @handle_attr_error

422 - def temperature(self):

423 - return self.root.find('temperature').find('gpu_temp').text.split()[0]

424 -

425 - @handle_attr_error

426 - def graphics_clock(self):

427 - return self.root.find('clocks').find('graphics_clock').text.split()[0]

428 -

429 - @handle_attr_error

430 - def video_clock(self):

431 - return self.root.find('clocks').find('video_clock').text.split()[0]

432 -

433 - @handle_attr_error

434 - def sm_clock(self):

435 - return self.root.find('clocks').find('sm_clock').text.split()[0]

436 -

437 - @handle_attr_error

438 - def mem_clock(self):

439 - return self.root.find('clocks').find('mem_clock').text.split()[0]

440 -

441 - @handle_attr_error

442 - def power_readings(self):

443 - elem = self.root.find('power_readings')

444 - return elem if elem else self.root.find('gpu_power_readings')

445 -

446 - @handle_attr_error

447 - def power_state(self):

448 - return str(self.power_readings().find('power_state').text.split()[0])

449 -

450 - @handle_value_error

451 - @handle_attr_error

452 - def power_draw(self):

453 - return float(self.power_readings().find('power_draw').text.split()[0]) * 100

454 -

455 - @handle_attr_error

456 - def processes(self):

457 - processes_info = self.root.find('processes').findall('process_info')

458 - if not processes_info:

459 - return list()

460 -

461 - passwd_file = read_passwd_file_safe()

462 - processes = list()

463 -

464 - for info in processes_info:

465 - pid = info.find('pid').text

466 - processes.append({

467 - 'pid': int(pid),

468 - 'process_name': info.find('process_name').text,

469 - 'used_memory': int(info.find('used_memory').text.split()[0]),

470 - 'username': get_username_by_pid_safe(pid, passwd_file),

471 - })

472 - return processes

473 -

474 - def data(self):

475 - data = {

476 - 'rx_util': self.rx_util(),

477 - 'tx_util': self.tx_util(),

478 - 'fan_speed': self.fan_speed(),

479 - 'gpu_util': self.gpu_util(),

480 - 'memory_util': self.memory_util(),

481 - 'encoder_util': self.encoder_util(),

482 - 'decoder_util': self.decoder_util(),

483 - 'fb_memory_used': self.fb_memory_used(),

484 - 'fb_memory_free': self.fb_memory_free(),

485 - 'bar1_memory_used': self.bar1_memory_used(),

486 - 'bar1_memory_free': self.bar1_memory_free(),

487 - 'gpu_temp': self.temperature(),

488 - 'graphics_clock': self.graphics_clock(),

489 - 'video_clock': self.video_clock(),

490 - 'sm_clock': self.sm_clock(),

491 - 'mem_clock': self.mem_clock(),

492 - 'power_draw': self.power_draw(),

493 - }

494 -

495 - if self.rx_util() != NOT_AVAILABLE and self.tx_util() != NOT_AVAILABLE:

496 - pci_bw_max = self.pci_bw_max()

497 - if not pci_bw_max:

498 - data['rx_util_percent'] = 0

499 - data['tx_util_percent'] = 0

500 - else:

501 - data['rx_util_percent'] = str(int(int(self.rx_util()) * 100 / self.pci_bw_max()))

502 - data['tx_util_percent'] = str(int(int(self.tx_util()) * 100 / self.pci_bw_max()))

503 -

504 - for v in POWER_STATES:

505 - data['power_state_' + v.lower()] = 0

506 - p_state = self.power_state()

507 - if p_state:

508 - data['power_state_' + p_state.lower()] = 1

509 -

510 - processes = self.processes() or []

511 - users = set()

512 - for p in processes:

513 - data['process_mem_{0}'.format(p['pid'])] = p['used_memory']

514 - if p['username']:

515 - if self.exclude_zero_memory_users and p['used_memory'] == 0:

516 - continue

517 - users.add(p['username'])

518 - key = 'user_mem_{0}'.format(p['username'])

519 - if key in data:

520 - data[key] += p['used_memory']

521 - else:

522 - data[key] = p['used_memory']

523 - data['user_num'] = len(users)

524 -

525 - return dict(('gpu{0}_{1}'.format(self.num, k), v) for k, v in data.items())

526 -

527 -

528 -class Service(SimpleService):

529 - def __init__(self, configuration=None, name=None):

530 - super(Service, self).__init__(configuration=configuration, name=name)

531 - self.order = list()

532 - self.definitions = dict()

533 - self.loop_mode = configuration.get('loop_mode', True)

534 - poll = int(configuration.get('poll_seconds', self.get_update_every()))

535 - self.exclude_zero_memory_users = configuration.get('exclude_zero_memory_users', False)

536 - self.poller = NvidiaSMIPoller(poll)

537 -

538 - def get_data_loop_mode(self):

539 - if not self.poller.is_started():

540 - self.poller.start()

541 -

542 - if not self.poller.is_alive():

543 - self.debug('poller is off')

544 - return None

545 -

546 - return self.poller.data()

547 -

548 - def get_data_normal_mode(self):

549 - return self.poller.run_once()

550 -

551 - def get_data(self):

552 - if self.loop_mode:

553 - last_data = self.get_data_loop_mode()

554 - else:

555 - last_data = self.get_data_normal_mode()

556 -

557 - if not last_data:

558 - return None

559 -

560 - parsed = self.parse_xml(last_data)

561 - if parsed is None:

562 - return None

563 -

564 - data = dict()

565 - for idx, root in enumerate(parsed.findall('gpu')):

566 - gpu = GPU(idx, root, self.exclude_zero_memory_users)

567 - gpu_data = gpu.data()

568 - # self.debug(gpu_data)

569 - gpu_data = dict((k, v) for k, v in gpu_data.items() if is_gpu_data_value_valid(v))

570 - data.update(gpu_data)

571 - self.update_processes_mem_chart(gpu)

572 - self.update_processes_user_mem_chart(gpu)

573 -

574 - return data or None

575 -

576 - def update_processes_mem_chart(self, gpu):

577 - ps = gpu.processes()

578 - if not ps:

579 - return

580 - chart = self.charts['gpu{0}_{1}'.format(gpu.num, PROCESSES_MEM)]

581 - active_dim_ids = []

582 - for p in ps:

583 - dim_id = 'gpu{0}_process_mem_{1}'.format(gpu.num, p['pid'])

584 - active_dim_ids.append(dim_id)

585 - if dim_id not in chart:

586 - chart.add_dimension([dim_id, '{0} {1}'.format(p['pid'], p['process_name'])])

587 - for dim in chart:

588 - if dim.id not in active_dim_ids:

589 - chart.del_dimension(dim.id, hide=False)

590 -

591 - def update_processes_user_mem_chart(self, gpu):

592 - ps = gpu.processes()

593 - if not ps:

594 - return

595 - chart = self.charts['gpu{0}_{1}'.format(gpu.num, USER_MEM)]

596 - active_dim_ids = []

597 - for p in ps:

598 - if not p.get('username'):

599 - continue

600 - dim_id = 'gpu{0}_user_mem_{1}'.format(gpu.num, p['username'])

601 - active_dim_ids.append(dim_id)

602 - if dim_id not in chart:

603 - chart.add_dimension([dim_id, '{0}'.format(p['username'])])

604 -

605 - for dim in chart:

606 - if dim.id not in active_dim_ids:

607 - chart.del_dimension(dim.id, hide=False)

608 -

609 - def check(self):

610 - if not self.poller.has_smi():

611 - self.error("couldn't find '{0}' binary".format(NVIDIA_SMI))

612 - return False

613 -

614 - raw_data = self.poller.run_once()

615 - if not raw_data:

616 - self.error("failed to invoke '{0}' binary".format(NVIDIA_SMI))

617 - return False

618 -

619 - parsed = self.parse_xml(raw_data)

620 - if parsed is None:

621 - return False

622 -

623 - gpus = parsed.findall('gpu')

624 - if not gpus:

625 - return False

626 -

627 - self.create_charts(gpus)

628 -

629 - return True

630 -

631 - def parse_xml(self, data):

632 - try:

633 - return et.fromstring(data)

634 - except et.ParseError as error:

635 - self.error('xml parse failed: "{0}", error: {1}'.format(data, error))

636 -

637 - return None

638 -

639 - def create_charts(self, gpus):

640 - for idx, root in enumerate(gpus):

641 - order, charts = gpu_charts(GPU(idx, root))

642 - self.order.extend(order)

643 - self.definitions.update(charts)

644 -

645 -

646 -def is_gpu_data_value_valid(value):

647 - try:

648 - int(value)

649 - except (TypeError, ValueError):

650 - return False

651 - return True

src/collectors/python.d.plugin/nvidia_smi/nvidia_smi.conf deleted

-68

@@ -1,68 +0,0 @@

1 -# netdata python.d.plugin configuration for nvidia_smi

2 -#

3 -# This file is in YaML format. Generally the format is:

4 -#

5 -# name: value

6 -#

7 -# There are 2 sections:

8 -# - global variables

9 -# - one or more JOBS

10 -#

11 -# JOBS allow you to collect values from multiple sources.

12 -# Each source will have its own set of charts.

13 -#

14 -# JOB parameters have to be indented (using spaces only, example below).

15 -

16 -# ----------------------------------------------------------------------

17 -# Global Variables

18 -# These variables set the defaults for all JOBs, however each JOB

19 -# may define its own, overriding the defaults.

20 -

21 -# update_every sets the default data collection frequency.

22 -# If unset, the python.d.plugin default is used.

23 -# update_every: 1

24 -

25 -# priority controls the order of charts at the netdata dashboard.

26 -# Lower numbers move the charts towards the top of the page.

27 -# If unset, the default for python.d.plugin is used.

28 -# priority: 60000

29 -

30 -# penalty indicates whether to apply penalty to update_every in case of failures.

31 -# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.

32 -# penalty: yes

33 -

34 -# autodetection_retry sets the job re-check interval in seconds.

35 -# The job is not deleted if check fails.

36 -# Attempts to start the job are made once every autodetection_retry.

37 -# This feature is disabled by default.

38 -# autodetection_retry: 0

39 -

40 -# ----------------------------------------------------------------------

41 -# JOBS (data collection sources)

42 -#

43 -# The default JOBS share the same *name*. JOBS with the same name

44 -# are mutually exclusive. Only one of them will be allowed running at

45 -# any time. This allows autodetection to try several alternatives and

46 -# pick the one that works.

47 -#

48 -# Any number of jobs is supported.

49 -#

50 -# All python.d.plugin JOBS (for all its modules) support a set of

51 -# predefined parameters. These are:

52 -#

53 -# job_name:

54 -# name: myname # the JOB's name as it will appear at the

55 -# # dashboard (by default is the job_name)

56 -# # JOBs sharing a name are mutually exclusive

57 -# update_every: 1 # the JOB's data collection frequency

58 -# priority: 60000 # the JOB's order on the dashboard

59 -# penalty: yes # the JOB's penalty

60 -# autodetection_retry: 0 # the JOB's re-check interval in seconds

61 -#

62 -# Additionally to the above, example also supports the following:

63 -#

64 -# loop_mode: yes/no # default is yes. If set to yes `nvidia-smi` is executed in a separate thread using `-l` option.

65 -# poll_seconds: SECONDS # default is 1. Sets the frequency of seconds the nvidia-smi tool is polled in loop mode.

66 -# exclude_zero_memory_users: yes/no # default is no. Whether to collect users metrics with 0Mb memory allocation.

67 -#

68 -# ----------------------------------------------------------------------

src/collectors/python.d.plugin/python.d.conf

+1 -1

@@ -36,7 +36,6 @@ example: no

36 go_expvar: no

37 # haproxy: yes

38 # monit: yes

39 -# nvidia_smi: yes

39 # openldap: yes

40 # oracledb: yes

41 # pandas: yes

@@ -73,6 +72,7 @@ mongodb: no # Removed (replaced with go.d/mongodb).

72 mysql: no # Removed (replaced with go.d/mysql).

73 nginx: no # Removed (replaced with go.d/nginx).

74 nsd: no # Removed (replaced with go.d/nsd).

75 +nvidia_smi: no # Removed (replaced with go.d/nvidia_smi).

76 postfix: no # Removed (replaced with go.d/postfix).

77 postgres: no # Removed (replaced with go.d/postgres).

78 proxysql: no # Removed (replaced with go.d/proxysql).