go.d nvidia_smi: add loop mode (#18313)
Ilya Mashchenko committed
Aug 12, 2024 at 13:47 UTC
63631b495b3961acfcd7aba57173b54d627a9fdc
8 files changed
+205
-10
src/go/plugin/go.d/modules/nvidia_smi/config_schema.json
+9
@@ -23,6 +23,12 @@
23
"type": "number",
24
"minimum": 0.5,
25
"default": 10
26
+ },
27
+ "loop_mode": {
28
+ "title": "Loop Mode",
29
+ "description": "When enabled, `nvidia-smi` is executed continuously in a separate thread using the `-l` option.",
30
+ "type": "boolean",
31
+ "default": true
32
}
33
},
34
"required": [
@@ -42,6 +48,9 @@
48
},
49
"timeout": {
50
"ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
51
+ },
52
+ "loop_mode": {
53
+ "ui:help": "In loop mode, `nvidia-smi` will repeatedly query GPU data at specified intervals, defined by the `-l SEC` or `--loop=SEC` parameter, rather than just running the query once. This enables ongoing performance tracking by putting the application to sleep between queries."
54
}
55
}
56
}
src/go/plugin/go.d/modules/nvidia_smi/exec.go
+173
-6
@@ -3,9 +3,14 @@
3
package nvidia_smi
4
5
import (
6
+ "bufio"
7
+ "bytes"
8
"context"
9
+ "errors"
10
"fmt"
11
"os/exec"
12
+ "strconv"
13
+ "sync"
14
"time"
15
16
"github.com/netdata/netdata/go/plugins/logger"
@@ -13,14 +18,30 @@ import (
18
19
type nvidiaSmiBinary interface {
20
queryGPUInfo() ([]byte, error)
21
+ stop() error
22
}
23
18
-func newNvidiaSmiExec(path string, cfg Config, log *logger.Logger) (*nvidiaSmiExec, error) {
19
- return &nvidiaSmiExec{
20
- Logger: log,
21
- binPath: path,
22
- timeout: cfg.Timeout.Duration(),
23
- }, nil
24
+func newNvidiaSmiBinary(path string, cfg Config, log *logger.Logger) (nvidiaSmiBinary, error) {
25
+ if !cfg.LoopMode {
26
+ return &nvidiaSmiExec{
27
+ Logger: log,
28
+ binPath: path,
29
+ timeout: cfg.Timeout.Duration(),
30
+ }, nil
31
+ }
32
+
33
+ smi := &nvidiaSmiLoopExec{
34
+ Logger: log,
35
+ binPath: path,
36
+ updateEvery: cfg.UpdateEvery,
37
+ firstSampleTimeout: time.Second * 3,
38
+ }
39
+
40
+ if err := smi.run(); err != nil {
41
+ return nil, err
42
+ }
43
+
44
+ return smi, nil
45
}
46
47
type nvidiaSmiExec struct {
@@ -44,3 +65,149 @@ func (e *nvidiaSmiExec) queryGPUInfo() ([]byte, error) {
65
66
return bs, nil
67
}
68
+
69
+func (e *nvidiaSmiExec) stop() error { return nil }
70
+
71
+type nvidiaSmiLoopExec struct {
72
+ *logger.Logger
73
+
74
+ binPath string
75
+
76
+ updateEvery int
77
+ firstSampleTimeout time.Duration
78
+
79
+ cmd *exec.Cmd
80
+ done chan struct{}
81
+
82
+ mux sync.Mutex
83
+ lastSample string
84
+}
85
+
86
+func (e *nvidiaSmiLoopExec) queryGPUInfo() ([]byte, error) {
87
+ select {
88
+ case <-e.done:
89
+ return nil, errors.New("process has already exited")
90
+ default:
91
+ }
92
+
93
+ e.mux.Lock()
94
+ defer e.mux.Unlock()
95
+
96
+ return []byte(e.lastSample), nil
97
+}
98
+
99
+func (e *nvidiaSmiLoopExec) run() error {
100
+ secs := 5
101
+ if e.updateEvery < secs {
102
+ secs = e.updateEvery
103
+ }
104
+
105
+ cmd := exec.Command(e.binPath, "-q", "-x", "-l", strconv.Itoa(secs))
106
+
107
+ e.Debugf("executing '%s'", cmd)
108
+
109
+ r, err := cmd.StdoutPipe()
110
+ if err != nil {
111
+ return err
112
+ }
113
+
114
+ if err := cmd.Start(); err != nil {
115
+ return err
116
+ }
117
+
118
+ firstSample := make(chan struct{}, 1)
119
+ done := make(chan struct{})
120
+ e.cmd = cmd
121
+ e.done = done
122
+
123
+ go func() {
124
+ defer close(done)
125
+
126
+ var buf bytes.Buffer
127
+ var insideLog bool
128
+ var emptyRows int64
129
+ var outsideLogRows int64
130
+
131
+ const unexpectedRowsLimit = 500
132
+
133
+ sc := bufio.NewScanner(r)
134
+
135
+ for sc.Scan() {
136
+ line := sc.Text()
137
+
138
+ if !insideLog {
139
+ outsideLogRows++
140
+ } else {
141
+ outsideLogRows = 0
142
+ }
143
+
144
+ if line == "" {
145
+ emptyRows++
146
+ } else {
147
+ emptyRows = 0
148
+ }
149
+
150
+ if outsideLogRows >= unexpectedRowsLimit || emptyRows >= unexpectedRowsLimit {
151
+ e.Errorf("unexpected output from nvidia-smi loop: outside log rows %d, empty rows %d", outsideLogRows, emptyRows)
152
+ break
153
+ }
154
+
155
+ switch {
156
+ case line == "<nvidia_smi_log>":
157
+ insideLog = true
158
+ buf.Reset()
159
+
160
+ buf.WriteString(line)
161
+ buf.WriteByte('\n')
162
+ case line == "</nvidia_smi_log>":
163
+ insideLog = false
164
+
165
+ buf.WriteString(line)
166
+
167
+ e.mux.Lock()
168
+ e.lastSample = buf.String()
169
+ e.mux.Unlock()
170
+
171
+ buf.Reset()
172
+
173
+ select {
174
+ case firstSample <- struct{}{}:
175
+ default:
176
+ }
177
+ case insideLog:
178
+ buf.WriteString(line)
179
+ buf.WriteByte('\n')
180
+ default:
181
+ continue
182
+ }
183
+ }
184
+ }()
185
+
186
+ select {
187
+ case <-e.done:
188
+ _ = e.stop()
189
+ return errors.New("process exited before the first sample was collected")
190
+ case <-time.After(e.firstSampleTimeout):
191
+ _ = e.stop()
192
+ return errors.New("timed out waiting for first sample")
193
+ case <-firstSample:
194
+ return nil
195
+ }
196
+}
197
+
198
+func (e *nvidiaSmiLoopExec) stop() error {
199
+ if e.cmd == nil || e.cmd.Process == nil {
200
+ return nil
201
+ }
202
+
203
+ _ = e.cmd.Process.Kill()
204
+ _ = e.cmd.Wait()
205
+ e.cmd = nil
206
+
207
+ select {
208
+ case <-e.done:
209
+ return nil
210
+ case <-time.After(time.Second * 2):
211
+ return errors.New("timed out waiting for process to exit")
212
+ }
213
+}
src/go/plugin/go.d/modules/nvidia_smi/init.go
+1
-1
@@ -18,5 +18,5 @@ func (nv *NvidiaSmi) initNvidiaSmiExec() (nvidiaSmiBinary, error) {
18
binPath = path
19
}
20
21
- return newNvidiaSmiExec(binPath, nv.Config, nv.Logger)
21
+ return newNvidiaSmiBinary(binPath, nv.Config, nv.Logger)
22
}
src/go/plugin/go.d/modules/nvidia_smi/metadata.yaml
+4
@@ -73,6 +73,10 @@ modules:
73
description: nvidia_smi binary execution timeout.
74
default_value: 2
75
required: false
76
+ - name: loop_mode
77
+ description: "When enabled, `nvidia-smi` is executed continuously in a separate thread using the `-l` option."
78
+ default_value: true
79
+ required: false
80
examples:
81
folding:
82
title: Config
src/go/plugin/go.d/modules/nvidia_smi/nvidia_smi.go
+11
-2
@@ -29,7 +29,8 @@ func init() {
29
func New() *NvidiaSmi {
30
return &NvidiaSmi{
31
Config: Config{
32
- Timeout: web.Duration(time.Second * 10),
32
+ Timeout: web.Duration(time.Second * 10),
33
+ LoopMode: true,
34
},
35
binName: "nvidia-smi",
36
charts: &module.Charts{},
@@ -43,6 +44,7 @@ type Config struct {
44
UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
45
Timeout web.Duration `yaml:"timeout,omitempty" json:"timeout"`
46
BinaryPath string `yaml:"binary_path" json:"binary_path"`
47
+ LoopMode bool `yaml:"loop_mode,omitempty" json:"loop_mode"`
48
}
49
50
type NvidiaSmi struct {
@@ -103,4 +105,11 @@ func (nv *NvidiaSmi) Collect() map[string]int64 {
105
return mx
106
}
107
106
-func (nv *NvidiaSmi) Cleanup() {}
108
+func (nv *NvidiaSmi) Cleanup() {
109
+ if nv.exec != nil {
110
+ if err := nv.exec.stop(); err != nil {
111
+ nv.Errorf("cleanup: %v", err)
112
+ }
113
+ nv.exec = nil
114
+ }
115
+}
src/go/plugin/go.d/modules/nvidia_smi/nvidia_smi_test.go
+4
@@ -418,6 +418,10 @@ func (m *mockNvidiaSmi) queryGPUInfo() ([]byte, error) {
418
return m.gpuInfo, nil
419
}
420
421
+func (m *mockNvidiaSmi) stop() error {
422
+ return nil
423
+}
424
+
425
func prepareCaseMIGA100(nv *NvidiaSmi) {
426
nv.exec = &mockNvidiaSmi{gpuInfo: dataXMLA100SXM4MIG}
427
}
src/go/plugin/go.d/modules/nvidia_smi/testdata/config.json
+2
-1
@@ -1,5 +1,6 @@
1
{
2
"update_every": 123,
3
"timeout": 123.123,
4
- "binary_path": "ok"
4
+ "binary_path": "ok",
5
+ "loop_mode": true
6
}
src/go/plugin/go.d/modules/nvidia_smi/testdata/config.yaml
+1
@@ -1,3 +1,4 @@
1
update_every: 123
2
timeout: 123.123
3
binary_path: "ok"
4
+loop_mode: true