@cryptotaxi247 / netdata-1 / commits / d8c362147

go.d beanstalk (#18263)

Ilya Mashchenko committed Aug 6, 2024 at 20:54 UTC d8c3621470dcaacced94b50561d5d0a94ada5571
20 files changed +1649 -28
src/go/plugin/go.d/README.md
+1
@@ -53,6 +53,7 @@ see the appropriate collector readme.
53 | [activemq](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/activemq) | ActiveMQ |
54 | [ap](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/ap) | Wireless AP |
55 | [apache](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/apache) | Apache |
56 +| [beanstalk](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/beanstalk) | Beanstalk |
57 | [bind](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/bind) | ISC Bind |
58 | [cassandra](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/cassandra) | Cassandra |
59 | [chrony](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/chrony) | Chrony |
src/go/plugin/go.d/config/go.d.conf
+1
@@ -19,6 +19,7 @@ modules:
19 # activemq: yes
20 # ap: yes
21 # apache: yes
22 +# beanstalk: yes
23 # bind: yes
24 # chrony: yes
25 # clickhouse: yes
src/go/plugin/go.d/config/go.d/beanstalk.conf new
+6
@@ -0,0 +1,6 @@
1 +## All available configuration options, their descriptions and default values:
2 +## https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/beanstalk#readme
3 +
4 +#jobs:
5 +# - name: local
6 +# address: 127.0.0.1:11300
src/go/plugin/go.d/config/go.d/sd/docker.conf
+7
@@ -26,6 +26,8 @@ classify:
26 match:
27 - tags: "apache"
28 expr: '{{ match "sp" .Image "httpd httpd:* */apache */apache:* */apache2 */apache2:*" }}'
29 + - tags: "beanstalk"
30 + expr: '{{ match "sp" .Image "*/beanstalkd */beanstalkd:*" }}'
31 - tags: "cockroachdb"
32 expr: '{{ match "sp" .Image "cockroachdb/cockroach cockroachdb/cockroach:*" }}'
33 - tags: "consul"
@@ -81,6 +83,11 @@ compose:
83 module: apache
84 name: docker_{{.Name}}
85 url: http://{{.Address}}/server-status?auto
86 + - selector: "beanstalk"
87 + template: |
88 + module: beanstalk
89 + name: docker_{{.Name}}
90 + address: {{.Address}}
91 - selector: "cockroachdb"
92 template: |
93 module: cockroachdb
src/go/plugin/go.d/config/go.d/sd/net_listeners.conf
+7
@@ -16,6 +16,8 @@ classify:
16 expr: '{{ and (eq .Port "8161") (eq .Comm "activemq") }}'
17 - tags: "apache"
18 expr: '{{ and (eq .Port "80" "8080") (eq .Comm "apache" "apache2" "httpd") }}'
19 + - tags: "beanstalk"
20 + expr: '{{ or (eq .Port "11300") (eq .Comm "beanstalkd") }}'
21 - tags: "bind"
22 expr: '{{ and (eq .Port "8653") (eq .Comm "bind" "named") }}'
23 - tags: "cassandra"
@@ -139,6 +141,11 @@ compose:
141 module: apache
142 name: local
143 url: http://{{.Address}}/server-status?auto
144 + - selector: "beanstalk"
145 + template: |
146 + module: beanstalk
147 + name: local
148 + address: {{.Address}}
149 - selector: "bind"
150 template: |
151 module: bind
src/go/plugin/go.d/modules/beanstalk/beanstalk.go new
+123
@@ -0,0 +1,123 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package beanstalk
4 +
5 +import (
6 + _ "embed"
7 + "errors"
8 + "fmt"
9 + "time"
10 +
11 + "github.com/netdata/netdata/go/plugins/logger"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/matcher"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"
15 +)
16 +
17 +//go:embed "config_schema.json"
18 +var configSchema string
19 +
20 +func init() {
21 + module.Register("beanstalk", module.Creator{
22 + JobConfigSchema: configSchema,
23 + Create: func() module.Module { return New() },
24 + Config: func() any { return &Config{} },
25 + })
26 +}
27 +
28 +func New() *Beanstalk {
29 + return &Beanstalk{
30 + Config: Config{
31 + Address: "127.0.0.1:11300",
32 + Timeout: web.Duration(time.Second * 1),
33 + TubeSelector: "*",
34 + },
35 +
36 + charts: statsCharts.Copy(),
37 + newConn: newBeanstalkConn,
38 + discoverTubesEvery: time.Minute * 1,
39 + tubeSr: matcher.TRUE(),
40 + seenTubes: make(map[string]bool),
41 + }
42 +}
43 +
44 +type Config struct {
45 + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
46 + Address string `yaml:"address" json:"address"`
47 + Timeout web.Duration `yaml:"timeout,omitempty" json:"timeout"`
48 + TubeSelector string `yaml:"tube_selector,omitempty" json:"tube_selector"`
49 +}
50 +
51 +type Beanstalk struct {
52 + module.Base
53 + Config `yaml:",inline" json:""`
54 +
55 + charts *module.Charts
56 +
57 + newConn func(Config, *logger.Logger) beanstalkConn
58 + conn beanstalkConn
59 +
60 + discoverTubesEvery time.Duration
61 + lastDiscoverTubesTime time.Time
62 + discoveredTubes []string
63 + tubeSr matcher.Matcher
64 + seenTubes map[string]bool
65 +}
66 +
67 +func (b *Beanstalk) Configuration() any {
68 + return b.Config
69 +}
70 +
71 +func (b *Beanstalk) Init() error {
72 + if err := b.validateConfig(); err != nil {
73 + return fmt.Errorf("config validation: %v", err)
74 + }
75 +
76 + sr, err := b.initTubeSelector()
77 + if err != nil {
78 + return fmt.Errorf("failed to init tube selector: %v", err)
79 + }
80 + b.tubeSr = sr
81 +
82 + return nil
83 +}
84 +
85 +func (b *Beanstalk) Check() error {
86 + mx, err := b.collect()
87 + if err != nil {
88 + b.Error(err)
89 + return err
90 + }
91 +
92 + if len(mx) == 0 {
93 + return errors.New("no metrics collected")
94 + }
95 +
96 + return nil
97 +}
98 +
99 +func (b *Beanstalk) Charts() *module.Charts {
100 + return b.charts
101 +}
102 +
103 +func (b *Beanstalk) Collect() map[string]int64 {
104 + mx, err := b.collect()
105 + if err != nil {
106 + b.Error(err)
107 + }
108 +
109 + if len(mx) == 0 {
110 + return nil
111 + }
112 +
113 + return mx
114 +}
115 +
116 +func (b *Beanstalk) Cleanup() {
117 + if b.conn != nil {
118 + if err := b.conn.disconnect(); err != nil {
119 + b.Warningf("error on disconnect: %s", err)
120 + }
121 + b.conn = nil
122 + }
123 +}
src/go/plugin/go.d/modules/beanstalk/beanstalk_test.go new
+384
@@ -0,0 +1,384 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package beanstalk
4 +
5 +import (
6 + "bufio"
7 + "errors"
8 + "fmt"
9 + "net"
10 + "os"
11 + "strings"
12 + "testing"
13 + "time"
14 +
15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
16 +
17 + "github.com/stretchr/testify/assert"
18 + "github.com/stretchr/testify/require"
19 +)
20 +
21 +var (
22 + dataConfigJSON, _ = os.ReadFile("testdata/config.json")
23 + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")
24 +
25 + dataStats, _ = os.ReadFile("testdata/stats.txt")
26 + dataListTubes, _ = os.ReadFile("testdata/list-tubes.txt")
27 + dataStatsTubeDefault, _ = os.ReadFile("testdata/stats-tube-default.txt")
28 +)
29 +
30 +func Test_testDataIsValid(t *testing.T) {
31 + for name, data := range map[string][]byte{
32 + "dataConfigJSON": dataConfigJSON,
33 + "dataConfigYAML": dataConfigYAML,
34 + "dataStats": dataStats,
35 + "dataListTubes": dataListTubes,
36 + "dataStatsTubeDefault": dataStatsTubeDefault,
37 + } {
38 + require.NotNil(t, data, name)
39 + }
40 +}
41 +
42 +func TestBeanstalk_ConfigurationSerialize(t *testing.T) {
43 + module.TestConfigurationSerialize(t, &Beanstalk{}, dataConfigJSON, dataConfigYAML)
44 +}
45 +
46 +func TestBeanstalk_Init(t *testing.T) {
47 + tests := map[string]struct {
48 + config Config
49 + wantFail bool
50 + }{
51 + "success with default config": {
52 + wantFail: false,
53 + config: New().Config,
54 + },
55 + "fails if address not set": {
56 + wantFail: true,
57 + config: func() Config {
58 + conf := New().Config
59 + conf.Address = ""
60 + return conf
61 + }(),
62 + },
63 + }
64 +
65 + for name, test := range tests {
66 + t.Run(name, func(t *testing.T) {
67 + beans := New()
68 + beans.Config = test.config
69 +
70 + if test.wantFail {
71 + assert.Error(t, beans.Init())
72 + } else {
73 + assert.NoError(t, beans.Init())
74 + }
75 + })
76 + }
77 +}
78 +
79 +func TestBeanstalk_Charts(t *testing.T) {
80 + assert.NotNil(t, New().Charts())
81 +}
82 +
83 +func TestBeanstalk_Check(t *testing.T) {
84 + tests := map[string]struct {
85 + prepare func() (*Beanstalk, *mockBeanstalkDaemon)
86 + wantFail bool
87 + }{
88 + "success on valid response": {
89 + wantFail: false,
90 + prepare: prepareCaseOk,
91 + },
92 + "fails on unexpected response": {
93 + wantFail: true,
94 + prepare: prepareCaseUnexpectedResponse,
95 + },
96 + "fails on connection refused": {
97 + wantFail: true,
98 + prepare: prepareCaseConnectionRefused,
99 + },
100 + }
101 + for name, test := range tests {
102 + t.Run(name, func(t *testing.T) {
103 + beanstalk, daemon := test.prepare()
104 +
105 + defer func() {
106 + assert.NoError(t, daemon.Close(), "daemon.Close()")
107 + }()
108 + go func() {
109 + assert.NoError(t, daemon.Run(), "daemon.Run()")
110 + }()
111 +
112 + select {
113 + case <-daemon.started:
114 + case <-time.After(time.Second * 3):
115 + t.Errorf("mock beanstalk daemon start timed out")
116 + }
117 +
118 + require.NoError(t, beanstalk.Init())
119 +
120 + if test.wantFail {
121 + assert.Error(t, beanstalk.Check())
122 + } else {
123 + assert.NoError(t, beanstalk.Check())
124 + }
125 +
126 + beanstalk.Cleanup()
127 +
128 + select {
129 + case <-daemon.stopped:
130 + case <-time.After(time.Second * 3):
131 + t.Errorf("mock beanstalk daemon stop timed out")
132 + }
133 + })
134 + }
135 +}
136 +
137 +func TestBeanstalk_Collect(t *testing.T) {
138 + tests := map[string]struct {
139 + prepare func() (*Beanstalk, *mockBeanstalkDaemon)
140 + wantMetrics map[string]int64
141 + wantCharts int
142 + }{
143 + "success on valid response": {
144 + prepare: prepareCaseOk,
145 + wantMetrics: map[string]int64{
146 + "binlog-records-migrated": 0,
147 + "binlog-records-written": 0,
148 + "cmd-bury": 0,
149 + "cmd-delete": 0,
150 + "cmd-ignore": 0,
151 + "cmd-kick": 0,
152 + "cmd-list-tube-used": 0,
153 + "cmd-list-tubes": 317,
154 + "cmd-list-tubes-watched": 0,
155 + "cmd-pause-tube": 0,
156 + "cmd-peek": 0,
157 + "cmd-peek-buried": 0,
158 + "cmd-peek-delayed": 0,
159 + "cmd-peek-ready": 0,
160 + "cmd-put": 0,
161 + "cmd-release": 0,
162 + "cmd-reserve": 0,
163 + "cmd-reserve-with-timeout": 0,
164 + "cmd-stats": 23619,
165 + "cmd-stats-job": 0,
166 + "cmd-stats-tube": 18964,
167 + "cmd-touch": 0,
168 + "cmd-use": 0,
169 + "cmd-watch": 0,
170 + "current-connections": 2,
171 + "current-jobs-buried": 0,
172 + "current-jobs-delayed": 0,
173 + "current-jobs-ready": 0,
174 + "current-jobs-reserved": 0,
175 + "current-jobs-urgent": 0,
176 + "current-producers": 0,
177 + "current-tubes": 1,
178 + "current-waiting": 0,
179 + "current-workers": 0,
180 + "job-timeouts": 0,
181 + "rusage-stime": 3922,
182 + "rusage-utime": 1602,
183 + "total-connections": 72,
184 + "total-jobs": 0,
185 + "tube_default_cmd-delete": 0,
186 + "tube_default_cmd-pause-tube": 0,
187 + "tube_default_current-jobs-buried": 0,
188 + "tube_default_current-jobs-delayed": 0,
189 + "tube_default_current-jobs-ready": 0,
190 + "tube_default_current-jobs-reserved": 0,
191 + "tube_default_current-jobs-urgent": 0,
192 + "tube_default_current-using": 2,
193 + "tube_default_current-waiting": 0,
194 + "tube_default_current-watching": 2,
195 + "tube_default_pause": 0,
196 + "tube_default_pause-time-left": 0,
197 + "tube_default_total-jobs": 0,
198 + "uptime": 105881,
199 + },
200 + wantCharts: len(statsCharts) + len(tubeChartsTmpl)*1,
201 + },
202 + "fails on unexpected response": {
203 + prepare: prepareCaseUnexpectedResponse,
204 + wantCharts: len(statsCharts),
205 + },
206 + "fails on connection refused": {
207 + prepare: prepareCaseConnectionRefused,
208 + wantCharts: len(statsCharts),
209 + },
210 + }
211 +
212 + for name, test := range tests {
213 + t.Run(name, func(t *testing.T) {
214 + beanstalk, daemon := test.prepare()
215 +
216 + defer func() {
217 + assert.NoError(t, daemon.Close(), "daemon.Close()")
218 + }()
219 + go func() {
220 + assert.NoError(t, daemon.Run(), "daemon.Run()")
221 + }()
222 +
223 + select {
224 + case <-daemon.started:
225 + case <-time.After(time.Second * 3):
226 + t.Errorf("mock beanstalk daemon start timed out")
227 + }
228 +
229 + require.NoError(t, beanstalk.Init())
230 +
231 + mx := beanstalk.Collect()
232 +
233 + require.Equal(t, test.wantMetrics, mx)
234 +
235 + assert.Equal(t, test.wantCharts, len(*beanstalk.Charts()), "want charts")
236 +
237 + if len(test.wantMetrics) > 0 {
238 + module.TestMetricsHasAllChartsDims(t, beanstalk.Charts(), mx)
239 + }
240 +
241 + beanstalk.Cleanup()
242 +
243 + select {
244 + case <-daemon.stopped:
245 + case <-time.After(time.Second * 3):
246 + t.Errorf("mock beanstalk daemon stop timed out")
247 + }
248 + })
249 + }
250 +}
251 +
252 +func prepareCaseOk() (*Beanstalk, *mockBeanstalkDaemon) {
253 + daemon := &mockBeanstalkDaemon{
254 + addr: "127.0.0.1:65001",
255 + started: make(chan struct{}),
256 + stopped: make(chan struct{}),
257 + dataStats: dataStats,
258 + dataListTubes: dataListTubes,
259 + dataStatsTube: dataStatsTubeDefault,
260 + }
261 +
262 + beanstalk := New()
263 + beanstalk.Address = daemon.addr
264 +
265 + return beanstalk, daemon
266 +}
267 +
268 +func prepareCaseUnexpectedResponse() (*Beanstalk, *mockBeanstalkDaemon) {
269 + daemon := &mockBeanstalkDaemon{
270 + addr: "127.0.0.1:65001",
271 + started: make(chan struct{}),
272 + stopped: make(chan struct{}),
273 + dataStats: []byte("INTERNAL_ERROR\n"),
274 + dataListTubes: []byte("INTERNAL_ERROR\n"),
275 + dataStatsTube: []byte("INTERNAL_ERROR\n"),
276 + }
277 +
278 + beanstalk := New()
279 + beanstalk.Address = daemon.addr
280 +
281 + return beanstalk, daemon
282 +}
283 +
284 +func prepareCaseConnectionRefused() (*Beanstalk, *mockBeanstalkDaemon) {
285 + ch := make(chan struct{})
286 + close(ch)
287 + daemon := &mockBeanstalkDaemon{
288 + addr: "127.0.0.1:65001",
289 + dontStart: true,
290 + started: ch,
291 + stopped: ch,
292 + }
293 +
294 + beanstalk := New()
295 + beanstalk.Address = daemon.addr
296 +
297 + return beanstalk, daemon
298 +}
299 +
300 +type mockBeanstalkDaemon struct {
301 + addr string
302 + srv net.Listener
303 + started chan struct{}
304 + stopped chan struct{}
305 + dontStart bool
306 +
307 + dataStats []byte
308 + dataListTubes []byte
309 + dataStatsTube []byte
310 +}
311 +
312 +func (m *mockBeanstalkDaemon) Run() error {
313 + if m.dontStart {
314 + return nil
315 + }
316 +
317 + srv, err := net.Listen("tcp", m.addr)
318 + if err != nil {
319 + return err
320 + }
321 +
322 + m.srv = srv
323 +
324 + close(m.started)
325 + defer close(m.stopped)
326 +
327 + return m.handleConnections()
328 +}
329 +
330 +func (m *mockBeanstalkDaemon) Close() error {
331 + if m.srv != nil {
332 + err := m.srv.Close()
333 + m.srv = nil
334 + return err
335 + }
336 + return nil
337 +}
338 +
339 +func (m *mockBeanstalkDaemon) handleConnections() error {
340 + conn, err := m.srv.Accept()
341 + if err != nil || conn == nil {
342 + return errors.New("could not accept connection")
343 + }
344 + return m.handleConnection(conn)
345 +}
346 +
347 +func (m *mockBeanstalkDaemon) handleConnection(conn net.Conn) error {
348 + defer func() { _ = conn.Close() }()
349 +
350 + rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
351 + var line string
352 + var err error
353 +
354 + for {
355 + if line, err = rw.ReadString('\n'); err != nil {
356 + return fmt.Errorf("error reading from connection: %v", err)
357 + }
358 +
359 + line = strings.TrimSpace(line)
360 +
361 + cmd, param, _ := strings.Cut(line, " ")
362 +
363 + switch cmd {
364 + case cmdQuit:
365 + return nil
366 + case cmdStats:
367 + _, err = rw.Write(m.dataStats)
368 + case cmdListTubes:
369 + _, err = rw.Write(m.dataListTubes)
370 + case cmdStatsTube:
371 + if param == "default" {
372 + _, err = rw.Write(m.dataStatsTube)
373 + } else {
374 + _, err = rw.WriteString("NOT_FOUND\n")
375 + }
376 + default:
377 + return fmt.Errorf("unexpected command: %s", line)
378 + }
379 + _ = rw.Flush()
380 + if err != nil {
381 + return err
382 + }
383 + }
384 +}
src/go/plugin/go.d/modules/beanstalk/charts.go new
+333
@@ -0,0 +1,333 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package beanstalk
4 +
5 +import (
6 + "fmt"
7 + "strings"
8 +
9 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
10 +)
11 +
12 +const (
13 + prioCurrentJobs = module.Priority + iota
14 + prioJobsRate
15 + prioJobsTimeouts
16 +
17 + prioCurrentTubes
18 +
19 + prioCommandsRate
20 +
21 + prioCurrentConnections
22 + prioConnectionsRate
23 +
24 + prioBinlogRecords
25 +
26 + prioCpuUsage
27 +
28 + prioUptime
29 +
30 + prioTubeCurrentJobs
31 + prioTubeJobsRate
32 +
33 + prioTubeCommands
34 +
35 + prioTubeCurrentConnections
36 +
37 + prioTubePauseTime
38 +)
39 +
40 +var (
41 + statsCharts = module.Charts{
42 + currentJobs.Copy(),
43 + jobsRateChart.Copy(),
44 + jobsTimeoutsChart.Copy(),
45 +
46 + currentTubesChart.Copy(),
47 +
48 + commandsRateChart.Copy(),
49 +
50 + currentConnectionsChart.Copy(),
51 + connectionsRateChart.Copy(),
52 +
53 + binlogRecordsChart.Copy(),
54 +
55 + cpuUsageChart.Copy(),
56 +
57 + uptimeChart.Copy(),
58 + }
59 +
60 + currentJobs = module.Chart{
61 + ID: "current_jobs",
62 + Title: "Current Jobs",
63 + Units: "jobs",
64 + Fam: "jobs",
65 + Ctx: "beanstalk.current_jobs",
66 + Type: module.Stacked,
67 + Priority: prioCurrentJobs,
68 + Dims: module.Dims{
69 + {ID: "current-jobs-ready", Name: "ready"},
70 + {ID: "current-jobs-buried", Name: "buried"},
71 + {ID: "current-jobs-urgent", Name: "urgent"},
72 + {ID: "current-jobs-delayed", Name: "delayed"},
73 + {ID: "current-jobs-reserved", Name: "reserved"},
74 + },
75 + }
76 + jobsRateChart = module.Chart{
77 + ID: "jobs_rate",
78 + Title: "Jobs Rate",
79 + Units: "jobs/s",
80 + Fam: "jobs",
81 + Ctx: "beanstalk.jobs_rate",
82 + Type: module.Line,
83 + Priority: prioJobsRate,
84 + Dims: module.Dims{
85 + {ID: "total-jobs", Name: "created", Algo: module.Incremental},
86 + },
87 + }
88 + jobsTimeoutsChart = module.Chart{
89 + ID: "jobs_timeouts",
90 + Title: "Timed Out Jobs",
91 + Units: "jobs/s",
92 + Fam: "jobs",
93 + Ctx: "beanstalk.jobs_timeouts",
94 + Type: module.Line,
95 + Priority: prioJobsTimeouts,
96 + Dims: module.Dims{
97 + {ID: "job-timeouts", Name: "timeouts", Algo: module.Incremental},
98 + },
99 + }
100 +
101 + currentTubesChart = module.Chart{
102 + ID: "current_tubes",
103 + Title: "Current Tubes",
104 + Units: "tubes",
105 + Fam: "tubes",
106 + Ctx: "beanstalk.current_tubes",
107 + Type: module.Line,
108 + Priority: prioCurrentTubes,
109 + Dims: module.Dims{
110 + {ID: "current-tubes", Name: "tubes"},
111 + },
112 + }
113 +
114 + commandsRateChart = module.Chart{
115 + ID: "commands_rate",
116 + Title: "Commands Rate",
117 + Units: "commands/s",
118 + Fam: "commands",
119 + Ctx: "beanstalk.commands_rate",
120 + Type: module.Stacked,
121 + Priority: prioCommandsRate,
122 + Dims: module.Dims{
123 + {ID: "cmd-put", Name: "put", Algo: module.Incremental},
124 + {ID: "cmd-peek", Name: "peek", Algo: module.Incremental},
125 + {ID: "cmd-peek-ready", Name: "peek-ready", Algo: module.Incremental},
126 + {ID: "cmd-peek-delayed", Name: "peek-delayed", Algo: module.Incremental},
127 + {ID: "cmd-peek-buried", Name: "peek-buried", Algo: module.Incremental},
128 + {ID: "cmd-reserve", Name: "reserve", Algo: module.Incremental},
129 + {ID: "cmd-reserve-with-timeout", Name: "reserve-with-timeout", Algo: module.Incremental},
130 + {ID: "cmd-touch", Name: "touch", Algo: module.Incremental},
131 + {ID: "cmd-use", Name: "use", Algo: module.Incremental},
132 + {ID: "cmd-watch", Name: "watch", Algo: module.Incremental},
133 + {ID: "cmd-ignore", Name: "ignore", Algo: module.Incremental},
134 + {ID: "cmd-delete", Name: "delete", Algo: module.Incremental},
135 + {ID: "cmd-release", Name: "release", Algo: module.Incremental},
136 + {ID: "cmd-bury", Name: "bury", Algo: module.Incremental},
137 + {ID: "cmd-kick", Name: "kick", Algo: module.Incremental},
138 + {ID: "cmd-stats", Name: "stats", Algo: module.Incremental},
139 + {ID: "cmd-stats-job", Name: "stats-job", Algo: module.Incremental},
140 + {ID: "cmd-stats-tube", Name: "stats-tube", Algo: module.Incremental},
141 + {ID: "cmd-list-tubes", Name: "list-tubes", Algo: module.Incremental},
142 + {ID: "cmd-list-tube-used", Name: "list-tube-used", Algo: module.Incremental},
143 + {ID: "cmd-list-tubes-watched", Name: "list-tubes-watched", Algo: module.Incremental},
144 + {ID: "cmd-pause-tube", Name: "pause-tube", Algo: module.Incremental},
145 + },
146 + }
147 +
148 + currentConnectionsChart = module.Chart{
149 + ID: "current_connections",
150 + Title: "Current Connections",
151 + Units: "connections",
152 + Fam: "connections",
153 + Ctx: "beanstalk.current_connections",
154 + Type: module.Line,
155 + Priority: prioCurrentConnections,
156 + Dims: module.Dims{
157 + {ID: "current-connections", Name: "open"},
158 + {ID: "current-producers", Name: "producers"},
159 + {ID: "current-workers", Name: "workers"},
160 + {ID: "current-waiting", Name: "waiting"},
161 + },
162 + }
163 + connectionsRateChart = module.Chart{
164 + ID: "connections_rate",
165 + Title: "Connections Rate",
166 + Units: "connections/s",
167 + Fam: "connections",
168 + Ctx: "beanstalk.connections_rate",
169 + Type: module.Line,
170 + Priority: prioConnectionsRate,
171 + Dims: module.Dims{
172 + {ID: "total-connections", Name: "created", Algo: module.Incremental},
173 + },
174 + }
175 +
176 + binlogRecordsChart = module.Chart{
177 + ID: "binlog_records",
178 + Title: "Binlog Records",
179 + Units: "records/s",
180 + Fam: "binlog",
181 + Ctx: "beanstalk.binlog_records",
182 + Type: module.Line,
183 + Priority: prioBinlogRecords,
184 + Dims: module.Dims{
185 + {ID: "binlog-records-written", Name: "written", Algo: module.Incremental},
186 + {ID: "binlog-records-migrated", Name: "migrated", Algo: module.Incremental},
187 + },
188 + }
189 +
190 + cpuUsageChart = module.Chart{
191 + ID: "cpu_usage",
192 + Title: "CPU Usage",
193 + Units: "percent",
194 + Fam: "cpu usage",
195 + Ctx: "beanstalk.cpu_usage",
196 + Type: module.Stacked,
197 + Priority: prioCpuUsage,
198 + Dims: module.Dims{
199 + {ID: "rusage-utime", Name: "user", Algo: module.Incremental, Mul: 100, Div: 1000},
200 + {ID: "rusage-stime", Name: "system", Algo: module.Incremental, Mul: 100, Div: 1000},
201 + },
202 + }
203 +
204 + uptimeChart = module.Chart{
205 + ID: "uptime",
206 + Title: "Uptime",
207 + Units: "seconds",
208 + Fam: "uptime",
209 + Ctx: "beanstalk.uptime",
210 + Type: module.Line,
211 + Priority: prioUptime,
212 + Dims: module.Dims{
213 + {ID: "uptime"},
214 + },
215 + }
216 +)
217 +
218 +var (
219 + tubeChartsTmpl = module.Charts{
220 + tubeCurrentJobsChartTmpl.Copy(),
221 + tubeJobsRateChartTmpl.Copy(),
222 +
223 + tubeCommandsRateChartTmpl.Copy(),
224 +
225 + tubeCurrentConnectionsChartTmpl.Copy(),
226 +
227 + tubePauseTimeChartTmpl.Copy(),
228 + }
229 +
230 + tubeCurrentJobsChartTmpl = module.Chart{
231 + ID: "tube_%s_current_jobs",
232 + Title: "Tube Current Jobs",
233 + Units: "jobs",
234 + Fam: "tube jobs",
235 + Ctx: "beanstalk.tube_current_jobs",
236 + Type: module.Stacked,
237 + Priority: prioTubeCurrentJobs,
238 + Dims: module.Dims{
239 + {ID: "tube_%s_current-jobs-ready", Name: "ready"},
240 + {ID: "tube_%s_current-jobs-buried", Name: "buried"},
241 + {ID: "tube_%s_current-jobs-urgent", Name: "urgent"},
242 + {ID: "tube_%s_current-jobs-delayed", Name: "delayed"},
243 + {ID: "tube_%s_current-jobs-reserved", Name: "reserved"},
244 + },
245 + }
246 + tubeJobsRateChartTmpl = module.Chart{
247 + ID: "tube_%s_jobs_rate",
248 + Title: "Tube Jobs Rate",
249 + Units: "jobs/s",
250 + Fam: "tube jobs",
251 + Ctx: "beanstalk.tube_jobs_rate",
252 + Type: module.Line,
253 + Priority: prioTubeJobsRate,
254 + Dims: module.Dims{
255 + {ID: "tube_%s_total-jobs", Name: "created", Algo: module.Incremental},
256 + },
257 + }
258 + tubeCommandsRateChartTmpl = module.Chart{
259 + ID: "tube_%s_commands_rate",
260 + Title: "Tube Commands",
261 + Units: "commands/s",
262 + Fam: "tube commands",
263 + Ctx: "beanstalk.tube_commands_rate",
264 + Type: module.Stacked,
265 + Priority: prioTubeCommands,
266 + Dims: module.Dims{
267 + {ID: "tube_%s_cmd-delete", Name: "delete", Algo: module.Incremental},
268 + {ID: "tube_%s_cmd-pause-tube", Name: "pause-tube", Algo: module.Incremental},
269 + },
270 + }
271 + tubeCurrentConnectionsChartTmpl = module.Chart{
272 + ID: "tube_%s_current_connections",
273 + Title: "Tube Current Connections",
274 + Units: "connections",
275 + Fam: "tube connections",
276 + Ctx: "beanstalk.tube_current_connections",
277 + Type: module.Stacked,
278 + Priority: prioTubeCurrentConnections,
279 + Dims: module.Dims{
280 + {ID: "tube_%s_current-using", Name: "using"},
281 + {ID: "tube_%s_current-waiting", Name: "waiting"},
282 + {ID: "tube_%s_current-watching", Name: "watching"},
283 + },
284 + }
285 + tubePauseTimeChartTmpl = module.Chart{
286 + ID: "tube_%s_pause_time",
287 + Title: "Tube Pause Time",
288 + Units: "seconds",
289 + Fam: "tube pause",
290 + Ctx: "beanstalk.tube_pause",
291 + Type: module.Line,
292 + Priority: prioTubePauseTime,
293 + Dims: module.Dims{
294 + {ID: "tube_%s_pause", Name: "since"},
295 + {ID: "tube_%s_pause-time-left", Name: "left"},
296 + },
297 + }
298 +)
299 +
300 +func (b *Beanstalk) addTubeCharts(name string) {
301 + charts := tubeChartsTmpl.Copy()
302 +
303 + for _, chart := range *charts {
304 + chart.ID = fmt.Sprintf(chart.ID, cleanTubeName(name))
305 + chart.Labels = []module.Label{
306 + {Key: "tube_name", Value: name},
307 + }
308 +
309 + for _, dim := range chart.Dims {
310 + dim.ID = fmt.Sprintf(dim.ID, name)
311 + }
312 + }
313 +
314 + if err := b.Charts().Add(*charts...); err != nil {
315 + b.Warning(err)
316 + }
317 +}
318 +
319 +func (b *Beanstalk) removeTubeCharts(name string) {
320 + px := fmt.Sprintf("tube_%s_", cleanTubeName(name))
321 +
322 + for _, chart := range *b.Charts() {
323 + if strings.HasPrefix(chart.ID, px) {
324 + chart.MarkRemove()
325 + chart.MarkNotCreated()
326 + }
327 + }
328 +}
329 +
330 +func cleanTubeName(name string) string {
331 + r := strings.NewReplacer(" ", "_", ".", "_", ",", "_")
332 + return r.Replace(name)
333 +}
src/go/plugin/go.d/modules/beanstalk/client.go new
+249
@@ -0,0 +1,249 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package beanstalk
4 +
5 +import (
6 + "errors"
7 + "fmt"
8 + "strconv"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/logger"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/socket"
13 +
14 + "gopkg.in/yaml.v2"
15 +)
16 +
17 +type beanstalkConn interface {
18 + connect() error
19 + disconnect() error
20 + queryStats() (*beanstalkdStats, error)
21 + queryListTubes() ([]string, error)
22 + queryStatsTube(string) (*tubeStats, error)
23 +}
24 +
25 +// https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L553
26 +type beanstalkdStats struct {
27 + CurrentJobsUrgent int64 `yaml:"current-jobs-urgent" stm:"current-jobs-urgent"`
28 + CurrentJobsReady int64 `yaml:"current-jobs-ready" stm:"current-jobs-ready"`
29 + CurrentJobsReserved int64 `yaml:"current-jobs-reserved" stm:"current-jobs-reserved"`
30 + CurrentJobsDelayed int64 `yaml:"current-jobs-delayed" stm:"current-jobs-delayed"`
31 + CurrentJobsBuried int64 `yaml:"current-jobs-buried" stm:"current-jobs-buried"`
32 + CmdPut int64 `yaml:"cmd-put" stm:"cmd-put"`
33 + CmdPeek int64 `yaml:"cmd-peek" stm:"cmd-peek"`
34 + CmdPeekReady int64 `yaml:"cmd-peek-ready" stm:"cmd-peek-ready"`
35 + CmdPeekDelayed int64 `yaml:"cmd-peek-delayed" stm:"cmd-peek-delayed"`
36 + CmdPeekBuried int64 `yaml:"cmd-peek-buried" stm:"cmd-peek-buried"`
37 + CmdReserve int64 `yaml:"cmd-reserve" stm:"cmd-reserve"`
38 + CmdReserveWithTimeout int64 `yaml:"cmd-reserve-with-timeout" stm:"cmd-reserve-with-timeout"`
39 + CmdTouch int64 `yaml:"cmd-touch" stm:"cmd-touch"`
40 + CmdUse int64 `yaml:"cmd-use" stm:"cmd-use"`
41 + CmdWatch int64 `yaml:"cmd-watch" stm:"cmd-watch"`
42 + CmdIgnore int64 `yaml:"cmd-ignore" stm:"cmd-ignore"`
43 + CmdDelete int64 `yaml:"cmd-delete" stm:"cmd-delete"`
44 + CmdRelease int64 `yaml:"cmd-release" stm:"cmd-release"`
45 + CmdBury int64 `yaml:"cmd-bury" stm:"cmd-bury"`
46 + CmdKick int64 `yaml:"cmd-kick" stm:"cmd-kick"`
47 + CmdStats int64 `yaml:"cmd-stats" stm:"cmd-stats"`
48 + CmdStatsJob int64 `yaml:"cmd-stats-job" stm:"cmd-stats-job"`
49 + CmdStatsTube int64 `yaml:"cmd-stats-tube" stm:"cmd-stats-tube"`
50 + CmdListTubes int64 `yaml:"cmd-list-tubes" stm:"cmd-list-tubes"`
51 + CmdListTubeUsed int64 `yaml:"cmd-list-tube-used" stm:"cmd-list-tube-used"`
52 + CmdListTubesWatched int64 `yaml:"cmd-list-tubes-watched" stm:"cmd-list-tubes-watched"`
53 + CmdPauseTube int64 `yaml:"cmd-pause-tube" stm:"cmd-pause-tube"`
54 + JobTimeouts int64 `yaml:"job-timeouts" stm:"job-timeouts"`
55 + TotalJobs int64 `yaml:"total-jobs" stm:"total-jobs"`
56 + CurrentTubes int64 `yaml:"current-tubes" stm:"current-tubes"`
57 + CurrentConnections int64 `yaml:"current-connections" stm:"current-connections"`
58 + CurrentProducers int64 `yaml:"current-producers" stm:"current-producers"`
59 + CurrentWorkers int64 `yaml:"current-workers" stm:"current-workers"`
60 + CurrentWaiting int64 `yaml:"current-waiting" stm:"current-waiting"`
61 + TotalConnections int64 `yaml:"total-connections" stm:"total-connections"`
62 + RusageUtime float64 `yaml:"rusage-utime" stm:"rusage-utime,1000,1"`
63 + RusageStime float64 `yaml:"rusage-stime" stm:"rusage-stime,1000,1"`
64 + Uptime int64 `yaml:"uptime" stm:"uptime"`
65 + BinlogRecordsWritten int64 `yaml:"binlog-records-written" stm:"binlog-records-written"`
66 + BinlogRecordsMigrated int64 `yaml:"binlog-records-migrated" stm:"binlog-records-migrated"`
67 +}
68 +
69 +// https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L497
70 +type tubeStats struct {
71 + Name string `yaml:"name"`
72 + CurrentJobsUrgent int64 `yaml:"current-jobs-urgent" stm:"current-jobs-urgent"`
73 + CurrentJobsReady int64 `yaml:"current-jobs-ready" stm:"current-jobs-ready"`
74 + CurrentJobsReserved int64 `yaml:"current-jobs-reserved" stm:"current-jobs-reserved"`
75 + CurrentJobsDelayed int64 `yaml:"current-jobs-delayed" stm:"current-jobs-delayed"`
76 + CurrentJobsBuried int64 `yaml:"current-jobs-buried" stm:"current-jobs-buried"`
77 + TotalJobs int64 `yaml:"total-jobs" stm:"total-jobs"`
78 + CurrentUsing int64 `yaml:"current-using" stm:"current-using"`
79 + CurrentWaiting int64 `yaml:"current-waiting" stm:"current-waiting"`
80 + CurrentWatching int64 `yaml:"current-watching" stm:"current-watching"`
81 + Pause float64 `yaml:"pause" stm:"pause"`
82 + CmdDelete int64 `yaml:"cmd-delete" stm:"cmd-delete"`
83 + CmdPauseTube int64 `yaml:"cmd-pause-tube" stm:"cmd-pause-tube"`
84 + PauseTimeLeft float64 `yaml:"pause-time-left" stm:"pause-time-left"`
85 +}
86 +
87 +func newBeanstalkConn(conf Config, log *logger.Logger) beanstalkConn {
88 + return &beanstalkClient{
89 + Logger: log,
90 + client: socket.New(socket.Config{
91 + Address: conf.Address,
92 + ConnectTimeout: conf.Timeout.Duration(),
93 + ReadTimeout: conf.Timeout.Duration(),
94 + WriteTimeout: conf.Timeout.Duration(),
95 + TLSConf: nil,
96 + }),
97 + }
98 +}
99 +
100 +const (
101 + cmdQuit = "quit"
102 + cmdStats = "stats"
103 + cmdListTubes = "list-tubes"
104 + cmdStatsTube = "stats-tube"
105 +)
106 +
107 +type beanstalkClient struct {
108 + *logger.Logger
109 +
110 + client socket.Client
111 +}
112 +
113 +func (c *beanstalkClient) connect() error {
114 + return c.client.Connect()
115 +}
116 +
117 +func (c *beanstalkClient) disconnect() error {
118 + _, _, _ = c.query(cmdQuit)
119 + return c.client.Disconnect()
120 +}
121 +
122 +func (c *beanstalkClient) queryStats() (*beanstalkdStats, error) {
123 + cmd := cmdStats
124 +
125 + resp, data, err := c.query(cmd)
126 + if err != nil {
127 + return nil, err
128 + }
129 + if resp != "OK" {
130 + return nil, fmt.Errorf("command '%s' bad response: %s", cmd, resp)
131 + }
132 +
133 + var stats beanstalkdStats
134 +
135 + if err := yaml.Unmarshal(data, &stats); err != nil {
136 + return nil, err
137 + }
138 +
139 + return &stats, nil
140 +}
141 +
142 +func (c *beanstalkClient) queryListTubes() ([]string, error) {
143 + cmd := cmdListTubes
144 +
145 + resp, data, err := c.query(cmd)
146 + if err != nil {
147 + return nil, err
148 + }
149 + if resp != "OK" {
150 + return nil, fmt.Errorf("command '%s' bad response: %s", cmd, resp)
151 + }
152 +
153 + var tubes []string
154 +
155 + if err := yaml.Unmarshal(data, &tubes); err != nil {
156 + return nil, err
157 + }
158 +
159 + return tubes, nil
160 +}
161 +
162 +func (c *beanstalkClient) queryStatsTube(tubeName string) (*tubeStats, error) {
163 + cmd := fmt.Sprintf("%s %s", cmdStatsTube, tubeName)
164 +
165 + resp, data, err := c.query(cmd)
166 + if err != nil {
167 + return nil, err
168 + }
169 + if resp == "NOT_FOUND" {
170 + return nil, nil
171 + }
172 + if resp != "OK" {
173 + return nil, fmt.Errorf("command '%s' bad response: %s", cmd, resp)
174 + }
175 +
176 + var stats tubeStats
177 + if err := yaml.Unmarshal(data, &stats); err != nil {
178 + return nil, err
179 + }
180 +
181 + return &stats, nil
182 +}
183 +
184 +func (c *beanstalkClient) query(command string) (string, []byte, error) {
185 + var resp string
186 + var length int
187 + var body []byte
188 + var err error
189 +
190 + c.Debugf("executing command: %s", command)
191 +
192 + const limitReadLines = 1000
193 + var num int
194 +
195 + clientErr := c.client.Command(command+"\r\n", func(line []byte) bool {
196 + if resp == "" {
197 + s := string(line)
198 + c.Debugf("command '%s' response: '%s'", command, s)
199 +
200 + resp, length, err = parseResponseLine(s)
201 + if err != nil {
202 + err = fmt.Errorf("command '%s' line '%s': %v", command, s, err)
203 + }
204 + return err == nil && resp == "OK"
205 + }
206 +
207 + if num++; num >= limitReadLines {
208 + err = fmt.Errorf("command '%s': read line limit exceeded (%d)", command, limitReadLines)
209 + return false
210 + }
211 +
212 + body = append(body, line...)
213 + body = append(body, '\n')
214 +
215 + return len(body) < length
216 + })
217 + if clientErr != nil {
218 + return "", nil, fmt.Errorf("command '%s' client error: %v", command, clientErr)
219 + }
220 + if err != nil {
221 + return "", nil, err
222 + }
223 +
224 + return resp, body, nil
225 +}
226 +
227 +func parseResponseLine(line string) (string, int, error) {
228 + parts := strings.Fields(line)
229 + if len(parts) == 0 {
230 + return "", 0, errors.New("empty response")
231 + }
232 +
233 + resp := parts[0]
234 +
235 + if resp != "OK" {
236 + return resp, 0, nil
237 + }
238 +
239 + if len(parts) < 2 {
240 + return "", 0, errors.New("missing bytes count")
241 + }
242 +
243 + length, err := strconv.Atoi(parts[1])
244 + if err != nil {
245 + return "", 0, errors.New("invalid bytes count")
246 + }
247 +
248 + return resp, length, nil
249 +}
src/go/plugin/go.d/modules/beanstalk/collect.go new
+118
@@ -0,0 +1,118 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package beanstalk
4 +
5 +import (
6 + "fmt"
7 + "slices"
8 + "time"
9 +
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/stm"
11 +)
12 +
13 +func (b *Beanstalk) collect() (map[string]int64, error) {
14 + if b.conn == nil {
15 + conn, err := b.establishConn()
16 + if err != nil {
17 + return nil, err
18 + }
19 + b.conn = conn
20 + }
21 +
22 + mx := make(map[string]int64)
23 +
24 + if err := b.collectStats(mx); err != nil {
25 + b.Cleanup()
26 + return nil, err
27 + }
28 + if err := b.collectTubesStats(mx); err != nil {
29 + return mx, err
30 + }
31 +
32 + return mx, nil
33 +}
34 +
35 +func (b *Beanstalk) collectStats(mx map[string]int64) error {
36 + stats, err := b.conn.queryStats()
37 + if err != nil {
38 + return err
39 + }
40 + for k, v := range stm.ToMap(stats) {
41 + mx[k] = v
42 + }
43 + return nil
44 +}
45 +
46 +func (b *Beanstalk) collectTubesStats(mx map[string]int64) error {
47 + now := time.Now()
48 +
49 + if now.Sub(b.lastDiscoverTubesTime) > b.discoverTubesEvery {
50 + tubes, err := b.conn.queryListTubes()
51 + if err != nil {
52 + return err
53 + }
54 +
55 + b.Debugf("discovered tubes (%d): %v", len(tubes), tubes)
56 + v := slices.DeleteFunc(tubes, func(s string) bool { return !b.tubeSr.MatchString(s) })
57 + if len(tubes) != len(v) {
58 + b.Debugf("discovered tubes after filtering (%d): %v", len(v), v)
59 + }
60 +
61 + b.discoveredTubes = v
62 + b.lastDiscoverTubesTime = now
63 + }
64 +
65 + seen := make(map[string]bool)
66 +
67 + for i, tube := range b.discoveredTubes {
68 + if tube == "" {
69 + continue
70 + }
71 +
72 + stats, err := b.conn.queryStatsTube(tube)
73 + if err != nil {
74 + return err
75 + }
76 +
77 + if stats == nil {
78 + b.Infof("tube '%s' stats object not found (tube does not exist)", tube)
79 + b.discoveredTubes[i] = ""
80 + continue
81 + }
82 + if stats.Name == "" {
83 + b.Debugf("tube '%s' stats object has an empty name, ignoring it", tube)
84 + b.discoveredTubes[i] = ""
85 + continue
86 + }
87 +
88 + seen[stats.Name] = true
89 + if !b.seenTubes[stats.Name] {
90 + b.seenTubes[stats.Name] = true
91 + b.addTubeCharts(stats.Name)
92 + }
93 +
94 + px := fmt.Sprintf("tube_%s_", stats.Name)
95 + for k, v := range stm.ToMap(stats) {
96 + mx[px+k] = v
97 + }
98 + }
99 +
100 + for tube := range b.seenTubes {
101 + if !seen[tube] {
102 + delete(b.seenTubes, tube)
103 + b.removeTubeCharts(tube)
104 + }
105 + }
106 +
107 + return nil
108 +}
109 +
110 +func (b *Beanstalk) establishConn() (beanstalkConn, error) {
111 + conn := b.newConn(b.Config, b.Logger)
112 +
113 + if err := conn.connect(); err != nil {
114 + return nil, err
115 + }
116 +
117 + return conn, nil
118 +}
src/go/plugin/go.d/modules/beanstalk/config_schema.json new
+54
@@ -0,0 +1,54 @@
1 +{
2 + "jsonSchema": {
3 + "$schema": "http://json-schema.org/draft-07/schema#",
4 + "title": "Beanstalk collector configuration.",
5 + "type": "object",
6 + "properties": {
7 + "update_every": {
8 + "title": "Update every",
9 + "description": "Data collection interval, measured in seconds.",
10 + "type": "integer",
11 + "minimum": 1,
12 + "default": 1
13 + },
14 + "address": {
15 + "title": "Address",
16 + "description": "The IP address and port where the Beanstalk service listens for connections.",
17 + "type": "string",
18 + "default": "127.0.0.1:11300"
19 + },
20 + "timeout": {
21 + "title": "Timeout",
22 + "description": "Timeout for establishing a connection and communication (reading and writing) in seconds.",
23 + "type": "number",
24 + "minimum": 0.5,
25 + "default": 1
26 + },
27 + "tube_selector": {
28 + "title": "Tube selector",
29 + "description": "Specifies a [pattern](https://github.com/netdata/netdata/tree/master/src/libnetdata/simple_pattern#readme) for which Beanstalk tubes Netdata will collect statistics. Only tubes whose names match the provided pattern will be included.",
30 + "type": "string",
31 + "minimum": 1,
32 + "default": "*"
33 + }
34 + },
35 + "required": [
36 + "address"
37 + ],
38 + "additionalProperties": false,
39 + "patternProperties": {
40 + "^name$": {}
41 + }
42 + },
43 + "uiSchema": {
44 + "uiOptions": {
45 + "fullPage": true
46 + },
47 + "timeout": {
48 + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
49 + },
50 + "tube_selector": {
51 + "ui:help": "Leave blank or use `*` to collect data for all tubes."
52 + }
53 + }
54 +}
src/go/plugin/go.d/modules/beanstalk/init.go new
+29
@@ -0,0 +1,29 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package beanstalk
4 +
5 +import (
6 + "errors"
7 +
8 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/matcher"
9 +)
10 +
11 +func (b *Beanstalk) validateConfig() error {
12 + if b.Address == "" {
13 + return errors.New("beanstalk address is required")
14 + }
15 + return nil
16 +}
17 +
18 +func (b *Beanstalk) initTubeSelector() (matcher.Matcher, error) {
19 + if b.TubeSelector == "" {
20 + return matcher.TRUE(), nil
21 + }
22 +
23 + m, err := matcher.NewSimplePatternsMatcher(b.TubeSelector)
24 + if err != nil {
25 + return nil, err
26 + }
27 +
28 + return m, nil
29 +}
src/go/plugin/go.d/modules/beanstalk/metadata.yaml new
+255
@@ -0,0 +1,255 @@
1 +plugin_name: go.d.plugin
2 +modules:
3 + - meta:
4 + id: collector-go.d.plugin-beanstalk
5 + plugin_name: go.d.plugin
6 + module_name: beanstalk
7 + monitored_instance:
8 + name: Beanstalk
9 + link: https://beanstalkd.github.io/
10 + categories:
11 + - data-collection.message-brokers
12 + icon_filename: "beanstalk.svg"
13 + related_resources:
14 + integrations:
15 + list: []
16 + info_provided_to_referring_integrations:
17 + description: ""
18 + keywords:
19 + - beanstalk
20 + - beanstalkd
21 + - message
22 + most_popular: false
23 + overview:
24 + data_collection:
25 + metrics_description: |
26 + This collector monitors Beanstalk server performance and provides detailed statistics for each tube.
27 + method_description: |
28 + Using the [beanstalkd protocol](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt), it communicates with the Beanstalk daemon to gather essential metrics that help understand the server's performance and activity.
29 + Executed commands:
30 +
31 + - [stats](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L553).
32 + - [list-tubes](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L688).
33 + - [stats-tube](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L497).
34 + supported_platforms:
35 + include: []
36 + exclude: []
37 + multi_instance: true
38 + additional_permissions:
39 + description: ""
40 + default_behavior:
41 + auto_detection:
42 + description: |
43 + By default, it detects Beanstalk instances running on localhost that are listening on port 11300.
44 + limits:
45 + description: ""
46 + performance_impact:
47 + description: ""
48 + setup:
49 + prerequisites:
50 + list: []
51 + configuration:
52 + file:
53 + name: go.d/beanstalk.conf
54 + options:
55 + description: |
56 + The following options can be defined globally: update_every, autodetection_retry.
57 + folding:
58 + title: Config options
59 + enabled: true
60 + list:
61 + - name: update_every
62 + description: Data collection frequency.
63 + default_value: 1
64 + required: false
65 + - name: autodetection_retry
66 + description: Recheck interval in seconds. Zero means no recheck will be scheduled.
67 + default_value: 0
68 + required: false
69 + - name: address
70 + description: The IP address and port where the Beanstalk service listens for connections.
71 + default_value: 127.0.0.1:11300
72 + required: true
73 + - name: timeout
74 + description: Connection, read, and write timeout duration in seconds. The timeout includes name resolution.
75 + default_value: 1
76 + required: false
77 + - name: tube_selector
78 + description: "Specifies a [pattern](https://github.com/netdata/netdata/tree/master/src/libnetdata/simple_pattern#readme) for which Beanstalk tubes Netdata will collect statistics."
79 + default_value: "*"
80 + required: false
81 + examples:
82 + folding:
83 + enabled: true
84 + title: Config
85 + list:
86 + - name: Basic
87 + description: A basic example configuration.
88 + config: |
89 + jobs:
90 + - name: local
91 + address: 127.0.0.1:11300
92 + - name: Multi-instance
93 + description: |
94 + > **Note**: When you define multiple jobs, their names must be unique.
95 +
96 + Collecting metrics from local and remote instances.
97 + config: |
98 + jobs:
99 + - name: local
100 + address: 127.0.0.1:11300
101 +
102 + - name: remote
103 + address: 203.0.113.0:11300
104 + troubleshooting:
105 + problems:
106 + list: []
107 + alerts:
108 + - name: beanstalk_server_buried_jobs
109 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/beanstalkd.conf
110 + metric: beanstalk.current_jobs
111 + info: number of buried jobs across all tubes. You need to manually kick them so they can be processed. Presence of buried jobs in a tube does not affect new jobs.
112 + metrics:
113 + folding:
114 + title: Metrics
115 + enabled: false
116 + description: ""
117 + availability: []
118 + scopes:
119 + - name: global
120 + description: "These metrics refer to the entire monitored application."
121 + labels: []
122 + metrics:
123 + - name: beanstalk.current_jobs
124 + description: Current Jobs
125 + unit: "jobs"
126 + chart_type: stacked
127 + dimensions:
128 + - name: ready
129 + - name: buried
130 + - name: urgent
131 + - name: delayed
132 + - name: reserved
133 + - name: beanstalk.jobs_rate
134 + description: Jobs Rate
135 + unit: "jobs/s"
136 + chart_type: line
137 + dimensions:
138 + - name: created
139 + - name: beanstalk.jobs_timeouts
140 + description: Timed Out Jobs
141 + unit: "jobs/s"
142 + chart_type: line
143 + dimensions:
144 + - name: timeouts
145 + - name: beanstalk.current_tubes
146 + description: Current Tubes
147 + unit: "tubes"
148 + chart_type: line
149 + dimensions:
150 + - name: tubes
151 + - name: beanstalk.commands_rate
152 + description: Commands Rate
153 + unit: "commands/s"
154 + chart_type: stacked
155 + dimensions:
156 + - name: put
157 + - name: peek
158 + - name: peek-ready
159 + - name: peek-delayed
160 + - name: peek-buried
161 + - name: reserve
162 + - name: reserve-with-timeout
163 + - name: touch
164 + - name: use
165 + - name: watch
166 + - name: ignore
167 + - name: delete
168 + - name: bury
169 + - name: kick
170 + - name: stats
171 + - name: stats-job
172 + - name: stats-tube
173 + - name: list-tubes
174 + - name: list-tube-used
175 + - name: list-tubes-watched
176 + - name: pause-tube
177 + - name: beanstalk.current_connections
178 + description: Current Connections
179 + unit: "connections"
180 + chart_type: line
181 + dimensions:
182 + - name: open
183 + - name: producers
184 + - name: workers
185 + - name: waiting
186 + - name: beanstalk.connections_rate
187 + description: Connections Rate
188 + unit: "connections/s"
189 + chart_type: area
190 + dimensions:
191 + - name: created
192 + - name: beanstalk.binlog_records
193 + description: Binlog Records
194 + unit: "records/s"
195 + chart_type: line
196 + dimensions:
197 + - name: written
198 + - name: migrated
199 + - name: beanstalk.cpu_usage
200 + description: Cpu Usage
201 + unit: "percent"
202 + chart_type: stacked
203 + dimensions:
204 + - name: user
205 + - name: system
206 + - name: beanstalk.uptime
207 + description: seconds
208 + unit: "seconds"
209 + chart_type: line
210 + dimensions:
211 + - name: uptime
212 + - name: tube
213 + description: "Metrics related to Beanstalk tubes. This set of metrics is provided for each tube."
214 + labels:
215 + - name: tube_name
216 + description: Tube name.
217 + metrics:
218 + - name: beanstalk.tube_current_jobs
219 + description: Tube Current Jobs
220 + unit: "jobs"
221 + chart_type: stacked
222 + dimensions:
223 + - name: ready
224 + - name: buried
225 + - name: urgent
226 + - name: delayed
227 + - name: reserved
228 + - name: beanstalk.tube_jobs_rate
229 + description: Tube Jobs Rate
230 + unit: "jobs/s"
231 + chart_type: line
232 + dimensions:
233 + - name: created
234 + - name: beanstalk.tube_commands_rate
235 + description: Tube Commands
236 + unit: "commands/s"
237 + chart_type: stacked
238 + dimensions:
239 + - name: delete
240 + - name: pause-tube
241 + - name: beanstalk.tube_current_connections
242 + description: Tube Current Connections
243 + unit: "connections"
244 + chart_type: stacked
245 + dimensions:
246 + - name: using
247 + - name: waiting
248 + - name: watching
249 + - name: beanstalk.tube_pause_time
250 + description: Tube Pause Time
251 + unit: "seconds"
252 + chart_type: line
253 + dimensions:
254 + - name: since
255 + - name: left
src/go/plugin/go.d/modules/beanstalk/testdata/config.json new
+6
@@ -0,0 +1,6 @@
1 +{
2 + "update_every": 123,
3 + "address": "ok",
4 + "timeout": 123.123,
5 + "tube_selector": "ok"
6 +}
src/go/plugin/go.d/modules/beanstalk/testdata/config.yaml new
+4
@@ -0,0 +1,4 @@
1 +update_every: 123
2 +address: "ok"
3 +timeout: 123.123
4 +tube_selector: "ok"
src/go/plugin/go.d/modules/beanstalk/testdata/list-tubes.txt new
+3
@@ -0,0 +1,3 @@
1 +OK 14
2 +---
3 +- default
src/go/plugin/go.d/modules/beanstalk/testdata/stats-tube-default.txt new
+16
@@ -0,0 +1,16 @@
1 +OK 265
2 +---
3 +name: default
4 +current-jobs-urgent: 0
5 +current-jobs-ready: 0
6 +current-jobs-reserved: 0
7 +current-jobs-delayed: 0
8 +current-jobs-buried: 0
9 +total-jobs: 0
10 +current-using: 2
11 +current-watching: 2
12 +current-waiting: 0
13 +cmd-delete: 0
14 +cmd-pause-tube: 0
15 +pause: 0
16 +pause-time-left: 0
src/go/plugin/go.d/modules/beanstalk/testdata/stats.txt new
+50
@@ -0,0 +1,50 @@
1 +OK 913
2 +---
3 +current-jobs-urgent: 0
4 +current-jobs-ready: 0
5 +current-jobs-reserved: 0
6 +current-jobs-delayed: 0
7 +current-jobs-buried: 0
8 +cmd-put: 0
9 +cmd-peek: 0
10 +cmd-peek-ready: 0
11 +cmd-peek-delayed: 0
12 +cmd-peek-buried: 0
13 +cmd-reserve: 0
14 +cmd-reserve-with-timeout: 0
15 +cmd-delete: 0
16 +cmd-release: 0
17 +cmd-use: 0
18 +cmd-watch: 0
19 +cmd-ignore: 0
20 +cmd-bury: 0
21 +cmd-kick: 0
22 +cmd-touch: 0
23 +cmd-stats: 23619
24 +cmd-stats-job: 0
25 +cmd-stats-tube: 18964
26 +cmd-list-tubes: 317
27 +cmd-list-tube-used: 0
28 +cmd-list-tubes-watched: 0
29 +cmd-pause-tube: 0
30 +job-timeouts: 0
31 +total-jobs: 0
32 +max-job-size: 65535
33 +current-tubes: 1
34 +current-connections: 2
35 +current-producers: 0
36 +current-workers: 0
37 +current-waiting: 0
38 +total-connections: 72
39 +pid: 1
40 +version: 1.10
41 +rusage-utime: 1.602079
42 +rusage-stime: 3.922748
43 +uptime: 105881
44 +binlog-oldest-index: 0
45 +binlog-current-index: 0
46 +binlog-records-migrated: 0
47 +binlog-records-written: 0
48 +binlog-max-size: 10485760
49 +id: 5a0667a881cd05e0
50 +hostname: c6796814b94b
src/go/plugin/go.d/modules/init.go
+1
@@ -7,6 +7,7 @@ import (
7 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/adaptecraid"
8 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/ap"
9 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/apache"
10 + _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/beanstalk"
11 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/bind"
12 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/cassandra"
13 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/chrony"
src/health/health.d/beanstalkd.conf
+2 -28
@@ -11,31 +11,5 @@ component: Beanstalk
11 warn: $this > 3
12 delay: up 0 down 5m multiplier 1.2 max 1h
13 summary: Beanstalk buried jobs
14 - info: Number of buried jobs across all tubes. \
15 - You need to manually kick them so they can be processed. \
16 - Presence of buried jobs in a tube does not affect new jobs.
17 - to: sysadmin
18 -
19 -# get the number of buried jobs per queue
20 -
21 -#template: beanstalk_tube_buried_jobs
22 -# on: beanstalk.jobs
23 -# calc: $buried
24 -# units: jobs
25 -# every: 10s
26 -# warn: $this > 0
27 -# crit: $this > 10
28 -# delay: up 0 down 5m multiplier 1.2 max 1h
29 -# info: the number of jobs buried per tube
30 -# to: sysadmin
31 -
32 -# get the current number of tubes
33 -
34 -#template: beanstalk_number_of_tubes
35 -# on: beanstalk.current_tubes
36 -# calc: $tubes
37 -# every: 10s
38 -# warn: $this < 5
39 -# delay: up 0 down 5m multiplier 1.2 max 1h
40 -# info: the current number of tubes on the server
41 -# to: sysadmin
14 + info: Number of buried jobs across all tubes.
15 + to: silent