Gearman plugin for Netdata (#6567)
Added Gearman plugin and alarms
Kyle Agronick committed
Sep 17, 2019 at 06:34 UTC
9c256c81189af3f95d186c207689b8d206d1e225
9 files changed
+383
collectors/python.d.plugin/Makefile.am
+1
@@ -54,6 +54,7 @@ include example/Makefile.inc
54
include exim/Makefile.inc
55
include fail2ban/Makefile.inc
56
include freeradius/Makefile.inc
57
+include gearman/Makefile.inc
58
include go_expvar/Makefile.inc
59
include haproxy/Makefile.inc
60
include hddtemp/Makefile.inc
collectors/python.d.plugin/gearman/Makefile.inc
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+# THIS IS NOT A COMPLETE Makefile
4
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
5
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
6
+
7
+# install these files
8
+dist_python_DATA += gearman/gearman.chart.py
9
+dist_pythonconfig_DATA += gearman/gearman.conf
10
+
11
+# do not install these files, but include them in the distribution
12
+dist_noinst_DATA += gearman/README.md gearman/Makefile.inc
collectors/python.d.plugin/gearman/README.md
new
+39
@@ -0,0 +1,39 @@
1
+# Gearman
2
+
3
+Module monitors Gearman worker statistics. A chart
4
+is shown for each job as well as one showing a summary
5
+of all workers.
6
+
7
+Note: Charts may show as a line graph rather than an area
8
+graph if you load Netdata with no jobs running. To change
9
+this go to "Settings" > "Which dimensions to show?" and
10
+select "All".
11
+
12
+Plugin can obtain data from tcp socket **OR** unix socket.
13
+
14
+**Requirement:**
15
+Socket MUST be readable by netdata user.
16
+
17
+It produces:
18
+
19
+ * Workers queued
20
+ * Workers idle
21
+ * Workers running
22
+
23
+### configuration
24
+
25
+```yaml
26
+localhost:
27
+ name : 'local'
28
+ host : 'localhost'
29
+ port : 4730
30
+
31
+ # TLS information can be provided as well
32
+ tls : no
33
+ cert : /path/to/cert
34
+ key : /path/to/key
35
+```
36
+
37
+When no configuration file is found, module tries to connect to TCP/IP socket: `localhost:4730`.
38
+
39
+---
collectors/python.d.plugin/gearman/gearman.chart.py
new
+229
@@ -0,0 +1,229 @@
1
+# Description: dovecot netdata python.d module
2
+# Author: Kyle Agronick (agronick)
3
+# SPDX-License-Identifier: GPL-3.0+
4
+
5
+# Gearman Netdata Plugin
6
+
7
+from bases.FrameworkServices.SocketService import SocketService
8
+from copy import deepcopy
9
+
10
+
11
+CHARTS = {
12
+ 'total_workers': {
13
+ 'options': [None, 'Total Jobs', 'Jobs', 'Total Jobs', 'gearman.total_jobs', 'line'],
14
+ 'lines': [
15
+ ['total_pending', 'Pending', 'absolute'],
16
+ ['total_running', 'Running', 'absolute'],
17
+ ]
18
+ },
19
+}
20
+
21
+
22
+def job_chart_template(job_name):
23
+ return {
24
+ 'options': [None, job_name, 'Jobs', 'Activity by Job', 'gearman.single_job', 'stacked'],
25
+ 'lines': [
26
+ ['{0}_pending'.format(job_name), 'Pending', 'absolute'],
27
+ ['{0}_idle'.format(job_name), 'Idle', 'absolute'],
28
+ ['{0}_running'.format(job_name), 'Running', 'absolute'],
29
+ ]
30
+ }
31
+
32
+def build_result_dict(job):
33
+ """
34
+ Get the status for each job
35
+ :return: dict
36
+ """
37
+
38
+ total, running, available = job['metrics']
39
+
40
+ idle = available - running
41
+ pending = total - running
42
+
43
+ return {
44
+ '{0}_pending'.format(job['job_name']): pending,
45
+ '{0}_idle'.format(job['job_name']): idle,
46
+ '{0}_running'.format(job['job_name']): running,
47
+ }
48
+
49
+def parse_worker_data(job):
50
+ job_name = job[0]
51
+ job_metrics = job[1:]
52
+
53
+ return {
54
+ 'job_name': job_name,
55
+ 'metrics': job_metrics,
56
+ }
57
+
58
+
59
+class GearmanReadException(BaseException):
60
+ pass
61
+
62
+
63
+class Service(SocketService):
64
+ def __init__(self, configuration=None, name=None):
65
+ super(Service, self).__init__(configuration=configuration, name=name)
66
+ self.request = "status\n"
67
+ self._keep_alive = True
68
+
69
+ self.host = self.configuration.get('host', 'localhost')
70
+ self.port = self.configuration.get('port', 4730)
71
+
72
+ self.tls = self.configuration.get('tls', False)
73
+ self.cert = self.configuration.get('cert', None)
74
+ self.key = self.configuration.get('key', None)
75
+
76
+ self.active_jobs = set()
77
+ self.definitions = deepcopy(CHARTS)
78
+ self.order = ['total_workers']
79
+
80
+ def _get_data(self):
81
+ """
82
+ Format data received from socket
83
+ :return: dict
84
+ """
85
+
86
+ try:
87
+ active_jobs = self.get_active_jobs()
88
+ except GearmanReadException:
89
+ return None
90
+
91
+ found_jobs, job_data = self.process_jobs(active_jobs)
92
+ self.remove_stale_jobs(found_jobs)
93
+ return job_data
94
+
95
+ def get_active_jobs(self):
96
+ active_jobs = []
97
+
98
+ for job in self.get_worker_data():
99
+ parsed_job = parse_worker_data(job)
100
+
101
+ # Gearman does not clean up old jobs
102
+ # We only care about jobs that have
103
+ # some relevant data
104
+ if not any(parsed_job['metrics']):
105
+ continue
106
+
107
+ active_jobs.append(parsed_job)
108
+
109
+ return active_jobs
110
+
111
+ def get_worker_data(self):
112
+ """
113
+ Split the data returned from Gearman
114
+ into a list of lists
115
+
116
+ This returns the same output that you
117
+ would get from a gearadmin --status
118
+ command.
119
+
120
+ Example output returned from
121
+ _get_raw_data():
122
+ generic_worker2 78 78 500
123
+ generic_worker3 0 0 760
124
+ generic_worker1 0 0 500
125
+
126
+ :return: list
127
+ """
128
+
129
+ try:
130
+ raw = self._get_raw_data()
131
+ except (ValueError, AttributeError):
132
+ raise GearmanReadException()
133
+
134
+ if raw is None:
135
+ self.debug("Gearman returned no data")
136
+ raise GearmanReadException()
137
+
138
+ job_lines = raw.splitlines()[:-1]
139
+ job_lines = [job.split() for job in sorted(job_lines)]
140
+
141
+ for line in job_lines:
142
+ line[1:] = map(int, line[1:])
143
+
144
+ return job_lines
145
+
146
+ def process_jobs(self, active_jobs):
147
+
148
+ output = {
149
+ 'total_pending': 0,
150
+ 'total_idle': 0,
151
+ 'total_running': 0,
152
+ }
153
+ found_jobs = set()
154
+
155
+ for parsed_job in active_jobs:
156
+
157
+ job_name = self.add_job(parsed_job)
158
+ found_jobs.add(job_name)
159
+ job_data = build_result_dict(parsed_job)
160
+
161
+ for sum_value in ('pending', 'running', 'idle'):
162
+ output['total_{0}'.format(sum_value)] += job_data['{0}_{1}'.format(job_name, sum_value)]
163
+
164
+ output.update(job_data)
165
+
166
+ return found_jobs, output
167
+
168
+ def remove_stale_jobs(self, active_job_list):
169
+ """
170
+ Removes jobs that have no workers, pending jobs,
171
+ or running jobs
172
+ :param active_job_list: The latest list of active jobs
173
+ :type active_job_list: iterable
174
+ :return: None
175
+ """
176
+
177
+ for to_remove in self.active_jobs - active_job_list:
178
+ self.remove_job(to_remove)
179
+
180
+ def add_job(self, parsed_job):
181
+ """
182
+ Adds a job to the list of active jobs
183
+ :param parsed_job: A parsed job dict
184
+ :type parsed_job: dict
185
+ :return: None
186
+ """
187
+
188
+ def add_chart(job_name):
189
+ """
190
+ Adds a new job chart
191
+ :param job_name: The name of the job to add
192
+ :type job_name: string
193
+ :return: None
194
+ """
195
+
196
+ job_key = 'job_{0}'.format(job_name)
197
+ template = job_chart_template(job_name)
198
+ new_chart = self.charts.add_chart([job_key] + template['options'])
199
+ for dimension in template['lines']:
200
+ new_chart.add_dimension(dimension)
201
+
202
+ if parsed_job['job_name'] not in self.active_jobs:
203
+ add_chart(parsed_job['job_name'])
204
+ self.active_jobs.add(parsed_job['job_name'])
205
+
206
+ return parsed_job['job_name']
207
+
208
+ def remove_job(self, job_name):
209
+ """
210
+ Removes a job to the list of active jobs
211
+ :param job_name: The name of the job to remove
212
+ :type job_name: string
213
+ :return: None
214
+ """
215
+
216
+ def remove_chart(job_name):
217
+ """
218
+ Removes a job chart
219
+ :param job_name: The name of the job to remove
220
+ :type job_name: string
221
+ :return: None
222
+ """
223
+
224
+ job_key = 'job_{0}'.format(job_name)
225
+ self.charts[job_key].obsolete()
226
+ del self.charts[job_key]
227
+
228
+ remove_chart(job_name)
229
+ self.active_jobs.remove(job_name)
collectors/python.d.plugin/gearman/gearman.conf
new
+72
@@ -0,0 +1,72 @@
1
+# netdata python.d.plugin configuration for gearman
2
+#
3
+# This file is in YaML format. Generally the format is:
4
+#
5
+# name: value
6
+#
7
+# There are 2 sections:
8
+# - global variables
9
+# - one or more JOBS
10
+#
11
+# JOBS allow you to collect values from multiple sources.
12
+# Each source will have its own set of charts.
13
+#
14
+# JOB parameters have to be indented (using spaces only, example below).
15
+
16
+# ----------------------------------------------------------------------
17
+# Global Variables
18
+# These variables set the defaults for all JOBs, however each JOB
19
+# may define its own, overriding the defaults.
20
+
21
+# update_every sets the default data collection frequency.
22
+# If unset, the python.d.plugin default is used.
23
+# update_every: 1
24
+
25
+# priority controls the order of charts at the netdata dashboard.
26
+# Lower numbers move the charts towards the top of the page.
27
+# If unset, the default for python.d.plugin is used.
28
+# priority: 60000
29
+
30
+# penalty indicates whether to apply penalty to update_every in case of failures.
31
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
32
+# penalty: yes
33
+
34
+# autodetection_retry sets the job re-check interval in seconds.
35
+# The job is not deleted if check fails.
36
+# Attempts to start the job are made once every autodetection_retry.
37
+# This feature is disabled by default.
38
+# autodetection_retry: 0
39
+
40
+# ----------------------------------------------------------------------
41
+# JOBS (data collection sources)
42
+#
43
+# The default JOBS share the same *name*. JOBS with the same name
44
+# are mutually exclusive. Only one of them will be allowed running at
45
+# any time. This allows autodetection to try several alternatives and
46
+# pick the one that works.
47
+#
48
+# Any number of jobs is supported.
49
+#
50
+# All python.d.plugin JOBS (for all its modules) support a set of
51
+# predefined parameters. These are:
52
+#
53
+# job_name:
54
+# name: myname # the JOB's name as it will appear at the
55
+# # dashboard (by default is the job_name)
56
+# # JOBs sharing a name are mutually exclusive
57
+# update_every: 1 # the JOB's data collection frequency
58
+# priority: 60000 # the JOB's order on the dashboard
59
+# penalty: yes # the JOB's penalty
60
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
61
+#
62
+# Additionally to the above, gearman also supports the following:
63
+#
64
+# hostname: localhost # The host running the Gearman server
65
+# port: 4730 # Port of the Gearman server
66
+# ----------------------------------------------------------------------
67
+# AUTO-DETECTION JOB
68
+
69
+localhost:
70
+ name : 'local'
71
+ host : 'localhost'
72
+ port : 4730
\ No newline at end of file
collectors/python.d.plugin/python.d.conf
+1
@@ -49,6 +49,7 @@ example: no
49
# exim: yes
50
# fail2ban: yes
51
# freeradius: yes
52
+# gearman: yes
53
go_expvar: no
54
55
# gunicorn_log has been replaced by web_log
health/Makefile.am
+1
@@ -42,6 +42,7 @@ dist_healthconfig_DATA = \
42
health.d/fping.conf \
43
health.d/ioping.conf \
44
health.d/fronius.conf \
45
+ health.d/gearman.conf \
46
health.d/haproxy.conf \
47
health.d/httpcheck.conf \
48
health.d/ipc.conf \
health/health.d/gearman.conf
new
+22
@@ -0,0 +1,22 @@
1
+# make sure Gearman is running
2
+template: gearman_last_collected_secs
3
+ on: gearman.total_jobs
4
+ calc: $now - $last_collected_t
5
+ units: seconds ago
6
+ every: 10s
7
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
8
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
+ delay: down 5m multiplier 1.5 max 1h
10
+ info: number of seconds since the last successful data collection
11
+ to: sysadmin
12
+
13
+template: gearman_workers_queued
14
+ on: gearman.single_job
15
+ lookup: average -10m unaligned match-names of Queued
16
+ units: workers
17
+ every: 10s
18
+ warn: $this > 30000
19
+ crit: $this > 100000
20
+ delay: down 5m multiplier 1.5 max 1h
21
+ info: number of queued jobs
22
+ to: sysadmin
\ No newline at end of file
web/gui/dashboard_info.js
+6
@@ -213,6 +213,12 @@ netdataDashboard.menu = {
213
info: 'Network latency statistics, via <b>fping</b>. <b>fping</b> is a program to send ICMP echo probes to network hosts, similar to <code>ping</code>, but much better performing when pinging multiple hosts. fping versions after 3.15 can be directly used as netdata plugins.'
214
},
215
216
+ 'gearman': {
217
+ title: 'Gearman',
218
+ icon: '<i class="fas fa-tasks"></i>',
219
+ info: 'Gearman is a job server that allows you to do work in parallel, to load balance processing, and to call functions between languages.'
220
+ },
221
+
222
'ioping': {
223
title: 'ioping',
224
icon: '<i class="fas fa-exchange-alt"></i>',