Added uwsgi plugin (#4404)
* Added uwsgi plugin * Added makefile for plugin and fixed some code style issues * Refactored generating dimensions to allow workers to be added at run-time. The dimension name is now the PID of the worker. exceptions, harikiris and respawns are now global and not per worker as this did not make any sense. * Code-review related formatting changes
robbert-ef committed
Oct 17, 2018 at 16:17 UTC
ad3ff17ac19546d24812daa6f0eca62301e296fa
6 files changed
+329
collectors/python.d.plugin/Makefile.am
+1
@@ -95,6 +95,7 @@ include squid/Makefile.inc
95
include tomcat/Makefile.inc
96
include traefik/Makefile.inc
97
include unbound/Makefile.inc
98
+include uwsgi/Makefile.inc
99
include varnish/Makefile.inc
100
include w1sensor/Makefile.inc
101
include web_log/Makefile.inc
collectors/python.d.plugin/python.d.conf
+1
@@ -90,6 +90,7 @@ nginx_log: no
90
# springboot: yes
91
# tomcat: yes
92
unbound: no
93
+# uwsgi: yes
94
# varnish: yes
95
# web_log: yes
96
# w1sensor: yes
collectors/python.d.plugin/uwsgi/Makefile.inc
new
+13
@@ -0,0 +1,13 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+# THIS IS NOT A COMPLETE Makefile
4
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
5
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
6
+
7
+# install these files
8
+dist_python_DATA += uwsgi/uwsgi.chart.py
9
+dist_pythonconfig_DATA += uwsgi/uwsgi.conf
10
+
11
+# do not install these files, but include them in the distribution
12
+dist_noinst_DATA += uwsgi/README.md uwsgi/Makefile.inc
13
+
collectors/python.d.plugin/uwsgi/README.md
new
+37
@@ -0,0 +1,37 @@
1
+# uwsgi
2
+
3
+Module monitor uwsgi performance metrics.
4
+
5
+https://uwsgi-docs.readthedocs.io/en/latest/StatsServer.html
6
+
7
+lines are creates dynamically based on how many workers are there
8
+
9
+Following charts are drawn:
10
+
11
+1. **Requests**
12
+ * requests per second
13
+ * transmitted data
14
+ * average request time
15
+
16
+2. **Memory**
17
+ * rss
18
+ * vsz
19
+
20
+3. **Exceptions**
21
+4. **Harakiris**
22
+5. **Respawns**
23
+
24
+### configuration
25
+
26
+```yaml
27
+socket:
28
+ name : 'local'
29
+ socket : '/tmp/stats.socket'
30
+
31
+localhost:
32
+ name : 'local'
33
+ host : 'localhost'
34
+ port : 1717
35
+```
36
+
37
+When no configuration file is found, module tries to connect to TCP/IP socket: `localhost:1717`.
collectors/python.d.plugin/uwsgi/uwsgi.chart.py
new
+183
@@ -0,0 +1,183 @@
1
+# -*- coding: utf-8 -*-
2
+# Description: uwsgi netdata python.d module
3
+# Author: Robbert Segeren (robbert-ef)
4
+# SPDX-License-Identifier: GPL-3.0-or-later
5
+
6
+import json
7
+from copy import deepcopy
8
+from bases.FrameworkServices.SocketService import SocketService
9
+
10
+# default module values (can be overridden per job in `config`)
11
+# update_every = 2
12
+priority = 60000
13
+retries = 60
14
+
15
+ORDER = [
16
+ 'requests',
17
+ 'tx',
18
+ 'avg_rt',
19
+ 'memory_rss',
20
+ 'memory_vsz',
21
+ 'exceptions',
22
+ 'harakiri',
23
+ 'respawn',
24
+]
25
+
26
+DYNAMIC_CHARTS = [
27
+ 'requests',
28
+ 'tx',
29
+ 'avg_rt',
30
+ 'memory_rss',
31
+ 'memory_vsz',
32
+]
33
+
34
+# NOTE: lines are created dynamically in `check()` method
35
+CHARTS = {
36
+ 'requests': {
37
+ 'options': [None, 'Requests', 'requests/s', 'requests', 'uwsgi.requests', 'stacked'],
38
+ 'lines': [
39
+ ['requests', 'requests', 'incremental']
40
+ ]
41
+ },
42
+ 'tx': {
43
+ 'options': [None, 'Transmitted data', 'KB/s', 'requests', 'uwsgi.tx', 'stacked'],
44
+ 'lines': [
45
+ ['tx', 'tx', 'incremental']
46
+ ]
47
+ },
48
+ 'avg_rt': {
49
+ 'options': [None, 'Average request time', 'ms', 'requests', 'uwsgi.avg_rt', 'line'],
50
+ 'lines': [
51
+ ['avg_rt', 'avg_rt', 'absolute']
52
+ ]
53
+ },
54
+ 'memory_rss': {
55
+ 'options': [None, 'RSS (Resident Set Size)', 'MB', 'memory', 'uwsgi.memory_rss', 'stacked'],
56
+ 'lines': [
57
+ ['memory_rss', 'memory_rss', 'absolute', 1, 1024 * 1024]
58
+ ]
59
+ },
60
+ 'memory_vsz': {
61
+ 'options': [None, 'VSZ (Virtual Memory Size)', 'MB', 'memory', 'uwsgi.memory_vsz', 'stacked'],
62
+ 'lines': [
63
+ ['memory_vsz', 'memory_vsz', 'absolute', 1, 1024 * 1024]
64
+ ]
65
+ },
66
+ 'exceptions': {
67
+ 'options': [None, 'Exceptions', 'exceptions', 'exceptions', 'uwsgi.exceptions', 'line'],
68
+ 'lines': [
69
+ ['exceptions', 'exceptions', 'incremental']
70
+ ]
71
+ },
72
+ 'harakiri': {
73
+ 'options': [None, 'Harakiris', 'harakiris', 'harakiris', 'uwsgi.harakiris', 'line'],
74
+ 'lines': [
75
+ ['harakiri_count', 'harakiris', 'incremental']
76
+ ]
77
+ },
78
+ 'respawn': {
79
+ 'options': [None, 'Respawns', 'respawns', 'respawns', 'uwsgi.respawns', 'line'],
80
+ 'lines': [
81
+ ['respawn_count', 'respawns', 'incremental']
82
+ ]
83
+ },
84
+}
85
+
86
+
87
+class Service(SocketService):
88
+ def __init__(self, configuration=None, name=None):
89
+ super(Service, self).__init__(configuration=configuration, name=name)
90
+ self.url = self.configuration.get('host', 'localhost')
91
+ self.port = self.configuration.get('port', 1717)
92
+ self.order = ORDER
93
+ self.definitions = deepcopy(CHARTS)
94
+
95
+ # Clear dynamic dimensions, these are added during `_get_data()` to allow adding workers at run-time
96
+ for chart in DYNAMIC_CHARTS:
97
+ self.definitions[chart]['lines'] = []
98
+
99
+ self.last_result = {}
100
+ self.workers = []
101
+
102
+ def read_data(self):
103
+ """
104
+ Read data from socket and parse as JSON.
105
+ :return: (dict) stats
106
+ """
107
+ raw_data = self._get_raw_data()
108
+ if not raw_data:
109
+ return None
110
+ try:
111
+ return json.loads(raw_data)
112
+ except ValueError as err:
113
+ self.error(err)
114
+ return None
115
+
116
+ def check(self):
117
+ """
118
+ Parse configuration and check if we can read data.
119
+ :return: boolean
120
+ """
121
+ self._parse_config()
122
+ return bool(self.read_data())
123
+
124
+ def add_worker_dimensions(self, key):
125
+ """
126
+ Helper to add dimensions for a worker.
127
+ :param key: (int or str) worker identifier
128
+ :return:
129
+ """
130
+ for chart in DYNAMIC_CHARTS:
131
+ for line in CHARTS[chart]['lines']:
132
+ dimension_id = '{}_{}'.format(line[0], key)
133
+ dimension_name = str(key)
134
+
135
+ dimension = [dimension_id, dimension_name] + line[2:]
136
+ self.charts[chart].add_dimension(dimension)
137
+
138
+ @staticmethod
139
+ def _check_raw_data(data):
140
+ # The server will close the connection when it's done sending
141
+ # data, so just keep looping until that happens.
142
+ return False
143
+
144
+ def _get_data(self):
145
+ """
146
+ Read data from socket
147
+ :return: dict
148
+ """
149
+ stats = self.read_data()
150
+ if not stats:
151
+ return None
152
+
153
+ result = {
154
+ 'exceptions': 0,
155
+ 'harakiri_count': 0,
156
+ 'respawn_count': 0,
157
+ }
158
+
159
+ for worker in stats['workers']:
160
+ key = worker['pid']
161
+
162
+ # Add dimensions for new workers
163
+ if key not in self.workers:
164
+ self.add_worker_dimensions(key)
165
+ self.workers.append(key)
166
+
167
+ result['requests_{}'.format(key)] = worker['requests']
168
+ result['tx_{}'.format(key)] = worker['tx']
169
+ result['avg_rt_{}'.format(key)] = worker['avg_rt']
170
+
171
+ # avg_rt is not reset by uwsgi, so reset here
172
+ if self.last_result.get('requests_{}'.format(key)) == worker['requests']:
173
+ result['avg_rt_{}'.format(key)] = 0
174
+
175
+ result['memory_rss_{}'.format(key)] = worker['rss']
176
+ result['memory_vsz_{}'.format(key)] = worker['vsz']
177
+
178
+ result['exceptions'] += worker['exceptions']
179
+ result['harakiri_count'] += worker['harakiri_count']
180
+ result['respawn_count'] += worker['respawn_count']
181
+
182
+ self.last_result = result
183
+ return result
collectors/python.d.plugin/uwsgi/uwsgi.conf
new
+94
@@ -0,0 +1,94 @@
1
+# netdata python.d.plugin configuration for uwsgi
2
+#
3
+# This file is in YaML format. Generally the format is:
4
+#
5
+# name: value
6
+#
7
+# There are 2 sections:
8
+# - global variables
9
+# - one or more JOBS
10
+#
11
+# JOBS allow you to collect values from multiple sources.
12
+# Each source will have its own set of charts.
13
+#
14
+# JOB parameters have to be indented (using spaces only, example below).
15
+
16
+# ----------------------------------------------------------------------
17
+# Global Variables
18
+# These variables set the defaults for all JOBs, however each JOB
19
+# may define its own, overriding the defaults.
20
+
21
+# update_every sets the default data collection frequency.
22
+# If unset, the python.d.plugin default is used.
23
+# update_every: 1
24
+
25
+# priority controls the order of charts at the netdata dashboard.
26
+# Lower numbers move the charts towards the top of the page.
27
+# If unset, the default for python.d.plugin is used.
28
+# priority: 60000
29
+
30
+# retries sets the number of retries to be made in case of failures.
31
+# If unset, the default for python.d.plugin is used.
32
+# Attempts to restore the service are made once every update_every
33
+# and only if the module has collected values in the past.
34
+# retries: 60
35
+
36
+# autodetection_retry sets the job re-check interval in seconds.
37
+# The job is not deleted if check fails.
38
+# Attempts to start the job are made once every autodetection_retry.
39
+# This feature is disabled by default.
40
+# autodetection_retry: 0
41
+
42
+# ----------------------------------------------------------------------
43
+# JOBS (data collection sources)
44
+#
45
+# The default JOBS share the same *name*. JOBS with the same name
46
+# are mutually exclusive. Only one of them will be allowed running at
47
+# any time. This allows autodetection to try several alternatives and
48
+# pick the one that works.
49
+#
50
+# Any number of jobs is supported.
51
+#
52
+# All python.d.plugin JOBS (for all its modules) support a set of
53
+# predefined parameters. These are:
54
+#
55
+# job_name:
56
+# name: myname # the JOB's name as it will appear at the
57
+# # dashboard (by default is the job_name)
58
+# # JOBs sharing a name are mutually exclusive
59
+# update_every: 1 # the JOB's data collection frequency
60
+# priority: 60000 # the JOB's order on the dashboard
61
+# retries: 60 # the JOB's number of restoration attempts
62
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
63
+#
64
+# Additionally to the above, uwsgi also supports the following:
65
+#
66
+# socket: 'path/to/uwsgistats.sock'
67
+#
68
+# or
69
+# host: 'IP or HOSTNAME' # the host to connect to
70
+# port: PORT # the port to connect to
71
+#
72
+# ----------------------------------------------------------------------
73
+# AUTO-DETECTION JOBS
74
+# only one of them will run (they have the same name)
75
+#
76
+
77
+socket:
78
+ name : 'local'
79
+ socket : '/tmp/stats.socket'
80
+
81
+localhost:
82
+ name : 'local'
83
+ host : 'localhost'
84
+ port : 1717
85
+
86
+localipv4:
87
+ name : 'local'
88
+ host : '127.0.0.1'
89
+ port : 1717
90
+
91
+localipv6:
92
+ name : 'local'
93
+ host : '::1'
94
+ port : 1717