@cryptotaxi247 / netdata-1 / commits / f7669380f

php_fpm_plugin: avg dimension for cpu,mem,dur charts added + bugfixes

Ilya committed Mar 4, 2017 at 19:09 UTC f7669380f30666f260600f287109ad9a38217e9e
1 file changed +106 -77
python.d/phpfpm.chart.py
+106 -77
@@ -4,6 +4,7 @@
4
5 from base import UrlService
6 import json
7 +import re
8
9 # default module values (can be overridden per job in `config`)
10 # update_every = 2
@@ -19,44 +20,75 @@ retries = 60
20 # }}
21
22 # charts order (can be overridden if you want less charts, or different order)
23 +
24 +POOL_INFO = [
25 + ('active processes', 'active'),
26 + ('max active processes', 'maxActive'),
27 + ('idle processes', 'idle'),
28 + ('accepted conn', 'requests'),
29 + ('max children reached', 'reached'),
30 + ('slow requests', 'slow')
31 +]
32 +
33 +PER_PROCESS_INFO = [
34 + ('request duration', 'ReqDur'),
35 + ('last request cpu', 'ReqCpu'),
36 + ('last request memory', 'ReqMem')
37 +]
38 +
39 +
40 +def average(collection):
41 + return sum(collection, 0.0) / max(len(collection), 1)
42 +
43 +CALC = [
44 + ('min', min),
45 + ('max', max),
46 + ('avg', average)
47 +]
48 +
49 ORDER = ['connections', 'requests', 'performance', 'request_duration', 'request_cpu', 'request_mem']
50
51 CHARTS = {
52 'connections': {
26 - 'options': [None, 'PHP-FPM Active Connections', 'connections', 'active connections', 'phpfpm.connections', 'line'],
53 + 'options': [None, 'PHP-FPM Active Connections', 'connections', 'active connections', 'phpfpm.connections',
54 + 'line'],
55 'lines': [
28 - ["active"],
29 - ["maxActive", 'max active'],
30 - ["idle"]
56 + ['active'],
57 + ['maxActive', 'max active'],
58 + ['idle']
59 ]},
60 'requests': {
61 'options': [None, 'PHP-FPM Requests', 'requests/s', 'requests', 'phpfpm.requests', 'line'],
62 'lines': [
35 - ["requests", None, "incremental"]
63 + ['requests', None, 'incremental']
64 ]},
65 'performance': {
66 'options': [None, 'PHP-FPM Performance', 'status', 'performance', 'phpfpm.performance', 'line'],
67 'lines': [
40 - ["reached", 'max children reached'],
41 - ["slow", 'slow requests']
68 + ['reached', 'max children reached'],
69 + ['slow', 'slow requests']
70 ]},
71 'request_duration': {
44 - 'options': [None, 'PHP-FPM Request Duration', 'milliseconds', 'request duration', 'phpfpm.request_duration', 'line'],
72 + 'options': [None, 'PHP-FPM Request Duration', 'milliseconds', 'request duration', 'phpfpm.request_duration',
73 + 'line'],
74 'lines': [
46 - ["maxReqDur", 'max request duration'],
47 - ["avgReqDur", 'average request duration']
75 + ['minReqDur', 'min', 'absolute', 1, 1000],
76 + ['maxReqDur', 'max', 'absolute', 1, 1000],
77 + ['avgReqDur', 'avg', 'absolute', 1, 1000]
78 ]},
79 'request_cpu': {
80 'options': [None, 'PHP-FPM Request CPU', 'percent', 'request CPU', 'phpfpm.request_cpu', 'line'],
81 'lines': [
52 - ["maxReqCPU", 'max request cpu'],
53 - ["avgReqCPU", 'average request cpu']
82 + ['minReqCpu', 'min'],
83 + ['maxReqCpu', 'max'],
84 + ['avgReqCpu', 'avg']
85 ]},
86 'request_mem': {
87 'options': [None, 'PHP-FPM Request Memory', 'kilobytes', 'request memory', 'phpfpm.request_mem', 'line'],
88 'lines': [
58 - ["maxReqMem", 'max request memory'],
59 - ["avgReqMem", 'average request memory']
89 + ['minReqMem', 'min', 'absolute', 1, 1024],
90 + ['maxReqMem', 'max', 'absolute', 1, 1024],
91 + ['avgReqMem', 'avg', 'absolute', 1, 1024]
92 ]}
93 }
94
@@ -64,76 +96,73 @@ CHARTS = {
96 class Service(UrlService):
97 def __init__(self, configuration=None, name=None):
98 UrlService.__init__(self, configuration=configuration, name=name)
67 - if len(self.url) == 0:
68 - self.url = "http://localhost/status?full&json"
99 + self.url = self.configuration.get('url', 'http://localhost/status?full&json')
100 self.order = ORDER
101 self.definitions = CHARTS
71 - self.assignment = {"active processes": 'active',
72 - "max active processes": 'maxActive',
73 - "idle processes": 'idle',
74 - "accepted conn": 'requests',
75 - "max children reached": 'reached',
76 - "slow requests": 'slow'}
77 - self.proc_assignment = {"request duration": 'ReqDur',
78 - "last request cpu": 'ReqCPU',
79 - "last request memory": 'ReqMem'}
102 + self.regex = re.compile(r'([a-z][a-z ]+): ([\d.]+)')
103 + self.json = '&json' in self.url or '?json' in self.url
104 + self.json_full = self.url.endswith(('?full&json', '?json&full'))
105 + self.if_all_processes_running = dict([(c_name + p_name, 0) for c_name, func in CALC
106 + for metric, p_name in PER_PROCESS_INFO])
107
108 def _get_data(self):
109 """
110 Format data received from http request
111 :return: dict
112 """
86 - try:
87 - raw = self._get_raw_data()
88 - except AttributeError:
113 + raw = self._get_raw_data()
114 + if not raw:
115 return None
116
91 - if '?json' in self.url or '&json' in self.url:
92 - try:
93 - raw_json = json.loads(raw)
94 - except ValueError:
95 - return None
96 - data = {}
97 - for k,v in raw_json.items():
98 - if k in self.assignment:
99 - data[self.assignment[k]] = v
100 -
101 - if '&full' in self.url or '?full' in self.url:
102 - c = 0
103 - sum_val = {}
104 - for proc in raw_json['processes']:
105 - if proc['state'] != 'Idle':
106 - continue
107 - c += 1
108 - for k, v in self.proc_assignment.items():
109 - d = proc[k]
110 - if v == 'ReqDur':
111 - d = d/1000
112 - if v == 'ReqMem':
113 - d = d/1024
114 - if 'max' + v not in data or data['max' + v] < d:
115 - data['max' + v] = d
116 - if 'avg' + v not in sum_val:
117 - sum_val['avg' + v] = 0
118 - data['avg' + v] = 0
119 - sum_val['avg' + v] += d
120 - if len(sum_val):
121 - for k, v in sum_val.items():
122 - data[k] = v/c
123 -
124 - if len(data) == 0:
125 - return None
126 - return data
127 -
128 - raw = raw.split('\n')
129 - data = {}
130 - for row in raw:
131 - tmp = row.split(":")
132 - if str(tmp[0]) in self.assignment:
133 - try:
134 - data[self.assignment[tmp[0]]] = int(tmp[1])
135 - except (IndexError, ValueError):
136 - pass
137 - if len(data) == 0:
138 - return None
139 - return data
117 + raw_json = parse_raw_data_(is_json=self.json, regex=self.regex, raw_data=raw)
118 +
119 + # Per Pool info: active connections, requests and performance charts
120 + to_netdata = fetch_data_(raw_data=raw_json, metrics_list=POOL_INFO)
121 +
122 + # Per Process Info: duration, cpu and memory charts (min, max, avg)
123 + if self.json_full:
124 + p_info = dict()
125 + to_netdata.update(self.if_all_processes_running) # If all processes are in running state
126 + # Metrics are always 0 if the process is not in Idle state because calculation is done
127 + # when the request processing has terminated
128 + for process in [p for p in raw_json['processes'] if p['state'] == 'Idle']:
129 + p_info.update(fetch_data_(raw_data=process, metrics_list=PER_PROCESS_INFO, pid=str(process['pid'])))
130 +
131 + if p_info:
132 + for new_name in PER_PROCESS_INFO:
133 + for name, function in CALC:
134 + to_netdata[name + new_name[1]] = function([p_info[k] for k in p_info if new_name[1] in k])
135 +
136 + return to_netdata or None
137 +
138 +
139 +def fetch_data_(raw_data, metrics_list, pid=''):
140 + """
141 + :param raw_data: dict
142 + :param metrics_list: list
143 + :param pid: str
144 + :return: dict
145 + """
146 + result = dict()
147 + for metric, new_name in metrics_list:
148 + if metric in raw_data:
149 + result[new_name + pid] = float(raw_data[metric])
150 + return result
151 +
152 +
153 +def parse_raw_data_(is_json, regex, raw_data):
154 + """
155 + :param is_json: bool
156 + :param regex: compiled regular expr
157 + :param raw_data: dict
158 + :return: dict
159 + """
160 + if is_json:
161 + try:
162 + return json.loads(raw_data)
163 + except ValueError:
164 + return dict()
165 + else:
166 + raw_data = ' '.join(raw_data.split())
167 + return dict(regex.findall(raw_data))
168 +