| 1 | # -*- coding: utf-8 -*- |
| 2 | # Description: haproxy netdata python.d module |
| 3 | # Author: ilyam8, ktarasz |
| 4 | # SPDX-License-Identifier: GPL-3.0-or-later |
| 5 | |
| 6 | from collections import defaultdict |
| 7 | from re import compile as re_compile |
| 8 | |
| 9 | try: |
| 10 | from urlparse import urlparse |
| 11 | except ImportError: |
| 12 | from urllib.parse import urlparse |
| 13 | |
| 14 | from bases.FrameworkServices.SocketService import SocketService |
| 15 | from bases.FrameworkServices.UrlService import UrlService |
| 16 | |
| 17 | # charts order (can be overridden if you want less charts, or different order) |
| 18 | ORDER = [ |
| 19 | 'fbin', |
| 20 | 'fbout', |
| 21 | 'fscur', |
| 22 | 'fqcur', |
| 23 | 'fhrsp_1xx', |
| 24 | 'fhrsp_2xx', |
| 25 | 'fhrsp_3xx', |
| 26 | 'fhrsp_4xx', |
| 27 | 'fhrsp_5xx', |
| 28 | 'fhrsp_other', |
| 29 | 'fhrsp_total', |
| 30 | 'bbin', |
| 31 | 'bbout', |
| 32 | 'bscur', |
| 33 | 'bqcur', |
| 34 | 'bhrsp_1xx', |
| 35 | 'bhrsp_2xx', |
| 36 | 'bhrsp_3xx', |
| 37 | 'bhrsp_4xx', |
| 38 | 'bhrsp_5xx', |
| 39 | 'bhrsp_other', |
| 40 | 'bhrsp_total', |
| 41 | 'bqtime', |
| 42 | 'bttime', |
| 43 | 'brtime', |
| 44 | 'bctime', |
| 45 | 'health_sup', |
| 46 | 'health_sdown', |
| 47 | 'health_smaint', |
| 48 | 'health_bdown', |
| 49 | 'health_idle' |
| 50 | ] |
| 51 | |
| 52 | CHARTS = { |
| 53 | 'fbin': { |
| 54 | 'options': [None, 'Kilobytes In', 'KiB/s', 'frontend', 'haproxy_f.bin', 'line'], |
| 55 | 'lines': [] |
| 56 | }, |
| 57 | 'fbout': { |
| 58 | 'options': [None, 'Kilobytes Out', 'KiB/s', 'frontend', 'haproxy_f.bout', 'line'], |
| 59 | 'lines': [] |
| 60 | }, |
| 61 | 'fscur': { |
| 62 | 'options': [None, 'Sessions Active', 'sessions', 'frontend', 'haproxy_f.scur', 'line'], |
| 63 | 'lines': [] |
| 64 | }, |
| 65 | 'fqcur': { |
| 66 | 'options': [None, 'Session In Queue', 'sessions', 'frontend', 'haproxy_f.qcur', 'line'], |
| 67 | 'lines': [] |
| 68 | }, |
| 69 | 'fhrsp_1xx': { |
| 70 | 'options': [None, 'HTTP responses with 1xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_1xx', 'line'], |
| 71 | 'lines': [] |
| 72 | }, |
| 73 | 'fhrsp_2xx': { |
| 74 | 'options': [None, 'HTTP responses with 2xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_2xx', 'line'], |
| 75 | 'lines': [] |
| 76 | }, |
| 77 | 'fhrsp_3xx': { |
| 78 | 'options': [None, 'HTTP responses with 3xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_3xx', 'line'], |
| 79 | 'lines': [] |
| 80 | }, |
| 81 | 'fhrsp_4xx': { |
| 82 | 'options': [None, 'HTTP responses with 4xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_4xx', 'line'], |
| 83 | 'lines': [] |
| 84 | }, |
| 85 | 'fhrsp_5xx': { |
| 86 | 'options': [None, 'HTTP responses with 5xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_5xx', 'line'], |
| 87 | 'lines': [] |
| 88 | }, |
| 89 | 'fhrsp_other': { |
| 90 | 'options': [None, 'HTTP responses with other codes (protocol error)', 'responses/s', 'frontend', |
| 91 | 'haproxy_f.hrsp_other', 'line'], |
| 92 | 'lines': [] |
| 93 | }, |
| 94 | 'fhrsp_total': { |
| 95 | 'options': [None, 'HTTP responses', 'responses', 'frontend', 'haproxy_f.hrsp_total', 'line'], |
| 96 | 'lines': [] |
| 97 | }, |
| 98 | 'bbin': { |
| 99 | 'options': [None, 'Kilobytes In', 'KiB/s', 'backend', 'haproxy_b.bin', 'line'], |
| 100 | 'lines': [] |
| 101 | }, |
| 102 | 'bbout': { |
| 103 | 'options': [None, 'Kilobytes Out', 'KiB/s', 'backend', 'haproxy_b.bout', 'line'], |
| 104 | 'lines': [] |
| 105 | }, |
| 106 | 'bscur': { |
| 107 | 'options': [None, 'Sessions Active', 'sessions', 'backend', 'haproxy_b.scur', 'line'], |
| 108 | 'lines': [] |
| 109 | }, |
| 110 | 'bqcur': { |
| 111 | 'options': [None, 'Sessions In Queue', 'sessions', 'backend', 'haproxy_b.qcur', 'line'], |
| 112 | 'lines': [] |
| 113 | }, |
| 114 | 'bhrsp_1xx': { |
| 115 | 'options': [None, 'HTTP responses with 1xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_1xx', 'line'], |
| 116 | 'lines': [] |
| 117 | }, |
| 118 | 'bhrsp_2xx': { |
| 119 | 'options': [None, 'HTTP responses with 2xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_2xx', 'line'], |
| 120 | 'lines': [] |
| 121 | }, |
| 122 | 'bhrsp_3xx': { |
| 123 | 'options': [None, 'HTTP responses with 3xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_3xx', 'line'], |
| 124 | 'lines': [] |
| 125 | }, |
| 126 | 'bhrsp_4xx': { |
| 127 | 'options': [None, 'HTTP responses with 4xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_4xx', 'line'], |
| 128 | 'lines': [] |
| 129 | }, |
| 130 | 'bhrsp_5xx': { |
| 131 | 'options': [None, 'HTTP responses with 5xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_5xx', 'line'], |
| 132 | 'lines': [] |
| 133 | }, |
| 134 | 'bhrsp_other': { |
| 135 | 'options': [None, 'HTTP responses with other codes (protocol error)', 'responses/s', 'backend', |
| 136 | 'haproxy_b.hrsp_other', 'line'], |
| 137 | 'lines': [] |
| 138 | }, |
| 139 | 'bhrsp_total': { |
| 140 | 'options': [None, 'HTTP responses (total)', 'responses/s', 'backend', 'haproxy_b.hrsp_total', 'line'], |
| 141 | 'lines': [] |
| 142 | }, |
| 143 | 'bqtime': { |
| 144 | 'options': [None, 'The average queue time over the 1024 last requests', 'milliseconds', 'backend', |
| 145 | 'haproxy_b.qtime', 'line'], |
| 146 | 'lines': [] |
| 147 | }, |
| 148 | 'bctime': { |
| 149 | 'options': [None, 'The average connect time over the 1024 last requests', 'milliseconds', 'backend', |
| 150 | 'haproxy_b.ctime', 'line'], |
| 151 | 'lines': [] |
| 152 | }, |
| 153 | 'brtime': { |
| 154 | 'options': [None, 'The average response time over the 1024 last requests', 'milliseconds', 'backend', |
| 155 | 'haproxy_b.rtime', 'line'], |
| 156 | 'lines': [] |
| 157 | }, |
| 158 | 'bttime': { |
| 159 | 'options': [None, 'The average total session time over the 1024 last requests', 'milliseconds', 'backend', |
| 160 | 'haproxy_b.ttime', 'line'], |
| 161 | 'lines': [] |
| 162 | }, |
| 163 | 'health_sdown': { |
| 164 | 'options': [None, 'Backend Servers In DOWN State', 'failed servers', 'health', 'haproxy_hs.down', 'line'], |
| 165 | 'lines': [] |
| 166 | }, |
| 167 | 'health_sup': { |
| 168 | 'options': [None, 'Backend Servers In UP State', 'health servers', 'health', 'haproxy_hs.up', 'line'], |
| 169 | 'lines': [] |
| 170 | }, |
| 171 | 'health_smaint': { |
| 172 | 'options': [None, 'Backend Servers In MAINT State', 'maintenance servers', 'health', 'haproxy_hs.maint', 'line'], |
| 173 | 'lines': [] |
| 174 | }, |
| 175 | 'health_bdown': { |
| 176 | 'options': [None, 'Is Backend Failed?', 'boolean', 'health', 'haproxy_hb.down', 'line'], |
| 177 | 'lines': [] |
| 178 | }, |
| 179 | 'health_idle': { |
| 180 | 'options': [None, 'The Ratio Of Polling Time Vs Total Time', 'percentage', 'health', 'haproxy.idle', 'line'], |
| 181 | 'lines': [ |
| 182 | ['idle', None, 'absolute'] |
| 183 | ] |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | METRICS = { |
| 188 | 'bin': {'algorithm': 'incremental', 'divisor': 1024}, |
| 189 | 'bout': {'algorithm': 'incremental', 'divisor': 1024}, |
| 190 | 'scur': {'algorithm': 'absolute', 'divisor': 1}, |
| 191 | 'qcur': {'algorithm': 'absolute', 'divisor': 1}, |
| 192 | 'hrsp_1xx': {'algorithm': 'incremental', 'divisor': 1}, |
| 193 | 'hrsp_2xx': {'algorithm': 'incremental', 'divisor': 1}, |
| 194 | 'hrsp_3xx': {'algorithm': 'incremental', 'divisor': 1}, |
| 195 | 'hrsp_4xx': {'algorithm': 'incremental', 'divisor': 1}, |
| 196 | 'hrsp_5xx': {'algorithm': 'incremental', 'divisor': 1}, |
| 197 | 'hrsp_other': {'algorithm': 'incremental', 'divisor': 1} |
| 198 | } |
| 199 | |
| 200 | BACKEND_METRICS = { |
| 201 | 'qtime': {'algorithm': 'absolute', 'divisor': 1}, |
| 202 | 'ctime': {'algorithm': 'absolute', 'divisor': 1}, |
| 203 | 'rtime': {'algorithm': 'absolute', 'divisor': 1}, |
| 204 | 'ttime': {'algorithm': 'absolute', 'divisor': 1} |
| 205 | } |
| 206 | |
| 207 | REGEX = dict(url=re_compile(r'idle = (?P<idle>[0-9]+)'), |
| 208 | socket=re_compile(r'Idle_pct: (?P<idle>[0-9]+)')) |
| 209 | |
| 210 | |
| 211 | # TODO: the code is unreadable |
| 212 | class Service(UrlService, SocketService): |
| 213 | def __init__(self, configuration=None, name=None): |
| 214 | if 'socket' in configuration: |
| 215 | SocketService.__init__(self, configuration=configuration, name=name) |
| 216 | self.poll = SocketService |
| 217 | self.options_ = dict(regex=REGEX['socket'], |
| 218 | stat='show stat\n'.encode(), |
| 219 | info='show info\n'.encode()) |
| 220 | else: |
| 221 | UrlService.__init__(self, configuration=configuration, name=name) |
| 222 | self.poll = UrlService |
| 223 | self.options_ = dict(regex=REGEX['url'], |
| 224 | stat=self.url, |
| 225 | info=url_remove_params(self.url)) |
| 226 | self.order = ORDER |
| 227 | self.definitions = CHARTS |
| 228 | |
| 229 | def check(self): |
| 230 | if self.poll.check(self): |
| 231 | self.create_charts() |
| 232 | self.info('We are using %s.' % self.poll.__name__) |
| 233 | return True |
| 234 | return False |
| 235 | |
| 236 | def _get_data(self): |
| 237 | to_netdata = dict() |
| 238 | self.request, self.url = self.options_['stat'], self.options_['stat'] |
| 239 | stat_data = self._get_stat_data() |
| 240 | self.request, self.url = self.options_['info'], self.options_['info'] |
| 241 | info_data = self._get_info_data(regex=self.options_['regex']) |
| 242 | |
| 243 | to_netdata.update(stat_data) |
| 244 | to_netdata.update(info_data) |
| 245 | return to_netdata or None |
| 246 | |
| 247 | def _get_stat_data(self): |
| 248 | """ |
| 249 | :return: dict |
| 250 | """ |
| 251 | raw_data = self.poll._get_raw_data(self) |
| 252 | |
| 253 | if not raw_data: |
| 254 | return dict() |
| 255 | |
| 256 | raw_data = raw_data.splitlines() |
| 257 | self.data = parse_data_([dict(zip(raw_data[0].split(','), raw_data[_].split(','))) |
| 258 | for _ in range(1, len(raw_data))]) |
| 259 | if not self.data: |
| 260 | return dict() |
| 261 | |
| 262 | stat_data = dict() |
| 263 | |
| 264 | for frontend in self.data['frontend']: |
| 265 | for metric in METRICS: |
| 266 | idx = frontend['# pxname'].replace('.', '_') |
| 267 | stat_data['_'.join(['frontend', metric, idx])] = frontend.get(metric) or 0 |
| 268 | |
| 269 | for backend in self.data['backend']: |
| 270 | name, idx = backend['# pxname'], backend['# pxname'].replace('.', '_') |
| 271 | stat_data['hsup_' + idx] = len([server for server in self.data['servers'] |
| 272 | if server_status(server, name, 'UP')]) |
| 273 | stat_data['hsdown_' + idx] = len([server for server in self.data['servers'] |
| 274 | if server_status(server, name, 'DOWN')]) |
| 275 | stat_data['hsmaint_' + idx] = len([server for server in self.data['servers'] |
| 276 | if server_status(server, name, 'MAINT')]) |
| 277 | stat_data['hbdown_' + idx] = 1 if backend.get('status') == 'DOWN' else 0 |
| 278 | for metric in BACKEND_METRICS: |
| 279 | stat_data['_'.join(['backend', metric, idx])] = backend.get(metric) or 0 |
| 280 | hrsp_total = 0 |
| 281 | for metric in METRICS: |
| 282 | stat_data['_'.join(['backend', metric, idx])] = backend.get(metric) or 0 |
| 283 | if metric.startswith('hrsp_'): |
| 284 | hrsp_total += int(backend.get(metric) or 0) |
| 285 | stat_data['_'.join(['backend', 'hrsp_total', idx])] = hrsp_total |
| 286 | return stat_data |
| 287 | |
| 288 | def _get_info_data(self, regex): |
| 289 | """ |
| 290 | :return: dict |
| 291 | """ |
| 292 | raw_data = self.poll._get_raw_data(self) |
| 293 | if not raw_data: |
| 294 | return dict() |
| 295 | |
| 296 | match = regex.search(raw_data) |
| 297 | return match.groupdict() if match else dict() |
| 298 | |
| 299 | @staticmethod |
| 300 | def _check_raw_data(data): |
| 301 | """ |
| 302 | Check if all data has been gathered from socket |
| 303 | :param data: str |
| 304 | :return: boolean |
| 305 | """ |
| 306 | return not bool(data) |
| 307 | |
| 308 | def create_charts(self): |
| 309 | for front in self.data['frontend']: |
| 310 | name, idx = front['# pxname'], front['# pxname'].replace('.', '_') |
| 311 | for metric in METRICS: |
| 312 | self.definitions['f' + metric]['lines'].append(['_'.join(['frontend', metric, idx]), |
| 313 | name, METRICS[metric]['algorithm'], 1, |
| 314 | METRICS[metric]['divisor']]) |
| 315 | self.definitions['fhrsp_total']['lines'].append(['_'.join(['frontend', 'hrsp_total', idx]), |
| 316 | name, 'incremental', 1, 1]) |
| 317 | for back in self.data['backend']: |
| 318 | name, idx = back['# pxname'], back['# pxname'].replace('.', '_') |
| 319 | for metric in METRICS: |
| 320 | self.definitions['b' + metric]['lines'].append(['_'.join(['backend', metric, idx]), |
| 321 | name, METRICS[metric]['algorithm'], 1, |
| 322 | METRICS[metric]['divisor']]) |
| 323 | self.definitions['bhrsp_total']['lines'].append(['_'.join(['backend', 'hrsp_total', idx]), |
| 324 | name, 'incremental', 1, 1]) |
| 325 | for metric in BACKEND_METRICS: |
| 326 | self.definitions['b' + metric]['lines'].append(['_'.join(['backend', metric, idx]), |
| 327 | name, BACKEND_METRICS[metric]['algorithm'], 1, |
| 328 | BACKEND_METRICS[metric]['divisor']]) |
| 329 | self.definitions['health_sup']['lines'].append(['hsup_' + idx, name, 'absolute']) |
| 330 | self.definitions['health_sdown']['lines'].append(['hsdown_' + idx, name, 'absolute']) |
| 331 | self.definitions['health_smaint']['lines'].append(['hsmaint_' + idx, name, 'absolute']) |
| 332 | self.definitions['health_bdown']['lines'].append(['hbdown_' + idx, name, 'absolute']) |
| 333 | |
| 334 | |
| 335 | def parse_data_(data): |
| 336 | def is_backend(backend): |
| 337 | return backend.get('svname') == 'BACKEND' and backend.get('# pxname') != 'stats' |
| 338 | |
| 339 | def is_frontend(frontend): |
| 340 | return frontend.get('svname') == 'FRONTEND' and frontend.get('# pxname') != 'stats' |
| 341 | |
| 342 | def is_server(server): |
| 343 | return not server.get('svname', '').startswith(('FRONTEND', 'BACKEND')) |
| 344 | |
| 345 | if not data: |
| 346 | return None |
| 347 | |
| 348 | result = defaultdict(list) |
| 349 | for elem in data: |
| 350 | if is_backend(elem): |
| 351 | result['backend'].append(elem) |
| 352 | continue |
| 353 | elif is_frontend(elem): |
| 354 | result['frontend'].append(elem) |
| 355 | continue |
| 356 | elif is_server(elem): |
| 357 | result['servers'].append(elem) |
| 358 | |
| 359 | return result or None |
| 360 | |
| 361 | |
| 362 | def server_status(server, backend_name, status='DOWN'): |
| 363 | return server.get('# pxname') == backend_name and server.get('status').partition(' ')[0] == status |
| 364 | |
| 365 | |
| 366 | def url_remove_params(url): |
| 367 | parsed = urlparse(url or str()) |
| 368 | return '{scheme}://{netloc}{path}'.format(scheme=parsed.scheme, netloc=parsed.netloc, path=parsed.path) |