d234e418e · cryptotaxi247/netdata-1

@@ -4,22 +4,20 @@

4

5 import bisect

6 import re

7 +import os

8

9 from collections import namedtuple, defaultdict

10 from copy import deepcopy

10 -from os import access, R_OK

11 -from os.path import getsize

11

12 try:

13 from itertools import filterfalse

14 except ImportError:

15 + from itertools import ifilter as filter

16 from itertools import ifilterfalse as filterfalse

17

18 -from base import LogService

19 -import msg

18 +from bases.collection import read_last_line

19 +from bases.FrameworkServices.LogService import LogService

20

21 -priority = 60000

22 -retries = 60

21

22 ORDER_APACHE_CACHE = ['apache_cache']

23

@@ -256,8 +254,9 @@ class Service(LogService):

254 :param name:

255 """

256 LogService.__init__(self, configuration=configuration, name=name)

259 - self.log_type = self.configuration.get('type', 'web')

257 + self.configuration = configuration

258 self.log_path = self.configuration.get('path')

259 + self.job = None

260

261 def check(self):

262 """

@@ -269,123 +268,43 @@ class Service(LogService):

268 4. other checks depends on log "type"

269 """

270

271 + log_type = self.configuration.get('type', 'web')

272 log_types = dict(web=Web, apache_cache=ApacheCache, squid=Squid)

273

274 - if self.log_type not in log_types:

275 - self.error('bad log type (%s). Supported types: %s' % (self.log_type, log_types.keys()))

274 + if log_type not in log_types:

275 + self.error("bad log type {log_type}. Supported types: {types}".format(log_type=log_type,

276 + types=log_types.keys()))

277 return False

278

279 if not self.log_path:

280 self.error('log path is not specified')

281 return False

282

282 - if not (self._find_recent_log_file() and access(self.log_path, R_OK)):

283 - self.error('%s not readable or not exist' % self.log_path)

283 + if not (self._find_recent_log_file() and os.access(self.log_path, os.R_OK)):

284 + self.error('{log_file} not readable or not exist'.format(log_file=self.log_path))

285 return False

286

286 - if not getsize(self.log_path):

287 - self.error('%s is empty' % self.log_path)

287 + if not os.path.getsize(self.log_path):

288 + self.error('{log_file} is empty'.format(log_file=self.log_path))

289 return False

290

290 - self.configuration['update_every'] = self.update_every

291 - self.configuration['name'] = self.name

292 - self.configuration['override_name'] = self.override_name

293 - self.configuration['_dimensions'] = self._dimensions

294 - self.configuration['path'] = self.log_path

295 -

296 - cls = log_types[self.log_type]

297 - self.Job = cls(configuration=self.configuration)

298 - if self.Job.check():

299 - self.order = self.Job.order

300 - self.definitions = self.Job.definitions

301 - self.info('Current log file: %s' % self.log_path)

291 + self.job = log_types[log_type](self)

292 + if self.job.check():

293 + self.order = self.job.order

294 + self.definitions = self.job.definitions

295 return True

296 return False

297

298 def _get_data(self):

306 - return self.Job.get_data(self._get_raw_data())

299 + return self.job.get_data(self._get_raw_data())

300

301

309 -class Mixin:

310 - def filter_data(self, raw_data):

311 - """

312 - :param raw_data: list

313 - :return:

314 - """

315 - if not self.pre_filter:

316 - return raw_data

317 - filtered = raw_data

318 - for elem in self.pre_filter:

319 - if elem.description == 'filter_include':

320 - filtered = filter(elem.func, filtered)

321 - elif elem.description == 'filter_exclude':

322 - filtered = filterfalse(elem.func, filtered)

323 - return filtered

324 -

325 - def add_new_dimension(self, dimension_id, chart_key, dimension=None,

326 - algorithm='incremental', multiplier=1, divisor=1):

327 - """

328 - :param dimension:

329 - :param chart_key:

330 - :param dimension_id:

331 - :param algorithm:

332 - :param multiplier:

333 - :param divisor:

334 - :return:

335 - """

336 -

337 - self.data[dimension_id] = 0

338 - # SET method check if dim in _dimensions

339 - self.conf['_dimensions'].append(dimension_id)

340 - # UPDATE method do SET only if dim in definitions

341 - dimension_list = list(map(str, [dimension_id,

342 - dimension if dimension else dimension_id,

343 - algorithm,

344 - multiplier,

345 - divisor]))

346 - self.definitions[chart_key]['lines'].append(dimension_list)

347 - job_name = find_job_name(self.conf['override_name'], self.conf['name'])

348 - opts = self.definitions[chart_key]['options']

349 - chart = 'CHART %s.%s "" "%s" %s "%s" %s %s 60000 %s\n' % (job_name, chart_key,

350 - opts[1], opts[2], opts[3],

351 - opts[4], opts[5], self.conf['update_every'])

352 - print(chart + "DIMENSION %s\n" % ' '.join(dimension_list))

353 -

354 - def get_last_line(self):

355 - """

356 - Reads last line from the log file

357 - :return: str:

358 - """

359 - # Read last line (or first if there is only one line)

360 - with open(self.conf['path'], 'rb') as logs:

361 - logs.seek(-2, 2)

362 - while logs.read(1) != b'\n':

363 - logs.seek(-2, 1)

364 - if logs.tell() == 0:

365 - break

366 - last_line = logs.readline()

367 - try:

368 - return last_line.decode()

369 - except UnicodeDecodeError:

370 - try:

371 - return last_line.decode(encoding='utf-8')

372 - except (TypeError, UnicodeDecodeError) as error:

373 - msg.error('web_log', str(error))

374 - return False

375 -

376 - @staticmethod

377 - def error(*params):

378 - msg.error('web_log', ' '.join(map(str, params)))

379 -

380 - @staticmethod

381 - def info(*params):

382 - msg.info('web_log', ' '.join(map(str, params)))

383 -

384 -

385 -class Web(Mixin):

386 - def __init__(self, configuration):

387 - self.conf = configuration

388 - self.pre_filter = check_patterns('filter', self.conf.get('filter'))

302 +class Web:

303 + def __init__(self, service):

304 + self.service = service

305 + self.order = ORDER_WEB[:]

306 + self.definitions = deepcopy(CHARTS_WEB)

307 + self.pre_filter = check_patterns('filter', self.configuration.get('filter'))

308 self.storage = dict()

309 self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0,

310 'resp_time_avg': 0, 'resp_time_upstream_min': 0, 'resp_time_upstream_max': 0,

@@ -394,23 +313,27 @@ class Web(Mixin):

313 'req_ipv6': 0, 'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0, 'successful_requests': 0,

314 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0, 'GET': 0}

315

316 + def __getattr__(self, item):

317 + return getattr(self.service, item)

318 +

319 def check(self):

398 - last_line = self.get_last_line()

320 + last_line = read_last_line(self.log_path)

321 if not last_line:

322 return False

323 # Custom_log_format or predefined log format.

402 - if self.conf.get('custom_log_format'):

324 + if self.configuration.get('custom_log_format'):

325 match_dict, error = self.find_regex_custom(last_line)

326 else:

327 match_dict, error = self.find_regex(last_line)

328

329 # "match_dict" is None if there are any problems

330 if match_dict is None:

409 - self.error(str(error))

331 + self.error(error)

332 return False

333 +

334 self.storage['unique_all_time'] = list()

412 - self.storage['url_pattern'] = check_patterns('url_pattern', self.conf.get('categories'))

413 - self.storage['user_pattern'] = check_patterns('user_pattern', self.conf.get('user_defined'))

335 + self.storage['url_pattern'] = check_patterns('url_pattern', self.configuration.get('categories'))

336 + self.storage['user_pattern'] = check_patterns('user_pattern', self.configuration.get('user_defined'))

337

338 self.create_web_charts(match_dict) # Create charts

339 self.info('Collected data: %s' % list(match_dict.keys()))

@@ -422,20 +345,21 @@ class Web(Mixin):

345 :return:

346 Create/remove additional charts depending on the 'match_dict' keys and configuration file options

347 """

425 - self.order = ORDER_WEB[:]

426 - self.definitions = deepcopy(CHARTS_WEB)

427 -

348 if 'resp_time' not in match_dict:

349 self.order.remove('response_time')

350 if 'resp_time_upstream' not in match_dict:

351 self.order.remove('response_time_upstream')

352

433 - if not self.conf.get('all_time', True):

353 + if not self.configuration.get('all_time', True):

354 self.order.remove('clients_all')

355

356 # Add 'detailed_response_codes' chart if specified in the configuration

437 - if self.conf.get('detailed_response_codes', True):

438 - codes = DET_RESP_AGGR[:1] if self.conf.get('detailed_response_aggregate', True) else DET_RESP_AGGR[1:]

357 + if self.configuration.get('detailed_response_codes', True):

358 + if self.configuration.get('detailed_response_aggregate', True):

359 + codes = DET_RESP_AGGR[:1]

360 + else:

361 + codes = DET_RESP_AGGR[1:]

362 +

363 for code in codes:

364 self.order.append('detailed_response_codes%s' % code)

365 self.definitions['detailed_response_codes%s' % code] \

@@ -446,9 +370,8 @@ class Web(Mixin):

370 # Add 'requests_per_url' chart if specified in the configuration

371 if self.storage['url_pattern']:

372 for elem in self.storage['url_pattern']:

449 - self.definitions['requests_per_url']['lines'].append([elem.description,

450 - elem.description[12:],

451 - 'incremental'])

373 + dim = [elem.description, elem.description[12:], 'incremental']

374 + self.definitions['requests_per_url']['lines'].append(dim)

375 self.data[elem.description] = 0

376 self.data['url_pattern_other'] = 0

377 else:

@@ -457,9 +380,8 @@ class Web(Mixin):

380 # Add 'requests_per_user_defined' chart if specified in the configuration

381 if self.storage['user_pattern'] and 'user_defined' in match_dict:

382 for elem in self.storage['user_pattern']:

460 - self.definitions['requests_per_user_defined']['lines'].append([elem.description,

461 - elem.description[13:],

462 - 'incremental'])

383 + dim = [elem.description, elem.description[13:], 'incremental']

384 + self.definitions['requests_per_user_defined']['lines'].append(dim)

385 self.data[elem.description] = 0

386 self.data['user_pattern_other'] = 0

387 else:

@@ -475,7 +397,7 @@ class Web(Mixin):

397 if not raw_data:

398 return None if raw_data is None else self.data

399

478 - filtered_data = self.filter_data(raw_data=raw_data)

400 + filtered_data = filter_data(raw_data=raw_data, pre_filter=self.pre_filter)

401

402 unique_current = set()

403 timings = defaultdict(lambda: dict(minimum=None, maximum=0, summary=0, count=0))

@@ -490,7 +412,7 @@ class Web(Mixin):

412 except KeyError:

413 self.data['0xx'] += 1

414 # detailed response code

493 - if self.conf.get('detailed_response_codes', True):

415 + if self.configuration.get('detailed_response_codes', True):

416 self.get_data_per_response_codes_detailed(code=match_dict['code'])

417 # response statuses

418 self.get_data_per_statuses(code=match_dict['code'])

@@ -518,7 +440,7 @@ class Web(Mixin):

440 proto = 'ipv6' if ':' in match_dict['address'] else 'ipv4'

441 self.data['req_' + proto] += 1

442 # unique clients ips

521 - if self.conf.get('all_time', True):

443 + if self.configuration.get('all_time', True):

444 if address_not_in_pool(pool=self.storage['unique_all_time'],

445 address=match_dict['address'],

446 pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):

@@ -644,14 +566,14 @@ class Web(Mixin):

566 ("resp_length" is integer or "-", "resp_time" is integer or float)

567

568 """

647 - if not hasattr(self.conf.get('custom_log_format'), 'keys'):

569 + if not hasattr(self.configuration.get('custom_log_format'), 'keys'):

570 return find_regex_return(msg='Custom log: "custom_log_format" is not a <dict>')

571

650 - pattern = self.conf.get('custom_log_format', dict()).get('pattern')

572 + pattern = self.configuration.get('custom_log_format', dict()).get('pattern')

573 if not (pattern and isinstance(pattern, str)):

574 return find_regex_return(msg='Custom log: "pattern" option is not specified or type is not <str>')

575

654 - resp_time_func = self.conf.get('custom_log_format', dict()).get('time_multiplier') or 0

576 + resp_time_func = self.configuration.get('custom_log_format', dict()).get('time_multiplier') or 0

577

578 if not isinstance(resp_time_func, int):

579 return find_regex_return(msg='Custom log: "time_multiplier" is not an integer')

@@ -660,6 +582,7 @@ class Web(Mixin):

582 regex = re.compile(pattern)

583 except re.error as error:

584 return find_regex_return(msg='Pattern compile error: %s' % str(error))

585 +

586 match = regex.search(last_line)

587 if not match:

588 return find_regex_return(msg='Custom log: pattern search FAILED')

@@ -716,16 +639,19 @@ class Web(Mixin):

639 # requests per http method

640 if match_dict.get('method'):

641 if match_dict['method'] not in self.data:

719 - self.add_new_dimension(dimension_id=match_dict['method'],

720 - chart_key='http_method')

642 + self.charts['http_method'].add_dimension_and_push_chart([match_dict['method'],

643 + match_dict['method'],

644 + 'incremental'])

645 + self.data[match_dict['method']] = 0

646 self.data[match_dict['method']] += 1

647 # requests per http version

648 if match_dict.get('http_version'):

649 dim_id = match_dict['http_version'].replace('.', '_')

650 if dim_id not in self.data:

726 - self.add_new_dimension(dimension_id=dim_id,

727 - chart_key='http_version',

728 - dimension=match_dict['http_version'])

651 + self.charts['http_version'].add_dimension_and_push_chart([dim_id,

652 + match_dict['http_version'],

653 + 'incremental'])

654 + self.data[dim_id] = 0

655 self.data[dim_id] += 1

656

657 def get_data_per_response_codes_detailed(self, code):

@@ -735,14 +661,14 @@ class Web(Mixin):

661 Calls add_new_dimension method If the value is found for the first time

662 """

663 if code not in self.data:

738 - if self.conf.get('detailed_response_aggregate', True):

739 - self.add_new_dimension(dimension_id=code,

740 - chart_key='detailed_response_codes')

664 + if self.configuration.get('detailed_response_aggregate', True):

665 + self.charts['detailed_response_codes'].add_dimension_and_push_chart([code, code, 'incremental'])

666 + self.data[code] = 0

667 else:

668 code_index = int(code[0]) if int(code[0]) < 6 else 6

669 chart_key = 'detailed_response_codes' + DET_RESP_AGGR[code_index]

744 - self.add_new_dimension(dimension_id=code,

745 - chart_key=chart_key)

670 + self.charts[chart_key].add_dimension_and_push_chart([code, code, 'incremental'])

671 + self.data[code] = 0

672 self.data[code] += 1

673

674 def get_data_per_pattern(self, row, other, pattern):

@@ -782,8 +708,8 @@ class Web(Mixin):

708

709

710 class ApacheCache:

785 - def __init__(self, configuration):

786 - self.conf = configuration

711 + def __init__(self, service):

712 + self.service = service

713 self.order = ORDER_APACHE_CACHE

714 self.definitions = CHARTS_APACHE_CACHE

715

@@ -807,12 +733,12 @@ class ApacheCache:

733 return data

734

735

810 -class Squid(Mixin):

811 - def __init__(self, configuration):

812 - self.conf = configuration

736 +class Squid:

737 + def __init__(self, service):

738 + self.service = service

739 self.order = ORDER_SQUID

740 self.definitions = CHARTS_SQUID

815 - self.pre_filter = check_patterns('filter', self.conf.get('filter'))

741 + self.pre_filter = check_patterns('filter', self.configuration.get('filter'))

742 self.storage = dict()

743 self.data = {'duration_max': 0, 'duration_avg': 0, 'duration_min': 0, 'bytes': 0,

744 '0xx': 0, '1xx': 0, '2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0,

@@ -821,8 +747,11 @@ class Squid(Mixin):

747 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0

748 }

749

750 + def __getattr__(self, item):

751 + return getattr(self.service, item)

752 +

753 def check(self):

825 - last_line = self.get_last_line()

754 + last_line = read_last_line(self.log_path)

755 if not last_line:

756 return False

757 self.storage['unique_all_time'] = list()

@@ -858,7 +787,7 @@ class Squid(Mixin):

787 'chart': 'squid_mime_type',

788 'func_dim_id': lambda v: v.split('/')[0],

789 'func_dim': None}}

861 - if not self.conf.get('all_time', True):

790 + if not self.configuration.get('all_time', True):

791 self.order.remove('squid_clients_all')

792 return True

793

@@ -866,7 +795,7 @@ class Squid(Mixin):

795 if not raw_data:

796 return None if raw_data is None else self.data

797

869 - filtered_data = self.filter_data(raw_data=raw_data)

798 + filtered_data = filter_data(raw_data=raw_data, pre_filter=self.pre_filter)

799

800 unique_ip = set()

801 timings = defaultdict(lambda: dict(minimum=None, maximum=0, summary=0, count=0))

@@ -890,7 +819,7 @@ class Squid(Mixin):

819

820 proto = 'ipv4' if '.' in match['client_address'] else 'ipv6'

821 # unique clients ips

893 - if self.conf.get('all_time', True):

822 + if self.configuration.get('all_time', True):

823 if address_not_in_pool(pool=self.storage['unique_all_time'],

824 address=match['client_address'],

825 pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):

@@ -906,9 +835,10 @@ class Squid(Mixin):

835 dimension_id = values['func_dim_id'](match[key]) if values['func_dim_id'] else match[key]

836 if dimension_id not in self.data:

837 dimension = values['func_dim'](match[key]) if values['func_dim'] else dimension_id

909 - self.add_new_dimension(dimension_id=dimension_id,

910 - chart_key=values['chart'],

911 - dimension=dimension)

838 + self.charts[values['chart']].add_dimension_and_push_chart([dimension_id,

839 + dimension,

840 + 'incremental'])

841 + self.data[dimension_id] = 0

842 self.data[dimension_id] += 1

843 else:

844 self.data['unmatched'] += 1

@@ -942,8 +872,8 @@ class Squid(Mixin):

872 :return:

873 """

874 if code not in self.data:

945 - self.add_new_dimension(dimension_id=code,

946 - chart_key='squid_code')

875 + self.charts['squid_code'].add_dimension_and_push_chart([code, code, 'incremental'])

876 + self.data[code] = 0

877 self.data[code] += 1

878

879 for tag in code.split('_'):

@@ -953,9 +883,8 @@ class Squid(Mixin):

883 continue

884 dimension_id = '_'.join(['code_detailed', tag])

885 if dimension_id not in self.data:

956 - self.add_new_dimension(dimension_id=dimension_id,

957 - dimension=tag,

958 - chart_key=chart_key)

886 + self.charts[chart_key].add_dimension_and_push_chart([dimension_id, tag, 'incremental'])

887 + self.data[dimension_id] = 0

888 self.data[dimension_id] += 1

889

890

@@ -1040,14 +969,19 @@ def check_patterns(string, dimension_regex_dict):

969 return result or None

970

971

1043 -def find_job_name(override_name, name):

972 +def filter_data(raw_data, pre_filter):

973 """

1045 - :param override_name: str: 'name' var from configuration file

1046 - :param name: str: 'job_name' from configuration file

1047 - :return: str: new job name

1048 - We need this for dynamic charts. Actually the same logic as in python.d.plugin.

974 + :param raw_data:

975 + :param pre_filter:

976 + :return:

977 """

1050 - add_to_name = override_name or name

1051 - if add_to_name:

1052 - return '_'.join(['web_log', re.sub('\s+', '_', add_to_name)])

1053 - return 'web_log'

978 +

979 + if not pre_filter:

980 + return raw_data

981 + filtered = raw_data

982 + for elem in pre_filter:

983 + if elem.description == 'filter_include':

984 + filtered = filter(elem.func, filtered)

985 + elif elem.description == 'filter_exclude':

986 + filtered = filterfalse(elem.func, filtered)

987 + return filtered

"web_log" fixes