57ce750f4 · cryptotaxi247/netdata-1

@@ -19,17 +19,20 @@ ORDER_WEB = ['response_statuses', 'response_codes', 'bandwidth', 'response_time'

19 'requests_per_user_defined', 'http_method', 'http_version', 'requests_per_ipproto',

20 'clients', 'clients_all']

21

22 +ORDER_SQUID = ['squid_response_codes', 'squid_code', 'http_code',

23 + 'hier_code', 'bytes', 'duration', 'method', 'clients']

24 +

25 CHARTS_WEB = {

26 'response_codes': {

27 'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response_codes', 'stacked'],

28 'lines': [

26 - ['2xx', '2xx', 'incremental'],

27 - ['5xx', '5xx', 'incremental'],

28 - ['3xx', '3xx', 'incremental'],

29 - ['4xx', '4xx', 'incremental'],

30 - ['1xx', '1xx', 'incremental'],

29 + ['2xx', None, 'incremental'],

30 + ['5xx', None, 'incremental'],

31 + ['3xx', None, 'incremental'],

32 + ['4xx', None, 'incremental'],

33 + ['1xx', None, 'incremental'],

34 ['0xx', 'other', 'incremental'],

32 - ['unmatched', 'unmatched', 'incremental']

35 + ['unmatched', None, 'incremental']

36 ]},

37 'bandwidth': {

38 'options': [None, 'Bandwidth', 'KB/s', 'bandwidth', 'web_log.bandwidth', 'area'],

@@ -98,7 +101,8 @@ CHARTS_WEB = {

101

102 CHARTS_APACHE_CACHE = {

103 'cache': {

101 - 'options': [None, 'Apache Cached Responses', 'percent cached', 'cached', 'web_log.apache_cache', 'stacked'],

104 + 'options': [None, 'Apache Cached Responses', 'percent cached', 'cached', 'web_log.apache_cache_cache',

105 + 'stacked'],

106 'lines': [

107 ["hit", 'cache', "percentage-of-absolute-row"],

108 ["miss", None, "percentage-of-absolute-row"],

@@ -106,10 +110,68 @@ CHARTS_APACHE_CACHE = {

110 ]}

111 }

112

113 +CHARTS_SQUID = {

114 + 'duration': {

115 + 'options': [None, 'Elapsed Time The Transaction Busied The Cache',

116 + 'seconds', 'timings', 'web_log.squid_duration', 'area'],

117 + 'lines': [

118 + ['duration_min', 'min', 'incremental', 1, 1000],

119 + ['duration_max', 'max', 'incremental', 1, 1000],

120 + ['duration_avg', 'avg', 'incremental', 1, 1000]

121 + ]},

122 + 'bytes': {

123 + 'options': [None, 'Amount Of Data Delivered To The Clients',

124 + 'KB', 'bandwidth', 'web_log.squid_bytes', 'line'],

125 + 'lines': [

126 + ['bytes', 'sent', 'incremental', 1, 1024]

127 + ]},

128 + 'squid_response_codes': {

129 + 'options': [None, 'Response Codes', 'responses/s', 'responses', 'web_log.squid_response_codes', 'stacked'],

130 + 'lines': [

131 + ['2xx', None, 'incremental'],

132 + ['5xx', None, 'incremental'],

133 + ['3xx', None, 'incremental'],

134 + ['4xx', None, 'incremental'],

135 + ['1xx', None, 'incremental'],

136 + ['0xx', None, 'incremental'],

137 + ['other', None, 'incremental'],

138 + ['unmatched', None, 'incremental']

139 + ]},

140 + 'squid_code': {

141 + 'options': [None, 'Responses Per Cache Result Of The Request',

142 + 'responses/s', 'responses', 'web_log.squid_responses_squid_code', 'stacked'],

143 + 'lines': [

144 + ]},

145 + 'http_code': {

146 + 'options': [None, 'Responses Per HTTP Code',

147 + 'responses/s', 'responses', 'web_log.squid_responses_http_code', 'stacked'],

148 + 'lines': [

149 + ]},

150 + 'hier_code': {

151 + 'options': [None, 'Responses Per Hierarchy Code',

152 + 'responses/s', 'responses', 'web_log.squid_responses_hier_code', 'stacked'],

153 + 'lines': [

154 + ]},

155 + 'method': {

156 + 'options': [None, 'Responses Per Request Method',

157 + 'responses/s', 'responses', 'web_log.squid_responses_request_method', 'stacked'],

158 + 'lines': [

159 + ]},

160 + 'clients': {

161 + 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'clients',

162 + 'web_log.squid_clients', 'stacked'],

163 + 'lines': [

164 + ['unique_ipv4', 'ipv4', 'incremental', 1, 1],

165 + ['unique_ipv6', 'ipv6', 'incremental', 1, 1]

166 + ]}

167 +}

168 +

169 NAMED_PATTERN = namedtuple('PATTERN', ['description', 'pattern'])

170

171 DET_RESP_AGGR = ['', '_1xx', '_2xx', '_3xx', '_4xx', '_5xx', '_Other']

172

173 +SQUID_DYNAMIC = ('squid_code', 'http_code', 'hier_code', 'method')

174 +

175

176 class Service(LogService):

177 def __init__(self, configuration=None, name=None):

@@ -131,21 +193,21 @@ class Service(LogService):

193 4. other checks depends on log "type"

194 """

195

134 - log_types = dict(web=Web, apache_cache=ApacheCache)

196 + log_types = dict(web=Web, apache_cache=ApacheCache, squid=Squid)

197

198 if self.log_type not in log_types:

199 self.error('bad log type (%s). Supported types: %s' % (self.log_type, log_types.keys()))

200 return False

201

140 - if not self.configuration.get('path'):

202 + if not self.log_path:

203 self.error('log path is not specified')

204 return False

205

144 - if not access(self.configuration['path'], R_OK):

206 + if not access(self.log_path, R_OK):

207 self.error('%s not readable or not exist' % self.configuration['path'])

208 return False

209

148 - if not getsize(self.configuration['path']):

210 + if not getsize(self.log_path):

211 self.error('%s is empty' % self.configuration['path'])

212 return False

213

@@ -167,19 +229,34 @@ class Service(LogService):

229

230

231 class Mixin:

170 - def add_new_dimension(self, dimension, dimension_list, key, chart):

232 + def add_new_dimension(self, dimension_id, chart_key, dimension=None,

233 + algorithm='incremental', multiplier=1, divisor=1):

234 """

172 - :param dimension: str: response status code. Ex.: '202', '499'

173 - :param dimension_list: list: Ex.: ['202', '202', 'incremental']

174 - :param key: str: CHARTS dict key (chart name). Ex.: 'response_time'

175 - :param chart: Current string we need to pass to netdata to rebuild the chart

235 + :param dimension:

236 + :param chart_key:

237 + :param dimension_id:

238 + :param algorithm:

239 + :param multiplier:

240 + :param divisor:

241 + :return:

242 """

177 - self.data[dimension] = 0

243 +

244 + self.data[dimension_id] = 0

245 # SET method check if dim in _dimensions

179 - self.conf['_dimensions'].append(dimension)

246 + self.conf['_dimensions'].append(dimension_id)

247 # UPDATE method do SET only if dim in definitions

181 - self.definitions[key]['lines'].append(dimension_list)

182 - print(chart + "%s %s\n" % ('DIMENSION', ' '.join(dimension_list)))

248 + dimension_list = list(map(str, [dimension_id,

249 + dimension if dimension else dimension_id,

250 + algorithm,

251 + multiplier,

252 + divisor]))

253 + self.definitions[chart_key]['lines'].append(dimension_list)

254 + job_name = find_job_name(self.conf['override_name'], self.conf['name'])

255 + opts = self.definitions[chart_key]['options']

256 + chart = 'CHART %s.%s "" "%s" %s "%s" %s %s 60000 %s\n' % (job_name, chart_key,

257 + opts[1], opts[2], opts[3],

258 + opts[4], opts[5], self.conf['update_every'])

259 + print(chart + "DIMENSION %s\n" % ' '.join(dimension_list))

260

261 def get_last_line(self):

262 """

@@ -204,13 +281,11 @@ class Mixin:

281

282 @staticmethod

283 def error(*params):

207 - params = map(str, params)

208 - msg.error('web_log', ' '.join(params))

284 + msg.error('web_log', ' '.join(map(str, params)))

285

286 @staticmethod

287 def info(*params):

212 - params = map(str, params)

213 - msg.info('web_log', ' '.join(params))

288 + msg.info('web_log', ' '.join(map(str, params)))

289

290

291 class Web(Mixin):

@@ -255,32 +330,14 @@ class Web(Mixin):

330 """

331 self.order = ORDER_WEB[:]

332 self.definitions = deepcopy(CHARTS_WEB)

258 - job_name = find_job_name(self.conf['override_name'], self.conf['name'])

259 -

260 - self.storage['chart_http_method'] = 'CHART %s.http_method' \

261 - ' "" "Requests Per HTTP Method" requests/s "http methods"' \

262 - ' web_log.http_method stacked 60000 %s\n' \

263 - 'DIMENSION GET GET incremental\n' \

264 - % (job_name, self.conf['update_every'])

265 - self.storage['chart_http_version'] = 'CHART %s.http_version' \

266 - ' "" "Requests Per HTTP Version" requests/s "http versions"' \

267 - ' web_log.http_version stacked 60000 %s\n' \

268 - % (job_name, self.conf['update_every'])

333

334 if 'resp_time' not in match_dict:

335 self.order.remove('response_time')

336 if not self.conf.get('all_time', True):

337 self.order.remove('clients_all')

338 +

339 # Add 'detailed_response_codes' chart if specified in the configuration

340 if self.conf.get('detailed_response_codes', True):

276 - self.storage['chart_detailed'] = list()

277 - for add_to_dim in DET_RESP_AGGR:

278 - self.storage['chart_detailed'].append('CHART %s.detailed_response_codes%s ""'

279 - ' "Detailed Response Codes %s" requests/s responses'

280 - ' web_log.detailed_response_codes%s stacked 60000 %s\n'

281 - % (job_name, add_to_dim, add_to_dim[1:],

282 - add_to_dim, self.conf['update_every']))

283 -

341 codes = DET_RESP_AGGR[:1] if self.conf.get('detailed_response_aggregate', True) else DET_RESP_AGGR[1:]

342 for code in codes:

343 self.order.append('detailed_response_codes%s' % code)

@@ -300,6 +357,7 @@ class Web(Mixin):

357 else:

358 self.order.remove('requests_per_url')

359

360 + # Add 'requests_per_user_defined' chart if specified in the configuration

361 if self.storage['user_pattern'] and 'user_defined' in match_dict:

362 for elem in self.storage['user_pattern']:

363 self.definitions['requests_per_user_defined']['lines'].append([elem.description,

@@ -334,22 +392,32 @@ class Web(Mixin):

392 self.data['0xx'] += 1

393 # detailed response code

394 if self.conf.get('detailed_response_codes', True):

337 - self.get_data_detailed_response_codes(code=match_dict['code'])

395 + self.get_data_per_response_codes_detailed(code=match_dict['code'])

396 # response statuses

339 - self.get_data_statuses(code=match_dict['code'])

397 + self.get_data_per_statuses(code=match_dict['code'])

398 # requests per url

399 if self.storage['url_pattern']:

342 - self.get_data_per_pattern(field=match_dict['url'], other='url_pattern_other',

400 + self.get_data_per_pattern(field=match_dict['url'],

401 + other='url_pattern_other',

402 pattern=self.storage['url_pattern'])

403 # requests per user defined pattern

404 if self.storage['user_pattern'] and 'user_defined' in match_dict:

346 - self.get_data_per_pattern(field=match_dict['user_defined'], other='user_pattern_other',

405 + self.get_data_per_pattern(field=match_dict['user_defined'],

406 + other='user_pattern_other',

407 pattern=self.storage['user_pattern'])

408 # requests per http method

349 - self.get_data_http_method(method=match_dict['method'])

409 + if match_dict['method'] not in self.data:

410 + self.add_new_dimension(dimension_id=match_dict['method'],

411 + chart_key='http_method')

412 + self.data[match_dict['method']] += 1

413 # requests per http version

414 if 'http_version' in match_dict:

352 - self.get_data_http_version(http_version=match_dict['http_version'])

415 + dim_id = match_dict['http_version'].replace('.', '_')

416 + if dim_id not in self.data:

417 + self.add_new_dimension(dimension_id=dim_id,

418 + chart_key='http_version',

419 + dimension=match_dict['http_version'])

420 + self.data[dim_id] += 1

421 # bandwidth sent

422 bytes_sent = match_dict['bytes_sent'] if '-' not in match_dict['bytes_sent'] else 0

423 self.data['bytes_sent'] += int(bytes_sent)

@@ -365,10 +433,12 @@ class Web(Mixin):

433 proto = 'ipv4' if '.' in match_dict['address'] else 'ipv6'

434 self.data['req_' + proto] += 1

435 # unique clients ips

368 - if address_not_in_pool(pool=self.storage['unique_all_time'], address=match_dict['address'],

436 + if address_not_in_pool(pool=self.storage['unique_all_time'],

437 + address=match_dict['address'],

438 pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']):

439 self.data['unique_tot_' + proto] += 1

371 - if address_not_in_pool(pool=unique_current, address=match_dict['address'],

440 + if address_not_in_pool(pool=unique_current,

441 + address=match_dict['address'],

442 pool_size=ip_address_counter['unique_cur_ip']):

443 self.data['unique_cur_' + proto] += 1

444 ip_address_counter['unique_cur_ip'] += 1

@@ -547,7 +617,7 @@ class Web(Mixin):

617 self.storage['regex'] = regex

618 return find_regex_return(match_dict=match_dict)

619

550 - def get_data_detailed_response_codes(self, code):

620 + def get_data_per_response_codes_detailed(self, code):

621 """

622 :param code: str: CODE from parsed line. Ex.: '202, '499'

623 :return:

@@ -555,39 +625,15 @@ class Web(Mixin):

625 """

626 if code not in self.data:

627 if self.conf.get('detailed_response_aggregate', True):

558 - self.add_new_dimension(dimension=code, dimension_list=list((code, code, 'incremental')),

559 - key='detailed_response_codes', chart=self.storage['chart_detailed'][0])

628 + self.add_new_dimension(dimension_id=code,

629 + chart_key='detailed_response_codes')

630 else:

631 code_index = int(code[0]) if int(code[0]) < 6 else 6

562 - chart_name = 'detailed_response_codes' + DET_RESP_AGGR[code_index]

563 - self.add_new_dimension(dimension=code, dimension_list=list((code, code, 'incremental')),

564 - key=chart_name, chart=self.storage['chart_detailed'][code_index])

632 + chart_key = 'detailed_response_codes' + DET_RESP_AGGR[code_index]

633 + self.add_new_dimension(dimension_id=code,

634 + chart_key=chart_key)

635 self.data[code] += 1

636

567 - def get_data_http_method(self, method):

568 - """

569 - :param method: str: METHOD from parsed line. Ex.: 'GET', 'POST'

570 - :return:

571 - Calls add_new_dimension method If the value is found for the first time

572 - """

573 - if method not in self.data:

574 - self.add_new_dimension(dimension=method, dimension_list=list((method, method, 'incremental')),

575 - key='http_method', chart=self.storage['chart_http_method'])

576 - self.data[method] += 1

577 -

578 - def get_data_http_version(self, http_version):

579 - """

580 - :param http_version: str: METHOD from parsed line. Ex.: '1.1', '1.0'

581 - :return:

582 - Calls add_new_dimension method If the value is found for the first time

583 - """

584 - http_version_dim_id = http_version.replace('.', '_')

585 - if http_version_dim_id not in self.data:

586 - self.add_new_dimension(dimension=http_version_dim_id,

587 - dimension_list=list((http_version_dim_id, http_version, 'incremental')),

588 - key='http_version', chart=self.storage['chart_http_version'])

589 - self.data[http_version_dim_id] += 1

590 -

637 def get_data_per_pattern(self, field, other, pattern):

638 """

639 :param field: str:

@@ -606,7 +652,7 @@ class Web(Mixin):

652 if not match:

653 self.data[other] += 1

654

609 - def get_data_statuses(self, code):

655 + def get_data_per_statuses(self, code):

656 """

657 :param code: str: response status code. Ex.: '202', '499'

658 :return:

@@ -650,6 +696,75 @@ class ApacheCache:

696 return data

697

698

699 +class Squid(Mixin):

700 + def __init__(self, configuration):

701 + self.conf = configuration

702 + self.order = ORDER_SQUID

703 + self.definitions = CHARTS_SQUID

704 + self.regex = re.compile(r'[0-9.]+\s+(?P<duration>[0-9]+)'

705 + r' (?P<client_address>[\da-f.:]+)'

706 + r' (?P<squid_code>[A-Z_]+)/'

707 + r'(?P<http_code>[0-9]+)'

708 + r' (?P<bytes>[0-9]+)'

709 + r' (?P<method>[A-Z]+)'

710 + r' (?P<url>[^ ]+)'

711 + r' (?P<user>[^ ]+)'

712 + r' (?P<hier_code>[A-Z_]+)/[0-9.-]+'

713 + r' (?P<mime_type>[^\n]+)')

714 + self.data = {'duration_max': 0, 'duration_avg': 0, 'duration_min': 0,

715 + 'bytes': 0, '0xx': 0, '1xx': 0, '2xx': 0, '3xx': 0, '4xx': 0,

716 + '5xx': 0, 'other': 0, 'unmatched': 0, 'unique_ipv4': 0, 'unique_ipv6': 0}

717 +

718 + def check(self):

719 + last_line = self.get_last_line()

720 + if not last_line:

721 + return False

722 + match = self.regex.search(last_line)

723 + if not match:

724 + self.error('Regex not matches (%s)' % self.regex.pattern)

725 + return False

726 + return True

727 +

728 + def get_data(self, raw_data=None):

729 + if raw_data is None:

730 + return None

731 +

732 + unique_ip, duration = set(), list()

733 +

734 + for row in raw_data:

735 + match = self.regex.search(row)

736 + if match:

737 + match = match.groupdict()

738 + duration.append(match['duration'])

739 + try:

740 + self.data[match['http_code'][0] + 'xx'] += 1

741 + except KeyError:

742 + self.data['other'] += 1

743 +

744 + self.data['bytes'] += int(match['bytes'])

745 +

746 + proto = 'ipv4' if '.' in match['client_address'] else 'ipv6'

747 + if match['client_address'] not in unique_ip:

748 + self.data['unique_' + proto] += 1

749 + unique_ip.add(match['client_address'])

750 +

751 + for elem in SQUID_DYNAMIC:

752 + if match[elem] not in self.data:

753 + self.add_new_dimension(dimension_id=match[elem],

754 + chart_key=elem)

755 + else:

756 + self.data[match[elem]] += 1

757 + else:

758 + self.data['unmatched'] += 1

759 +

760 + if duration:

761 + length, duration = len(duration), map(float, duration)

762 + self.data['duration_min'] += min(duration)

763 + self.data['duration_max'] += max(duration)

764 + self.data['duration_avg'] += sum(duration) / length

765 + return self.data

766 +

767 +

768 def address_not_in_pool(pool, address, pool_size):

769 """

770 :param pool: list of ip addresses

web_log plugin: initial support for squid logs added