40236a6a3 · cryptotaxi247/netdata-1

@@ -100,7 +100,6 @@ class Service(LogService):

100 self.unique_all_time = list() # sorted list of unique IPs

101 self.regex = None # will be assigned in 'find_regex' or 'find_regex_custom' method

102 self.resp_time_func = None # will be assigned in 'find_regex' or 'find_regex_custom' method

103 - # if there is no new logs this dict returned to netdata

103 self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0,

104 'resp_time_avg': 0, 'unique_cur_ipv4': 0, 'unique_cur_ipv6': 0, '2xx': 0,

105 '5xx': 0, '3xx': 0, '4xx': 0, '1xx': 0, '0xx': 0, 'unmatched': 0, 'req_ipv4': 0,

@@ -108,16 +107,25 @@ class Service(LogService):

107 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0}

108

109 def check(self):

110 + """

111 + :return: bool

112 +

113 + We need to make sure:

114 + 1. "log_path" is specified in the module configuration file

115 + 2. "log_path" must be readable by netdata user and must exist

116 + 3. "log_path' must not be empty. We need at least 1 line to find appropriate pattern to parse

117 + 4. Plugin can work using predefined patterns (OK for nginx, apache default log format) or user defined

118 + pattern. So we need to check if we can parse last line from log file with user pattern OR module patterns.

119 + 5. All patterns for per_url_request_counter feature are valid regex expressions

120 + """

121 if not self.log_path:

122 self.error('log path is not specified')

123 return False

124

115 - # log_path must be readable

125 if not access(self.log_path, R_OK):

126 self.error('%s not readable or not exist' % self.log_path)

127 return False

128

120 - # log_path file should not be empty

129 if not getsize(self.log_path):

130 self.error('%s is empty' % self.log_path)

131 return False

@@ -140,19 +148,26 @@ class Service(LogService):

148 self.error(str(error))

149 return False

150

143 - # Custom_log_format is preferable

151 + # Custom_log_format or predefined log format.

152 if self.custom_log_format:

153 match_dict, log_name, error = self.find_regex_custom(last_line)

154 else:

155 match_dict, log_name, error = self.find_regex(last_line)

156

157 + # "match_dict" is None if there are any problems

158 if match_dict is None:

159 self.error(str(error))

160 return False

161

162 + # self.url_pattern check

163 + if self.url_pattern:

164 + self.url_pattern = check_req_per_url_pattern(self.url_pattern)

165 +

166 + # Double check

167 if not (self.regex and self.resp_time_func):

168 self.error('That can not happen, but it happened. "regex" or "resp_time_func" is None')

169

170 + # All is ok. We are about to start.

171 if log_name == 'web_access':

172 self.create_access_charts(match_dict) # Create charts

173 self._get_data = self._get_access_data

@@ -191,52 +206,61 @@ class Service(LogService):

206 ("resp_length" is integer or "-", "resp_time" is integer or float)

207

208 """

194 - try:

195 - self.custom_log_format.keys()

196 - except AttributeError:

197 - return None, None, 'Custom log: "custom_log_format" is not a <dict>'

209 + if not is_dict(self.custom_log_format):

210 + return find_regex_return(msg='Custom log: "custom_log_format" is not a <dict>')

211

212 pattern = self.custom_log_format.get('pattern')

213 if not (pattern and isinstance(pattern, str)):

201 - return None, None, 'Custom log: "pattern" option is not specified or type is not <str>'

214 + return find_regex_return(msg='Custom log: "pattern" option is not specified or type is not <str>')

215

216 resp_time_func = self.custom_log_format.get('time_multiplier') or 0

217

218 if not isinstance(resp_time_func, int):

206 - return None, None, 'Custom log: "time_multiplier" is not an integer'

219 + return find_regex_return(msg='Custom log: "time_multiplier" is not an integer')

220 +

221 + try:

222 + regex = re.compile(pattern)

223 + except re.error as error:

224 + return find_regex_return(msg='Pattern compile error: %s' % str(error))

225

208 - regex = re.compile(pattern)

226 match = regex.search(last_line)

227 if match:

228 match_dict = match.groupdict() or None

229 else:

213 - return None, None, 'Custom log: pattern search FAILED'

230 + return find_regex_return(msg='Custom log: pattern search FAILED')

231

232 if match_dict is None:

216 - return None, None, 'Custom log: search OK but contains no named subgroups' \

217 - ' (you need to use ?P<subgroup_name>)'

233 + find_regex_return(msg='Custom log: search OK but contains no named subgroups'

234 + ' (you need to use ?P<subgroup_name>)')

235 else:

236 basic_values = {'address', 'method', 'url', 'code', 'bytes_sent'} - set(match_dict)

237

238 if basic_values:

222 - return None, None, 'Custom log: search OK but some mandatory keys (%s) are missing' % list(basic_values)

239 + return find_regex_return(msg='Custom log: search OK but some mandatory keys (%s) are missing'

240 + % list(basic_values))

241 else:

242 if not re.search(r'[\da-f.:]+', match_dict['address']):

225 - return None, None, 'Custom log: can\'t parse "address": %s' % match_dict['address']

243 + return find_regex_return(msg='Custom log: can\'t parse "address": %s'

244 + % match_dict['address'])

245 if not re.search(r'[1-9]\d{2}', match_dict['code']):

227 - return None, None, 'Custom log: can\'t parse "code": %s' % match_dict['code']

246 + return find_regex_return(msg='Custom log: can\'t parse "code": %s'

247 + % match_dict['code'])

248 if not re.search(r'[A-Z]+', match_dict['method']):

229 - return None, None, 'Custom log: can\'t parse "method": %s' % match_dict['method']

249 + return find_regex_return(msg='Custom log: can\'t parse "method": %s'

250 + % match_dict['method'])

251 if not re.search(r'\d+|-', match_dict['bytes_sent']):

231 - return None, None, 'Custom log: can\'t parse "bytes_sent": %s' % match_dict['bytes_sent']

252 + return find_regex_return(msg='Custom log: can\'t parse "bytes_sent": %s'

253 + % match_dict['bytes_sent'])

254

255 if 'resp_length' in match_dict:

234 - if not re.search(r'\d+', match_dict.get('resp_length', '')):

235 - return None, None, 'Custom log: can\'t parse "resp_length": %s' % match_dict['resp_length']

256 + if not re.search(r'\d+', match_dict['resp_length']):

257 + return find_regex_return(msg='Custom log: can\'t parse "resp_length": %s'

258 + % match_dict['resp_length'])

259

260 if 'resp_time' in match_dict:

238 - if not re.search(r'[\d.]+', match_dict.get('resp_length', '')):

239 - return None, None, 'Custom log: can\'t parse "resp_time": %s' % match_dict['resp_time']

261 + if not re.search(r'[\d.]+', match_dict['resp_length']):

262 + return find_regex_return(msg='Custom log: can\'t parse "resp_time": %s'

263 + % match_dict['resp_time'])

264 else:

265 if '.' in match_dict['resp_time']:

266 self.resp_time_func = lambda time: time * (resp_time_func or 1000000)

@@ -244,7 +268,9 @@ class Service(LogService):

268 self.resp_time_func = lambda time: time * (resp_time_func or 1)

269

270 self.regex = regex

247 - return match_dict, 'web_access', 'Custom log: we are fine'

271 + return find_regex_return(match_dict=match_dict,

272 + log_name='web_access',

273 + msg='We are fine')

274

275 def find_regex(self, last_line):

276 """

@@ -320,8 +346,9 @@ class Service(LogService):

346 match_dict = match.groupdict()

347 break

348

323 - return match_dict or None, 'web_access', 'Unknown log format. Plugin still can work for you.' \

324 - ' Read about the "custom_log_format" feature in the conf file'

349 + return find_regex_return(match_dict=match_dict or None,

350 + log_name='web_access',

351 + msg='Unknown log format. You need to use "custom_log_format" feature.')

352

353 def create_access_charts(self, match_dict):

354 """

@@ -371,8 +398,6 @@ class Service(LogService):

398

399 # Add 'requests_per_url' chart if specified in the configuration

400 if self.url_pattern:

374 - self.url_pattern = [NAMED_URL_PATTERN(description=k, pattern=re.compile(v)) for k, v

375 - in self.url_pattern.items()]

401 self.definitions['requests_per_url'] = {'options': [None, 'Requests Per Url', 'requests/s',

402 'urls', 'web_log.requests_per_url', 'stacked'],

403 'lines': [['other_url', 'other', 'incremental']]}

@@ -436,7 +461,8 @@ class Service(LogService):

461 # requests per http method

462 self._get_data_http_method(match_dict['method'])

463 # bandwidth sent

439 - self.data['bytes_sent'] += int(match_dict['bytes_sent'] if '-' not in match_dict['bytes_sent'] else 0)

464 + bytes_sent = match_dict['bytes_sent'] if '-' not in match_dict['bytes_sent'] else 0

465 + self.data['bytes_sent'] += int(bytes_sent)

466 # request processing time and bandwidth received

467 if 'resp_length' in match_dict:

468 self.data['resp_length'] += int(match_dict['resp_length'])

@@ -527,8 +553,8 @@ def address_not_in_pool(pool, address, pool_size):

553 """

554 :param pool: list of ip addresses

555 :param address: ip address

530 - :param pool_size: current size of pool

531 - :return: True if address not in pool. False if address in pool

556 + :param pool_size: current pool size

557 + :return: True if address not in pool. False if address in pool.

558 """

559 index = bisect.bisect_left(pool, address)

560 if index < pool_size:

@@ -540,3 +566,61 @@ def address_not_in_pool(pool, address, pool_size):

566 else:

567 bisect.insort_left(pool, address)

568 return True

569 +

570 +

571 +def find_regex_return(match_dict=None, log_name=None, msg='Generic error message'):

572 + """

573 + :param match_dict: dict: re.search.groupdict() or None

574 + :param log_name: str: log name

575 + :param msg: str: error description

576 + :return: tuple:

577 + """

578 + return match_dict, log_name, msg

579 +

580 +

581 +def check_req_per_url_pattern(url_pattern):

582 + """

583 + :param url_pattern: dict: ex. {'dim1': 'pattern1>', 'dim2': '<pattern2>'}

584 + :return: list of named tuples or None:

585 + We need to make sure all patterns are valid regular expressions

586 + """

587 + if not is_dict(url_pattern):

588 + return None

589 +

590 + result = list()

591 +

592 + def is_valid_pattern(pattern):

593 + """

594 + :param pattern: str

595 + :return: re.compile(pattern) or None

596 + """

597 + if not isinstance(pattern, str):

598 + return False

599 + else:

600 + try:

601 + compile_pattern = re.compile(pattern)

602 + except re.error:

603 + return False

604 + else:

605 + return compile_pattern

606 +

607 + for dimension, regex in url_pattern.items():

608 + valid_pattern = is_valid_pattern(regex)

609 + if isinstance(dimension, str) and valid_pattern:

610 + result.append(NAMED_URL_PATTERN(description=dimension, pattern=valid_pattern))

611 +

612 + return result or None

613 +

614 +

615 +def is_dict(obj):

616 + """

617 + :param obj: dict:

618 + :return: True or False

619 + obj can be <dict> or <OrderedDict>

620 + """

621 + try:

622 + obj.keys()

623 + except AttributeError:

624 + return False

625 + else:

626 + return True

web_log plugin: user defined patterns check added, other minor fixes