07c7809af · cryptotaxi247/netdata-1

@@ -87,20 +87,15 @@ class Service(LogService):

87 # self._get_data = None # will be assigned in 'check' method.

88 # self.order = None # will be assigned in 'create_*_method' method.

89 # self.definitions = None # will be assigned in 'create_*_method' method.

90 - # self.detailed_chart = None # will be assigned in 'create_*_method' method.

91 - # self.http_method_chart = None # will be assigned in 'create_*_method' method.

90 """

91 LogService.__init__(self, configuration=configuration, name=name)

92 # Variables from module configuration file

93 + self.type = self.configuration.get('type', 'web_access')

94 self.log_path = self.configuration.get('path')

96 - self.detailed_response_codes = self.configuration.get('detailed_response_codes', True)

97 - self.all_time = self.configuration.get('all_time', True)

95 self.url_pattern = self.configuration.get('categories') # dict

96 self.custom_log_format = self.configuration.get('custom_log_format') # dict

97 # Instance variables

101 - self.unique_all_time = list() # sorted list of unique IPs

98 self.regex = None # will be assigned in 'find_regex' or 'find_regex_custom' method

103 - self.resp_time_func = None # will be assigned in 'find_regex' or 'find_regex_custom' method

99 self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0,

100 'resp_time_avg': 0, 'unique_cur_ipv4': 0, 'unique_cur_ipv6': 0, '2xx': 0,

101 '5xx': 0, '3xx': 0, '4xx': 0, '1xx': 0, '0xx': 0, 'unmatched': 0, 'req_ipv4': 0,

@@ -111,13 +106,10 @@ class Service(LogService):

106 """

107 :return: bool

108

114 - We need to make sure:

109 1. "log_path" is specified in the module configuration file

110 2. "log_path" must be readable by netdata user and must exist

111 3. "log_path' must not be empty. We need at least 1 line to find appropriate pattern to parse

118 - 4. Plugin can work using predefined patterns (OK for nginx, apache default log format) or user defined

119 - pattern. So we need to check if we can parse last line from log file with user pattern OR module patterns.

120 - 5. All patterns for per_url_request_counter feature are valid regex expressions

112 + 4. other checks depends on log "type"

113 """

114 if not self.log_path:

115 self.error('log path is not specified')

@@ -149,42 +141,45 @@ class Service(LogService):

141 self.error(str(error))

142 return False

143

152 - # Custom_log_format or predefined log format.

153 - if self.custom_log_format:

154 - match_dict, log_name, error = self.find_regex_custom(last_line)

155 - else:

156 - match_dict, log_name, error = self.find_regex(last_line)

144 + if self.type == 'web_access':

145 + self.unique_all_time = list() # sorted list of unique IPs

146 + self.detailed_response_codes = self.configuration.get('detailed_response_codes', True)

147 + self.all_time = self.configuration.get('all_time', True)

148

158 - # "match_dict" is None if there are any problems

159 - if match_dict is None:

160 - self.error(str(error))

161 - return False

149 + # Custom_log_format or predefined log format.

150 + if self.custom_log_format:

151 + match_dict, error = self.find_regex_custom(last_line)

152 + else:

153 + match_dict, error = self.find_regex(last_line)

154

163 - # self.url_pattern check

164 - if self.url_pattern:

165 - self.url_pattern = check_req_per_url_pattern(self.url_pattern)

155 + # "match_dict" is None if there are any problems

156 + if match_dict is None:

157 + self.error(str(error))

158 + return False

159

167 - # Double check

168 - if not (self.regex and self.resp_time_func):

169 - self.error('That can not happen, but it happened. "regex" or "resp_time_func" is None')

160 + # self.url_pattern check

161 + if self.url_pattern:

162 + self.url_pattern = check_req_per_url_pattern('rpu', self.url_pattern)

163

171 - # All is ok. We are about to start.

172 - if log_name == 'web_access':

164 self.create_access_charts(match_dict) # Create charts

174 - self._get_data = self._get_access_data

175 - self.info('Collected data: %s' % list(match_dict.keys()))

176 - return True

165 + self._get_data = self._get_access_data # _get_data assignment

166 else:

178 - # If it's not access_logs.. Not used at the moment

167 + self.error('Not implemented')

168 return False

169

170 + # Double check

171 + if not self.regex:

172 + self.error('That can not happen, but it happened. "regex" is None')

173 +

174 + self.info('Collected data: %s' % list(match_dict.keys()))

175 + return True

176 +

177 def find_regex_custom(self, last_line):

178 """

179 :param last_line: str: literally last line from log file

180 :return: tuple where:

181 [0]: dict or None: match_dict or None

186 - [1]: str or None: log_name or None

187 - [2]: str: error description

182 + [1]: str: error description

183

184 We are here only if "custom_log_format" is in logs. We need to make sure:

185 1. "custom_log_format" is a dict

@@ -264,16 +259,14 @@ class Service(LogService):

259 self.resp_time_func = lambda time: time * (resp_time_func or 1)

260

261 self.regex = regex

267 - return find_regex_return(match_dict=match_dict,

268 - log_name='web_access')

262 + return find_regex_return(match_dict=match_dict)

263

264 def find_regex(self, last_line):

265 """

266 :param last_line: str: literally last line from log file

267 :return: tuple where:

268 [0]: dict or None: match_dict or None

275 - [1]: str or None: log_name or None

276 - [2]: str: error description

269 + [1]: str: error description

270 We need to find appropriate pattern for current log file

271 All logic is do a regex search through the string for all predefined patterns

272 until we find something or fail.

@@ -342,7 +335,6 @@ class Service(LogService):

335 break

336

337 return find_regex_return(match_dict=match_dict or None,

345 - log_name='web_access',

338 msg='Unknown log format. You need to use "custom_log_format" feature.')

339

340 def create_access_charts(self, match_dict):

@@ -396,12 +388,12 @@ class Service(LogService):

388 if self.url_pattern:

389 self.definitions['requests_per_url'] = {'options': [None, 'Requests Per Url', 'requests/s',

390 'urls', 'web_log.requests_per_url', 'stacked'],

399 - 'lines': [['pur_other', 'other', 'incremental']]}

391 + 'lines': [['rpu_other', 'other', 'incremental']]}

392 for elem in self.url_pattern:

393 self.definitions['requests_per_url']['lines'].append([elem.description, elem.description[4:],

394 'incremental'])

395 self.data.update({elem.description: 0})

404 - self.data.update({'pur_other': 0})

396 + self.data.update({'rpu_other': 0})

397 else:

398 self.order.remove('requests_per_url')

399

@@ -525,7 +517,7 @@ class Service(LogService):

517 match = True

518 break

519 if not match:

528 - self.data['pur_other'] += 1

520 + self.data['rpu_other'] += 1

521

522 def _get_data_statuses(self, code):

523 """

@@ -564,18 +556,18 @@ def address_not_in_pool(pool, address, pool_size):

556 return True

557

558

567 -def find_regex_return(match_dict=None, log_name=None, msg='Generic error message'):

559 +def find_regex_return(match_dict=None, msg='Generic error message'):

560 """

561 :param match_dict: dict: re.search.groupdict() or None

570 - :param log_name: str: log name

562 :param msg: str: error description

563 :return: tuple:

564 """

574 - return match_dict, log_name, msg

565 + return match_dict, msg

566

567

577 -def check_req_per_url_pattern(url_pattern):

568 +def check_req_per_url_pattern(string, url_pattern):

569 """

570 + :param string: str:

571 :param url_pattern: dict: ex. {'dim1': 'pattern1>', 'dim2': '<pattern2>'}

572 :return: list of named tuples or None:

573 We need to make sure all patterns are valid regular expressions

@@ -603,7 +595,7 @@ def check_req_per_url_pattern(url_pattern):

595 for dimension, regex in url_pattern.items():

596 valid_pattern = is_valid_pattern(regex)

597 if isinstance(dimension, str) and valid_pattern:

606 - result.append(NAMED_URL_PATTERN(description='_'.join(['pur', dimension]), pattern=valid_pattern))

598 + result.append(NAMED_URL_PATTERN(description='_'.join([string, dimension]), pattern=valid_pattern))

599

600 return result or None

601

web_log plugin: some optimization