400f2876a · cryptotaxi247/netdata-1

@@ -78,7 +78,13 @@ class Service(LogService):

78 self.detailed_response_codes = self.configuration.get('detailed_response_codes', True)

79 self.all_time = self.configuration.get('all_time', True)

80 self.url_pattern = self.configuration.get('categories') # dict

81 - self.regex = None

81 + self.regex = None # will be assigned in 'find_regex' method

82 + self.resp_time_func = None # will be assigned in 'find_regex' method

83 + self._get_data = None # will be assigned in 'check' method.

84 + self.order = None # will be assigned in 'create_*_method' method.

85 + self.definitions = None # will be assigned in 'create_*_method' method.

86 + self.detailed_chart = None # will be assigned in 'create_*_method' method.

87 + self.http_method_chart = None # will be assigned in 'create_*_method' method.

88 # sorted list of unique IPs

89 self.unique_all_time = list()

90 # dict for values that should not be zeroed every poll

@@ -114,20 +120,26 @@ class Service(LogService):

120 last_line = logs.readline().decode(encoding='utf-8')

121

122 # Parse last line

117 - parsed_line, regex_name = self.find_regex(last_line)

118 - if not parsed_line:

119 - self.error('Can\'t parse output')

123 + regex_name = self.find_regex(last_line)

124 + if not regex_name:

125 + self.error('Can\'t parse %s' % self.log_path)

126 return False

127

122 - self.create_charts(parsed_line[0], regex_name)

123 - if len(parsed_line[0]) == 5:

124 - self.info('Not all data collected. You need to modify LogFormat.')

125 - return True

128 + if regex_name.startswith('access_'):

129 + self.create_access_charts(regex_name)

130 + if regex_name == 'access_default':

131 + self.info('Not all data collected. You need to modify LogFormat.')

132 + self._get_data = self._get_access_data

133 + self.info('Used regex: %s' % regex_name)

134 + return True

135 + else:

136 + # If it's not access_logs.. Not used at the moment

137 + return False

138

139 def find_regex(self, last_line):

140 """

141 :param last_line: str: literally last line from log file

130 - :return: parsed line, regex name (the one that matches) OR None, None

142 + :return: regex_name

143 It's sad but different web servers has different logs formats

144 We need to find appropriate regex for current log file

145 All logic is do a regex search through the string for all patterns

@@ -135,47 +147,42 @@ class Service(LogService):

147 """

148 # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code

149 # 5. Bytes sent 6. Response length 7. Response process time

138 - default = re.compile(r'([\da-f.:]+)'

139 - r' -.*?"([A-Z]+)'

140 - r' (.*?)"'

141 - r' ([1-9]\d{2})'

142 - r' (\d+)')

143 -

144 - apache_extended = re.compile(r'([\da-f.:]+)'

150 + access_default = re.compile(r'([\da-f.:]+)'

151 r' -.*?"([A-Z]+)'

152 r' (.*?)"'

153 r' ([1-9]\d{2})'

148 - r' (\d+)'

149 - r' (\d+)'

150 - r' (\d+) ')

151 -

152 - nginx_extended = re.compile(r'([\da-f.:]+)'

153 - r' -.*?"([A-Z]+)'

154 - r' (.*?)"'

155 - r' ([1-9]\d{2})'

156 - r' (\d+)'

157 - r' (\d+)'

158 - r' ([\d.]+) ')

159 -

160 - regex_function = zip([apache_extended, nginx_extended, default],

154 + r' (\d+)')

155 +

156 + access_apache_ext = re.compile(r'([\da-f.:]+)'

157 + r' -.*?"([A-Z]+)'

158 + r' (.*?)"'

159 + r' ([1-9]\d{2})'

160 + r' (\d+)'

161 + r' (\d+)'

162 + r' (\d+) ')

163 +

164 + access_nginx_ext = re.compile(r'([\da-f.:]+)'

165 + r' -.*?"([A-Z]+)'

166 + r' (.*?)"'

167 + r' ([1-9]\d{2})'

168 + r' (\d+)'

169 + r' (\d+)'

170 + r' ([\d.]+) ')

171 +

172 + regex_function = zip([access_apache_ext, access_nginx_ext, access_default],

173 [lambda x: x, lambda x: x * 1000, lambda x: x],

162 - ['apache_extended', 'nginx_extended', 'default'])

163 -

174 + ['access_apache_ext', 'access_nginx_ext', 'access_default'])

175 + regex_name = None

176 for regex, function, name in regex_function:

177 if regex.search(last_line):

178 self.regex = regex

179 self.resp_time_func = function

180 regex_name = name

181 break

182 + return regex_name

183

171 - if self.regex:

172 - return self.regex.findall(last_line), regex_name

173 - else:

174 - return None, None

175 -

176 - def create_charts(self, parsed_line, regex_name):

184 + def create_access_charts(self, regex_name):

185 """

178 - :param parsed_line: list: re.findall result.

186 :param regex_name: str: regex name from 'find_regex' method. Ex.: 'apache_extended', 'nginx_extended'

187 :return:

188 Create additional charts depending on the 'find_regex' result (parsed_line) and configuration file

@@ -207,13 +214,13 @@ class Service(LogService):

214 ' "" "HTTP Methods" requests/s requests' \

215 ' web_log.http_method stacked 2 %s\n' % (job_name, self.update_every)

216

210 - if 'apache' in regex_name:

217 + if regex_name == 'access_apache_ext':

218 self.definitions['response_time']['lines'][0][4] = 1000

219 self.definitions['response_time']['lines'][1][4] = 1000

220 self.definitions['response_time']['lines'][2][4] = 1000

221

222 # Remove 'request_time' chart from ORDER if request_time not in logs

216 - if len(parsed_line) < 7:

223 + if regex_name == 'access_default':

224 self.order.remove('response_time')

225 # Remove 'clients_all' chart from ORDER if specified in the configuration

226 if not self.all_time:

@@ -246,7 +253,7 @@ class Service(LogService):

253 :param key: str: CHARTS dict key (chart name). Ex.: 'response_time'

254 :return: str: new chart string = previous + new dimensions

255 """

249 - self.storage.update({dimension: 0})

256 + self.data.update({dimension: 0})

257 # SET method check if dim in _dimensions

258 self._dimensions.append(dimension)

259 # UPDATE method do SET only if dim in definitions

@@ -256,7 +263,7 @@ class Service(LogService):

263 print(chart)

264 return chart

265

259 - def _get_data(self):

266 + def _get_access_data(self):

267 """

268 Parse new log lines

269 :return: dict OR None

@@ -276,7 +283,8 @@ class Service(LogService):

283 for line in raw:

284 match = self.regex.search(line)

285 if match:

279 - match_dict = dict(zip_longest('address method url code sent resp_length resp_time'.split(), match.groups()))

286 + match_dict = dict(zip_longest('address method url code sent resp_length resp_time'.split(),

287 + match.groups()))

288 try:

289 code = ''.join([match_dict['code'][0], 'xx'])

290 to_netdata[code] += 1

@@ -290,9 +298,9 @@ class Service(LogService):

298 self._get_data_per_url(match_dict['url'], default_dict)

299 # requests per http method

300 self._get_data_http_method(match_dict['method'], default_dict)

293 -

301 + # bandwidth sent

302 to_netdata['bytes_sent'] += int(match_dict['sent'])

295 -

303 + # request processing time and bandwidth received

304 if match_dict['resp_length'] and match_dict['resp_time']:

305 to_netdata['resp_length'] += int(match_dict['resp_length'])

306 resp_time = self.resp_time_func(float(match_dict['resp_time']))

@@ -300,12 +308,8 @@ class Service(LogService):

308 request_counter['count'] += 1

309 request_counter['sum'] += resp_time

310 # requests per ip proto

303 - if '.' in match_dict['address']:

304 - proto = 'ipv4'

305 - to_netdata['req_ipv4'] += 1

306 - else:

307 - proto = 'ipv6'

308 - to_netdata['req_ipv6'] += 1

311 + proto = 'ipv4' if '.' in match_dict['address'] else 'ipv6'

312 + to_netdata['req_' + proto] += 1

313 # unique clients ips

314 if address_not_in_pool(self.unique_all_time, match_dict['address'],

315 self.storage['unique_tot_ipv4'] + self.storage['unique_tot_ipv6']):

@@ -332,7 +336,7 @@ class Service(LogService):

336 :return:

337 Calls add_new_dimension method If the value is found for the first time

338 """

335 - if code not in self.storage:

339 + if code not in self.data:

340 chart_string_copy = self.detailed_chart

341 self.detailed_chart = self.add_new_dimension(code, [code, code, 'absolute'],

342 chart_string_copy, 'detailed_response_codes')

@@ -345,7 +349,7 @@ class Service(LogService):

349 :return:

350 Calls add_new_dimension method If the value is found for the first time

351 """

348 - if method not in self.storage:

352 + if method not in self.data:

353 chart_string_copy = self.http_method_chart

354 self.http_method_chart = self.add_new_dimension(method, [method, method, 'absolute'],

355 chart_string_copy, 'http_method')

web_log plugin: some optimization