c4c7b74f0 · cryptotaxi247/netdata-1

@@ -9,10 +9,6 @@ from os import access, R_OK

9 from os.path import getsize

10 from collections import namedtuple

11 from copy import deepcopy

12 -try:

13 - from itertools import zip_longest

14 -except ImportError:

15 - from itertools import izip_longest as zip_longest

12

13 priority = 60000

14 retries = 60

@@ -84,21 +80,26 @@ NAMED_URL_PATTERN = namedtuple('URL_PATTERN', ['description', 'pattern'])

80

81 class Service(LogService):

82 def __init__(self, configuration=None, name=None):

83 + """

84 + :param configuration:

85 + :param name:

86 + # self._get_data = None # will be assigned in 'check' method.

87 + # self.order = None # will be assigned in 'create_*_method' method.

88 + # self.definitions = None # will be assigned in 'create_*_method' method.

89 + # self.detailed_chart = None # will be assigned in 'create_*_method' method.

90 + # self.http_method_chart = None # will be assigned in 'create_*_method' method.

91 + """

92 LogService.__init__(self, configuration=configuration, name=name)

93 # Variables from module configuration file

94 self.log_path = self.configuration.get('path')

95 self.detailed_response_codes = self.configuration.get('detailed_response_codes', True)

96 self.all_time = self.configuration.get('all_time', True)

97 self.url_pattern = self.configuration.get('categories') # dict

93 - self.regex = None # will be assigned in 'find_regex' method

94 - self.resp_time_func = None # will be assigned in 'find_regex' method

95 - self._get_data = None # will be assigned in 'check' method.

96 - self.order = None # will be assigned in 'create_*_method' method.

97 - self.definitions = None # will be assigned in 'create_*_method' method.

98 - self.detailed_chart = None # will be assigned in 'create_*_method' method.

99 - self.http_method_chart = None # will be assigned in 'create_*_method' method.

100 - # sorted list of unique IPs

101 - self.unique_all_time = list()

98 + self.custom_log_format = self.configuration.get('custom_log_format') # dict

99 + # Instance variables

100 + self.unique_all_time = list() # sorted list of unique IPs

101 + self.regex = None # will be assigned in 'find_regex' or 'find_regex_custom' method

102 + self.resp_time_func = None # will be assigned in 'find_regex' or 'find_regex_custom' method

103 # if there is no new logs this dict returned to netdata

104 self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0,

105 'resp_time_avg': 0, 'unique_cur_ipv4': 0, 'unique_cur_ipv6': 0, '2xx': 0,

@@ -139,97 +140,195 @@ class Service(LogService):

140 self.error(str(error))

141 return False

142

142 - # Parse last line

143 - regex_name = self.find_regex(last_line)

144 - if not regex_name:

145 - self.error('Unknown log format. Can\'t parse %s' % self.log_path)

143 + # Custom_log_format is preferable

144 + if self.custom_log_format:

145 + match_dict, log_name, error = self.find_regex_custom(last_line)

146 + else:

147 + match_dict, log_name, error = self.find_regex(last_line)

148 +

149 + if match_dict is None:

150 + self.error(str(error))

151 return False

152

148 - if regex_name.startswith('acs_'):

149 - self.create_access_charts(regex_name)

150 - if regex_name == 'acs_default':

151 - self.info('Not all data collected. You need to modify LogFormat.')

153 + if not (self.regex and self.resp_time_func):

154 + self.error('That can not happen, but it happened. "regex" or "resp_time_func" is None')

155 +

156 + if log_name == 'web_access':

157 + self.create_access_charts(match_dict) # Create charts

158 self._get_data = self._get_access_data

153 - self.info('Used regex: %s' % regex_name)

159 + self.info('Collected data: %s' % list(match_dict.keys()))

160 return True

161 else:

162 # If it's not access_logs.. Not used at the moment

163 return False

164

165 + def find_regex_custom(self, last_line):

166 + """

167 + :param last_line: str: literally last line from log file

168 + :return: tuple where:

169 + [0]: dict or None: match_dict or None

170 + [1]: str or None: log_name or None

171 + [2]: str: error description

172 +

173 + We are here only if "custom_log_format" is in logs. We need to make sure:

174 + 1. "custom_log_format" is a dict

175 + 2. "pattern" in "custom_log_format" and pattern is <str> instance

176 + 3. if "time_multiplier" is in "custom_log_format" it must be <int> instance

177 +

178 + If all parameters is ok we need to make sure:

179 + 1. Pattern search is success

180 + 2. Pattern search contains named subgroups (?P<subgroup_name>) (= "match_dict")

181 +

182 + If pattern search is success we need to make sure:

183 + 1. All mandatory keys ['address', 'code', 'bytes_sent', 'method', 'url'] are in "match_dict"

184 +

185 + If this is True we need to make sure:

186 + 1. All mandatory key values from "match_dict" have the correct format

187 + ("code" is integer, "method" is uppercase word, etc)

188 +

189 + If non mandatory keys in "match_dict" we need to make sure:

190 + 1. All non mandatory key values from match_dict ['resp_length', 'resp_time'] have the correct format

191 + ("resp_length" is integer or "-", "resp_time" is integer or float)

192 +

193 + """

194 + try:

195 + self.custom_log_format.keys()

196 + except AttributeError:

197 + return None, None, 'Custom log: "custom_log_format" is not a <dict>'

198 +

199 + pattern = self.custom_log_format.get('pattern')

200 + if not (pattern and isinstance(pattern, str)):

201 + return None, None, 'Custom log: "pattern" option is not specified or type is not <str>'

202 +

203 + resp_time_func = self.custom_log_format.get('time_multiplier') or 0

204 +

205 + if not isinstance(resp_time_func, int):

206 + return None, None, 'Custom log: "time_multiplier" is not an integer'

207 +

208 + regex = re.compile(pattern)

209 + match = regex.search(last_line)

210 + if match:

211 + match_dict = match.groupdict() or None

212 + else:

213 + return None, None, 'Custom log: pattern search FAILED'

214 +

215 + if match_dict is None:

216 + return None, None, 'Custom log: search OK but contains no named subgroups' \

217 + ' (you need to use ?P<subgroup_name>)'

218 + else:

219 + basic_values = {'address', 'method', 'url', 'code', 'bytes_sent'} - set(match_dict)

220 +

221 + if basic_values:

222 + return None, None, 'Custom log: search OK but some mandatory keys (%s) are missing' % list(basic_values)

223 + else:

224 + if not re.search(r'[\da-f.:]+', match_dict['address']):

225 + return None, None, 'Custom log: can\'t parse "address": %s' % match_dict['address']

226 + if not re.search(r'[1-9]\d{2}', match_dict['code']):

227 + return None, None, 'Custom log: can\'t parse "code": %s' % match_dict['code']

228 + if not re.search(r'[A-Z]+', match_dict['method']):

229 + return None, None, 'Custom log: can\'t parse "method": %s' % match_dict['method']

230 + if not re.search(r'\d+|-', match_dict['bytes_sent']):

231 + return None, None, 'Custom log: can\'t parse "bytes_sent": %s' % match_dict['bytes_sent']

232 +

233 + if 'resp_length' in match_dict:

234 + if not re.search(r'\d+', match_dict.get('resp_length', '')):

235 + return None, None, 'Custom log: can\'t parse "resp_length": %s' % match_dict['resp_length']

236 +

237 + if 'resp_time' in match_dict:

238 + if not re.search(r'[\d.]+', match_dict.get('resp_length', '')):

239 + return None, None, 'Custom log: can\'t parse "resp_time": %s' % match_dict['resp_time']

240 + else:

241 + if '.' in match_dict['resp_time']:

242 + self.resp_time_func = lambda time: time * (resp_time_func or 1000000)

243 + else:

244 + self.resp_time_func = lambda time: time * (resp_time_func or 1)

245 +

246 + self.regex = regex

247 + return match_dict, 'web_access', 'Custom log: we are fine'

248 +

249 def find_regex(self, last_line):

250 """

251 :param last_line: str: literally last line from log file

162 - :return: regex_name

163 - It's sad but different web servers has different logs formats

164 - We need to find appropriate regex for current log file

165 - All logic is do a regex search through the string for all patterns

252 + :return: tuple where:

253 + [0]: dict or None: match_dict or None

254 + [1]: str or None: log_name or None

255 + [2]: str: error description

256 + We need to find appropriate pattern for current log file

257 + All logic is do a regex search through the string for all predefined patterns

258 until we find something or fail.

259 """

260 # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code

261 # 5. Bytes sent 6. Response length 7. Response process time

170 - acs_default = re.compile(r'([\da-f.:]+)'

171 - r' -.*?"([A-Z]+)'

172 - r' (.*?)"'

173 - r' ([1-9]\d{2})'

174 - r' (\d+|-)')

175 -

176 - acs_apache_ext_insert = re.compile(r'([\da-f.:]+)'

177 - r' -.*?"([A-Z]+)'

178 - r' (.*?)"'

179 - r' ([1-9]\d{2})'

180 - r' (\d+|-)'

181 - r' (\d+)'

182 - r' (\d+) ')

183 -

184 - acs_apache_ext_append = re.compile(r'([\da-f.:]+)'

185 - r' -.*?"([A-Z]+)'

186 - r' (.*?)"'

187 - r' ([1-9]\d{2})'

188 - r' (\d+|-)'

262 + acs_default = re.compile(r'(?P<address>[\da-f.:]+)'

263 + r' -.*?"(?P<method>[A-Z]+)'

264 + r' (?P<url>.*?)"'

265 + r' (?P<code>[1-9]\d{2})'

266 + r' (?P<bytes_sent>\d+|-)')

267 +

268 + acs_apache_ext_insert = re.compile(r'(?P<address>[\da-f.:]+)'

269 + r' -.*?"(?P<method>[A-Z]+)'

270 + r' (?P<url>.*?)"'

271 + r' (?P<code>[1-9]\d{2})'

272 + r' (?P<bytes_sent>\d+|-)'

273 + r' (?P<resp_length>\d+)'

274 + r' (?P<resp_time>\d+) ')

275 +

276 + acs_apache_ext_append = re.compile(r'(?P<address>[\da-f.:]+)'

277 + r' -.*?"(?P<method>[A-Z]+)'

278 + r' (?P<url>.*?)"'

279 + r' (?P<code>[1-9]\d{2})'

280 + r' (?P<bytes_sent>\d+|-)'

281 r' .*?'

190 - r' (\d+)'

191 - r' (\d+)'

282 + r' (?P<resp_length>\d+)'

283 + r' (?P<resp_time>\d+)'

284 r'(?: |$)')

285

194 - acs_nginx_ext_insert = re.compile(r'([\da-f.:]+)'

195 - r' -.*?"([A-Z]+)'

196 - r' (.*?)"'

197 - r' ([1-9]\d{2})'

198 - r' (\d+)'

199 - r' (\d+)'

200 - r' (\d\.\d+) ')

201 -

202 - acs_nginx_ext_append = re.compile(r'([\da-f.:]+)'

203 - r' -.*?"([A-Z]+)'

204 - r' (.*?)"'

205 - r' ([1-9]\d{2})'

206 - r' (\d+)'

286 + acs_nginx_ext_insert = re.compile(r'(?P<address>[\da-f.:]+)'

287 + r' -.*?"(?P<method>[A-Z]+)'

288 + r' (?P<url>.*?)"'

289 + r' (?P<code>[1-9]\d{2})'

290 + r' (?P<bytes_sent>\d+)'

291 + r' (?P<resp_length>\d+)'

292 + r' (?P<resp_time>\d\.\d+) ')

293 +

294 + acs_nginx_ext_append = re.compile(r'(?P<address>[\da-f.:]+)'

295 + r' -.*?"(?P<method>[A-Z]+)'

296 + r' (?P<url>.*?)"'

297 + r' (?P<code>[1-9]\d{2})'

298 + r' (?P<bytes_sent>\d+)'

299 r' .*?'

208 - r' (\d+)'

209 - r' (\d\.\d+)')

300 + r' (?P<resp_length>\d+)'

301 + r' (?P<resp_time>\d\.\d+)')

302 +

303 + def func_usec(time):

304 + return time

305 +

306 + def func_sec(time):

307 + return time * 1000000

308

309 r_regex = [acs_apache_ext_insert, acs_apache_ext_append, acs_nginx_ext_insert,

310 acs_nginx_ext_append, acs_default]

213 - r_function = [lambda x: x, lambda x: x, lambda x: x * 1000000, lambda x: x * 1000000, lambda x: x]

214 - r_name = ['acs_apache_ext_insert', 'acs_apache_ext_append', 'acs_nginx_ext_insert',

215 - 'acs_nginx_ext_append', 'acs_default']

216 - regex_function_name = zip(r_regex, r_function, r_name)

217 -

218 - regex_name = None

219 - for regex, function, name in regex_function_name:

220 - if regex.search(last_line):

311 + r_function = [func_usec, func_usec, func_sec, func_sec, func_usec]

312 + regex_function = zip(r_regex, r_function)

313 +

314 + match_dict = dict()

315 + for regex, function in regex_function:

316 + match = regex.search(last_line)

317 + if match:

318 self.regex = regex

319 self.resp_time_func = function

223 - regex_name = name

320 + match_dict = match.groupdict()

321 break

225 - return regex_name

322

227 - def create_access_charts(self, regex_name):

323 + return match_dict or None, 'web_access', 'Unknown log format. Plugin still can work for you.' \

324 + ' Read about the "custom_log_format" feature in the conf file'

325 +

326 + def create_access_charts(self, match_dict):

327 """

229 - :param regex_name: str: regex name from 'find_regex' method. Ex.: 'apache_extended', 'nginx_extended'

328 + :param match_dict: dict: regex.search.groupdict(). Ex. {'address': '127.0.0.1', 'code': '200', 'method': 'GET'}

329 :return:

231 - Create additional charts depending on the 'find_regex' result (parsed_line) and configuration file

232 - 1. 'time_response' chart is removed if there is no 'time_response' in logs.

330 + Create additional charts depending on the 'match_dict' keys and configuration file options

331 + 1. 'time_response' chart is removed if there is no 'resp_time' in match_dict.

332 2. Other stuff is just remove/add chart depending on yes/no in conf

333 """

334 def find_job_name(override_name, name):

@@ -256,8 +355,8 @@ class Service(LogService):

355 ' "" "Requests Per HTTP Method" requests/s "http methods"' \

356 ' web_log.http_method stacked 2 %s\n' % (job_name, self.update_every)

357

259 - # Remove 'request_time' chart from ORDER if request_time not in logs

260 - if regex_name == 'acs_default':

358 + # Remove 'request_time' chart from ORDER if resp_time not in match_dict

359 + if 'resp_time' not in match_dict:

360 self.order.remove('response_time')

361 # Remove 'clients_all' chart from ORDER if specified in the configuration

362 if not self.all_time:

@@ -320,8 +419,7 @@ class Service(LogService):

419 for line in raw:

420 match = self.regex.search(line)

421 if match:

323 - match_dict = dict(zip_longest('address method url code sent resp_length resp_time'.split(),

324 - match.groups()))

422 + match_dict = match.groupdict()

423 try:

424 code = ''.join([match_dict['code'][0], 'xx'])

425 self.data[code] += 1

@@ -338,10 +436,11 @@ class Service(LogService):

436 # requests per http method

437 self._get_data_http_method(match_dict['method'])

438 # bandwidth sent

341 - self.data['bytes_sent'] += int(match_dict['sent'] if '-' not in match_dict['sent'] else 0)

439 + self.data['bytes_sent'] += int(match_dict['bytes_sent'] if '-' not in match_dict['bytes_sent'] else 0)

440 # request processing time and bandwidth received

343 - if match_dict['resp_length'] and match_dict['resp_time']:

441 + if 'resp_length' in match_dict:

442 self.data['resp_length'] += int(match_dict['resp_length'])

443 + if 'resp_time' in match_dict:

444 resp_time = self.resp_time_func(float(match_dict['resp_time']))

445 bisect.insort_left(request_time, resp_time)

446 request_counter['count'] += 1

web_log plugin: initial "custom_log_format" feature support added"