MUCH faster version of log reader
paulfantom committed
Jul 1, 2016 at 18:03 UTC
65e51fa32a76e1bd6b1f1cdb92745e8569380dc0
1 file changed
+8
-18
python.d/python_modules/base.py
+8
-18
@@ -425,7 +425,7 @@ class LogService(SimpleService):
425
# 'chart_name_in_netdata' : [ charts['chart_name_in_netdata']['lines']['name'] ]
426
# }
427
self.log_path = ""
428
- self._last_line = -1
428
+ self._last_position = 0
429
# self._log_reader = None
430
SimpleService.__init__(self, configuration=configuration, name=name)
431
# FIXME Remove preventing of frequent log parsing
@@ -434,27 +434,17 @@ class LogService(SimpleService):
434
self.retries = 100000 # basically always retry
435
436
def _get_data(self):
437
- # FIXME find faster solution of reading data. Maybe implement reading in subprocess?
438
- # if self._log_reader is None:
439
- # self._log_reader = Popen(['tail', '-F', self.log_path], stdout=PIPE, stderr=STDOUT)
440
- # if self._log_reader.poll() is not None:
441
- # self._log_reader = Popen(['tail', '-F', self.log_path], stdout=PIPE, stderr=STDOUT)
437
lines = []
443
- last = -1
444
- total = 0
438
try:
446
- with open(self.log_path) as fp:
439
+ if os.path.getsize(self.log_path) < self._last_position:
440
+ self._last_position = 0
441
+ with open(self.log_path, "r") as fp:
442
+ fp.seek(self._last_position)
443
for i, line in enumerate(fp):
448
- if i > self._last_line:
449
- lines.append(line)
450
- last = i
451
- total += 1
444
+ lines.append(line)
445
+ self._last_position = fp.tell()
446
except Exception as e:
447
msg.error(self.__module__, str(e))
454
- if last != -1:
455
- self._last_line = last
456
- if self._last_line > total:
457
- self._last_line = -1
448
449
if len(lines) != 0:
450
return lines
@@ -478,6 +468,6 @@ class LogService(SimpleService):
468
469
def create(self):
470
status = SimpleService.create(self)
481
- self._last_line = -1
471
+ self._last_position = 0
472
return status
473