07a151cab · cryptotaxi247/netdata-1

@@ -2,32 +2,37 @@

2 # Description: fail2ban log netdata python.d module

3 # Author: l2isbad

4

5 -from base import LogService

5 from re import compile as r_compile

6 from os import access as is_accessible, R_OK

8 -from os.path import isdir

7 +from os.path import isdir, getsize

8 from glob import glob

9 import bisect

10 +from base import LogService

11

12 priority = 60000

13 retries = 60

14 REGEX_JAILS = r_compile(r'\[([A-Za-z-_]+)][^\[\]]*?(?<!# )enabled = (?:(true|false))')

15 -REGEX_DATA = r_compile(r'\[(?P<jail>[a-z]+)\] (?P<ban>[A-Z])[a-z]+ (?P<ipaddr>\d{1,3}(?:\.\d{1,3}){3})')

15 +REGEX_DATA = r_compile(r'\[(?P<jail>[a-z]+)\] (?P<action>[A-Z])[a-z]+ (?P<ipaddr>\d{1,3}(?:\.\d{1,3}){3})')

16 ORDER = ['jails_bans', 'jails_in_jail']

17

18

19 class Service(LogService):

20 + """

21 + fail2ban log class

22 + Reads logs line by line

23 + Jail auto detection included

24 + It produces following charts:

25 + * Bans per second for every jail

26 + * Banned IPs for every jail (since the last restart of netdata)

27 + """

28 def __init__(self, configuration=None, name=None):

29 LogService.__init__(self, configuration=configuration, name=name)

30 self.order = ORDER

31 + self.definitions = dict()

32 self.log_path = self.configuration.get('log_path', '/var/log/fail2ban.log')

33 self.conf_path = self.configuration.get('conf_path', '/etc/fail2ban/jail.local')

34 self.conf_dir = self.configuration.get('conf_dir', '/etc/fail2ban/jail.d/')

26 - self.bans = dict()

27 - try:

28 - self.exclude = self.configuration['exclude'].split()

29 - except (KeyError, AttributeError):

30 - self.exclude = list()

35 + self.exclude = self.configuration.get('exclude')

36

37 def _get_data(self):

38 """

@@ -38,7 +43,7 @@ class Service(LogService):

43 if raw is None:

44 return None

45 elif not raw:

41 - return self.data

46 + return self.to_netdata

47

48 # Fail2ban logs looks like

49 # 2016-12-25 12:36:04,711 fail2ban.actions[2455]: WARNING [ssh] Ban 178.156.32.231

@@ -46,31 +51,46 @@ class Service(LogService):

51 match = REGEX_DATA.search(row)

52 if match:

53 match_dict = match.groupdict()

49 - jail, ban, ipaddr = match_dict['jail'], match_dict['ban'], match_dict['ipaddr']

54 + jail, action, ipaddr = match_dict['jail'], match_dict['action'], match_dict['ipaddr']

55 if jail in self.jails_list:

51 - if ban == 'B':

52 - self.data[jail] += 1

53 - if address_not_in_jail(self.bans[jail], ipaddr, self.data[jail + '_in_jail']):

54 - self.data[jail + '_in_jail'] += 1

56 + if action == 'B':

57 + self.to_netdata[jail] += 1

58 + if address_not_in_jail(self.banned_ips[jail], ipaddr, self.to_netdata[jail + '_in_jail']):

59 + self.to_netdata[jail + '_in_jail'] += 1

60 else:

56 - if ipaddr in self.bans[jail]:

57 - self.bans[jail].remove(ipaddr)

58 - self.data[jail + '_in_jail'] -= 1

61 + if ipaddr in self.banned_ips[jail]:

62 + self.banned_ips[jail].remove(ipaddr)

63 + self.to_netdata[jail + '_in_jail'] -= 1

64

60 - return self.data

65 + return self.to_netdata

66

67 def check(self):

68 + """

69 + :return: bool

70 +

71 + Check if the "log_path" is not empty and readable

72 + """

73

64 - # Check "log_path" is accessible.

65 - # If NOT STOP plugin

66 - if not is_accessible(self.log_path, R_OK):

67 - self.error('Cannot access file %s' % self.log_path)

74 + if not (is_accessible(self.log_path, R_OK) and getsize(self.log_path) != 0):

75 + self.error('%s is not readable or empty' % self.log_path)

76 return False

77 + self.jails_list, self.to_netdata, self.banned_ips = self.jails_auto_detection_()

78 + self.definitions = create_definitions_(self.jails_list)

79 + self.info('Jails: %s' % self.jails_list)

80 + return True

81 +

82 + def jails_auto_detection_(self):

83 + """

84 + return: <tuple>

85

86 + * jails_list - list of enabled jails (['ssh', 'apache', ...])

87 + * to_netdata - dict ({'ssh': 0, 'ssh_in_jail': 0, ...})

88 + * banned_ips - here will be stored all the banned ips ({'ssh': ['1.2.3.4', '5.6.7.8', ...], ...})

89 + """

90 raw_jails_list = list()

91 jails_list = list()

92

73 - for raw_jail in parse_configuration_files(self.conf_path, self.conf_dir, self.error):

93 + for raw_jail in parse_configuration_files_(self.conf_path, self.conf_dir, self.error):

94 raw_jails_list.extend(raw_jail)

95

96 for jail, status in raw_jails_list:

@@ -80,30 +100,52 @@ class Service(LogService):

100 jails_list.remove(jail)

101

102 # If for some reason parse failed we still can START with default jails_list.

83 - self.jails_list = list(set(jails_list) - set(self.exclude)) or ['ssh']

84 -

85 - self.data = dict([(jail, 0) for jail in self.jails_list])

86 - self.data.update(dict([(jail + '_in_jail', 0) for jail in self.jails_list]))

87 - self.bans = dict([(jail, list()) for jail in self.jails_list])

88 -

89 - self._data_from_check = self.data

90 - self.create_dimensions()

91 - self.info('Plugin successfully started. Jails: %s' % self.jails_list)

92 - return True

93 -

94 - def create_dimensions(self):

95 - self.definitions = {

96 - 'jails_bans': {'options': [None, 'Jails Ban Statistics', 'bans/s', 'bans', 'jail.bans', 'line'],

97 - 'lines': []},

98 - 'jails_in_jail': {'options': [None, 'Banned IPs (since the last restart of netdata)', 'IPs',

99 - 'in jail', 'jail.in_jail', 'line'],

100 - 'lines': []},

101 - }

102 - for jail in self.jails_list:

103 - self.definitions['jails_bans']['lines'].append([jail, jail, 'incremental'])

104 - self.definitions['jails_in_jail']['lines'].append([jail + '_in_jail', jail, 'absolute'])

105 -

106 -def parse_configuration_files(jails_conf_path, jails_conf_dir, print_error):

103 + jails_list = list(set(jails_list) - set(self.exclude.split()

104 + if isinstance(self.exclude, str) else list())) or ['ssh']

105 +

106 + to_netdata = dict([(jail, 0) for jail in jails_list])

107 + to_netdata.update(dict([(jail + '_in_jail', 0) for jail in jails_list]))

108 + banned_ips = dict([(jail, list()) for jail in jails_list])

109 +

110 + return jails_list, to_netdata, banned_ips

111 +

112 +

113 +def create_definitions_(jails_list):

114 + """

115 + Chart definitions creating

116 + """

117 +

118 + definitions = {

119 + 'jails_bans': {'options': [None, 'Jails Ban Statistics', 'bans/s', 'bans', 'jail.bans', 'line'],

120 + 'lines': []},

121 + 'jails_in_jail': {'options': [None, 'Banned IPs (since the last restart of netdata)', 'IPs',

122 + 'in jail', 'jail.in_jail', 'line'],

123 + 'lines': []}}

124 + for jail in jails_list:

125 + definitions['jails_bans']['lines'].append([jail, jail, 'incremental'])

126 + definitions['jails_in_jail']['lines'].append([jail + '_in_jail', jail, 'absolute'])

127 +

128 + return definitions

129 +

130 +

131 +def parse_configuration_files_(jails_conf_path, jails_conf_dir, print_error):

132 + """

133 + :param jails_conf_path: <str>

134 + :param jails_conf_dir: <str>

135 + :param print_error: <function>

136 + :return: <tuple>

137 +

138 + Uses "find_jails_in_files" function to find all jails in the "jails_conf_dir" directory

139 + and in the "jails_conf_path"

140 +

141 + All files must endswith ".local" or ".conf"

142 + Return order is important.

143 + According man jail.conf it should be

144 + * jail.conf

145 + * jail.d/*.conf (in alphabetical order)

146 + * jail.local

147 + * jail.d/*.local (in alphabetical order)

148 + """

149 path_conf, path_local, dir_conf, dir_local = list(), list(), list(), list()

150

151 # Parse files in the directory

@@ -116,22 +158,31 @@ def parse_configuration_files(jails_conf_path, jails_conf_dir, print_error):

158 print_error('%s is empty or not readable' % jails_conf_dir)

159 else:

160 dir_conf, dir_local = (find_jails_in_files(dir_conf, print_error),

119 - find_jails_in_files(dir_local, print_error))

161 + find_jails_in_files(dir_local, print_error))

162

163 # Parse .conf and .local files

122 - if (isinstance(jails_conf_path, str) and jails_conf_path.endswith(('.local', '.conf'))):

164 + if isinstance(jails_conf_path, str) and jails_conf_path.endswith(('.local', '.conf')):

165 path_conf, path_local = (find_jails_in_files([jails_conf_path.split('.')[0] + '.conf'], print_error),

124 - find_jails_in_files([jails_conf_path.split('.')[0] + '.local'], print_error))

166 + find_jails_in_files([jails_conf_path.split('.')[0] + '.local'], print_error))

167

168 return path_conf, dir_conf, path_local, dir_local

169

170

171 def find_jails_in_files(list_of_files, print_error):

172 + """

173 + :param list_of_files: <list>

174 + :param print_error: <function>

175 + :return: <list>

176 +

177 + Open a file and parse it to find all (enabled and disabled) jails

178 + The output is a list of tuples:

179 + [('ssh', 'true'), ('apache', 'false'), ...]

180 + """

181 jails_list = list()

182 for conf in list_of_files:

183 if is_accessible(conf, R_OK):

133 - with open(conf, 'rt') as f:

134 - raw_data = f.read()

184 + with open(conf, 'rt') as conf:

185 + raw_data = conf.read()

186 data = ' '.join(raw_data.split())

187 jails_list.extend(REGEX_JAILS.findall(data))

188 else:

@@ -140,13 +191,21 @@ def find_jails_in_files(list_of_files, print_error):

191

192

193 def address_not_in_jail(pool, address, pool_size):

194 + """

195 + :param pool: <list>

196 + :param address: <str>

197 + :param pool_size: <int>

198 + :return: bool

199 +

200 + Checks if the address is in the pool.

201 + If not address will be added

202 + """

203 index = bisect.bisect_left(pool, address)

204 if index < pool_size:

205 if pool[index] == address:

206 return False

147 - else:

148 - bisect.insort_left(pool, address)

149 - return True

207 + bisect.insort_left(pool, address)

208 + return True

209 else:

210 bisect.insort_left(pool, address)

211 return True

fail2ban plugin: docstrings added + minor fixes