0d7b7affb · cryptotaxi247/netdata-1

@@ -3,212 +3,198 @@

3 # Author: l2isbad

4 # SPDX-License-Identifier: GPL-3.0+

5

6 -import bisect

6 +import re

7 +import os

8

9 +from collections import defaultdict

10 from glob import glob

9 -from re import compile as r_compile

10 -from os import access as is_accessible, R_OK

11 -from os.path import isdir, getsize

12 -

11

12 from bases.FrameworkServices.LogService import LogService

13

16 -priority = 60000

17 -retries = 60

18 -REGEX_JAILS = r_compile(r'\[([a-zA-Z0-9_-]+)\][^\[\]]+?enabled\s+= (true|false)')

19 -REGEX_DATA = r_compile(r'\[(?P<jail>[A-Za-z-_0-9]+)\] (?P<action>U|B)[a-z]+ (?P<ipaddr>\d{1,3}(?:\.\d{1,3}){3})')

20 -ORDER = ['jails_bans', 'jails_in_jail']

14

15 +ORDER = [

16 + 'jails_bans',

17 + 'jails_in_jail',

18 +]

19

23 -class Service(LogService):

20 +

21 +def charts(jails):

22 """

25 - fail2ban log class

26 - Reads logs line by line

27 - Jail auto detection included

28 - It produces following charts:

29 - * Bans per second for every jail

30 - * Banned IPs for every jail (since the last restart of netdata)

23 + Chart definitions creating

24 """

25 +

26 + ch = {

27 + ORDER[0]:

28 + {

29 + 'options':

30 + [None, 'Jails Ban Rate', 'bans/s', 'bans', 'jail.bans', 'line'],

31 + 'lines': []

32 + },

33 + ORDER[1]:

34 + {

35 + 'options':

36 + [None, 'Banned IPs (since the last restart of netdata)', 'IPs','in jail', 'jail.in_jail', 'line'],

37 + 'lines':

38 + []

39 + },

40 + }

41 + for jail in jails:

42 + ch[ORDER[0]]['lines'].append([jail, jail, 'incremental'])

43 + ch[ORDER[1]]['lines'].append(['{0}_in_jail'.format(jail), jail, 'absolute'])

44 +

45 + return ch

46 +

47 +

48 +RE_JAILS = re.compile(r'\[([a-zA-Z0-9_-]+)\][^\[\]]+?enabled\s+= (true|false)')

49 +

50 +# Example:

51 +# 2018-09-12 11:45:53,715 fail2ban.actions[25029]: WARNING [ssh] Unban 195.201.88.33

52 +# 2018-09-12 11:45:58,727 fail2ban.actions[25029]: WARNING [ssh] Ban 217.59.246.27

53 +RE_DATA = re.compile(r'\[(?P<jail>[A-Za-z-_0-9]+)\] (?P<action>Unban|Ban) (?P<ip>[a-f0-9.:]+)')

54 +

55 +DEFAULT_JAILS = [

56 + "ssh",

57 +]

58 +

59 +

60 +class Service(LogService):

61 def __init__(self, configuration=None, name=None):

62 LogService.__init__(self, configuration=configuration, name=name)

63 self.order = ORDER

64 self.definitions = dict()

65 +

66 self.log_path = self.configuration.get('log_path', '/var/log/fail2ban.log')

67 self.conf_path = self.configuration.get('conf_path', '/etc/fail2ban/jail.local')

68 self.conf_dir = self.configuration.get('conf_dir', '/etc/fail2ban/jail.d/')

39 - self.exclude = self.configuration.get('exclude')

69 + self.exclude = self.configuration.get('exclude', str())

70

41 - def _get_data(self):

42 - """

43 - Parse new log lines

44 - :return: dict

45 - """

46 - raw = self._get_raw_data()

47 - if raw is None:

48 - return None

49 - elif not raw:

50 - return self.to_netdata

51 -

52 - # Fail2ban logs looks like

53 - # 2016-12-25 12:36:04,711 fail2ban.actions[2455]: WARNING [ssh] Ban 178.156.32.231

54 - for row in raw:

55 - match = REGEX_DATA.search(row)

56 - if match:

57 - match_dict = match.groupdict()

58 - jail, action, ipaddr = match_dict['jail'], match_dict['action'], match_dict['ipaddr']

59 - if jail in self.jails_list:

60 - if action == 'B':

61 - self.to_netdata[jail] += 1

62 - if address_not_in_jail(self.banned_ips[jail], ipaddr, self.to_netdata[jail + '_in_jail']):

63 - self.to_netdata[jail + '_in_jail'] += 1

64 - else:

65 - if ipaddr in self.banned_ips[jail]:

66 - self.banned_ips[jail].remove(ipaddr)

67 - self.to_netdata[jail + '_in_jail'] -= 1

68 -

69 - return self.to_netdata

71 + self.monitoring_jails = list()

72 + self.banned_ips = defaultdict(set)

73 + self.data = dict()

74

75 def check(self):

76 """

77 :return: bool

74 -

75 - Check if the "log_path" is not empty and readable

78 """

79 + if not self.conf_path.endswith((".conf", ".local")):

80 + self.error("{0} is a wrong conf path name, must be *.conf or *.local".format(self.conf_path))

81 + return False

82 +

83 + if not os.access(self.log_path, os.R_OK):

84 + self.error('{0} is not readable'.format(self.log_path))

85 + return False

86

78 - if not (is_accessible(self.log_path, R_OK) and getsize(self.log_path) != 0):

79 - self.error('%s is not readable or empty' % self.log_path)

87 + if os.path.getsize(self.log_path) == 0:

88 + self.error('{0} is empty'.format(self.log_path))

89 return False

81 - self.jails_list, self.to_netdata, self.banned_ips = self.jails_auto_detection_()

82 - self.definitions = create_definitions_(self.jails_list)

83 - self.info('Jails: %s' % self.jails_list)

90 +

91 + self.monitoring_jails = self.jails_auto_detection()

92 + for jail in self.monitoring_jails:

93 + self.data[jail] = 0

94 + self.data["{0}_in_jail".format(jail)] = 0

95 +

96 + self.definitions = charts(self.monitoring_jails)

97 + self.info('monitoring jails: {0}'.format(self.monitoring_jails))

98 +

99 return True

100

86 - def jails_auto_detection_(self):

101 + def get_data(self):

102 """

88 - return: <tuple>

89 -

90 - * jails_list - list of enabled jails (['ssh', 'apache', ...])

91 - * to_netdata - dict ({'ssh': 0, 'ssh_in_jail': 0, ...})

92 - * banned_ips - here will be stored all the banned ips ({'ssh': ['1.2.3.4', '5.6.7.8', ...], ...})

103 + :return: dict

104 """

94 - raw_jails_list = list()

95 - jails_list = list()

105 + raw = self._get_raw_data()

106

97 - for raw_jail in parse_configuration_files_(self.conf_path, self.conf_dir, self.error):

98 - raw_jails_list.extend(raw_jail)

107 + if not raw:

108 + return None if raw is None else self.data

109

100 - for jail, status in raw_jails_list:

101 - if status == 'true' and jail not in jails_list:

102 - jails_list.append(jail)

103 - elif status == 'false' and jail in jails_list:

104 - jails_list.remove(jail)

105 - # If for some reason parse failed we still can START with default jails_list.

106 - jails_list = list(set(jails_list) - set(self.exclude.split()

107 - if isinstance(self.exclude, str) else list())) or ['ssh']

110 + for row in raw:

111 + match = RE_DATA.search(row)

112

109 - to_netdata = dict([(jail, 0) for jail in jails_list])

110 - to_netdata.update(dict([(jail + '_in_jail', 0) for jail in jails_list]))

111 - banned_ips = dict([(jail, list()) for jail in jails_list])

113 + if not match:

114 + continue

115

113 - return jails_list, to_netdata, banned_ips

116 + match = match.groupdict()

117

118 + if match["jail"] not in self.monitoring_jails:

119 + continue

120

116 -def create_definitions_(jails_list):

117 - """

118 - Chart definitions creating

119 - """

121 + jail, action, ip = match['jail'], match['action'], match['ip']

122

121 - definitions = {

122 - 'jails_bans': {'options': [None, 'Jails Ban Statistics', 'bans/s', 'bans', 'jail.bans', 'line'],

123 - 'lines': []},

124 - 'jails_in_jail': {'options': [None, 'Banned IPs (since the last restart of netdata)', 'IPs',

125 - 'in jail', 'jail.in_jail', 'line'],

126 - 'lines': []}}

127 - for jail in jails_list:

128 - definitions['jails_bans']['lines'].append([jail, jail, 'incremental'])

129 - definitions['jails_in_jail']['lines'].append([jail + '_in_jail', jail, 'absolute'])

123 + if action == "Ban":

124 + self.data[jail] += 1

125 + if ip not in self.banned_ips[jail]:

126 + self.banned_ips[jail].add(ip)

127 + self.data["{0}_in_jail".format(jail)] += 1

128 + else:

129 + if ip in self.banned_ips[jail]:

130 + self.banned_ips[jail].remove(ip)

131 + self.data["{0}_in_jail".format(jail)] -= 1

132

131 - return definitions

133 + return self.data

134

135 + def get_files_from_dir(self, dir_path, suffix):

136 + """

137 + :return: list

138 + """

139 + if not os.path.isdir(dir_path):

140 + self.error("{0} is not a directory".format(dir_path))

141 + return list()

142

134 -def parse_configuration_files_(jails_conf_path, jails_conf_dir, print_error):

135 - """

136 - :param jails_conf_path: <str>

137 - :param jails_conf_dir: <str>

138 - :param print_error: <function>

139 - :return: <tuple>

140 -

141 - Uses "find_jails_in_files" function to find all jails in the "jails_conf_dir" directory

142 - and in the "jails_conf_path"

143 -

144 - All files must endswith ".local" or ".conf"

145 - Return order is important.

146 - According man jail.conf it should be

147 - * jail.conf

148 - * jail.d/*.conf (in alphabetical order)

149 - * jail.local

150 - * jail.d/*.local (in alphabetical order)

151 - """

152 - path_conf, path_local, dir_conf, dir_local = list(), list(), list(), list()

143 + return glob("{0}/*.{1}".format(self.conf_dir, suffix))

144

154 - # Parse files in the directory

155 - if not (isinstance(jails_conf_dir, str) and isdir(jails_conf_dir)):

156 - print_error('%s is not a directory' % jails_conf_dir)

157 - else:

158 - dir_conf = list(filter(lambda conf: is_accessible(conf, R_OK), glob(jails_conf_dir + '/*.conf')))

159 - dir_local = list(filter(lambda local: is_accessible(local, R_OK), glob(jails_conf_dir + '/*.local')))

160 - if not (dir_conf or dir_local):

161 - print_error('%s is empty or not readable' % jails_conf_dir)

162 - else:

163 - dir_conf, dir_local = (find_jails_in_files(dir_conf, print_error),

164 - find_jails_in_files(dir_local, print_error))

145 + def get_jails_from_file(self, file_path):

146 + """

147 + :return: list

148 + """

149 + if not os.access(file_path, os.R_OK):

150 + self.error("{0} is not readable or not exist".format(file_path))

151 + return list()

152

166 - # Parse .conf and .local files

167 - if isinstance(jails_conf_path, str) and jails_conf_path.endswith(('.local', '.conf')):

168 - path_conf, path_local = (find_jails_in_files([jails_conf_path.split('.')[0] + '.conf'], print_error),

169 - find_jails_in_files([jails_conf_path.split('.')[0] + '.local'], print_error))

153 + with open(file_path, 'rt') as f:

154 + lines = f.readlines()

155 + raw = " ".join(line for line in lines if line.startswith(('[', 'enabled')))

156

171 - return path_conf, dir_conf, path_local, dir_local

157 + match = RE_JAILS.findall(raw)

158 + # Result: [('ssh', 'true'), ('dropbear', 'true'), ('pam-generic', 'true'), ...]

159

160 + if not match:

161 + self.debug("{0} parse failed".format(file_path))

162 + return list()

163

174 -def find_jails_in_files(list_of_files, print_error):

175 - """

176 - :param list_of_files: <list>

177 - :param print_error: <function>

178 - :return: <list>

164 + return match

165

180 - Open a file and parse it to find all (enabled and disabled) jails

181 - The output is a list of tuples:

182 - [('ssh', 'true'), ('apache', 'false'), ...]

183 - """

184 - jails_list = list()

185 - for conf in list_of_files:

186 - if is_accessible(conf, R_OK):

187 - with open(conf, 'rt') as f:

188 - raw_data = f.readlines()

189 - data = ' '.join(line for line in raw_data if line.startswith(('[', 'enabled')))

190 - jails_list.extend(REGEX_JAILS.findall(data))

191 - else:

192 - print_error('%s is not readable or not exist' % conf)

193 - return jails_list

194 -

195 -

196 -def address_not_in_jail(pool, address, pool_size):

197 - """

198 - :param pool: <list>

199 - :param address: <str>

200 - :param pool_size: <int>

201 - :return: bool

166 + def jails_auto_detection(self):

167 + """

168 + :return: list

169 +

170 + Parses jail configuration files. Returns list of enabled jails.

171 + According man jail.conf parse order must be

172 + * jail.conf

173 + * jail.d/*.conf (in alphabetical order)

174 + * jail.local

175 + * jail.d/*.local (in alphabetical order)

176 + """

177 + jails_files, all_jails, active_jails = list(), list(), list()

178

203 - Checks if the address is in the pool.

204 - If not address will be added

205 - """

206 - index = bisect.bisect_left(pool, address)

207 - if index < pool_size:

208 - if pool[index] == address:

209 - return False

210 - bisect.insort_left(pool, address)

211 - return True

212 - else:

213 - bisect.insort_left(pool, address)

214 - return True

179 + jails_files.append("{0}.conf".format(self.conf_path.rsplit(".")[0]))

180 + jails_files.extend(self.get_files_from_dir(self.conf_dir, "conf"))

181 + jails_files.append("{0}.local".format(self.conf_path.rsplit(".")[0]))

182 + jails_files.extend(self.get_files_from_dir(self.conf_dir, "local"))

183 +

184 + self.debug("config files to parse: {0}".format(jails_files))

185 +

186 + for f in jails_files:

187 + all_jails.extend(self.get_jails_from_file(f))

188 +

189 + exclude = self.exclude.split()

190 +

191 + for name, status in all_jails:

192 + if name in exclude:

193 + continue

194 +

195 + if status == "true" and name not in active_jails:

196 + active_jails.append(name)

197 + elif status == "false" and name in active_jails:

198 + active_jails.remove(name)

199 +

200 + return active_jails or DEFAULT_JAILS

[WIP] fail2ban: ipv6 support added + module simplification (#4168)