@cryptotaxi247 / netdata-1 / commits / 204e0d099

added information on the dashboard for web_log plugin

Costa Tsaousis (ktsaou) committed Feb 11, 2017 at 13:27 UTC 204e0d099d9e5f679bd72f581209a485c83d6189
4 files changed +238 -45
conf.d/python.d/web_log.conf
+35 -11
@@ -42,6 +42,9 @@
42 # pick the one that works.
43 #
44 # Any number of jobs is supported.
45 +
46 +# ----------------------------------------------------------------------
47 +# PLUGIN CONFIGURATION
48 #
49 # All python.d.plugin JOBS (for all its modules) support a set of
50 # predefined parameters. These are:
@@ -63,22 +66,34 @@
66 # cacti: 'cacti.*' # name(dimension): REGEX to match
67 # observium: 'observium.*' # name(dimension): REGEX to match
68 # stub_status: 'stub_status' # name(dimension): REGEX to match
69 +
70 +# ----------------------------------------------------------------------
71 +# WEB SERVER CONFIGURATION
72 +#
73 +# Make sure the log directory and file can be read by user 'netdata'.
74 #
75 # Preferable Log Format. You need to change to this to collect all metrics.
68 -# Nginx:
69 -# log_format netdata '$remote_addr - $remote_user [$time_local] '
70 -# '"$request" $status $body_bytes_sent '
71 -# '$request_length $request_time '
72 -# '"$http_referer" "$http_user_agent"';
73 -# access_log /var/log/nginx/access.log netdata;
74 -# Apache:
75 -# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
76 -# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" combined
76 +#
77 +# nginx:
78 +# log_format netdata '$remote_addr - $remote_user [$time_local] '
79 +# '"$request" $status $body_bytes_sent '
80 +# '$request_length $request_time '
81 +# '"$http_referer" "$http_user_agent"';
82 +# access_log /var/log/nginx/access.log netdata;
83 +#
84 +# apache:
85 +# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
86 +# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" combined
87 +
88 # ----------------------------------------------------------------------
89 # AUTO-DETECTION JOBS
79 -# only one of them will run (if they have the same name)
90 +# only one of them per web server will run (when they have the same name)
91
81 -# debian
92 +
93 +# -------------------------------------------
94 +# nginx log on various distros
95 +
96 +# debian, arch
97 nginx_log:
98 name: 'nginx'
99 path: '/var/log/nginx/access.log'
@@ -88,6 +103,10 @@ nginx_log2:
103 name: 'nginx'
104 path: '/var/log/nginx/localhost.access_log'
105
106 +
107 +# -------------------------------------------
108 +# apache log on various distros
109 +
110 # debian
111 apache_log:
112 name: 'apache'
@@ -103,10 +122,15 @@ apache_log3:
122 name: 'apache'
123 path: '/var/log/httpd/access_log'
124
125 +# debian
126 apache_vhosts_log:
127 name: 'apache_vhosts'
128 path: '/var/log/apache2/other_vhosts_access.log'
129
130 +
131 +# -------------------------------------------
132 +# gunicorn log on various distros
133 +
134 gunicorn_log:
135 name: 'gunicorn'
136 path: '/var/log/gunicorn/access.log'
python.d/web_log.chart.py
+7 -7
@@ -17,7 +17,7 @@ except ImportError:
17 priority = 60000
18 retries = 60
19
20 -ORDER = ['response_codes', 'response_time', 'requests_per_url', 'http_method', 'requests_per_ipproto', 'bandwidth', 'clients', 'clients_all']
20 +ORDER = ['response_codes', 'bandwidth', 'response_time', 'requests_per_url', 'http_method', 'requests_per_ipproto', 'clients', 'clients_all']
21 CHARTS = {
22 'response_codes': {
23 'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response_codes', 'stacked'],
@@ -44,23 +44,23 @@ CHARTS = {
44 ['resp_time_avg', 'avg', 'absolute', 1, 1]
45 ]},
46 'clients': {
47 - 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients', 'stacked'],
47 + 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'clients', 'web_log.clients', 'stacked'],
48 'lines': [
49 ['unique_cur_ipv4', 'ipv4', 'absolute', 1, 1],
50 ['unique_cur_ipv6', 'ipv6', 'absolute', 1, 1]
51 ]},
52 'clients_all': {
53 - 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients_all', 'stacked'],
53 + 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'clients', 'web_log.clients_all', 'stacked'],
54 'lines': [
55 ['unique_tot_ipv4', 'ipv4', 'absolute', 1, 1],
56 ['unique_tot_ipv6', 'ipv6', 'absolute', 1, 1]
57 ]},
58 'http_method': {
59 - 'options': [None, 'Requests Per HTTP Method', 'requests/s', 'requests', 'web_log.http_method', 'stacked'],
59 + 'options': [None, 'Requests Per HTTP Method', 'requests/s', 'http methods', 'web_log.http_method', 'stacked'],
60 'lines': [
61 ]},
62 'requests_per_ipproto': {
63 - 'options': [None, 'Requests Per IP Protocol', 'requests/s', 'requests', 'web_log.requests_per_ipproto', 'stacked'],
63 + 'options': [None, 'Requests Per IP Protocol', 'requests/s', 'ip protocols', 'web_log.requests_per_ipproto', 'stacked'],
64 'lines': [
65 ['req_ipv4', 'ipv4', 'absolute', 1, 1],
66 ['req_ipv6', 'ipv6', 'absolute', 1, 1]
@@ -229,14 +229,14 @@ class Service(LogService):
229 if self.detailed_response_codes:
230 self.order.append('detailed_response_codes')
231 self.definitions['detailed_response_codes'] = {'options': [None, 'Detailed Response Codes', 'requests/s',
232 - 'responses', 'web_log.detailed_resp', 'stacked'],
232 + 'responses', 'web_log.detailed_response_codes', 'stacked'],
233 'lines': []}
234
235 # Add 'requests_per_url' chart if specified in the configuration
236 if self.url_pattern:
237 self.url_pattern = [NAMED_URL_PATTERN(description=k, pattern=re.compile(v)) for k, v in self.url_pattern.items()]
238 self.definitions['requests_per_url'] = {'options': [None, 'Requests Per Url', 'requests/s',
239 - 'requests', 'web_log.url_pattern', 'stacked'],
239 + 'urls', 'web_log.requests_per_url', 'stacked'],
240 'lines': [['other_url', 'other', 'absolute']]}
241 for elem in self.url_pattern:
242 self.definitions['requests_per_url']['lines'].append([elem.description, elem.description, 'absolute'])
web/dashboard_info.js
+126 -2
@@ -1,7 +1,9 @@
1
2 var netdataDashboard = window.netdataDashboard || {};
3
4 -// menu
4 +// ----------------------------------------------------------------------------
5 +// menus
6 +
7 // information about the main menus
8
9 netdataDashboard.menu = {
@@ -217,6 +219,12 @@ netdataDashboard.menu = {
219 info: undefined
220 },
221
222 + 'web_log': {
223 + title: undefined,
224 + icon: '<i class="fa fa-file-text-o" aria-hidden="true"></i>',
225 + info: 'Information extracted from the web server log file. netdata <code>python.d/web_log</code> plugin incrementally parses web server log files to provide a real-time break down of key web server performance metrics. A special log file format is also supported (for <code>nginx</code> and <code>apache</code>) that allows netdata to extract timing information for the web server responses and bandwidth for both requests and responses. The <code>web_log</code> plugin can also be configured to provide a break down of requests per URL pattern (check <a href="https://github.com/firehol/netdata/blob/master/conf.d/python.d/web_log.conf" target="_blank"><code>/etc/netdata/python.d/web_log.conf</code></a>). netdata provides also several alarms based on the information on these charts, such as <b>too many bad requests</b>, <b>too many redirects</b>, <b>too many internal errros</b>, <b>unreasonably slow responses</b>, <b>too many requests</b> and <b>too few requests</b>.'
226 + },
227 +
228 'named': {
229 title: 'named',
230 icon: '<i class="fa fa-tag" aria-hidden="true"></i>',
@@ -254,9 +262,31 @@ netdataDashboard.menu = {
262 }
263 };
264
257 -// submenu
265 +
266 +
267 +// ----------------------------------------------------------------------------
268 +// submenus
269 +
270 +// information to be shown, just below each submenu
271 +
272 // information about the submenus
273 netdataDashboard.submenu = {
274 + 'web_log.bandwidth': {
275 + info: 'Bandwidth of requests (<code>received</code>) and responses (<code>sent</code>). <code>received</code> requires a special file format (without it, the web server log does not have this information). This chart may present unusual spikes, since the whole bandwidth will be accounted at the time the log line is saved by the web server, even if the time needed to serve it spans across a longer duration. We suggest to use QoS (e.g. <a href="http://firehol.org/#fireqos" target="_blank">FireQOS</a>) for accurate accounting of the web server bandwidth.'
276 + },
277 +
278 + 'web_log.urls': {
279 + info: 'Number of requests for each URL <code>category</code> (URL pattern) defined in <a href="https://github.com/firehol/netdata/blob/master/conf.d/python.d/web_log.conf" target="_blank"><code>/etc/netdata/python.d/web_log.conf</code></a>. This chart counts all requests matching the URL patterns defined, independently of the web server response codes (i.e. both successful and unsuccessful).'
280 + },
281 +
282 + 'web_log.clients': {
283 + info: 'Charts showing the number of unique client IPs, accessing the web server.'
284 + },
285 +
286 + 'web_log.timings': {
287 + info: 'Web server response timings - the time the web server needed to prepare and respond to requests. This requires a special log format and its meaning is web server specific. For most web servers this accounts the time from the reception of a complete request, to the dispatch of the last byte of the response. So, it includes the network delays of responses, but it does not include the network delays of requests.'
288 + },
289 +
290 'mem.ksm': {
291 title: 'Memory Deduper',
292 info: 'Kernel Same-page Merging (KSM) performance monitoring, read from several files in <code>/sys/kernel/mm/ksm/</code>. KSM is a memory-saving de-duplication feature in the Linux kernel (since version 2.6.32). The KSM daemon ksmd periodically scans those areas of user memory which have been registered with it, looking for pages of identical content which can be replaced by a single write-protected page (which is automatically copied if a process later wants to update its content). KSM was originally developed for use with KVM (where it was known as Kernel Shared Memory), to fit more virtual machines into physical memory, by sharing the data common between them. But it can be useful to any application which generates many instances of the same data.'
@@ -306,7 +336,11 @@ netdataDashboard.submenu = {
336 }
337 };
338
339 +
340 +
341 +// ----------------------------------------------------------------------------
342 // chart
343 +
344 // information works on the context of a chart
345 // Its purpose is to set:
346 //
@@ -811,6 +845,96 @@ netdataDashboard.context = {
845
846 'fping.packets': {
847 height: 0.5
848 + },
849 +
850 + 'web_log.response_codes': {
851 + info: 'Break down of web server responses by response code type. <code>1xx</code> are informational responses, <code>2xx</code> are successful responses, <code>3xx</code> are redirects, <code>4xx</code> are bad requests, <code>5xx</code> are internal server errors, <code>other</code> are non-standard responses, <code>unmatched</code> counts the lines in the log file that are not matched by the plugin (please <a href="https://github.com/firehol/netdata/issues/new?title=web_log%20reports%20unmatched%20lines&body=web_log%20plugin%20reports%20unmatched%20lines.%0A%0AThis%20is%20my%20log:%0A%0A%60%60%60txt%0A%0Aplease%20paste%20your%20web%20server%20log%20here%0A%0A%60%60%60" target="_blank">open a github issue</a> to help us fix it, if you have any unmatched lines).',
852 +
853 + mainheads: [
854 + function(os, id) {
855 + void(os);
856 + return '<div data-netdata="' + id + '"'
857 + + ' data-dimensions="2xx"'
858 + + ' data-chart-library="gauge"'
859 + + ' data-title="Successful"'
860 + + ' data-units="requests"'
861 + + ' data-gauge-adjust="width"'
862 + + ' data-width="12%"'
863 + + ' data-before="0"'
864 + + ' data-after="-CHART_DURATION"'
865 + + ' data-points="CHART_DURATION"'
866 + + ' data-common-max="' + id + '"'
867 + + ' data-colors="' + NETDATA.colors[0] + '"'
868 + + ' role="application"></div>';
869 + },
870 +
871 + function(os, id) {
872 + void(os);
873 + return '<div data-netdata="' + id + '"'
874 + + ' data-dimensions="3xx"'
875 + + ' data-chart-library="gauge"'
876 + + ' data-title="Redirects"'
877 + + ' data-units="requests"'
878 + + ' data-gauge-adjust="width"'
879 + + ' data-width="12%"'
880 + + ' data-before="0"'
881 + + ' data-after="-CHART_DURATION"'
882 + + ' data-points="CHART_DURATION"'
883 + + ' data-common-max="' + id + '"'
884 + + ' data-colors="' + NETDATA.colors[2] + '"'
885 + + ' role="application"></div>';
886 + },
887 +
888 + function(os, id) {
889 + void(os);
890 + return '<div data-netdata="' + id + '"'
891 + + ' data-dimensions="4xx"'
892 + + ' data-chart-library="gauge"'
893 + + ' data-title="Bad Requests"'
894 + + ' data-units="requests"'
895 + + ' data-gauge-adjust="width"'
896 + + ' data-width="12%"'
897 + + ' data-before="0"'
898 + + ' data-after="-CHART_DURATION"'
899 + + ' data-points="CHART_DURATION"'
900 + + ' data-common-max="' + id + '"'
901 + + ' data-colors="' + NETDATA.colors[3] + '"'
902 + + ' role="application"></div>';
903 + },
904 +
905 + function(os, id) {
906 + void(os);
907 + return '<div data-netdata="' + id + '"'
908 + + ' data-dimensions="5xx"'
909 + + ' data-chart-library="gauge"'
910 + + ' data-title="Server Errors"'
911 + + ' data-units="requests"'
912 + + ' data-gauge-adjust="width"'
913 + + ' data-width="12%"'
914 + + ' data-before="0"'
915 + + ' data-after="-CHART_DURATION"'
916 + + ' data-points="CHART_DURATION"'
917 + + ' data-common-max="' + id + '"'
918 + + ' data-colors="' + NETDATA.colors[1] + '"'
919 + + ' role="application"></div>';
920 + }
921 + ]
922 + },
923 +
924 + 'web_log.detailed_response_codes': {
925 + info: 'Number of responses for each response code.'
926 + },
927 +
928 + 'web_log.requests_per_ipproto': {
929 + info: 'Web server requests received per IP protocol version.'
930 + },
931 +
932 + 'web_log.clients': {
933 + info: 'Unique client IPs accessing the web server, within each data collection iteration. So, if data collection is <b>per second</b>, this chart shows <b>unique client IPs per second</b>.'
934 + },
935 +
936 + 'web_log.clients_all': {
937 + info: 'Unique client IPs accessing the web server since the last restart of netdata. This plugin keeps in memory all the unique IPs that have accessed the web server. On very busy web servers (several millions of unique IPs) you may want to disable this chart (check <a href="https://github.com/firehol/netdata/blob/master/conf.d/python.d/web_log.conf" target="_blank"><code>/etc/netdata/python.d/web_log.conf</code></a>).'
938 }
939
940 };
web/index.html
+70 -25
@@ -1046,6 +1046,7 @@
1046
1047 submenuTitle: function(menu, submenu) {
1048 var key = menu + '.' + submenu;
1049 + // console.log(key);
1050 var title = this.anyAttribute(this.submenu, 'title', key, submenu).toString().replace(/_/g, ' ');
1051 if(title.length > 28) {
1052 var a = title.substring(0, 13);
@@ -1093,8 +1094,10 @@
1094
1095 // enrich the data structure returned by netdata
1096 // to reflect our menu system and content
1097 + // FIXME: this is a shame - we should fix charts naming (issue #807)
1098 function enrichChartData(chart) {
1097 - var tmp = chart.type.split('_')[0];
1099 + var parts = chart.type.split('_');
1100 + var tmp = parts[0];
1101
1102 switch(tmp) {
1103 case 'ap':
@@ -1103,6 +1106,18 @@
1106 chart.menu = tmp;
1107 break;
1108
1109 + case 'apache':
1110 + chart.menu = chart.type;
1111 + if(parts.length > 2 && parts[1] === 'cache')
1112 + chart.menu_pattern = tmp + '_' + parts[1];
1113 + break;
1114 +
1115 + case 'bind':
1116 + chart.menu = chart.type;
1117 + if(parts.length > 2 && parts[1] === 'rndc')
1118 + chart.menu_pattern = tmp + '_' + parts[1];
1119 + break;
1120 +
1121 case 'cgroup':
1122 chart.menu = chart.type;
1123 if(chart.id.match(/.*[\._\/-:]qemu[\._\/-:]*/) || chart.id.match(/.*[\._\/-:]kvm[\._\/-:]*/))
@@ -1111,9 +1126,27 @@
1126 chart.menu_pattern = 'cgroup';
1127 break;
1128
1114 - case 'apache':
1115 - case 'exim':
1129 + case 'isc':
1130 + chart.menu = chart.type;
1131 + if(parts.length > 2 && parts[1] === 'dhcpd')
1132 + chart.menu_pattern = tmp + '_' + parts[1];
1133 + break;
1134 +
1135 + case 'ovpn':
1136 + chart.menu = chart.type;
1137 + if(parts.length > 3 && parts[1] === 'status' && parts[2] === 'log')
1138 + chart.menu_pattern = tmp + '_' + parts[1];
1139 + break;
1140 +
1141 + case 'smartd':
1142 + case 'web':
1143 + chart.menu = chart.type;
1144 + if(parts.length > 2 && parts[1] === 'log')
1145 + chart.menu_pattern = tmp + '_' + parts[1];
1146 + break;
1147 +
1148 case 'dovecot':
1149 + case 'exim':
1150 case 'hddtemp':
1151 case 'ipfs':
1152 case 'memcached':
@@ -1397,48 +1430,59 @@
1430 function renderChartsAndMenu(data) {
1431 var menus = options.menus;
1432 var charts = data.charts;
1433 + var m, menu_key;
1434
1435 for(var c in charts) {
1436 if(!charts.hasOwnProperty(c)) continue;
1437
1404 - enrichChartData(charts[c]);
1438 + var chart = charts[c];
1439 + enrichChartData(chart);
1440 + m = chart.menu;
1441
1442 // create the menu
1407 - if(typeof menus[charts[c].menu] === 'undefined') {
1408 - menus[charts[c].menu] = {
1409 - priority: charts[c].priority,
1443 + if(typeof menus[m] === 'undefined') {
1444 + menus[m] = {
1445 + menu_pattern: chart.menu_pattern,
1446 + priority: chart.priority,
1447 submenus: {},
1411 - title: netdataDashboard.menuTitle(charts[c]),
1412 - icon: netdataDashboard.menuIcon(charts[c]),
1413 - info: netdataDashboard.menuInfo(charts[c]),
1414 - height: netdataDashboard.menuHeight(charts[c]) * options.chartsHeight
1448 + title: netdataDashboard.menuTitle(chart),
1449 + icon: netdataDashboard.menuIcon(chart),
1450 + info: netdataDashboard.menuInfo(chart),
1451 + height: netdataDashboard.menuHeight(chart) * options.chartsHeight
1452 };
1453 }
1454 + else {
1455 + if(typeof(menus[m].menu_pattern) === 'undefined')
1456 + menus[m].menu_pattern = chart.menu_pattern;
1457 +
1458 + if(chart.priority < menus[m].priority)
1459 + menus[m].priority = chart.priority;
1460 + }
1461
1418 - if(charts[c].priority < menus[charts[c].menu].priority)
1419 - menus[charts[c].menu].priority = charts[c].priority;
1462 + menu_key = (typeof(menus[m].menu_pattern) !== 'undefined')?menus[m].menu_pattern:m;
1463
1464 // create the submenu
1422 - if(typeof menus[charts[c].menu].submenus[charts[c].submenu] === 'undefined') {
1423 - menus[charts[c].menu].submenus[charts[c].submenu] = {
1424 - priority: charts[c].priority,
1465 + if(typeof menus[m].submenus[chart.submenu] === 'undefined') {
1466 + menus[m].submenus[chart.submenu] = {
1467 + priority: chart.priority,
1468 charts: [],
1469 title: null,
1427 - info: netdataDashboard.submenuInfo(charts[c].menu, charts[c].submenu),
1428 - height: netdataDashboard.submenuHeight(charts[c].menu, charts[c].submenu, menus[charts[c].menu].height)
1470 + info: netdataDashboard.submenuInfo(menu_key, chart.submenu),
1471 + height: netdataDashboard.submenuHeight(menu_key, chart.submenu, menus[m].height)
1472 };
1473 }
1431 -
1432 - if(charts[c].priority < menus[charts[c].menu].submenus[charts[c].submenu].priority)
1433 - menus[charts[c].menu].submenus[charts[c].submenu].priority = charts[c].priority;
1474 + else {
1475 + if (chart.priority < menus[m].submenus[chart.submenu].priority)
1476 + menus[m].submenus[chart.submenu].priority = chart.priority;
1477 + }
1478
1479 // index the chart in the menu/submenu
1436 - menus[charts[c].menu].submenus[charts[c].submenu].charts.push(charts[c]);
1480 + menus[m].submenus[chart.submenu].charts.push(chart);
1481 }
1482
1483 // propagate the descriptive subname given to QoS
1484 // to all the other submenus with the same name
1441 - for(var m in menus) {
1485 + for(m in menus) {
1486 if(!menus.hasOwnProperty(m)) continue;
1487
1488 for(var s in menus[m].submenus) {
@@ -1449,7 +1493,8 @@
1493 menus[m].submenus[s].title = s + ' (' + options.submenu_names[s] + ')';
1494 }
1495 else {
1452 - menus[m].submenus[s].title = netdataDashboard.submenuTitle(m, s);
1496 + menu_key = (typeof(menus[m].menu_pattern) !== 'undefined')?menus[m].menu_pattern:m;
1497 + menus[m].submenus[s].title = netdataDashboard.submenuTitle(menu_key, s);
1498 }
1499 }
1500 }
@@ -2755,7 +2800,7 @@
2800 });
2801
2802 NETDATA.requiredJs.push({
2758 - url: NETDATA.serverDefault + 'dashboard_info.js?v20170208-8',
2803 + url: NETDATA.serverDefault + 'dashboard_info.js?v20170211-18',
2804 async: false,
2805 isAlreadyLoaded: function() { return false; }
2806 });