properly handle the exit status of plugins to avoid infinite restart attempts; plugins that report failure are now restarted up to 10 times if they have collected values in the past; apache fixed to work with latest apache; example plugin is now disabled by default to prevent starting bash in systems that dont have any other bash plugin used
Costa Tsaousis (ktsaou) committed
Jun 9, 2016 at 22:49 UTC
bb8cdaf9dd25372bbbd09d916e9afe7f907ec1d6
6 files changed
+103
-44
charts.d/apache.chart.sh
+10
-13
@@ -67,19 +67,14 @@ apache_detect() {
67
68
# we will not check of the Conns*
69
# keys, since these are apache 2.4 specific
70
- if [ -z "${apache_key_accesses}" \
71
- -o -z "${apache_key_kbytes}" \
72
- -o -z "${apache_key_reqpersec}" \
73
- -o -z "${apache_key_bytespersec}" \
74
- -o -z "${apache_key_bytesperreq}" \
75
- -o -z "${apache_key_busyworkers}" \
76
- -o -z "${apache_key_idleworkers}" \
77
- -o -z "${apache_key_scoreboard}" \
78
- ]
79
- then
80
- echo >&2 "apache: Invalid response or missing keys from apache server: ${*}"
81
- return 1
82
- fi
70
+ [ -z "${apache_key_accesses}" ] && echo >&2 "apache: missing 'Total Accesses' from apache server: ${*}" && return 1
71
+ [ -z "${apache_key_kbytes}" ] && echo >&2 "apache: missing 'Total kBytes' from apache server: ${*}" && return 1
72
+ [ -z "${apache_key_reqpersec}" ] && echo >&2 "apache: missing 'ReqPerSec' from apache server: ${*}" && return 1
73
+ [ -z "${apache_key_bytespersec}" ] && echo >&2 "apache: missing 'BytesPerSec' from apache server: ${*}" && return 1
74
+ [ -z "${apache_key_bytesperreq}" ] && echo >&2 "apache: missing 'BytesPerReq' from apache server: ${*}" && return 1
75
+ [ -z "${apache_key_busyworkers}" ] && echo >&2 "apache: missing 'BusyWorkers' from apache server: ${*}" && return 1
76
+ [ -z "${apache_key_idleworkers}" ] && echo >&2 "apache: missing 'IdleWorkers' from apache server: ${*}" && return 1
77
+ [ -z "${apache_key_scoreboard}" ] && echo >&2 "apache: missing 'Scoreboard' from apache server: ${*}" && return 1
78
79
if [ ! -z "${apache_key_connstotal}" \
80
-a ! -z "${apache_key_connsasyncwriting}" \
@@ -88,6 +83,8 @@ apache_detect() {
83
]
84
then
85
apache_has_conns=1
86
+ else
87
+ apache_has_conns=0
88
fi
89
90
return 0
charts.d/example.chart.sh
+7
@@ -7,14 +7,21 @@
7
# between the calls of the _update() function
8
example_update_every=
9
10
+# the priority is used to sort the charts on the dashboard
11
+# 1 = the first chart
12
example_priority=150000
13
14
+# to enable this chart, you have to set this to 12345
15
+# (just a demonstration for something that needs to be checked)
16
+example_magic_number=
17
+
18
# _check is called once, to find out if this chart should be enabled or not
19
example_check() {
20
# this should return:
21
# - 0 to enable the chart
22
# - 1 to disable the chart
23
24
+ [ "${example_magic_number}" != "12345" ] && return 1
25
return 0
26
}
27
plugins.d/charts.d.plugin
+19
-6
@@ -504,7 +504,7 @@ if [ -z "$run_charts" ]
504
exit 1
505
fi
506
507
-declare -A charts_last_update=() charts_update_every=() charts_next_update=() charts_run_counter=()
507
+declare -A charts_last_update=() charts_update_every=() charts_next_update=() charts_run_counter=() charts_serial_failures=()
508
global_update() {
509
local exit_at \
510
c=0 dt ret last_ms exec_start_ms exec_end_ms \
@@ -522,13 +522,14 @@ global_update() {
522
charts_last_update[$chart]=$((now_ms - (now_ms % (charts_update_every[$chart] * 1000) ) ))
523
charts_next_update[$chart]=$(( charts_last_update[$chart] + (charts_update_every[$chart] * 1000) ))
524
charts_run_counter[$chart]=0
525
+ charts_serial_failures[$chart]=0
526
527
echo "CHART netdata.plugin_chartsd_$chart '' 'Execution time for $chart plugin' 'milliseconds / run' charts.d netdata.plugin_charts area 145000 ${charts_update_every[$chart]}"
528
echo "DIMENSION run_time 'run time' absolute 1 1"
529
done
530
531
# the main loop
531
- while [ 1 ]
532
+ while [ "${#next_charts[@]}" -gt 0 ]
533
do
534
c=$((c + 1))
535
now_charts=("${next_charts[@]}")
@@ -570,15 +571,24 @@ global_update() {
571
current_time_ms; exec_end_ms=$now_ms
572
573
echo "BEGIN netdata.plugin_chartsd_$chart $dt"
574
+ echo "SET run_time = $(( exec_end_ms - exec_start_ms ))"
575
+ echo "END"
576
+
577
if [ $ret -eq 0 ]
578
then
575
- echo "SET run_time = $(( exec_end_ms - exec_start_ms ))"
579
+ charts_serial_failures[$chart]=0
580
next_charts+=($chart)
581
else
578
- echo "SET run_time = $(( (exec_end_ms - exec_start_ms) * -1 ))"
579
- echo >&2 "$PROGRAM_NAME: chart '$chart' update() function reports failure. Disabling it."
582
+ charts_serial_failures[$chart]=$(( charts_serial_failures[$chart] + 1 ))
583
+
584
+ if [ charts_serial_failures[$chart] -gt 10 ]
585
+ then
586
+ echo >&2 "$PROGRAM_NAME: chart '$chart' update() function reported failure ${charts_serial_failures[$chart]} times. Disabling it."
587
+ else
588
+ echo >&2 "$PROGRAM_NAME: chart '$chart' update() function reports failure. Will keep trying for a while."
589
+ next_charts+=($chart)
590
+ fi
591
fi
581
- echo "END"
592
else
593
next_charts+=($chart)
594
fi
@@ -601,6 +611,9 @@ global_update() {
611
612
test ${now_ms} -ge ${exit_at} && exit 0
613
done
614
+
615
+ echo >&2 "$PROGRAM_NAME: Nothing left to do. Disabling charts.d.plugin."
616
+ echo "DISABLE"
617
}
618
619
global_update
src/plugins_d.c
+52
-16
@@ -125,6 +125,8 @@ void *pluginsd_worker_thread(void *arg)
125
uint32_t STOPPING_WAKE_ME_UP_PLEASE_HASH = simple_hash("STOPPING_WAKE_ME_UP_PLEASE");
126
#endif
127
128
+ size_t count = 0;
129
+
130
while(likely(1)) {
131
if(unlikely(netdata_exit)) break;
132
@@ -137,7 +139,6 @@ void *pluginsd_worker_thread(void *arg)
139
info("PLUGINSD: '%s' running on pid %d", cd->fullfilename, cd->pid);
140
141
RRDSET *st = NULL;
140
- unsigned long long count = 0;
142
char *s;
143
uint32_t hash;
144
@@ -182,8 +183,6 @@ void *pluginsd_worker_thread(void *arg)
183
if(unlikely(st->debug)) debug(D_PLUGINSD, "PLUGINSD: '%s' is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value?value:"<nothing>");
184
185
if(value) rrddim_set(st, dimension, strtoll(value, NULL, 0));
185
-
186
- count++;
186
}
187
else if(likely(hash == BEGIN_HASH && !strcmp(s, "BEGIN"))) {
188
char *id = words[1];
@@ -223,6 +222,8 @@ void *pluginsd_worker_thread(void *arg)
222
223
rrdset_done(st);
224
st = NULL;
225
+
226
+ count++;
227
}
228
else if(likely(hash == FLUSH_HASH && !strcmp(s, "FLUSH"))) {
229
debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename);
@@ -386,17 +387,17 @@ void *pluginsd_worker_thread(void *arg)
387
break;
388
}
389
}
390
+ if(likely(count)) {
391
+ cd->successful_collections += count;
392
+ cd->serial_failures = 0;
393
+ }
394
+ else
395
+ cd->serial_failures++;
396
390
- info("PLUGINSD: '%s' on pid %d stopped.", cd->fullfilename, cd->pid);
397
+ info("PLUGINSD: '%s' on pid %d stopped after %zu successful data collections (ENDs).", cd->fullfilename, cd->pid, count);
398
392
- // fgets() failed or loop broke
399
+ // get the return code
400
int code = mypclose(fp, cd->pid);
394
- if(code == 1 || code == 127) {
395
- // 1 = DISABLE
396
- // 127 = cannot even run it
397
- error("PLUGINSD: '%s' (pid %d) exited with code %d. Disabling it.", cd->fullfilename, cd->pid, code);
398
- cd->enabled = 0;
399
- }
401
402
if(netdata_exit) {
403
cd->pid = 0;
@@ -406,14 +407,49 @@ void *pluginsd_worker_thread(void *arg)
407
return NULL;
408
}
409
409
- if(unlikely(!count && cd->enabled)) {
410
- error("PLUGINSD: '%s' (pid %d) does not generate useful output. Waiting a bit before starting it again.", cd->fullfilename, cd->pid);
411
- sleep((unsigned int) (cd->update_every * 10));
410
+ if(code != 0) {
411
+ // the plugin reports failure
412
+
413
+ if(likely(!cd->successful_collections)) {
414
+ // nothing collected - disable it
415
+ error("PLUGINSD: '%s' exited with error code %d. Disabling it.", cd->fullfilename, code);
416
+ cd->enabled = 0;
417
+ }
418
+ else {
419
+ // we have collected something
420
+
421
+ if(likely(cd->serial_failures <= 10)) {
422
+ error("PLUGINSD: '%s' exited with error code %d, but has given useful output in the past (%zu times). Waiting a bit before starting it again.", cd->fullfilename, code, cd->successful_collections);
423
+ sleep((unsigned int) (cd->update_every * 10));
424
+ }
425
+ else {
426
+ error("PLUGINSD: '%s' exited with error code %d, but has given useful output in the past (%zu times). We tried %d times to restart it, but it failed to generate data. Disabling it.", cd->fullfilename, code, cd->successful_collections, cd->serial_failures);
427
+ cd->enabled = 0;
428
+ }
429
+ }
430
}
431
+ else {
432
+ // the plugin reports success
433
434
+ if(unlikely(!cd->successful_collections)) {
435
+ // we have collected nothing so far
436
+
437
+ if(likely(cd->serial_failures <= 10)) {
438
+ error("PLUGINSD: '%s' (pid %d) does not generate useful output but it reports success (exits with 0). Waiting a bit before starting it again.", cd->fullfilename, cd->pid);
439
+ sleep((unsigned int) (cd->update_every * 10));
440
+ }
441
+ else {
442
+ error("PLUGINSD: '%s' (pid %d) does not generate useful output, although it reports success (exits with 0), but we have tried %d times to collect something. Disabling it.", cd->fullfilename, cd->pid, cd->serial_failures);
443
+ cd->enabled = 0;
444
+ }
445
+ }
446
+ else
447
+ sleep((unsigned int) cd->update_every);
448
+ }
449
cd->pid = 0;
415
- if(likely(cd->enabled)) sleep((unsigned int) cd->update_every);
416
- else break;
450
+
451
+ if(unlikely(!cd->enabled))
452
+ break;
453
}
454
455
cd->obsolete = 1;
src/plugins_d.h
+9
-3
@@ -20,9 +20,15 @@ struct plugind {
20
pid_t pid;
21
pthread_t thread;
22
23
- int update_every;
24
- int obsolete;
25
- int enabled;
23
+ size_t successful_collections; // the number of times we have seen
24
+ // values collected from this plugin
25
+
26
+ size_t serial_failures; // the number of times the plugin started
27
+ // without collecting values
28
+
29
+ int update_every; // the plugin default data collection frequency
30
+ int obsolete; // do not touch this structure after setting this to 1
31
+ int enabled; // if this is enabled or not
32
33
time_t started_t;
34
web/demo2.html
+6
-6
@@ -20,7 +20,7 @@
20
<meta property="og:description" content="Stunning real-time dashboards, blazingly fast and extremely interactive. Zero configuration, zero dependencies, zero maintenance." />
21
22
<script>var netdataTheme = 'slate';</script>
23
- <script type="text/javascript" src="dashboard.js?v39"></script>
23
+ <script type="text/javascript" src="http://my-netdata.io/dashboard.js?v39"></script>
24
</head>
25
<body>
26
@@ -63,8 +63,8 @@
63
data-gauge-max-value="32767"
64
data-width="45%"
65
data-after="-600"
66
- data-points="40"
67
- data-title="Updates Every 15 Sec"
66
+ data-points="60"
67
+ data-title="Updates Every 10 Sec"
68
data-units="important metric"
69
data-colors="#C55"
70
></div>
@@ -106,8 +106,8 @@
106
data-easypiechart-max-value="32767"
107
data-width="45%"
108
data-after="-600"
109
- data-points="40"
110
- data-title="Updates Every 15 Sec (<a href='https://github.com/OpenTSDB/opentsdb.net/blob/gh-pages/docs/source/user_guide/utilities/tcollector.rst#collecting-lots-of-metrics-with-tcollector' target='_blank'>OpenTSDB</a>)"
109
+ data-points="60"
110
+ data-title="Updates Every 10 Sec"
111
data-units="important metric"
112
data-colors="#C55"
113
></div>
@@ -118,7 +118,7 @@
118
data-width="45%"
119
data-after="-600"
120
data-points="2"
121
- data-title="Updates Every 5 Mins (your NMS)"
121
+ data-title="Updates Every 5 Mins"
122
data-units="important metric"
123
data-colors="#C55"
124
></div>