alarms log now includes creation and removal of alarms; the dashboard does not poll for alarms if health monitoring is disabled on netdata - fixes #908
Costa Tsaousis (ktsaou) committed
Sep 7, 2016 at 23:51 UTC
966bc1afe5aedabe63fbd04a633661650cebcd64
3 files changed
+180
-135
src/health.c
+129
-85
@@ -6,6 +6,82 @@ static const char *health_default_exec = PLUGINS_DIR "/alarm-email.sh";
6
static const char *health_default_recipient = "root";
7
int health_enabled = 1;
8
9
+// ----------------------------------------------------------------------------
10
+// Health Alarms Log Management
11
+
12
+static inline void health_alarm_log(RRDHOST *host,
13
+ uint32_t alarm_id, uint32_t alarm_event_id,
14
+ time_t when,
15
+ const char *name, const char *chart, const char *family,
16
+ const char *exec, const char *recipient, time_t duration,
17
+ calculated_number old_value, calculated_number new_value,
18
+ int old_status, int new_status,
19
+ const char *source,
20
+ const char *units,
21
+ const char *info
22
+) {
23
+ debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
24
+
25
+ ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
26
+ ae->name = strdupz(name);
27
+ ae->hash_name = simple_hash(ae->name);
28
+
29
+ if(chart) {
30
+ ae->chart = strdupz(chart);
31
+ ae->hash_chart = simple_hash(ae->chart);
32
+ }
33
+
34
+ if(family)
35
+ ae->family = strdupz(family);
36
+
37
+ if(exec) ae->exec = strdupz(exec);
38
+ if(recipient) ae->recipient = strdupz(recipient);
39
+ if(source) ae->source = strdupz(source);
40
+ if(units) ae->units = strdupz(units);
41
+ if(info) ae->info = strdupz(info);
42
+
43
+ ae->unique_id = host->health_log.next_log_id++;
44
+ ae->alarm_id = alarm_id;
45
+ ae->alarm_event_id = alarm_event_id;
46
+ ae->when = when;
47
+ ae->old_value = old_value;
48
+ ae->new_value = new_value;
49
+ ae->old_status = old_status;
50
+ ae->new_status = new_status;
51
+ ae->duration = duration;
52
+
53
+ if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
54
+ ae->non_clear_duration += ae->duration;
55
+
56
+ // link it
57
+ pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
58
+ ae->next = host->health_log.alarms;
59
+ host->health_log.alarms = ae;
60
+ host->health_log.count++;
61
+ pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
62
+
63
+ // match previous alarms
64
+ pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
65
+ ALARM_ENTRY *t;
66
+ for(t = host->health_log.alarms ; t ; t = t->next) {
67
+ if(t != ae && t->alarm_id == ae->alarm_id) {
68
+ if(!(t->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) && !t->updated_by) {
69
+ t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
70
+ t->updated_by = ae;
71
+
72
+ if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
73
+ (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
74
+ ae->non_clear_duration += t->non_clear_duration;
75
+ }
76
+ else {
77
+ // no need to continue
78
+ break;
79
+ }
80
+ }
81
+ }
82
+ pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
83
+}
84
+
85
// ----------------------------------------------------------------------------
86
// RRDVAR management
87
@@ -411,12 +487,15 @@ void rrddimvar_free(RRDDIMVAR *rs) {
487
488
static inline const char *rrdcalc_status2string(int status) {
489
switch(status) {
414
- case RRDCALC_STATUS_UNINITIALIZED:
415
- return "UNINITIALIZED";
490
+ case RRDCALC_STATUS_REMOVED:
491
+ return "REMOVED";
492
493
case RRDCALC_STATUS_UNDEFINED:
494
return "UNDEFINED";
495
496
+ case RRDCALC_STATUS_UNINITIALIZED:
497
+ return "UNINITIALIZED";
498
+
499
case RRDCALC_STATUS_CLEAR:
500
return "CLEAR";
501
@@ -430,6 +509,7 @@ static inline const char *rrdcalc_status2string(int status) {
509
return "CRITICAL";
510
511
default:
512
+ error("Unknown alarm status %d", status);
513
return "UNKNOWN";
514
}
515
}
@@ -474,6 +554,11 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
554
rc->hostname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
555
556
if(!rc->units) rc->units = strdupz(st->units);
557
+
558
+ {
559
+ time_t now = time(NULL);
560
+ health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_UNINITIALIZED, rc->source, rc->units, rc->info);
561
+ }
562
}
563
564
static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {
@@ -508,6 +593,11 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
593
return;
594
}
595
596
+ {
597
+ time_t now = time(NULL);
598
+ health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info);
599
+ }
600
+
601
RRDHOST *host = st->rrdhost;
602
603
debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
@@ -578,6 +668,22 @@ static inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *n
668
return 0;
669
}
670
671
+static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name) {
672
+ if(chart && name) {
673
+ uint32_t hash_chart = simple_hash(chart);
674
+ uint32_t hash_name = simple_hash(name);
675
+
676
+ // re-use old IDs, by looking them up in the alarm log
677
+ ALARM_ENTRY *ae;
678
+ for(ae = host->health_log.alarms; ae ;ae = ae->next) {
679
+ if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart)))
680
+ return ae->alarm_id;
681
+ }
682
+ }
683
+
684
+ return host->health_log.next_alarm_id++;
685
+}
686
+
687
static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
688
rrdhost_check_rdlock(host);
689
@@ -636,7 +742,7 @@ static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const cha
742
return NULL;
743
744
RRDCALC *rc = callocz(1, sizeof(RRDCALC));
639
- rc->id = host->health_log.next_alarm_id++;
745
+ rc->id = rrdcalc_get_unique_id(host, chart, name);
746
rc->next_event_id = 1;
747
rc->name = strdupz(name);
748
rc->hash = simple_hash(rc->name);
@@ -840,9 +946,12 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
946
if (rrdcalc_exists(host, rc->chart, rc->name, rc->hash_chart, rc->hash))
947
return 0;
948
843
- debug(D_HEALTH, "Health configuration adding alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
949
+ rc->id = rrdcalc_get_unique_id(&localhost, rc->chart, rc->name);
950
+
951
+ debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
952
rc->chart?rc->chart:"NOCHART",
953
rc->name,
954
+ rc->id,
955
(rc->exec)?rc->exec:"DEFAULT",
956
(rc->recipient)?rc->recipient:"DEFAULT",
957
rc->green,
@@ -1166,7 +1275,6 @@ int health_readfile(const char *path, const char *filename) {
1275
}
1276
1277
rc = callocz(1, sizeof(RRDCALC));
1169
- rc->id = localhost.health_log.next_alarm_id++;
1278
rc->next_event_id = 1;
1279
rc->name = strdupz(value);
1280
rc->hash = simple_hash(rc->name);
@@ -1698,9 +1806,18 @@ static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1806
1807
void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
1808
int i;
1809
+
1810
rrdhost_rdlock(&localhost);
1811
+ buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\","
1812
+ "\n\t\"latest_alarm_log_unique_id\": %u,"
1813
+ "\n\t\"status\": %s,"
1814
+ "\n\t\"now\": %lu,"
1815
+ "\n\t\"alarms\": {\n",
1816
+ host->hostname,
1817
+ (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
1818
+ health_enabled?"true":"false",
1819
+ (unsigned long)time(NULL));
1820
1703
- buffer_strcat(wb, "{\n\t\"alarms\": {\n");
1821
RRDCALC *rc;
1822
for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
1823
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
@@ -1715,12 +1832,11 @@ void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
1832
}
1833
1834
// buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
1718
-
1835
// RRDCALCTEMPLATE *rt;
1836
// for(rt = host->templates; rt ; rt = rt->next)
1837
// health_rrdcalctemplate2json_nolock(wb, rt);
1838
1723
- buffer_sprintf(wb, "\n\t},\n\t\"now\": %lu\n}\n", (unsigned long)time(NULL));
1839
+ buffer_strcat(wb, "\n\t}\n}\n");
1840
rrdhost_unlock(&localhost);
1841
}
1842
@@ -1752,7 +1868,8 @@ void health_reload(void) {
1868
// invalidate all previous entries in the alarm log
1869
ALARM_ENTRY *t;
1870
for(t = localhost.health_log.alarms ; t ; t = t->next) {
1755
- t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1871
+ if(t->new_status != RRDCALC_STATUS_REMOVED)
1872
+ t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1873
}
1874
1875
// reset all thresholds to all charts
@@ -1853,82 +1970,6 @@ static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae)
1970
health_alarm_execute(host, ae);
1971
}
1972
1856
-static inline void health_alarm_log(RRDHOST *host,
1857
- uint32_t alarm_id, uint32_t alarm_event_id,
1858
- time_t when,
1859
- const char *name, const char *chart, const char *family,
1860
- const char *exec, const char *recipient, time_t duration,
1861
- calculated_number old_value, calculated_number new_value,
1862
- int old_status, int new_status,
1863
- const char *source,
1864
- const char *units,
1865
- const char *info
1866
-) {
1867
- ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
1868
- ae->name = strdupz(name);
1869
- ae->hash_name = simple_hash(ae->name);
1870
-
1871
- if(chart) {
1872
- ae->chart = strdupz(chart);
1873
- ae->hash_chart = simple_hash(ae->chart);
1874
- }
1875
-
1876
- if(family)
1877
- ae->family = strdupz(family);
1878
-
1879
- if(exec) ae->exec = strdupz(exec);
1880
- if(recipient) ae->recipient = strdupz(recipient);
1881
- if(source) ae->source = strdupz(source);
1882
- if(units) ae->units = strdupz(units);
1883
- if(info) ae->info = strdupz(info);
1884
-
1885
- ae->unique_id = host->health_log.next_log_id++;
1886
- ae->alarm_id = alarm_id;
1887
- ae->alarm_event_id = alarm_event_id;
1888
- ae->when = when;
1889
- ae->old_value = old_value;
1890
- ae->new_value = new_value;
1891
- ae->old_status = old_status;
1892
- ae->new_status = new_status;
1893
- ae->duration = duration;
1894
-
1895
- if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
1896
- ae->non_clear_duration += ae->duration;
1897
-
1898
- // link it
1899
- pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
1900
- ae->next = host->health_log.alarms;
1901
- host->health_log.alarms = ae;
1902
- host->health_log.count++;
1903
- pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1904
-
1905
- // match previous alarms
1906
- pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1907
- ALARM_ENTRY *t;
1908
- for(t = host->health_log.alarms ; t ; t = t->next) {
1909
- if(t != ae &&
1910
- t->hash_name == ae->hash_name &&
1911
- t->hash_chart == ae->hash_chart &&
1912
- !strcmp(t->name, ae->name) &&
1913
- t->chart && ae->chart && !strcmp(t->chart, ae->chart)) {
1914
-
1915
- if(!(t->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) && !t->updated_by) {
1916
- t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1917
- t->updated_by = ae;
1918
-
1919
- if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
1920
- (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
1921
- ae->non_clear_duration += t->non_clear_duration;
1922
- }
1923
- else {
1924
- // no need to continue
1925
- break;
1926
- }
1927
- }
1928
- }
1929
- pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1930
-}
1931
-
1973
static inline void health_alarm_log_process(RRDHOST *host) {
1974
static uint32_t last_processed = 0;
1975
ALARM_ENTRY *ae;
@@ -1966,6 +2007,8 @@ static inline void health_alarm_log_process(RRDHOST *host) {
2007
ae = NULL;
2008
2009
while(ae) {
2010
+ debug(D_HEALTH, "Health removing alarm log entry with id: %u", ae->unique_id);
2011
+
2012
ALARM_ENTRY *t = ae->next;
2013
2014
freez(ae->name);
@@ -1979,6 +2022,7 @@ static inline void health_alarm_log_process(RRDHOST *host) {
2022
freez(ae);
2023
2024
ae = t;
2025
+ host->health_log.count--;
2026
}
2027
2028
pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
src/health.h
+2
-1
@@ -105,8 +105,9 @@ typedef struct rrddimvar {
105
// having as RRDSET.calculations the RRDCALC to be processed
106
// next.
107
108
-#define RRDCALC_STATUS_UNINITIALIZED 0
108
+#define RRDCALC_STATUS_REMOVED -2
109
#define RRDCALC_STATUS_UNDEFINED -1
110
+#define RRDCALC_STATUS_UNINITIALIZED 0
111
#define RRDCALC_STATUS_CLEAR 1
112
#define RRDCALC_STATUS_RAISED 2
113
#define RRDCALC_STATUS_WARNING 3
web/dashboard.js
+49
-49
@@ -5587,12 +5587,10 @@
5587
NETDATA.alarms = {
5588
notifications: false, // when true, the browser supports notifications (may not be granted though)
5589
last_notification_id: 0, // the id of the last alarm_log we have raised an alarm for
5590
- notifications_shown: new Array(),
5590
+ // notifications_shown: new Array(),
5591
5592
server: null, // the server to connect to for fetching alarms
5593
- current: null, // the list of raised alarms
5594
- notified: { alarms: {} }, // the list of raised alarms for which we have shown notifications
5595
-
5593
+ current: null, // the list of raised alarms - updated in the background
5594
callback: null, // a callback function to call every time the list of raised alarms is refreshed
5595
5596
notify: function(entry) {
@@ -5610,14 +5608,19 @@
5608
var icon = 'images/seo-performance-128.png';
5609
var interaction = false;
5610
var data = entry;
5611
+ var show = true;
5612
+
5613
+ // console.log('alarm ' + entry.unique_id + ' ' + entry.chart + '.' + entry.name + ' is ' + entry.status);
5614
5615
switch(entry.status) {
5615
- case 'UNINITIALIZED':
5616
- // console.log('alarm ' + entry.unique_id + ' is UNINITIALIZED');
5617
- return;
5616
+ case 'REMOVED':
5617
+ show = false;
5618
+ break;
5619
5620
case 'UNDEFINED':
5620
- // console.log('alarm ' + entry.unique_id + ' is UNDEFINED');
5621
+ return;
5622
+
5623
+ case 'UNINITIALIZED':
5624
return;
5625
5626
case 'CLEAR':
@@ -5655,14 +5658,14 @@
5658
return;
5659
}
5660
5658
- var body = entry.hostname + ' - ' + entry.chart + ' (' + entry.family + ') - ' + status + ': ' + entry.info;
5659
-
5661
+ /*
5662
// cleanup old notifications with the same alarm_id as this one
5663
+ // FIXME: it does not seem to work on any web browser!
5664
var len = NETDATA.alarms.notifications_shown.length;
5665
while(len--) {
5666
var n = NETDATA.alarms.notifications_shown[len];
5667
if(n.data.alarm_id === entry.alarm_id) {
5665
- // console.log('removing old alarm ' + n.data.unique_id);
5668
+ console.log('removing old alarm ' + n.data.unique_id);
5669
5670
// close the notification
5671
n.close.bind(n);
@@ -5672,25 +5675,32 @@
5675
len = NETDATA.alarms.notifications_shown.length;
5676
}
5677
}
5678
+ */
5679
+
5680
+ if(show === true) {
5681
+ // show this notification
5682
+ // console.log('new notification: ' + title);
5683
+ var n = new Notification(title, {
5684
+ body: entry.hostname + ' - ' + entry.chart + ' (' + entry.family + ') - ' + status + ': ' + entry.info,
5685
+ tag: tag,
5686
+ requireInteraction: interaction,
5687
+ icon: NETDATA.serverDefault + icon,
5688
+ data: data
5689
+ // FIXME: does not seem to work on any web browser
5690
+ // onclick: function(event) {
5691
+ // event.preventDefault();
5692
+ // console.log('clicked: ');
5693
+ // console.log(event);
5694
+ //}
5695
+ });
5696
5676
- // show this notification
5677
- // console.log('new notification: ' + title);
5678
- var n = new Notification(title, {
5679
- body: body,
5680
- tag: tag,
5681
- requireInteraction: interaction,
5682
- icon: NETDATA.serverDefault + icon,
5683
- data: data
5684
- });
5685
-
5686
- // console.log(n);
5687
- NETDATA.alarms.notifications_shown.push(n);
5688
- // console.log(entry);
5697
+ // console.log(n);
5698
+ // NETDATA.alarms.notifications_shown.push(n);
5699
+ // console.log(entry);
5700
+ }
5701
},
5702
5703
notifyAll: function() {
5692
- NETDATA.alarms.notified = NETDATA.alarms.current;
5693
-
5704
// console.log('FETCHING ALARM LOG');
5705
NETDATA.alarms.get_log(NETDATA.alarms.last_notification_id, function(data) {
5706
// console.log('ALARM LOG FETCHED');
@@ -5705,6 +5715,8 @@
5715
return;
5716
}
5717
5718
+ // console.log('received alarm log of ' + data.length + ' entries, from ' + data[data.length - 1].unique_id.toString() + ' to ' + data[0].unique_id.toString());
5719
+
5720
data.sort(function(a, b) {
5721
if(a.unique_id > b.unique_id) return -1;
5722
if(a.unique_id < b.unique_id) return 1;
@@ -5716,9 +5728,12 @@
5728
if(data[len].unique_id > NETDATA.alarms.last_notification_id) {
5729
NETDATA.alarms.notify(data[len]);
5730
}
5731
+ //else
5732
+ // console.log('ignoring alarm (older) with id ' + data[len].unique_id.toString());
5733
}
5734
5735
NETDATA.alarms.last_notification_id = data[0].unique_id;
5736
+ // console.log('last notification id = ' + NETDATA.alarms.last_notification_id);
5737
})
5738
},
5739
@@ -5736,31 +5751,13 @@
5751
}
5752
5753
if(typeof NETDATA.alarms.current !== 'undefined' && typeof NETDATA.alarms.current.alarms === 'object') {
5739
- // console.log('can do alarms');
5754
+ // console.log('can do alarms: old id = ' + NETDATA.alarms.last_notification_id + ' new id = ' + NETDATA.alarms.current.latest_alarm_log_unique_id);
5755
5741
- var current = NETDATA.alarms.current.alarms;
5742
- var old = NETDATA.alarms.notified.alarms;
5743
- var x;
5744
-
5745
- if(Object.keys(current).length !== Object.keys(old).length) {
5746
- // console.log('alarm count differs');
5756
+ if(NETDATA.alarms.current.latest_alarm_log_unique_id > NETDATA.alarms.last_notification_id) {
5757
+ // console.log('new alarms detected');
5758
return true;
5759
}
5749
- else {
5750
- for(x in current) {
5751
- if(!(x in old)) {
5752
- // console.log('new alarm detected: ' + x);
5753
- return true;
5754
- }
5755
-
5756
- if(current[x].status !== old[x].status) {
5757
- // console.log('alarm changed state: ' + x);
5758
- return true;
5759
- }
5760
- }
5761
- }
5762
-
5763
- //console.log('alarms did not change');
5760
+ //else console.log('no new alarms');
5761
}
5762
// else console.log('cannot process alarms');
5763
@@ -5789,7 +5786,6 @@
5786
update_forever: function() {
5787
NETDATA.alarms.get('active', function(data) {
5788
if(data !== null) {
5792
-
5789
NETDATA.alarms.current = data;
5790
5791
if(NETDATA.alarms.check_notifications() === true) {
@@ -5799,6 +5795,9 @@
5795
if (typeof NETDATA.alarms.callback === 'function') {
5796
NETDATA.alarms.callback(data);
5797
}
5798
+
5799
+ // Health monitoring is disabled on this netdata
5800
+ if(data.status === false) return;
5801
}
5802
5803
setTimeout(NETDATA.alarms.update_forever, 10000);
@@ -5806,6 +5805,7 @@
5805
},
5806
5807
get_log: function(last_id, callback) {
5808
+ // console.log('fetching all log after ' + last_id.toString());
5809
$.ajax({
5810
url: NETDATA.alarms.server + '/api/v1/alarm_log?after=' + last_id.toString(),
5811
async: true,