@cryptotaxi247 / netdata-1 / commits / c803114d3

context index converted to family index; alarm.sh now knows the total time an alarm is problematic

Costa Tsaousis committed Aug 20, 2016 at 21:17 UTC c803114d3a754aa95379ed13c18df2c34fa652c4
5 files changed +87 -58
plugins.d/alarm.sh
+14 -1
@@ -29,6 +29,7 @@ value="${5}" # the current value
29 old_value="${6}" # the previous value
30 src="${7}" # the line number and file the alarm has been configured
31 duration="${8}" # the duration in seconds the previous state took
32 +non_clear_duration="${9}" # the total duration in seconds this is non-clear
33
34 # get the system hostname
35 hostname="$(hostname)"
@@ -84,7 +85,7 @@ duration4human() {
85 }
86
87 severity="${status}"
87 -raised_for="<br/>(was in ${old_status,,} for $(duration4human ${duration}))"
88 +raised_for="<br/>(was ${old_status,,} for $(duration4human ${duration}))"
89 status_message="status unknown"
90 color="grey"
91 alarm="${name} = ${value}"
@@ -124,14 +125,26 @@ then
125 elif [ "${status}" = "CLEAR" ]
126 then
127 severity="Recovered from ${old_status}"
128 + if [ $non_clear_duration > $duration ]
129 + then
130 + raised_for="<br/>(had issues for $(duration4human ${non_clear_duration}))"
131 + fi
132
133 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
134 then
135 severity="Escalated to ${status}"
136 + if [ $non_clear_duration > $duration ]
137 + then
138 + raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
139 + fi
140
141 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
142 then
143 severity="Demoted to ${status}"
144 + if [ $non_clear_duration > $duration ]
145 + then
146 + raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
147 + fi
148
149 else
150 raised_for=
src/health.c
+34 -19
@@ -156,7 +156,7 @@ int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, cal
156 return 1;
157 }
158
159 - rv = rrdvar_index_find(&st->rrdcontext->variables_root_index, variable, hash);
159 + rv = rrdvar_index_find(&st->rrdfamily->variables_root_index, variable, hash);
160 if(rv) {
161 *result = rrdvar2number(rv);
162 return 1;
@@ -171,8 +171,8 @@ int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, cal
171 debug(D_HEALTH, "Available local chart '%s' variables:", st->id);
172 avl_traverse_lock(&st->variables_root_index, dump_variable);
173
174 - debug(D_HEALTH, "Available context '%s' variables:", st->rrdcontext->id);
175 - avl_traverse_lock(&st->rrdcontext->variables_root_index, dump_variable);
174 + debug(D_HEALTH, "Available context '%s' variables:", st->rrdfamily->family);
175 + avl_traverse_lock(&st->rrdfamily->variables_root_index, dump_variable);
176
177 debug(D_HEALTH, "Available host '%s' variables:", st->rrdhost->hostname);
178 avl_traverse_lock(&st->rrdhost->variables_root_index, dump_variable);
@@ -202,9 +202,9 @@ RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *va
202 rs->rrdset = st;
203
204 rs->local = rrdvar_create_and_index("local", &st->variables_root_index, rs->variable, rs->type, rs->value);
205 - rs->context = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->fullid, rs->type, rs->value);
205 + rs->context = rrdvar_create_and_index("context", &st->rrdfamily->variables_root_index, rs->fullid, rs->type, rs->value);
206 rs->host = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullid, rs->type, rs->value);
207 - rs->context_name = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->fullname, rs->type, rs->value);
207 + rs->context_name = rrdvar_create_and_index("context", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
208 rs->host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
209
210 rs->next = st->variables;
@@ -229,12 +229,12 @@ void rrdsetvar_rename_all(RRDSET *st) {
229
230 if (strcmp(buffer, rs->fullname)) {
231 // name changed
232 - rrdvar_free(st->rrdhost, &st->rrdcontext->variables_root_index, rs->context_name);
232 + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->context_name);
233 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
234
235 freez(rs->fullname);
236 rs->fullname = strdupz(st->name);
237 - rs->context_name = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->fullname, rs->type, rs->value);
237 + rs->context_name = rrdvar_create_and_index("context", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
238 rs->host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
239 }
240 }
@@ -247,9 +247,9 @@ void rrdsetvar_free(RRDSETVAR *rs) {
247 debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable);
248
249 rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local);
250 - rrdvar_free(st->rrdhost, &st->rrdcontext->variables_root_index, rs->context);
250 + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->context);
251 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host);
252 - rrdvar_free(st->rrdhost, &st->rrdcontext->variables_root_index, rs->context_name);
252 + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->context_name);
253 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
254
255 if(st->variables == rs) {
@@ -313,8 +313,8 @@ RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char
313 rs->local_id = rrdvar_create_and_index("local", &st->variables_root_index, rs->id, rs->type, rs->value);
314 rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
315
316 - rs->context_id = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->id, rs->type, rs->value);
317 - rs->context_name = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->name, rs->type, rs->value);
316 + rs->context_id = rrdvar_create_and_index("context", &st->rrdfamily->variables_root_index, rs->id, rs->type, rs->value);
317 + rs->context_name = rrdvar_create_and_index("context", &st->rrdfamily->variables_root_index, rs->name, rs->type, rs->value);
318
319 rs->host_fullidid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidid, rs->type, rs->value);
320 rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidname, rs->type, rs->value);
@@ -380,8 +380,8 @@ void rrddimvar_free(RRDDIMVAR *rs) {
380 rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_id);
381 rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
382
383 - rrdvar_free(st->rrdhost, &st->rrdcontext->variables_root_index, rs->context_id);
384 - rrdvar_free(st->rrdhost, &st->rrdcontext->variables_root_index, rs->context_name);
383 + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->context_id);
384 + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->context_name);
385
386 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidid);
387 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
@@ -426,7 +426,7 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
426 st->red = rc->red;
427
428 rc->local = rrdvar_create_and_index("local", &st->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
429 - rc->context = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
429 + rc->context = rrdvar_create_and_index("context", &st->rrdfamily->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
430
431 char fullname[RRDVAR_MAX_LENGTH + 1];
432 snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);
@@ -485,7 +485,7 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
485 rrdvar_free(st->rrdhost, &st->variables_root_index, rc->local);
486 rc->local = NULL;
487
488 - rrdvar_free(st->rrdhost, &st->rrdcontext->variables_root_index, rc->context);
488 + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rc->context);
489 rc->context = NULL;
490
491 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostid);
@@ -1446,7 +1446,7 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1446 const char *exec = ae->exec;
1447 if(!exec) exec = health_default_exec;
1448
1449 - snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u'",
1449 + snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u'",
1450 exec,
1451 ae->name,
1452 ae->chart?ae->chart:"NOCAHRT",
@@ -1455,7 +1455,8 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1455 ae->new_value,
1456 ae->old_value,
1457 ae->source?ae->source:"UNKNOWN",
1458 - (uint32_t)ae->duration
1458 + (uint32_t)ae->duration,
1459 + (uint32_t)ae->non_clear_duration
1460 );
1461
1462 debug(D_HEALTH, "executing command '%s'", buffer);
@@ -1510,6 +1511,9 @@ static inline void health_alarm_log(time_t when,
1511 ae->new_status = new_status;
1512 ae->duration = duration;
1513
1514 + if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
1515 + ae->non_clear_duration += ae->duration;
1516 +
1517 // link it
1518 ae->next = health_log.alarms;
1519 health_log.alarms = ae;
@@ -1523,8 +1527,19 @@ static inline void health_alarm_log(time_t when,
1527 t->hash_chart == ae->hash_chart &&
1528 !strcmp(t->name, ae->name) &&
1529 t->chart && ae->chart && !strcmp(t->chart, ae->chart)) {
1526 - t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1527 - t->updated_by = ae;
1530 +
1531 + if(!(t->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) && !t->updated_by) {
1532 + t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1533 + t->updated_by = ae;
1534 +
1535 + if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
1536 + (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
1537 + ae->non_clear_duration += t->non_clear_duration;
1538 + }
1539 + else {
1540 + // no need to continue
1541 + break;
1542 + }
1543 }
1544 }
1545 }
src/health.h
+2 -1
@@ -13,7 +13,7 @@ extern int rrdvar_compare(void *a, void *b);
13 // the variables as stored in the variables indexes
14 // there are 3 indexes:
15 // 1. at each chart (RRDSET.variables_root_index)
16 -// 2. at each context (RRDCONTEXT.variables_root_index)
16 +// 2. at each context (RRDFAMILY.variables_root_index)
17 // 3. at each host (RRDHOST.variables_root_index)
18 typedef struct rrdvar {
19 avl avl;
@@ -214,6 +214,7 @@ typedef struct alarm_entry {
214
215 time_t when;
216 time_t duration;
217 + time_t non_clear_duration;
218
219 char *name;
220 uint32_t hash_name;
src/rrd.c
+31 -31
@@ -16,7 +16,7 @@ int rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
16
17 static int rrdset_compare(void* a, void* b);
18 static int rrdset_compare_name(void* a, void* b);
19 -static int rrdcontext_compare(void* a, void* b);
19 +static int rrdfamily_compare(void *a, void *b);
20
21 // ----------------------------------------------------------------------------
22 // RRDHOST
@@ -33,8 +33,8 @@ RRDHOST localhost = {
33 { NULL, rrdset_compare_name },
34 AVL_LOCK_INITIALIZER
35 },
36 - .rrdcontext_root_index = {
37 - { NULL, rrdcontext_compare },
36 + .rrdfamily_root_index = {
37 + { NULL, rrdfamily_compare },
38 AVL_LOCK_INITIALIZER
39 },
40 .variables_root_index = {
@@ -70,56 +70,56 @@ void rrdhost_check_wrlock_int(RRDHOST *host, const char *file, const char *funct
70 }
71
72 // ----------------------------------------------------------------------------
73 -// RRDCONTEXT index
73 +// RRDFAMILY index
74
75 -static int rrdcontext_compare(void* a, void* b) {
76 - if(((RRDCONTEXT *)a)->hash < ((RRDCONTEXT *)b)->hash) return -1;
77 - else if(((RRDCONTEXT *)a)->hash > ((RRDCONTEXT *)b)->hash) return 1;
78 - else return strcmp(((RRDCONTEXT *)a)->id, ((RRDCONTEXT *)b)->id);
75 +static int rrdfamily_compare(void *a, void *b) {
76 + if(((RRDFAMILY *)a)->hash_family < ((RRDFAMILY *)b)->hash_family) return -1;
77 + else if(((RRDFAMILY *)a)->hash_family > ((RRDFAMILY *)b)->hash_family) return 1;
78 + else return strcmp(((RRDFAMILY *)a)->family, ((RRDFAMILY *)b)->family);
79 }
80
81 -#define rrdcontext_index_add(host, rc) (RRDCONTEXT *)avl_insert_lock(&((host)->rrdcontext_root_index), (avl *)(rc))
82 -#define rrdcontext_index_del(host, rc) (RRDCONTEXT *)avl_remove_lock(&((host)->rrdcontext_root_index), (avl *)(rc))
81 +#define rrdfamily_index_add(host, rc) (RRDFAMILY *)avl_insert_lock(&((host)->rrdfamily_root_index), (avl *)(rc))
82 +#define rrdfamily_index_del(host, rc) (RRDFAMILY *)avl_remove_lock(&((host)->rrdfamily_root_index), (avl *)(rc))
83
84 -static RRDCONTEXT *rrdcontext_index_find(RRDHOST *host, const char *id, uint32_t hash) {
85 - RRDCONTEXT tmp;
86 - tmp.id = id;
87 - tmp.hash = (hash)?hash:simple_hash(tmp.id);
84 +static RRDFAMILY *rrdfamily_index_find(RRDHOST *host, const char *id, uint32_t hash) {
85 + RRDFAMILY tmp;
86 + tmp.family = id;
87 + tmp.hash_family = (hash)?hash:simple_hash(tmp.family);
88
89 - return (RRDCONTEXT *)avl_search_lock(&(host->rrdcontext_root_index), (avl *) &tmp);
89 + return (RRDFAMILY *)avl_search_lock(&(host->rrdfamily_root_index), (avl *) &tmp);
90 }
91
92 -RRDCONTEXT *rrdcontext_create(const char *id) {
93 - RRDCONTEXT *rc = rrdcontext_index_find(&localhost, id, 0);
92 +RRDFAMILY *rrdfamily_create(const char *id) {
93 + RRDFAMILY *rc = rrdfamily_index_find(&localhost, id, 0);
94 if(!rc) {
95 - rc = callocz(1, sizeof(RRDCONTEXT));
95 + rc = callocz(1, sizeof(RRDFAMILY));
96
97 - rc->id = strdupz(id);
98 - rc->hash = simple_hash(rc->id);
97 + rc->family = strdupz(id);
98 + rc->hash_family = simple_hash(rc->family);
99
100 // initialize the variables index
101 avl_init_lock(&rc->variables_root_index, rrdvar_compare);
102
103 - RRDCONTEXT *ret = rrdcontext_index_add(&localhost, rc);
103 + RRDFAMILY *ret = rrdfamily_index_add(&localhost, rc);
104 if(ret != rc)
105 - fatal("INTERNAL ERROR: Expected to INSERT RRDCONTEXT '%s' into index, but inserted '%s'.", rc->id, (ret)?ret->id:"NONE");
105 + fatal("INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", rc->family, (ret)?ret->family:"NONE");
106 }
107
108 rc->use_count++;
109 return rc;
110 }
111
112 -void rrdcontext_free(RRDCONTEXT *rc) {
112 +void rrdfamily_free(RRDFAMILY *rc) {
113 rc->use_count--;
114 if(!rc->use_count) {
115 - RRDCONTEXT *ret = rrdcontext_index_del(&localhost, rc);
115 + RRDFAMILY *ret = rrdfamily_index_del(&localhost, rc);
116 if(ret != rc)
117 - fatal("INTERNAL ERROR: Expected to DELETE RRDCONTEXT '%s' from index, but deleted '%s'.", rc->id, (ret)?ret->id:"NONE");
117 + fatal("INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", rc->family, (ret)?ret->family:"NONE");
118
119 if(rc->variables_root_index.avl_tree.root != NULL)
120 - fatal("INTERNAL ERROR: Variables index of RRDCONTEXT '%s' that is freed, is not empty.", rc->id);
120 + fatal("INTERNAL ERROR: Variables index of RRDFAMILY '%s' that is freed, is not empty.", rc->family);
121
122 - freez((void *)rc->id);
122 + freez((void *)rc->family);
123 freez(rc);
124 }
125 }
@@ -547,7 +547,7 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const
547 st->title = config_get(st->id, "title", varvalue);
548 }
549
550 - st->rrdcontext = rrdcontext_create(st->context);
550 + st->rrdfamily = rrdfamily_create(st->family);
551 st->rrdhost = &localhost;
552
553 st->next = localhost.rrdset_root;
@@ -779,9 +779,9 @@ void rrdset_free_all(void)
779
780 rrdset_index_del(&localhost, st);
781
782 - st->rrdcontext->use_count--;
783 - if(!st->rrdcontext->use_count)
784 - rrdcontext_free(st->rrdcontext);
782 + st->rrdfamily->use_count--;
783 + if(!st->rrdfamily->use_count)
784 + rrdfamily_free(st->rrdfamily);
785
786 pthread_rwlock_unlock(&st->rwlock);
787
src/rrd.h
+6 -6
@@ -78,17 +78,17 @@ extern const char *rrddim_algorithm_name(int chart_type);
78 // ----------------------------------------------------------------------------
79 // RRD CONTEXT
80
81 -struct rrdcontext {
81 +struct rrdfamily {
82 avl avl;
83
84 - const char *id;
85 - uint32_t hash;
84 + const char *family;
85 + uint32_t hash_family;
86
87 size_t use_count;
88
89 avl_tree_lock variables_root_index;
90 };
91 -typedef struct rrdcontext RRDCONTEXT;
91 +typedef struct rrdfamily RRDFAMILY;
92
93 // ----------------------------------------------------------------------------
94 // RRD DIMENSION
@@ -252,7 +252,7 @@ struct rrdset {
252 total_number collected_total; // used internally to calculate percentages
253 total_number last_collected_total; // used internally to calculate percentages
254
255 - RRDCONTEXT *rrdcontext;
255 + RRDFAMILY *rrdfamily;
256 struct rrdhost *rrdhost;
257
258 struct rrdset *next; // linking of rrdsets
@@ -297,7 +297,7 @@ struct rrdhost {
297 avl_tree_lock rrdset_root_index;
298 avl_tree_lock rrdset_root_index_name;
299
300 - avl_tree_lock rrdcontext_root_index;
300 + avl_tree_lock rrdfamily_root_index;
301 avl_tree_lock variables_root_index;
302
303 // all RRDCALCs are primarily allocated and linked here