operational health templatizer
Costa Tsaousis committed
Aug 14, 2016 at 18:31 UTC
489abaae741457be5ae88e4e7ca35225c0bc079e
7 files changed
+143
-51
src/eval.c
+2
-2
@@ -829,11 +829,10 @@ int expression_evaluate(EVAL_EXPRESSION *exp) {
829
}
830
831
EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error) {
832
- const char *s;
832
+ const char *s = string;
833
int err = EVAL_ERROR_OK;
834
unsigned long pos = 0;
835
836
- s = string;
836
EVAL_NODE *op = parse_full_expression(&s, &err);
837
838
if(*s) {
@@ -864,6 +863,7 @@ EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, in
863
864
EVAL_EXPRESSION *exp = callocz(1, sizeof(EVAL_EXPRESSION));
865
866
+ exp->source = strdupz(string);
867
exp->parsed_as = strdupz(buffer_tostring(out));
868
buffer_free(out);
869
src/health.c
+123
-44
@@ -512,12 +512,14 @@ static inline void rrdcalc_unlinked_optimize_rrdhost(RRDHOST *host, RRDCALC *rc)
512
}
513
514
static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
515
+ debug(D_HEALTH, "Health linking alarm '%s' from chart '%s' of host '%s'", rc->name, st->id, st->rrdhost->hostname);
516
+
517
rc->rrdset = st;
518
517
- if(rc->green)
519
+ if(rc->green && !st->green)
520
st->green = rc->green;
521
520
- if(rc->red)
522
+ if(rc->red && !st->red)
523
st->red = rc->red;
524
525
rc->local = rrdvar_create_and_index("local", &st->variables_root_index, rc->name, rc->hash, RRDVAR_TYPE_CALCULATED, &rc->value);
@@ -560,6 +562,8 @@ void rrdsetcalc_unlink(RRDCALC *rc) {
562
563
RRDHOST *host = st->rrdhost;
564
565
+ debug(D_HEALTH, "Health unlinking alarm '%s' from chart '%s' of host '%s'", rc->name, st->id, host->hostname);
566
+
567
// unlink it
568
if(rc->rrdset_prev)
569
rc->rrdset_prev->rrdset_next = rc->rrdset_next;
@@ -603,7 +607,7 @@ static inline int rrdcalc_exists(RRDHOST *host, const char *name, uint32_t hash)
607
// make sure it does not already exist
608
for(rc = host->calculations; rc ; rc = rc->next) {
609
if (rc->hash == hash && !strcmp(name, rc->name)) {
606
- error("Attempted to create RRDCAL '%s' in host '%s', but it already exists.", name, host->hostname);
610
+ error("alarm '%s' already exists in host '%s'.", name, host->hostname);
611
return 1;
612
}
613
}
@@ -626,7 +630,10 @@ void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
630
}
631
}
632
629
-RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, const char *dimensions, int group_method, uint32_t after, uint32_t before, int update_every, uint32_t options) {
633
+RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, const char *dimensions, int group_method,
634
+ int after, int before, int update_every, uint32_t options,
635
+ calculated_number green, calculated_number red,
636
+ const char *exec, const char *calc, const char *warn, const char *crit) {
637
uint32_t hash = simple_hash(name);
638
639
if(rrdcalc_exists(host, name, hash))
@@ -648,6 +655,43 @@ RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, cons
655
rc->update_every = update_every;
656
rc->options = options;
657
658
+ rc->green = green;
659
+ rc->red = red;
660
+ if(exec) rc->exec = strdupz(exec);
661
+ if(calc) {
662
+ rc->calculation = expression_parse(calc, NULL, NULL);
663
+ if(!rc->calculation)
664
+ error("Failed to parse calculation expression '%s'", calc);
665
+ }
666
+ if(warn) {
667
+ rc->warning = expression_parse(warn, NULL, NULL);
668
+ if(!rc->warning)
669
+ error("Failed to re-parse warning expression '%s'", warn);
670
+ }
671
+ if(crit) {
672
+ rc->critical = expression_parse(crit, NULL, NULL);
673
+ if(!rc->critical)
674
+ error("Failed to re-parse critical expression '%s'", crit);
675
+ }
676
+
677
+ debug(D_HEALTH, "Health runtime added alarm '%s': chart '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
678
+ rc->name,
679
+ (rc->chart)?rc->chart:"NONE",
680
+ (rc->exec)?rc->exec:"DEFAULT",
681
+ rc->green,
682
+ rc->red,
683
+ rc->group,
684
+ rc->after,
685
+ rc->before,
686
+ rc->options,
687
+ (rc->dimensions)?rc->dimensions:"NONE",
688
+ rc->update_every,
689
+ (rc->calculation)?rc->calculation->parsed_as:"NONE",
690
+ (rc->warning)?rc->warning->parsed_as:"NONE",
691
+ (rc->critical)?rc->critical->parsed_as:"NONE",
692
+ rc->source
693
+ );
694
+
695
rrdcalc_create_part2(host, rc);
696
return rc;
697
}
@@ -655,6 +699,8 @@ RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, cons
699
void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
700
if(!rc) return;
701
702
+ debug(D_HEALTH, "Health removing alarm '%s' of host '%s'", rc->name, host->hostname);
703
+
704
// unlink it from RRDSET
705
if(rc->rrdset) rrdsetcalc_unlink(rc);
706
@@ -689,7 +735,34 @@ void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
735
// ----------------------------------------------------------------------------
736
// RRDCALCTEMPLATE management
737
738
+void rrdcalctemplate_link_matching(RRDSET *st) {
739
+ RRDCALCTEMPLATE *rt;
740
+
741
+ for(rt = st->rrdhost->templates; rt ; rt = rt->next) {
742
+ if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) {
743
+ char *s, buffer[RRDSETVAR_ID_MAX + 1];
744
+ snprintfz(buffer, RRDSETVAR_ID_MAX, "%s.%s", st->family, rt->name);
745
+ s = buffer;
746
+ while(*s) {
747
+ if (!isalnum(*s) && *s != '.' && *s != '_')
748
+ *s++ = '_';
749
+ else
750
+ s++;
751
+ }
752
+
753
+ rrdcalc_create(st->rrdhost, buffer, st->id,
754
+ rt->dimensions, rt->group, rt->after, rt->before, rt->update_every, rt->options,
755
+ rt->green, rt->red, rt->exec,
756
+ (rt->calculation)?rt->calculation->source:NULL,
757
+ (rt->warning)?rt->warning->source:NULL,
758
+ (rt->critical)?rt->critical->source:NULL);
759
+ }
760
+ }
761
+}
762
+
763
static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
764
+ debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname);
765
+
766
if(host->templates) {
767
if(host->templates == rt) {
768
host->templates = rt->next;
@@ -735,24 +808,6 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
808
#define HEALTH_EXEC_KEY "exec"
809
810
static inline int rrdcalc_add(RRDHOST *host, RRDCALC *rc) {
738
- info("Health configuration examining alarm '%s': chart '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
739
- rc->name,
740
- (rc->chart)?rc->chart:"NONE",
741
- (rc->exec)?rc->exec:"DEFAULT",
742
- rc->green,
743
- rc->red,
744
- rc->group,
745
- rc->after,
746
- rc->before,
747
- rc->options,
748
- (rc->dimensions)?rc->dimensions:"NONE",
749
- rc->update_every,
750
- (rc->calculation)?rc->calculation->parsed_as:"NONE",
751
- (rc->warning)?rc->warning->parsed_as:"NONE",
752
- (rc->critical)?rc->critical->parsed_as:"NONE",
753
- rc->source
754
- );
755
-
811
if(rrdcalc_exists(host, rc->name, rc->hash))
812
return 0;
813
@@ -766,29 +821,29 @@ static inline int rrdcalc_add(RRDHOST *host, RRDCALC *rc) {
821
return 0;
822
}
823
824
+ debug(D_HEALTH, "Health configuration adding alarm '%s': chart '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
825
+ rc->name,
826
+ (rc->chart)?rc->chart:"NONE",
827
+ (rc->exec)?rc->exec:"DEFAULT",
828
+ rc->green,
829
+ rc->red,
830
+ rc->group,
831
+ rc->after,
832
+ rc->before,
833
+ rc->options,
834
+ (rc->dimensions)?rc->dimensions:"NONE",
835
+ rc->update_every,
836
+ (rc->calculation)?rc->calculation->parsed_as:"NONE",
837
+ (rc->warning)?rc->warning->parsed_as:"NONE",
838
+ (rc->critical)?rc->critical->parsed_as:"NONE",
839
+ rc->source
840
+ );
841
+
842
rrdcalc_create_part2(host, rc);
843
return 1;
844
}
845
846
static inline int rrdcalctemplate_add(RRDHOST *host, RRDCALCTEMPLATE *rt) {
774
- info("Health configuration examining template '%s': context '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
775
- rt->name,
776
- (rt->context)?rt->context:"NONE",
777
- (rt->exec)?rt->exec:"DEFAULT",
778
- rt->green,
779
- rt->red,
780
- rt->group,
781
- rt->after,
782
- rt->before,
783
- rt->options,
784
- (rt->dimensions)?rt->dimensions:"NONE",
785
- rt->update_every,
786
- (rt->calculation)?rt->calculation->parsed_as:"NONE",
787
- (rt->warning)?rt->warning->parsed_as:"NONE",
788
- (rt->critical)?rt->critical->parsed_as:"NONE",
789
- rt->source
790
- );
791
-
847
if(!rt->context) {
848
error("Health configuration for template '%s' does not have a context", rt->name);
849
return 0;
@@ -807,6 +862,24 @@ static inline int rrdcalctemplate_add(RRDHOST *host, RRDCALCTEMPLATE *rt) {
862
}
863
}
864
865
+ debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
866
+ rt->name,
867
+ (rt->context)?rt->context:"NONE",
868
+ (rt->exec)?rt->exec:"DEFAULT",
869
+ rt->green,
870
+ rt->red,
871
+ rt->group,
872
+ rt->after,
873
+ rt->before,
874
+ rt->options,
875
+ (rt->dimensions)?rt->dimensions:"NONE",
876
+ rt->update_every,
877
+ (rt->calculation)?rt->calculation->parsed_as:"NONE",
878
+ (rt->warning)?rt->warning->parsed_as:"NONE",
879
+ (rt->critical)?rt->critical->parsed_as:"NONE",
880
+ rt->source
881
+ );
882
+
883
rt->next = host->templates;
884
host->templates = rt;
885
return 1;
@@ -957,6 +1030,8 @@ static inline char *health_source_file(int line, const char *path, const char *f
1030
}
1031
1032
int health_readfile(const char *path, const char *filename) {
1033
+ debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1034
+
1035
static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0;
1036
char buffer[HEALTH_CONF_MAX_LINE + 1];
1037
@@ -974,8 +1049,6 @@ int health_readfile(const char *path, const char *filename) {
1049
hash_every = simple_uhash(HEALTH_EVERY_KEY);
1050
}
1051
977
- // info("Reading file '%s/%s'", path, filename);
978
-
1052
snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
1053
FILE *fp = fopen(buffer, "r");
1054
if(!fp) {
@@ -1241,7 +1314,7 @@ int health_readfile(const char *path, const char *filename) {
1314
void health_readdir(const char *path) {
1315
size_t pathlen = strlen(path);
1316
1244
- info("Reading directory '%s'", path);
1317
+ debug(D_HEALTH, "Health configuration reading directory '%s'", path);
1318
1319
DIR *dir = opendir(path);
1320
if (!dir) {
@@ -1278,9 +1351,15 @@ void health_readdir(const char *path) {
1351
}
1352
1353
void health_init(void) {
1354
+ debug(D_HEALTH, "Health configuration initializing");
1355
+
1356
char *path;
1357
1283
- // FIXME: allow the user to enable/disable health monitoring
1358
+ if(!config_get_boolean("health", "enabled", 1)) {
1359
+ debug(D_HEALTH, "Health is disabled.");
1360
+ return;
1361
+ }
1362
+
1363
{
1364
char buffer[FILENAME_MAX + 1];
1365
snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
src/health.h
+2
-1
@@ -150,7 +150,7 @@ typedef struct rrdcalc {
150
151
char *exec;
152
153
- char *chart; // the chart name
153
+ char *chart; // the chart id this should be linked to
154
uint32_t hash_chart;
155
156
char *source; // the source of this calculation
@@ -235,6 +235,7 @@ extern void rrddimvar_free(RRDDIMVAR *rs);
235
236
extern void rrdsetcalc_link_matching(RRDSET *st);
237
extern void rrdsetcalc_unlink(RRDCALC *rc);
238
+extern void rrdcalctemplate_link_matching(RRDSET *st);
239
240
extern void health_init(void);
241
src/log.h
+1
@@ -24,6 +24,7 @@
24
#define D_CGROUP 0x00100000
25
#define D_REGISTRY 0x00200000
26
#define D_VARIABLES 0x00400000
27
+#define D_HEALTH 0x00800000
28
29
//#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)
30
//#define DEBUG 0xffffffff
src/rrd.c
+6
-1
@@ -504,9 +504,11 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const
504
st->chart_type = rrdset_type_id(config_get(st->id, "chart type", rrdset_type_name(chart_type)));
505
st->type = config_get(st->id, "type", type);
506
st->family = config_get(st->id, "family", family?family:st->type);
507
- st->context = config_get(st->id, "context", context?context:st->id);
507
st->units = config_get(st->id, "units", units?units:"");
508
509
+ st->context = config_get(st->id, "context", context?context:st->id);
510
+ st->hash_context = simple_hash(st->context);
511
+
512
st->priority = config_get_number(st->id, "priority", priority);
513
st->enabled = enabled;
514
@@ -543,10 +545,13 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const
545
546
rrdsetvar_create(st, "last_collected", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, 0);
547
rrdsetvar_create(st, "raw_total", RRDVAR_TYPE_TOTAL, &st->collected_total, 0);
548
+ rrdsetvar_create(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, 0);
549
+ rrdsetvar_create(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, 0);
550
551
rrdset_index_add(&localhost, st);
552
553
rrdsetcalc_link_matching(st);
554
+ rrdcalctemplate_link_matching(st);
555
556
pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
557
src/rrd.h
+3
-1
@@ -197,10 +197,12 @@ struct rrdset {
197
198
char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options)
199
char *family; // grouping sets under the same family
200
- char *context; // the template of this data set
200
char *title; // title shown to user
201
char *units; // units of measurement
202
203
+ char *context; // the template of this data set
204
+ uint32_t hash_context;
205
+
206
int chart_type;
207
208
int update_every; // every how many seconds is this updated?
src/rrd2json.c
+6
-2
@@ -62,9 +62,13 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
62
c++;
63
}
64
65
+ buffer_strcat(wb, "\n\t\t\t},\n\t\t\t\"green\": ");
66
+ buffer_rrd_value(wb, st->green);
67
+ buffer_strcat(wb, ",\n\t\t\t\"red\": ");
68
+ buffer_rrd_value(wb, st->red);
69
+
70
buffer_sprintf(wb,
66
- "\n\t\t\t}\n"
67
- "\t\t}"
71
+ "\n\t\t}"
72
);
73
74
pthread_rwlock_unlock(&st->rwlock);