@cryptotaxi247 / netdata-1 / commits / f0575ccc9

operational health monitoring - we got alarms! - no notifications yet though

Costa Tsaousis committed Aug 15, 2016 at 02:39 UTC f0575ccc983e41843bd19616b7ef0a729297ceba

15 files changed +408 -77

src/appconfig.c

+6 -2

@@ -478,8 +478,12 @@ void generate_config(BUFFER *wb, int only_changed)

478

479 config_global_write_lock();

480 for(co = config_root; co ; co = co->next) {

481 - if(strcmp(co->name, "global") == 0 || strcmp(co->name, "plugins") == 0 || strcmp(co->name, "registry") == 0) pri = 0;

482 - else if(strncmp(co->name, "plugin:", 7) == 0) pri = 1;

481 + if(!strcmp(co->name, "global") ||

482 + !strcmp(co->name, "plugins") ||

483 + !strcmp(co->name, "registry") ||

484 + !strcmp(co->name, "health"))

485 + pri = 0;

486 + else if(!strncmp(co->name, "plugin:", 7)) pri = 1;

487 else pri = 2;

488

489 if(i == pri) {

src/avl.c

+30 -2

@@ -278,7 +278,27 @@ avl *avl_remove(avl_tree *tree, avl *item) {

278 }

279

280 /* ------------------------------------------------------------------------- */

281 -// these functions are (C) Costa Tsaousis

281 +// below are functions by (C) Costa Tsaousis

282 +

283 +// ---------------------------

284 +// traversing

285 +

286 +void avl_walker(avl *node, void (*callback)(void *)) {

287 + if(node->avl_link[0])

288 + avl_walker(node->avl_link[0], callback);

289 +

290 + callback(node);

291 +

292 + if(node->avl_link[1])

293 + avl_walker(node->avl_link[1], callback);

294 +}

295 +

296 +void avl_traverse(avl_tree *t, void (*callback)(void *)) {

297 + avl_walker(t->root, callback);

298 +}

299 +

300 +// ---------------------------

301 +// locks

302

303 void avl_read_lock(avl_tree_lock *t) {

304 #ifndef AVL_WITHOUT_PTHREADS

@@ -310,7 +330,8 @@ void avl_unlock(avl_tree_lock *t) {

330 #endif /* AVL_WITHOUT_PTHREADS */

331 }

332

313 -/* ------------------------------------------------------------------------- */

333 +// ---------------------------

334 +// operations with locking

335

336 void avl_init_lock(avl_tree_lock *t, int (*compar)(void *a, void *b)) {

337 avl_init(&t->avl_tree, compar);

@@ -351,8 +372,15 @@ avl *avl_insert_lock(avl_tree_lock *t, avl *a) {

372 return ret;

373 }

374

375 +void avl_traverse_lock(avl_tree_lock *t, void (*callback)(void *)) {

376 + avl_read_lock(t);

377 + avl_traverse(&t->avl_tree, callback);

378 + avl_unlock(t);

379 +}

380 +

381 void avl_init(avl_tree *t, int (*compar)(void *a, void *b)) {

382 t->root = NULL;

383 t->compar = compar;

384 }

385

386 +// ------------------

\ No newline at end of file

src/avl.h

@@ -79,4 +79,8 @@ avl *avl_search(avl_tree *t, avl *a);

79 void avl_init_lock(avl_tree_lock *t, int (*compar)(void *a, void *b));

80 void avl_init(avl_tree *t, int (*compar)(void *a, void *b));

82 +

83 +void avl_traverse_lock(avl_tree_lock *t, void (*callback)(void *));

84 +void avl_traverse(avl_tree *t, void (*callback)(void *));

85 +

86 #endif /* avl.h */

src/eval.c

+32 -5

@@ -52,6 +52,7 @@ static inline EVAL_NODE *parse_full_expression(const char **string, int *error);

52 static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error);

53 static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);

54 static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error);

55 +static inline void print_parsed_as_constant(BUFFER *out, calculated_number n);

57 // ----------------------------------------------------------------------------

58 // evaluation of expressions

@@ -71,13 +72,28 @@ static inline calculated_number eval_check_number(calculated_number n, int *erro

72 }

74 static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) {

74 - // FIXME: do the variable look up here

75 + static uint32_t this_hash = 0;

76 +

77 + if(unlikely(this_hash == 0))

78 + this_hash = simple_hash("this");

79 +

80 + if(exp->this && v->hash == this_hash && !strcmp(v->name, "this")) {

81 + buffer_strcat(exp->error_msg, "[ $this = ");

82 + print_parsed_as_constant(exp->error_msg, *exp->this);

83 + buffer_strcat(exp->error_msg, " ] ");

84 + return *exp->this;

85 + }

86 +

87 + calculated_number n;

88 + if(exp->rrdcalc && health_variable_lookup(v->name, v->hash, exp->rrdcalc, &n)) {

89 + buffer_sprintf(exp->error_msg, "[ $%s = ", v->name);

90 + print_parsed_as_constant(exp->error_msg, n);

91 + buffer_strcat(exp->error_msg, " ] ");

92 + return n;

93 + }

76 -// if(!exp->data) {

95 *error = EVAL_ERROR_UNKNOWN_VARIABLE;

96 buffer_sprintf(exp->error_msg, "unknown variable '%s'", v->name);

79 -// }

80 -

97 return 0;

98 }

@@ -208,6 +224,16 @@ static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *

224 }

225

226 static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) {

227 + if(unlikely(isnan(n))) {

228 + buffer_strcat(out, "NaN");

229 + return;

230 + }

231 +

232 + if(unlikely(isinf(n))) {

233 + buffer_strcat(out, "INFINITE");

234 + return;

235 + }

236 +

237 char b[100+1], *s;

238 snprintfz(b, 100, CALCULATED_NUMBER_FORMAT, n);

239

@@ -641,6 +667,7 @@ static inline void eval_node_set_value_to_variable(EVAL_NODE *op, int pos, const

667 op->ops[pos].type = EVAL_VALUE_VARIABLE;

668 op->ops[pos].variable = callocz(1, sizeof(EVAL_VARIABLE));

669 op->ops[pos].variable->name = strdupz(variable);

670 + op->ops[pos].variable->hash = simple_hash(op->ops[pos].variable->name);

671 }

672

673 static inline void eval_variable_free(EVAL_VARIABLE *v) {

@@ -910,7 +937,7 @@ const char *expression_strerror(int error) {

937 return "wrong number of operands for operation - internal error";

938

939 case EVAL_ERROR_VALUE_IS_NAN:

913 - return "value or variable is missing or is not a number";

940 + return "value is unset";

941

942 case EVAL_ERROR_VALUE_IS_INFINITE:

943 return "computed value is infinite";

src/eval.h

+3 -1

@@ -5,6 +5,7 @@

6 typedef struct eval_variable {

7 char *name;

8 + uint32_t hash;

9 struct rrdvar *rrdvar;

10 struct eval_variable *next;

11 } EVAL_VARIABLE;

@@ -13,6 +14,7 @@ typedef struct eval_expression {

14 const char *source;

15 const char *parsed_as;

17 + calculated_number *this;

18 calculated_number result;

20 int error;

@@ -22,7 +24,7 @@ typedef struct eval_expression {

24 void *nodes;

26 // custom data to be used for looking up variables

25 - void *data;

27 + struct rrdcalc *rrdcalc;

28 } EVAL_EXPRESSION;

30 #define EVAL_VALUE_INVALID 0

src/health.c

+274 -23

@@ -80,21 +80,83 @@ static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *

80 rv = NULL;

81 }

83 - /*

84 - * check

83 + return rv;

84 +}

85 +

86 +// ----------------------------------------------------------------------------

87 +// RRDVAR lookup

88 +

89 +calculated_number rrdvar2number(RRDVAR *rv) {

90 + switch(rv->type) {

91 + case RRDVAR_TYPE_CALCULATED: {

92 + calculated_number *n = (calculated_number *)rv->value;

93 + return *n;

94 + }

95 + break;

96 +

97 + case RRDVAR_TYPE_TIME_T: {

98 + time_t *n = (time_t *)rv->value;

99 + return *n;

100 + }

101 + break;

102 +

103 + case RRDVAR_TYPE_COLLECTED: {

104 + collected_number *n = (collected_number *)rv->value;

105 + return *n;

106 + }

107 + break;

108 +

109 + case RRDVAR_TYPE_TOTAL: {

110 + total_number *n = (total_number *)rv->value;

111 + return *n;

112 + }

113 +

114 + default:

115 + error("I don't know how to convert RRDVAR type %d to calculated_number", rv->type);

116 + return NAN;

117 + break;

118 + }

119 +}

120 +

121 +void dump_variable(void *data) {

122 + RRDVAR *rv = (RRDVAR *)data;

123 + debug(D_HEALTH, "%30s : " CALCULATED_NUMBER_FORMAT, rv->name, rrdvar2number(rv));

124 +}

125 +

126 +int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) {

127 + RRDSET *st = rc->rrdset;

128 + RRDVAR *rv;

129 +

130 + if(!st) return 0;

131 +

132 + rv = rrdvar_index_find(&st->variables_root_index, variable, hash);

133 if(rv) {

86 - RRDVAR *ret = rrdvar_index_find(tree, name, hash);

87 - if(ret != rv) fatal("oops! 1");

134 + *result = rrdvar2number(rv);

135 + return 1;

136 + }

137

89 - ret = rrdvar_index_del(tree, rv);

90 - if(ret != rv) fatal("oops! 2");

138 + rv = rrdvar_index_find(&st->rrdcontext->variables_root_index, variable, hash);

139 + if(rv) {

140 + *result = rrdvar2number(rv);

141 + return 1;

142 + }

143

92 - ret = rrdvar_index_add(tree, rv);

93 - if(ret != rv) fatal("oops! 3");

144 + rv = rrdvar_index_find(&st->rrdhost->variables_root_index, variable, hash);

145 + if(rv) {

146 + *result = rrdvar2number(rv);

147 + return 1;

148 }

95 - */

149

97 - return rv;

150 + debug(D_HEALTH, "Available local chart '%s' variables:", st->id);

151 + avl_traverse_lock(&st->variables_root_index, dump_variable);

152 +

153 + debug(D_HEALTH, "Available context '%s' variables:", st->rrdcontext->id);

154 + avl_traverse_lock(&st->rrdcontext->variables_root_index, dump_variable);

155 +

156 + debug(D_HEALTH, "Available host '%s' variables:", st->rrdhost->hostname);

157 + avl_traverse_lock(&st->rrdhost->variables_root_index, dump_variable);

158 +

159 + return 0;

160 }

161

162 // ----------------------------------------------------------------------------

@@ -446,7 +508,7 @@ void rrddimvar_free(RRDDIMVAR *rs) {

508

509 // ----------------------------------------------------------------------------

510 // RRDCALC management

449 -

511 +/*

512 // this has to be called while the caller has locked

513 // the RRDHOST

514 static inline void rrdset_linked_optimize_rrdhost(RRDHOST *host, RRDCALC *rc) {

@@ -511,6 +573,7 @@ static inline void rrdcalc_unlinked_optimize_rrdhost(RRDHOST *host, RRDCALC *rc)

573 host->calculations = rc;

574 }

575 }

576 +*/

577

578 static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {

579 debug(D_HEALTH, "Health linking alarm '%s' from chart '%s' of host '%s'", rc->name, st->id, st->rrdhost->hostname);

@@ -527,12 +590,12 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {

590 rc->context = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rc->name, rc->hash, RRDVAR_TYPE_CALCULATED, &rc->value);

591 rc->host = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rc->name, rc->hash, RRDVAR_TYPE_CALCULATED, &rc->value);

592

530 - rrdset_linked_optimize_rrdhost(st->rrdhost, rc);

593 + // rrdset_linked_optimize_rrdhost(st->rrdhost, rc);

594 }

595

596 static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {

534 - if((rc->hash_chart == st->hash && !strcmp(rc->name, st->id)) ||

535 - (rc->hash_chart == st->hash_name && !strcmp(rc->name, st->name)))

597 + if( (rc->hash_chart == st->hash && !strcmp(rc->chart, st->id)) ||

598 + (rc->hash_chart == st->hash_name && !strcmp(rc->chart, st->name)))

599 return 1;

600

601 return 0;

@@ -540,12 +603,11 @@ static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {

603

604 // this has to be called while the RRDHOST is locked

605 void rrdsetcalc_link_matching(RRDSET *st) {

543 - RRDCALC *rc;

606 + // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id);

607

608 + RRDCALC *rc;

609 for(rc = st->rrdhost->calculations; rc ; rc = rc->next) {

546 - // since unlinked ones are in front and linked at the end

547 - // we stop on the first linked RRDCALC

548 - if(rc->rrdset != NULL) break;

610 + if(rc->rrdset) continue;

611

612 if(rrdcalc_is_matching_this_rrdset(rc, st))

613 rrdsetcalc_link(st, rc);

@@ -599,7 +661,7 @@ void rrdsetcalc_unlink(RRDCALC *rc) {

661 // so that if the matching chart is found in the future

662 // it will be applied automatically

663

602 - rrdcalc_unlinked_optimize_rrdhost(host, rc);

664 + // rrdcalc_unlinked_optimize_rrdhost(host, rc);

665 }

666

667 static inline int rrdcalc_exists(RRDHOST *host, const char *name, uint32_t hash) {

@@ -617,6 +679,23 @@ static inline int rrdcalc_exists(RRDHOST *host, const char *name, uint32_t hash)

679 }

680

681 void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {

682 + rrdhost_check_rdlock(host);

683 +

684 + if(rc->calculation) {

685 + rc->calculation->this = &rc->value;

686 + rc->calculation->rrdcalc = rc;

687 + }

688 +

689 + if(rc->warning) {

690 + rc->warning->this = &rc->value;

691 + rc->warning->rrdcalc = rc;

692 + }

693 +

694 + if(rc->critical) {

695 + rc->critical->this = &rc->value;

696 + rc->critical->rrdcalc = rc;

697 + }

698 +

699 // link it to the host

700 rc->next = host->calculations;

701 host->calculations = rc;

@@ -634,7 +713,8 @@ void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {

713 RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, const char *dimensions, int group_method,

714 int after, int before, int update_every, uint32_t options,

715 calculated_number green, calculated_number red,

637 - const char *exec, const char *calc, const char *warn, const char *crit) {

716 + const char *exec, const char *source,

717 + const char *calc, const char *warn, const char *crit) {

718 uint32_t hash = simple_hash(name);

719

720 if(rrdcalc_exists(host, name, hash))

@@ -658,7 +738,10 @@ RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, cons

738

739 rc->green = green;

740 rc->red = red;

741 +

742 if(exec) rc->exec = strdupz(exec);

743 + if(source) rc->source = strdupz(source);

744 +

745 if(calc) {

746 rc->calculation = expression_parse(calc, NULL, NULL);

747 if(!rc->calculation)

@@ -758,12 +841,16 @@ void rrdcalctemplate_link_matching(RRDSET *st) {

841 char buffer[RRDSETVAR_ID_MAX + 1];

842 snprintfz(buffer, RRDSETVAR_ID_MAX, "%s.%s", st->family, rt->name);

843 variable_fix_name(buffer);

761 - rrdcalc_create(st->rrdhost, buffer, st->id,

844 + RRDCALC *rc = rrdcalc_create(st->rrdhost, buffer, st->id,

845 rt->dimensions, rt->group, rt->after, rt->before, rt->update_every, rt->options,

763 - rt->green, rt->red, rt->exec,

846 + rt->green, rt->red, rt->exec, rt->source,

847 (rt->calculation)?rt->calculation->source:NULL,

848 (rt->warning)?rt->warning->source:NULL,

849 (rt->critical)?rt->critical->source:NULL);

850 +

851 + // FIXME

852 + if(rc->rrdset != st)

853 + fatal("RRDCAL '%s' is not linked to chart '%s'", rc->name, st->id);

854 }

855 }

856 }

@@ -1360,6 +1447,8 @@ void health_readdir(const char *path) {

1447 health_readfile(path, de->d_name);

1448 }

1449 }

1450 +

1451 + closedir(dir);

1452 }

1453

1454 void health_init(void) {

@@ -1375,11 +1464,173 @@ void health_init(void) {

1464 {

1465 char buffer[FILENAME_MAX + 1];

1466 snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));

1378 - path = config_get("health", "configuration files in directory", buffer);

1467 + path = config_get("health", "health configuration directory", buffer);

1468

1469 snprintfz(buffer, FILENAME_MAX, "%s/alarm.sh", config_get("global", "plugins directory", PLUGINS_DIR));

1470 health_default_exec = config_get("health", "script to execute on alarm", buffer);

1471 }

1472

1473 + rrdhost_rwlock(&localhost);

1474 health_readdir(path);

1475 + rrdhost_unlock(&localhost);

1476 +}

1477 +

1478 +static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {

1479 + if (unlikely(!rc->rrdset)) {

1480 + debug(D_HEALTH, "Health not running alarm '%s'. It is not linked to a chart.", rc->name);

1481 + return 0;

1482 + }

1483 +

1484 + if (unlikely(!rc->update_every)) {

1485 + debug(D_HEALTH, "Health not running alarm '%s'. It does not have an update frequency", rc->name);

1486 + return 0;

1487 + }

1488 +

1489 + if (unlikely(rc->next_update > now)) {

1490 + if (*next_run > rc->next_update)

1491 + *next_run = rc->next_update;

1492 +

1493 + debug(D_HEALTH, "Health not examining alarm '%s' yet (will do in %d secs).", rc->name,

1494 + (int) (rc->next_update - now));

1495 + return 0;

1496 + }

1497 +

1498 + return 1;

1499 +}

1500 +

1501 +void *health_main(void *ptr) {

1502 + (void)ptr;

1503 +

1504 + info("HEALTH thread created with task id %d", gettid());

1505 +

1506 + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)

1507 + error("Cannot set pthread cancel type to DEFERRED.");

1508 +

1509 + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)

1510 + error("Cannot set pthread cancel state to ENABLE.");

1511 +

1512 + int min_run_every = (int)config_get_number("health", "run at least every seconds", 10);

1513 + if(min_run_every < 1) min_run_every = 1;

1514 +

1515 + BUFFER *wb = buffer_create(100);

1516 +

1517 + unsigned int loop = 0;

1518 + while(health_enabled) {

1519 + loop++;

1520 + debug(D_HEALTH, "Health monitoring iteration no %u started", loop);

1521 +

1522 + int oldstate, runnable = 0;

1523 + time_t now = time(NULL);

1524 + time_t next_run = now + min_run_every;

1525 + RRDCALC *rc;

1526 +

1527 + if (unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))

1528 + error("Cannot set pthread cancel state to DISABLE.");

1529 +

1530 + rrdhost_rdlock(&localhost);

1531 + for (rc = localhost.calculations; rc; rc = rc->next) {

1532 + if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))

1533 + continue;

1534 +

1535 + runnable++;

1536 + debug(D_HEALTH, "Health running alarm '%s'", rc->name);

1537 +

1538 + // 1. if there is database lookup, do it

1539 + // 1b. if the lookup has a calculation expression run it

1540 + // 2. if there is warning expression, do it

1541 + // 3. if there is critical expression, do it

1542 +

1543 + if (rc->after) {

1544 + time_t latest_timestamp;

1545 + int value_is_null;

1546 + int ret = rrd2value(rc->rrdset, wb, &rc->value, rc->dimensions, 1, rc->after, rc->before, rc->group,

1547 + rc->options, &latest_timestamp, &value_is_null);

1548 + if (ret != 200) {

1549 + error("Health for alarm '%s', database lookup returned error %d", rc->name, ret);

1550 + }

1551 + else {

1552 + if(value_is_null) {

1553 + rc->value = NAN;

1554 + error("Health for alarm '%s', database lookup returned empty value (possibly value is not collected yet)", rc->name);

1555 + }

1556 + else {

1557 + debug(D_HEALTH, "Health for alarm '%s', database lookup gave value "

1558 + CALCULATED_NUMBER_FORMAT, rc->name, rc->value);

1559 +

1560 + if (rc->calculation) {

1561 + if (!expression_evaluate(rc->calculation)) {

1562 + error("Health for alarm '%s', failed to evaluate calculation with error: %s", rc->name,

1563 + buffer_tostring(rc->calculation->error_msg));

1564 + } else {

1565 + debug(D_HEALTH, "Health for alarm '%s', calculation gave value "

1566 + CALCULATED_NUMBER_FORMAT, rc->name, rc->calculation->result);

1567 + rc->value = rc->calculation->result;

1568 + }

1569 + }

1570 + }

1571 + }

1572 + }

1573 + }

1574 + rrdhost_unlock(&localhost);

1575 +

1576 + if (runnable) {

1577 + rrdhost_rdlock(&localhost);

1578 +

1579 + for (rc = localhost.calculations; rc; rc = rc->next) {

1580 + if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))

1581 + continue;

1582 +

1583 + if(rc->warning) {

1584 + if (!expression_evaluate(rc->warning)) {

1585 + error("Health for alarm '%s', failed to evaluate warning expression with error: %s", rc->name,

1586 + buffer_tostring(rc->warning->error_msg));

1587 + }

1588 + else {

1589 + debug(D_HEALTH, "Health for alarm '%s', warning expression gave value "

1590 + CALCULATED_NUMBER_FORMAT ": %s",

1591 + rc->name, rc->warning->result,

1592 + buffer_tostring(rc->warning->error_msg)

1593 + );

1594 + }

1595 + }

1596 +

1597 + if(rc->critical) {

1598 + if (!expression_evaluate(rc->critical)) {

1599 + error("Health for alarm '%s', failed to evaluate critical expression with error: %s", rc->name,

1600 + buffer_tostring(rc->critical->error_msg));

1601 + }

1602 + else {

1603 + debug(D_HEALTH, "Health for alarm '%s', critical expression gave value "

1604 + CALCULATED_NUMBER_FORMAT ": %s",

1605 + rc->name, rc->critical->result,

1606 + buffer_tostring(rc->critical->error_msg)

1607 + );

1608 + }

1609 + }

1610 +

1611 + rc->last_updated = now;

1612 + rc->next_update = now + rc->update_every;

1613 +

1614 + if (next_run > rc->next_update)

1615 + next_run = rc->next_update;

1616 + }

1617 +

1618 + rrdhost_unlock(&localhost);

1619 + }

1620 +

1621 +

1622 + if (unlikely(pthread_setcancelstate(oldstate, NULL) != 0))

1623 + error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);

1624 +

1625 + debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",

1626 + loop, (int) (next_run - now));

1627 +

1628 + sleep_usec(1000000 * (unsigned long long) (next_run - now));

1629 + }

1630 +

1631 + buffer_free(wb);

1632 +

1633 + info("HEALTH thread exiting");

1634 + pthread_exit(NULL);

1635 + return NULL;

1636 }

src/health.h

@@ -218,5 +218,8 @@ extern void rrdsetcalc_unlink(RRDCALC *rc);

218 extern void rrdcalctemplate_link_matching(RRDSET *st);

219

220 extern void health_init(void);

221 +extern void *health_main(void *ptr);

222 +

223 +extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result);

224

225 #endif //NETDATA_HEALTH_H

src/log.c

+4 -15

@@ -75,24 +75,13 @@ int open_log_file(int fd, FILE **fp, const char *filename, int *enabled_syslog)

75 }

76 else fd = f;

78 - if(fp && *fp == NULL) {

79 - // info("fdopen(%d) on filename '%s'", fd, filename);

80 -

81 - FILE *n = fdopen(fd, "a");

82 - if (!n)

78 + if(fp && !*fp) {

79 + *fp = fdopen(fd, "a");

80 + if (!*fp)

81 error("Cannot fdopen() fd %d ('%s')", fd, filename);

84 -

82 else {

86 - if (setvbuf(n, NULL, _IOLBF, 0) != 0)

83 + if (setvbuf(*fp, NULL, _IOLBF, 0) != 0)

84 error("Cannot set line buffering on fd %d ('%s')", fd, filename);

88 -

89 - if(!*fp)

90 - *fp = n;

91 - else {

92 - FILE *o = *fp;

93 - *fp = n;

94 - fclose(o);

95 - }

85 }

86 }

src/main.c

+2 -1

@@ -47,8 +47,9 @@ struct netdata_static_thread {

47 {"idlejitter", "plugins", "idlejitter", 1, NULL, NULL, cpuidlejitter_main},

48 {"proc", "plugins", "proc", 1, NULL, NULL, proc_main},

49 {"cgroups", "plugins", "cgroups", 1, NULL, NULL, cgroups_main},

50 + {"check", "plugins", "checks", 0, NULL, NULL, checks_main},

51 + {"health", NULL, NULL, 1, NULL, NULL, health_main},

52 {"plugins.d", NULL, NULL, 1, NULL, NULL, pluginsd_main},

51 - {"check", "plugins", "checks", 0, NULL, NULL, checks_main},

53 {"web", NULL, NULL, 1, NULL, NULL, socket_listen_main_multi_threaded},

54 {"web-single-threaded", NULL, NULL, 0, NULL, NULL, socket_listen_main_single_threaded},

55 {NULL, NULL, NULL, 0, NULL, NULL, NULL}

src/plugins_d.c

+2 -3

@@ -439,9 +439,8 @@ void *pluginsd_worker_thread(void *arg)

439 return NULL;

440 }

441

442 -void *pluginsd_main(void *ptr)

443 -{

444 - if(ptr) { ; }

442 +void *pluginsd_main(void *ptr) {

443 + (void)ptr;

444

445 info("PLUGINS.D thread created with task id %d", gettid());

446

src/rrd.c

+27 -14

@@ -55,10 +55,17 @@ void rrdhost_unlock(RRDHOST *host) {

55 pthread_rwlock_unlock(&host->rrdset_root_rwlock);

56 }

58 +void rrdhost_check_rdlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line) {

59 + int ret = pthread_rwlock_trywrlock(&host->rrdset_root_rwlock);

60 +

61 + if(ret == 0)

62 + fatal("RRDHOST '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", host->hostname, function, line, file);

63 +}

64 +

65 void rrdhost_check_wrlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line) {

66 int ret = pthread_rwlock_tryrdlock(&host->rrdset_root_rwlock);

61 - if(ret != 0)

68 + if(ret == 0)

69 fatal("RRDHOST '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", host->hostname, function, line, file);

70 }

@@ -526,7 +533,7 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const

533 avl_init_lock(&st->variables_root_index, rrdvar_compare);

534

535 pthread_rwlock_init(&st->rwlock, NULL);

529 - pthread_rwlock_wrlock(&localhost.rrdset_root_rwlock);

536 + rrdhost_rwlock(&localhost);

537

538 if(name && *name) rrdset_set_name(st, name);

539 else rrdset_set_name(st, id);

@@ -543,17 +550,19 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const

550 st->next = localhost.rrdset_root;

551 localhost.rrdset_root = st;

552

546 - rrdsetvar_create(st, "last_collected", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, 0);

547 - rrdsetvar_create(st, "raw_total", RRDVAR_TYPE_TOTAL, &st->collected_total, 0);

548 - rrdsetvar_create(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, 0);

549 - rrdsetvar_create(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, 0);

553 + if(health_enabled) {

554 + rrdsetvar_create(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, 0);

555 + rrdsetvar_create(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, 0);

556 + rrdsetvar_create(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, 0);

557 + rrdsetvar_create(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, 0);

558 + }

559

560 rrdset_index_add(&localhost, st);

561

562 rrdsetcalc_link_matching(st);

563 rrdcalctemplate_link_matching(st);

564

556 - pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);

565 + rrdhost_unlock(&localhost);

566

567 return(st);

568 }

@@ -681,9 +690,11 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier

690 td->next = rd;

691 }

692

684 - rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->calculated_value, 0);

685 - rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->collected_value, 0);

686 - rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected", &rd->last_collected_time.tv_sec, 0);

693 + if(health_enabled) {

694 + rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, 0);

695 + rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, 0);

696 + rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, 0);

697 + }

698

699 pthread_rwlock_unlock(&st->rwlock);

700

@@ -746,7 +757,7 @@ void rrdset_free_all(void)

757 {

758 info("Freeing all memory...");

759

749 - pthread_rwlock_wrlock(&localhost.rrdset_root_rwlock);

760 + rrdhost_rwlock(&localhost);

761

762 RRDSET *st;

763 for(st = localhost.rrdset_root; st ;) {

@@ -789,7 +800,7 @@ void rrdset_free_all(void)

800 }

801 localhost.rrdset_root = NULL;

802

792 - pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);

803 + rrdhost_unlock(&localhost);

804

805 info("Memory cleanup completed...");

806 }

@@ -800,7 +811,7 @@ void rrdset_save_all(void) {

811 RRDSET *st;

812 RRDDIM *rd;

813

803 - pthread_rwlock_wrlock(&localhost.rrdset_root_rwlock);

814 + rrdhost_rwlock(&localhost);

815 for(st = localhost.rrdset_root; st ; st = st->next) {

816 pthread_rwlock_wrlock(&st->rwlock);

817

@@ -818,7 +829,7 @@ void rrdset_save_all(void) {

829

830 pthread_rwlock_unlock(&st->rwlock);

831 }

821 - pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);

832 + rrdhost_unlock(&localhost);

833 }

834

835

@@ -1326,6 +1337,7 @@ unsigned long long rrdset_done(RRDSET *st)

1337

1338 if(likely(rd->updated && rd->counter > 1 && iterations < st->gap_when_lost_iterations_above)) {

1339 rd->values[st->current_entry] = pack_storage_number(new_value, storage_flags );

1340 + rd->last_stored_value = new_value;

1341

1342 if(unlikely(st->debug))

1343 debug(D_RRD_STATS, "%s/%s: STORE[%ld] "

@@ -1341,6 +1353,7 @@ unsigned long long rrdset_done(RRDSET *st)

1353 , st->current_entry

1354 );

1355 rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);

1356 + rd->last_stored_value = 0;

1357 }

1358

1359 stored_entries++;

src/rrd.h

+14 -4

@@ -138,11 +138,13 @@ struct rrddim {

138 // this is actual date time we updated the last_collected_value

139 // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET

140

141 - calculated_number calculated_value; // the current calculated value, after applying the algorithm

142 - calculated_number last_calculated_value; // the last calculated value

141 + calculated_number calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used

142 + calculated_number last_calculated_value; // the last calculated value processed

143

144 - collected_number collected_value; // the current value, as collected

145 - collected_number last_collected_value; // the last value that was collected

144 + calculated_number last_stored_value; // the last value as stored in the database (after interpolation)

145 +

146 + collected_number collected_value; // the current value, as collected - resets to 0 after being used

147 + collected_number last_collected_value; // the last value that was collected, after being processed

148

149 // the *_volume members are used to calculate the accuracy of the rounding done by the

150 // storage number - they are printed to debug.log when debug is enabled for a set.

@@ -315,10 +317,18 @@ extern RRDHOST localhost;

317

318 #ifdef NETDATA_INTERNAL_CHECKS

319 #define rrdhost_check_wrlock(host) rrdhost_check_wrlock_int(host, __FILE__, __FUNCTION__, __LINE__)

320 +#define rrdhost_check_rdlock(host) rrdhost_check_rdlock_int(host, __FILE__, __FUNCTION__, __LINE__)

321 #else

322 +#define rrdhost_check_rdlock(host) (void)0

323 #define rrdhost_check_wrlock(host) (void)0

324 #endif

325 +

326 extern void rrdhost_check_wrlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line);

327 +extern void rrdhost_check_rdlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line);

328 +

329 +extern void rrdhost_rwlock(RRDHOST *host);

330 +extern void rrdhost_rdlock(RRDHOST *host);

331 +extern void rrdhost_unlock(RRDHOST *host);

332

333 // ----------------------------------------------------------------------------

334 // RRD SET functions

src/rrd2json.c

+5 -5

@@ -1039,12 +1039,12 @@ inline static calculated_number rrdr2value(RRDR *r, long i, uint32_t options, in

1039 }

1040

1041 if(unlikely(all_null)) {

1042 - if(likely(*all_values_are_null))

1042 + if(likely(all_values_are_null))

1043 *all_values_are_null = 1;

1044 return 0;

1045 }

1046 else {

1047 - if(likely(*all_values_are_null))

1047 + if(likely(all_values_are_null))

1048 *all_values_are_null = 0;

1049 }

1050

@@ -1524,7 +1524,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g

1524 return r;

1525 }

1526

1527 -int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, BUFFER *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp, int *value_is_null)

1527 +int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp, int *value_is_null)

1528 {

1529 RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));

1530 if(!r) {

@@ -1543,10 +1543,10 @@ int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, BUFFER *dimensions,

1543 else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)

1544 wb->options |= WB_CONTENT_CACHEABLE;

1545

1546 - options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);

1546 + options = rrdr_check_options(r, options, dimensions);

1547

1548 if(dimensions)

1549 - rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));

1549 + rrdr_disable_not_selected_dimensions(r, dimensions);

1550

1551 if(latest_timestamp)

1552 *latest_timestamp = r->before;

src/rrd2json.h

+1 -1

@@ -61,6 +61,6 @@ extern void rrd_stats_all_json(BUFFER *wb);

61 extern time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long entries_to_show, long group, int group_method, time_t after, time_t before, int only_non_zero);

63 extern int rrd2format(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp);

64 -extern int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, BUFFER *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp, int *value_is_null);

64 +extern int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp, int *value_is_null);

66 #endif /* NETDATA_RRD2JSON_H */

src/web_client.c

+1 -1

@@ -759,7 +759,7 @@ int web_client_api_v1_badge(struct web_client *w, char *url) {

759

760 // if the collected value is too old, don't calculate its value

761 if(rrdset_last_entry_t(st) >= (time(NULL) - (st->update_every * st->gap_when_lost_iterations_above)))

762 - ret = rrd2value(st, w->response.data, &n, dimensions, points, after, before, group, options, &latest_timestamp, &value_is_null);

762 + ret = rrd2value(st, w->response.data, &n, (dimensions)?buffer_tostring(dimensions):NULL, points, after, before, group, options, &latest_timestamp, &value_is_null);

763

764 // if the value cannot be calculated, show empty badge

765 if(ret != 200) {