@cryptotaxi247 / netdata-1 / commits / dd73f3e0c

Repeating alarm notifications (#6309)

* Alarm_repeat mergin the original! * Alarm_repeat binary tree! * Alarm_repeat binary tree finished! * Alarm_repeat move function and format string * Alarms bringing a new Binary tree * Alarms fixing the last two * Alarm_repeat useless var! * Alarm fix format and repeat alarm! * Alarm_backend steps! * Alarm_repeat stopping to test cloud! * Alarm_repeat stopping to test cloud 2! * Alarm_repeat fixing when restart!

thiagoftsm committed Jul 1, 2019 at 11:55 UTC dd73f3e0cd0d9a1797f0c82de0fae5ccf97e9464

20 files changed +513 -105

daemon/common.h

@@ -14,6 +14,7 @@

14 #define config_get_float(section, name, value) appconfig_get_float(&netdata_config, section, name, value)

15 #define config_get_boolean(section, name, value) appconfig_get_boolean(&netdata_config, section, name, value)

16 #define config_get_boolean_ondemand(section, name, value) appconfig_get_boolean_ondemand(&netdata_config, section, name, value)

17 +#define config_get_duration(section, name, value) appconfig_get_duration(&netdata_config, section, name, value)

19 #define config_set(section, name, default_value) appconfig_set(&netdata_config, section, name, default_value)

20 #define config_set_default(section, name, value) appconfig_set_default(&netdata_config, section, name, value)

daemon/main.c

+1 -1

@@ -1217,7 +1217,7 @@ int main(int argc, char **argv) {

1217 info("netdata initialization completed. Enjoy real-time performance monitoring!");

1218 netdata_ready = 1;

1219

1220 - send_statistics("START","-", "-");

1220 + send_statistics("START", "-", "-");

1221

1222 // ------------------------------------------------------------------------

1223 // unblock signals

database/rrd.c

+1 -1

@@ -132,7 +132,6 @@ const char *rrdset_type_name(RRDSET_TYPE chart_type) {

132 }

133 }

134

135 -

135 // ----------------------------------------------------------------------------

136 // RRD - cache directory

137

@@ -154,3 +153,4 @@ char *rrdset_cache_dir(RRDHOST *host, const char *id, const char *config_section

153

154 return ret;

155 }

156 +

database/rrd.h

+13

@@ -572,6 +572,8 @@ struct alarm_entry {

572 uint32_t updated_by_id;

573 uint32_t updates_id;

574

575 + time_t last_repeat;

576 +

577 struct alarm_entry *next;

578 };

579

@@ -686,11 +688,16 @@ struct rrdhost {

688 char *health_log_filename; // the alarms event log filename

689 size_t health_log_entries_written; // the number of alarm events writtern to the alarms event log

690 FILE *health_log_fp; // the FILE pointer to the open alarms event log file

691 + uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications

692 + uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications

693 +

694

695 // all RRDCALCs are primarily allocated and linked here

696 // RRDCALCs may be linked to charts at any point

697 // (charts may or may not exist when these are loaded)

698 RRDCALC *alarms;

699 + avl_tree_lock alarms_idx_health_log;

700 + avl_tree_lock alarms_idx_name;

701

702 ALARM_LOG health_log; // alarms historical events (event log)

703 uint32_t health_last_processed_id; // the last processed health id from the log

@@ -1021,6 +1028,12 @@ extern collected_number rrddim_set(RRDSET *st, const char *id, collected_number

1028

1029 extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);

1030

1031 +// ----------------------------------------------------------------------------

1032 +// Miscellaneous functions

1033 +

1034 +extern int alarm_compare_id(void *a, void *b);

1035 +extern int alarm_compare_name(void *a, void *b);

1036 +

1037 // ----------------------------------------------------------------------------

1038 // RRD internal functions

1039

database/rrdcalc.c

+91 -11

@@ -81,9 +81,9 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {

82 if(!rc->units) rc->units = strdupz(st->units);

84 - {

84 + if(!rrdcalc_isrepeating(rc)) {

85 time_t now = now_realtime_sec();

86 - health_alarm_log(

86 + ALARM_ENTRY *ae = health_create_alarm_entry(

87 host,

88 rc->id,

89 rc->next_event_id++,

@@ -104,6 +104,7 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {

104 0,

105 0

106 );

107 + health_alarm_log(host, ae);

108 }

109 }

110

@@ -142,9 +143,9 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {

143

144 RRDHOST *host = st->rrdhost;

145

145 - {

146 + if(!rrdcalc_isrepeating(rc)) {

147 time_t now = now_realtime_sec();

147 - health_alarm_log(

148 + ALARM_ENTRY *ae = health_create_alarm_entry(

149 host,

150 rc->id,

151 rc->next_event_id++,

@@ -165,6 +166,7 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {

166 0,

167 0

168 );

169 + health_alarm_log(host, ae);

170 }

171

172 debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);

@@ -253,7 +255,7 @@ inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const ch

255 return host->health_log.next_alarm_id++;

256 }

257

256 -inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {

258 +inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) {

259 rrdhost_check_rdlock(host);

260

261 if(rc->calculation) {

@@ -301,8 +303,7 @@ inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {

303 }

304 }

305

304 -inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) {

305 -

306 +inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) {

307 debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rt->name);

308

309 if(rrdcalc_exists(host, chart, rt->name, 0, 0))

@@ -328,6 +329,10 @@ inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *c

329 rc->delay_max_duration = rt->delay_max_duration;

330 rc->delay_multiplier = rt->delay_multiplier;

331

332 + rc->last_repeat = 0;

333 + rc->warn_repeat_every = rt->warn_repeat_every;

334 + rc->crit_repeat_every = rt->crit_repeat_every;

335 +

336 rc->group = rt->group;

337 rc->after = rt->after;

338 rc->before = rt->before;

@@ -356,7 +361,7 @@ inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *c

361 error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source);

362 }

363

359 - debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",

364 + debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",

365 (rc->chart)?rc->chart:"NOCHART",

366 rc->name,

367 (rc->exec)?rc->exec:"DEFAULT",

@@ -376,16 +381,24 @@ inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *c

381 rc->delay_up_duration,

382 rc->delay_down_duration,

383 rc->delay_max_duration,

379 - rc->delay_multiplier

384 + rc->delay_multiplier,

385 + rc->warn_repeat_every,

386 + rc->crit_repeat_every

387 );

388

382 - rrdcalc_create_part2(host, rc);

389 + rrdcalc_add_to_host(host, rc);

390 + RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl *)rc);

391 + if (rdcmp != rc) {

392 + error("Cannot insert the alarm index ID %s",rc->name);

393 + }

394 +

395 return rc;

396 }

397

398 void rrdcalc_free(RRDCALC *rc) {

399 if(unlikely(!rc)) return;

400

401 +

402 expression_free(rc->calculation);

403 expression_free(rc->warning);

404 expression_free(rc->critical);

@@ -413,7 +426,6 @@ void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) {

426 // unlink it from RRDHOST

427 if(unlikely(rc == host->alarms))

428 host->alarms = rc->next;

416 -

429 else {

430 RRDCALC *t;

431 for(t = host->alarms; t && t->next != rc; t = t->next) ;

@@ -425,5 +437,73 @@ void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) {

437 error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);

438 }

439

440 + if (rc) {

441 + RRDCALC *rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_health_log, (avl *)rc);

442 + if (!rdcmp) {

443 + error("Cannot remove the health alarm index");

444 + }

445 +

446 + rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_name, (avl *)rc);

447 + if (!rdcmp) {

448 + error("Cannot remove the health alarm index");

449 + }

450 + }

451 +

452 rrdcalc_free(rc);

453 }

454 +

455 +// ----------------------------------------------------------------------------

456 +// Alarm

457 +

458 +

459 +/**

460 + * Alarm is repeating

461 + *

462 + * Is this alarm repeating ?

463 + *

464 + * @param host The structure that has the binary tree

465 + * @param alarm_id the id of the alarm to search

466 + *

467 + * @return It returns 1 case it is repeating and 0 otherwise

468 + */

469 +int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id) {

470 + RRDCALC findme;

471 + findme.id = alarm_id;

472 + RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_health_log, (avl *)&findme);

473 + if (!rc) {

474 + return 0;

475 + }

476 + return rrdcalc_isrepeating(rc);

477 +}

478 +

479 +/**

480 + * Entry is repeating

481 + *

482 + * Check whether the id of alarm entry is yet present in the host structure

483 + *

484 + * @param host The structure that has the binary tree

485 + * @param ae the alarm entry

486 + *

487 + * @return It returns 1 case it is repeating and 0 otherwise

488 + */

489 +int alarm_entry_isrepeating(RRDHOST *host, ALARM_ENTRY *ae) {

490 + return alarm_isrepeating(host, ae->alarm_id);

491 +}

492 +

493 +/**

494 + * Max last repeat

495 + *

496 + * Check the maximum last_repeat for the alarms associated a host

497 + *

498 + * @param host The structure that has the binary tree

499 + *

500 + * @return It returns 1 case it is repeating and 0 otherwise

501 + */

502 +RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name,uint32_t hash) {

503 + RRDCALC findme;

504 + findme.name = alarm_name;

505 + findme.hash = hash;

506 + RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_name, (avl *)&findme);

507 +

508 + return rc;

509 +}

database/rrdcalc.h

+23 -2

@@ -29,7 +29,9 @@

29 #define RRDCALC_FLAG_SILENCED 0x00000100

30 #define RRDCALC_FLAG_NO_CLEAR_NOTIFICATION 0x80000000

32 +

33 struct rrdcalc {

34 + avl avl; // the index, with key the id - this has to be first!

35 uint32_t id; // the unique id of this alarm

36 uint32_t next_event_id; // the next event id that will be used for this alarm

@@ -77,9 +79,16 @@ struct rrdcalc {

79 float delay_multiplier; // multiplier for all delays when alarms switch status

80 // while now < delay_up_to

82 + // ------------------------------------------------------------------------

83 + // notification repeat settings

84 +

85 + uint32_t warn_repeat_every; // interval between repeating warning notifications

86 + uint32_t crit_repeat_every; // interval between repeating critical notifications

87 +

88 // ------------------------------------------------------------------------

89 // runtime information

91 + RRDCALC_STATUS old_status; // the old status of the alarm

92 RRDCALC_STATUS status; // the current status of the alarm

94 calculated_number value; // the current value of the alarm

@@ -90,6 +99,7 @@ struct rrdcalc {

99 time_t last_updated; // the last update timestamp of the alarm

100 time_t next_update; // the next update timestamp of the alarm

101 time_t last_status_change; // the timestamp of the last time this alarm changed status

102 + time_t last_repeat; // the last time the alarm got repeated

103

104 time_t db_after; // the first timestamp evaluated by the db lookup

105 time_t db_before; // the last timestamp evaluated by the db lookup

@@ -119,6 +129,10 @@ struct rrdcalc {

129 struct rrdcalc *next;

130 };

131

132 +extern int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id);

133 +extern int alarm_entry_isrepeating(RRDHOST *host, ALARM_ENTRY *ae);

134 +extern RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name, uint32_t hash);

135 +

136 #define RRDCALC_HAS_DB_LOOKUP(rc) ((rc)->after)

137

138 extern void rrdsetcalc_link_matching(RRDSET *st);

@@ -132,7 +146,14 @@ extern void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc);

146

147 extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name);

148 extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id);

135 -extern RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart);

136 -extern void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc);

149 +extern RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart);

150 +extern void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc);

151 +

152 +static inline int rrdcalc_isrepeating(RRDCALC *rc) {

153 + if (unlikely(rc->warn_repeat_every > 0 || rc->crit_repeat_every > 0)) {

154 + return 1;

155 + }

156 + return 0;

157 +}

158

159 #endif //NETDATA_RRDCALC_H

database/rrdcalctemplate.c

+1 -1

@@ -13,7 +13,7 @@ void rrdcalctemplate_link_matching(RRDSET *st) {

13 for(rt = host->templates; rt ; rt = rt->next) {

14 if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)

15 && (!rt->family_pattern || simple_pattern_matches(rt->family_pattern, st->family))) {

16 - RRDCALC *rc = rrdcalc_create(host, rt, st->id);

16 + RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);

17 if(unlikely(!rc))

18 info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, host->hostname);

database/rrdcalctemplate.h

@@ -48,6 +48,12 @@ struct rrdcalctemplate {

48 int delay_max_duration; // the absolute max delay to apply to this alarm

49 float delay_multiplier; // multiplier for all delays when alarms switch status

51 + // ------------------------------------------------------------------------

52 + // notification repeat settings

53 +

54 + uint32_t warn_repeat_every; // interval between repeating warning notifications

55 + uint32_t crit_repeat_every; // interval between repeating critical notifications

56 +

57 // ------------------------------------------------------------------------

58 // expressions related to the alarm

database/rrdhost.c

+47 -3

@@ -179,6 +179,10 @@ RRDHOST *rrdhost_create(const char *hostname,

179 if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost)

180 rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST);

181

182 + host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");

183 + host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");

184 + avl_init_lock(&(host->alarms_idx_health_log), alarm_compare_id);

185 + avl_init_lock(&(host->alarms_idx_name), alarm_compare_name);

186

187 // ------------------------------------------------------------------------

188 // initialize health variables

@@ -274,12 +278,12 @@ RRDHOST *rrdhost_create(const char *hostname,

278 // load health configuration

279

280 if(host->health_enabled) {

277 - health_alarm_log_load(host);

278 - health_alarm_log_open(host);

279 -

281 rrdhost_wrlock(host);

282 health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);

283 rrdhost_unlock(host);

284 +

285 + health_alarm_log_load(host);

286 + health_alarm_log_open(host);

287 }

288

289

@@ -876,3 +880,43 @@ int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, ch

880

881 return res;

882 }

883 +

884 +/**

885 + * Alarm Compare ID

886 + *

887 + * Callback function used with the binary trees to compare the id of RRDCALC

888 + *

889 + * @param a a pointer to the RRDCAL item to insert,compare or update the binary tree

890 + * @param b the pointer to the binary tree.

891 + *

892 + * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b.

893 + */

894 +int alarm_compare_id(void *a, void *b) {

895 + register uint32_t hash1 = ((RRDCALC *)a)->id;

896 + register uint32_t hash2 = ((RRDCALC *)b)->id;

897 +

898 + if(hash1 < hash2) return -1;

899 + else if(hash1 > hash2) return 1;

900 +

901 + return 0;

902 +}

903 +

904 +/**

905 + * Alarm Compare NAME

906 + *

907 + * Callback function used with the binary trees to compare the name of RRDCALC

908 + *

909 + * @param a a pointer to the RRDCAL item to insert,compare or update the binary tree

910 + * @param b the pointer to the binary tree.

911 + *

912 + * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b.

913 + */

914 +int alarm_compare_name(void *a, void *b) {

915 + RRDCALC *in1 = (RRDCALC *)a;

916 + RRDCALC *in2 = (RRDCALC *)b;

917 +

918 + if(in1->hash < in2->hash) return -1;

919 + else if(in1->hash > in2->hash) return 1;

920 +

921 + return strcmp(in1->name,in2->name);

922 +}

health/README.md

+21 -1

@@ -11,7 +11,6 @@ packet dropped).

12 Netdata also supports alarm **templates**, so that an alarm can be attached to all the charts of the same context (i.e. all network interfaces, or all disks, or all mysql servers, etc.).

14 -

14 Each alarm can execute a single query to the database using statistical algorithms against past data,

15 but alarms can be combined. So, if you need 2 queries in the database, you can combine

16 2 alarms together (both will run a query to the database, and the results can be combined).

@@ -342,6 +341,24 @@ delay: [[[up U] [down D] multiplier M] max X]

341 their matching one) and a delay is in place.

342 - All are reset to their defaults when the alarm switches state without a delay in place.

343

344 +---

345 +

346 +#### Alarm line `repeat`

347 +

348 +Defines the interval between repeating notifications for the alarms in CRITICAL or WARNING mode. This will override the default interval settings inherited from health settings in `netdata.conf`. The default settings for repeating notifications are `default repeat warning = DURATION` and `default repeat critical = DURATION` which can be found in health stock configuration.

349 +

350 +Format:

351 +

352 +```

353 +repeat: [off] [warning DURATION] [critical DURATION]

354 +```

355 +

356 +* `off`: Turns off the repeating feature for the current alarm. This is effective when the default repeat settings has been enabled in health configuration.

357 +* `warning DURATION`: Defines the interval when the alarm is in WARNING state. Use `0s` to turn off the repeating notification for WARNING mode.

358 +* `critical DURATION`: Defines the interval when the alarm is in CRITICAL state. Use `0s` to turn off the repeating notification for CRITICAL mode.

359 +

360 +---

361 +

362 #### Alarm line `option`

363

364 The only possible value for the `option` line is

@@ -567,12 +584,15 @@ template: disk_full_percent

584 every: 1m

585 warn: $this > 80

586 crit: $this > 95

587 + repeat: warning 120s critical 10s

588 ```

589

590 `$used` and `$avail` are the `used` and `avail` chart dimensions as shown on the dashboard.

591

592 So, the `calc` line finds the percentage of used space. `$this` resolves to this percentage.

593

594 +This is a repeating alarm and if the alarm becomes CRITICAL it repeats the notifications every 10 seconds. It also repeats notifications every 2 minutes if the alarm goes into WARNING mode.

595 +

596 ### Example 3

597

598 Predict if any disk will run out of space in the near future.

health/health.c

+60 -24

@@ -255,17 +255,18 @@ static inline void health_alarm_log_process(RRDHOST *host) {

255 netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);

256

257 ALARM_ENTRY *ae;

258 - for(ae = host->health_log.alarms; ae && ae->unique_id >= host->health_last_processed_id ; ae = ae->next) {

259 - if(unlikely(

260 - !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) &&

261 - !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED)

258 + for(ae = host->health_log.alarms; ae && ae->unique_id >= host->health_last_processed_id; ae = ae->next) {

259 + if(likely(!alarm_entry_isrepeating(host, ae))) {

260 + if(unlikely(

261 + !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) &&

262 + !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED)

263 )) {

264 + if(unlikely(ae->unique_id < first_waiting))

265 + first_waiting = ae->unique_id;

266

264 - if(unlikely(ae->unique_id < first_waiting))

265 - first_waiting = ae->unique_id;

266 -

267 - if(likely(now >= ae->delay_up_to_timestamp))

268 - health_process_notifications(host, ae);

267 + if(likely(now >= ae->delay_up_to_timestamp))

268 + health_process_notifications(host, ae);

269 + }

270 }

271 }

272

@@ -295,6 +296,10 @@ static inline void health_alarm_log_process(RRDHOST *host) {

296 ALARM_ENTRY *t = ae->next;

297

298 health_alarm_log_free_one_nochecks_nounlink(ae);

299 + if(likely(!alarm_entry_isrepeating(host, ae))) {

300 + health_alarm_log_free_one_nochecks_nounlink(ae);

301 + host->health_log.count--;

302 + }

303

304 ae = t;

305 host->health_log.count--;

@@ -411,7 +416,7 @@ SILENCE_TYPE check_silenced(RRDCALC *rc, char* host, SILENCERS *silencers) {

416 debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rc->name);

417 } else {

418 debug(D_HEALTH, "Alarm %s via the command API - name:%s context:%s chart:%s host:%s family:%s"

414 - , (silencers->stype==STYPE_DISABLE_ALARMS)?"Disabled":"Silenced"

419 + , (silencers->stype == STYPE_DISABLE_ALARMS)?"Disabled":"Silenced"

420 , rc->name

421 , (rc->rrdset)?rc->rrdset->context:""

422 , rc->chart

@@ -756,20 +761,22 @@ void *health_main(void *ptr) {

761 rc->delay_last = delay;

762 rc->delay_up_to_timestamp = now + delay;

763

759 - health_alarm_log(

760 - host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id,

761 - rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change,

762 - rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info,

763 - rc->delay_last,

764 - (

765 - ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |

766 - ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0)

767 - )

768 -

769 - );

770 -

771 - rc->last_status_change = now;

772 - rc->status = status;

764 + if(likely(!rrdcalc_isrepeating(rc))) {

765 + ALARM_ENTRY *ae = health_create_alarm_entry(

766 + host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id,

767 + rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change,

768 + rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info,

769 + rc->delay_last,

770 + (

771 + ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |

772 + ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0)

773 + )

774 + );

775 + health_alarm_log(host, ae);

776 + }

777 + rc->last_status_change = now;

778 + rc->old_status = rc->status;

779 + rc->status = status;

780 }

781

782 rc->last_updated = now;

@@ -779,6 +786,35 @@ void *health_main(void *ptr) {

786 next_run = rc->next_update;

787 }

788

789 + // process repeating alarms

790 + RRDCALC *rc;

791 + for(rc = host->alarms; rc ; rc = rc->next) {

792 + int repeat_every = 0;

793 + if(unlikely(rrdcalc_isrepeating(rc))) {

794 + if(unlikely(rc->status == RRDCALC_STATUS_WARNING))

795 + repeat_every = rc->warn_repeat_every;

796 + else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL))

797 + repeat_every = rc->crit_repeat_every;

798 + }

799 + if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {

800 + rc->last_repeat = now;

801 + ALARM_ENTRY *ae = health_create_alarm_entry(

802 + host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id,

803 + rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change,

804 + rc->old_value, rc->value, rc->old_status, rc->status, rc->source, rc->units, rc->info,

805 + rc->delay_last,

806 + (

807 + ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |

808 + ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0)

809 + )

810 + );

811 + ae->last_repeat = rc->last_repeat;

812 + health_process_notifications(host, ae);

813 + debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);

814 + health_alarm_log_free_one_nochecks_nounlink(ae);

815 + }

816 + }

817 +

818 rrdhost_unlock(host);

819 }

820

health/health.h

+3 -1

@@ -108,7 +108,7 @@ extern void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);

108 extern ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename);

109 extern void health_alarm_log_load(RRDHOST *host);

110

111 -extern void health_alarm_log(

111 +extern ALARM_ENTRY* health_create_alarm_entry(

112 RRDHOST *host,

113 uint32_t alarm_id,

114 uint32_t alarm_event_id,

@@ -129,6 +129,8 @@ extern void health_alarm_log(

129 int delay,

130 uint32_t flags);

131

132 +extern void health_alarm_log(RRDHOST *host, ALARM_ENTRY *ae);

133 +

134 extern void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath);

135 extern char *health_user_config_dir(void);

136 extern char *health_stock_config_dir(void);

health/health_config.c

+80 -56

@@ -23,6 +23,7 @@

23 #define HEALTH_INFO_KEY "info"

24 #define HEALTH_DELAY_KEY "delay"

25 #define HEALTH_OPTIONS_KEY "options"

26 +#define HEALTH_REPEAT_KEY "repeat"

28 static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {

29 if(!rc->chart) {

@@ -45,7 +46,7 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {

47 rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id);

48 - debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",

49 + debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",

50 rc->chart?rc->chart:"NOCHART",

51 rc->name,

52 rc->id,

@@ -66,10 +67,12 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {

67 rc->delay_up_duration,

68 rc->delay_down_duration,

69 rc->delay_max_duration,

69 - rc->delay_multiplier

70 + rc->delay_multiplier,

71 + rc->warn_repeat_every,

72 + rc->crit_repeat_every

73 );

72 - rrdcalc_create_part2(host, rc);

75 + rrdcalc_add_to_host(host, rc);

76 return 1;

77 }

@@ -100,7 +103,7 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL

103 }

104 }

105

103 - debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",

106 + debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",

107 rt->name,

108 (rt->context)?rt->context:"NONE",

109 (rt->exec)?rt->exec:"DEFAULT",

@@ -120,7 +123,9 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL

123 rt->delay_up_duration,

124 rt->delay_down_duration,

125 rt->delay_max_duration,

123 - rt->delay_multiplier

126 + rt->delay_multiplier,

127 + rt->warn_repeat_every,

128 + rt->crit_repeat_every

129 );

130

131 if(likely(last)) {

@@ -134,48 +139,6 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL

139 return 1;

140 }

141

137 -static inline int health_parse_duration(char *string, int *result) {

138 - // make sure it is a number

139 - if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {

140 - *result = 0;

141 - return 0;

142 - }

143 -

144 - char *e = NULL;

145 - calculated_number n = str2ld(string, &e);

146 - if(e && *e) {

147 - switch (*e) {

148 - case 'Y':

149 - *result = (int) (n * 86400 * 365);

150 - break;

151 - case 'M':

152 - *result = (int) (n * 86400 * 30);

153 - break;

154 - case 'w':

155 - *result = (int) (n * 86400 * 7);

156 - break;

157 - case 'd':

158 - *result = (int) (n * 86400);

159 - break;

160 - case 'h':

161 - *result = (int) (n * 3600);

162 - break;

163 - case 'm':

164 - *result = (int) (n * 60);

165 - break;

166 -

167 - default:

168 - case 's':

169 - *result = (int) (n);

170 - break;

171 - }

172 - }

173 - else

174 - *result = (int)(n);

175 -

176 - return 1;

177 -}

178 -

142 static inline int health_parse_delay(

143 size_t line, const char *filename, char *string,

144 int *delay_up_duration,

@@ -202,14 +165,14 @@ static inline int health_parse_delay(

165 while(*s && isspace(*s)) *s++ = '\0';

166

167 if(!strcasecmp(key, "up")) {

205 - if (!health_parse_duration(value, delay_up_duration)) {

168 + if (!config_parse_duration(value, delay_up_duration)) {

169 error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",

170 line, filename, value, key);

171 }

172 else given_up = 1;

173 }

174 else if(!strcasecmp(key, "down")) {

212 - if (!health_parse_duration(value, delay_down_duration)) {

175 + if (!config_parse_duration(value, delay_down_duration)) {

176 error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",

177 line, filename, value, key);

178 }

@@ -224,7 +187,7 @@ static inline int health_parse_delay(

187 else given_multiplier = 1;

188 }

189 else if(!strcasecmp(key, "max")) {

227 - if (!health_parse_duration(value, delay_max_duration)) {

190 + if (!config_parse_duration(value, delay_max_duration)) {

191 error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",

192 line, filename, value, key);

193 }

@@ -285,6 +248,50 @@ static inline uint32_t health_parse_options(const char *s) {

248 return options;

249 }

250

251 +static inline int health_parse_repeat(

252 + size_t line,

253 + const char *file,

254 + char *string,

255 + uint32_t *warn_repeat_every,

256 + uint32_t *crit_repeat_every

257 +) {

258 +

259 + char *s = string;

260 + while(*s) {

261 + char *key = s;

262 +

263 + while(*s && !isspace(*s)) s++;

264 + while(*s && isspace(*s)) *s++ = '\0';

265 +

266 + if(!*key) break;

267 +

268 + char *value = s;

269 + while(*s && !isspace(*s)) s++;

270 + while(*s && isspace(*s)) *s++ = '\0';

271 +

272 + if(!strcasecmp(key, "off")) {

273 + *warn_repeat_every = 0;

274 + *crit_repeat_every = 0;

275 + return 1;

276 + }

277 + if(!strcasecmp(key, "warning")) {

278 + if (!config_parse_duration(value, (int*)warn_repeat_every)) {

279 + error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",

280 + line, file, value, key);

281 + }

282 + }

283 + else if(!strcasecmp(key, "critical")) {

284 + if (!config_parse_duration(value, (int*)crit_repeat_every)) {

285 + error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",

286 + line, file, value, key);

287 + }

288 + }

289 + }

290 +

291 + return 1;

292 +}

293 +

294 +

295 static inline int health_parse_db_lookup(

296 size_t line, const char *filename, char *string,

297 RRDR_GROUPING *group_method, int *after, int *before, int *every,

@@ -322,7 +329,7 @@ static inline int health_parse_db_lookup(

329 while(*s && !isspace(*s)) s++;

330 while(*s && isspace(*s)) *s++ = '\0';

331

325 - if(!health_parse_duration(key, after)) {

332 + if(!config_parse_duration(key, after)) {

333 error("Health configuration at line %zu of file '%s': invalid duration '%s' after group method",

334 line, filename, key);

335 return 0;

@@ -343,7 +350,7 @@ static inline int health_parse_db_lookup(

350 while(*s && !isspace(*s)) s++;

351 while(*s && isspace(*s)) *s++ = '\0';

352

346 - if (!health_parse_duration(value, before)) {

353 + if (!config_parse_duration(value, before)) {

354 error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword",

355 line, filename, value, key);

356 }

@@ -353,7 +360,7 @@ static inline int health_parse_db_lookup(

360 while(*s && !isspace(*s)) s++;

361 while(*s && isspace(*s)) *s++ = '\0';

362

356 - if (!health_parse_duration(value, every)) {

363 + if (!config_parse_duration(value, every)) {

364 error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword",

365 line, filename, value, key);

366 }

@@ -430,7 +437,8 @@ static int health_readfile(const char *filename, void *data) {

437 hash_info = 0,

438 hash_recipient = 0,

439 hash_delay = 0,

433 - hash_options = 0;

440 + hash_options = 0,

441 + hash_repeat = 0;

442

443 char buffer[HEALTH_CONF_MAX_LINE + 1];

444

@@ -454,6 +462,7 @@ static int health_readfile(const char *filename, void *data) {

462 hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);

463 hash_delay = simple_uhash(HEALTH_DELAY_KEY);

464 hash_options = simple_uhash(HEALTH_OPTIONS_KEY);

465 + hash_repeat = simple_uhash(HEALTH_REPEAT_KEY);

466 }

467

468 FILE *fp = fopen(filename, "r");

@@ -532,6 +541,9 @@ static int health_readfile(const char *filename, void *data) {

541 rc->value = NAN;

542 rc->old_value = NAN;

543 rc->delay_multiplier = 1.0;

544 + rc->old_status = RRDCALC_STATUS_UNINITIALIZED;

545 + rc->warn_repeat_every = host->health_default_warn_repeat_every;

546 + rc->crit_repeat_every = host->health_default_crit_repeat_every;

547

548 if(rrdvar_fix_name(rc->name))

549 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);

@@ -556,6 +568,8 @@ static int health_readfile(const char *filename, void *data) {

568 rt->green = NAN;

569 rt->red = NAN;

570 rt->delay_multiplier = 1.0;

571 + rt->warn_repeat_every = host->health_default_warn_repeat_every;

572 + rt->crit_repeat_every = host->health_default_crit_repeat_every;

573

574 if(rrdvar_fix_name(rt->name))

575 error("Health configuration renamed template '%s' to '%s'", value, rt->name);

@@ -612,7 +626,7 @@ static int health_readfile(const char *filename, void *data) {

626 &rc->options, &rc->dimensions);

627 }

628 else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {

615 - if(!health_parse_duration(value, &rc->update_every))

629 + if(!config_parse_duration(value, &rc->update_every))

630 error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",

631 line, filename, rc->name, key, value);

632 }

@@ -707,6 +721,11 @@ static int health_readfile(const char *filename, void *data) {

721 else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {

722 rc->options |= health_parse_options(value);

723 }

724 + else if(hash == hash_repeat && !strcasecmp(key, HEALTH_REPEAT_KEY)){

725 + health_parse_repeat(line, filename, value,

726 + &rc->warn_repeat_every,

727 + &rc->crit_repeat_every);

728 + }

729 else {

730 error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.",

731 line, filename, rc->name, key);

@@ -736,7 +755,7 @@ static int health_readfile(const char *filename, void *data) {

755 &rt->update_every, &rt->options, &rt->dimensions);

756 }

757 else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {

739 - if(!health_parse_duration(value, &rt->update_every))

758 + if(!config_parse_duration(value, &rt->update_every))

759 error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' cannot parse duration: '%s'.",

760 line, filename, rt->name, key, value);

761 }

@@ -831,6 +850,11 @@ static int health_readfile(const char *filename, void *data) {

850 else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {

851 rt->options |= health_parse_options(value);

852 }

853 + else if(hash == hash_repeat && !strcasecmp(key, HEALTH_REPEAT_KEY)){

854 + health_parse_repeat(line, filename, value,

855 + &rt->warn_repeat_every,

856 + &rt->crit_repeat_every);

857 + }

858 else {

859 error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",

860 line, filename, rt->name, key);

health/health_json.c

@@ -140,6 +140,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC

140 "\t\t\t\"delay_multiplier\": %f,\n"

141 "\t\t\t\"delay\": %d,\n"

142 "\t\t\t\"delay_up_to_timestamp\": %lu,\n"

143 + "\t\t\t\"warn_repeat_every\": \"%u\",\n"

144 + "\t\t\t\"crit_repeat_every\": \"%u\",\n"

145 "\t\t\t\"value_string\": \"%s\",\n"

146 , rc->chart, rc->name

147 , (unsigned long)rc->id

@@ -165,6 +167,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC

167 , rc->delay_multiplier

168 , rc->delay_last

169 , (unsigned long)rc->delay_up_to_timestamp

170 + , rc->warn_repeat_every

171 + , rc->crit_repeat_every

172 , value_string

173 );

174

health/health_log.c

+53 -4

@@ -79,6 +79,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {

79 "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"

80 "\t%d\t%d\t%d\t%d"

81 "\t" CALCULATED_NUMBER_FORMAT_AUTO "\t" CALCULATED_NUMBER_FORMAT_AUTO

82 + "\t%016lx"

83 "\n"

84 , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'

85 , host->hostname

@@ -112,6 +113,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {

113

114 , ae->new_value

115 , ae->old_value

116 + , (uint64_t)ae->last_repeat

117 ) < 0))

118 error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", host->hostname, host->health_log_filename);

119 else {

@@ -174,10 +176,40 @@ inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filena

176 continue;

177 }

178

179 + // Check if we got last_repeat field

180 + time_t last_repeat = 0;

181 + if(entries > 27) {

182 + char* alarm_name = pointers[13];

183 + last_repeat = (time_t)strtoul(pointers[27], NULL, 16);

184 +

185 + RRDCALC *rc = alarm_max_last_repeat(host, alarm_name,simple_hash(alarm_name));

186 + if (!rc) {

187 + for(rc = host->alarms; rc ; rc = rc->next) {

188 + RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_name, (avl *)rc);

189 + if(rdcmp != rc) {

190 + error("Cannot insert the alarm index ID using log %s", rc->name);

191 + }

192 + }

193 +

194 + rc = alarm_max_last_repeat(host, alarm_name,simple_hash(alarm_name));

195 + }

196 +

197 + if(unlikely(rc)) {

198 + if (rrdcalc_isrepeating(rc)) {

199 + rc->last_repeat = last_repeat;

200 + // We iterate through repeating alarm entries only to

201 + // find the latest last_repeat timestamp. Otherwise,

202 + // there is no need to keep them in memory.

203 + continue;

204 + }

205 + }

206 + }

207 +

208 if(unlikely(*pointers[0] == 'A')) {

209 // make sure it is properly numbered

210 if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) {

180 - error("HEALTH [%s]: line %zu of file '%s' has alarm log entry %u in wrong order. Ignoring it.", host->hostname, line, filename, unique_id);

211 + error( "HEALTH [%s]: line %zu of file '%s' has alarm log entry %u in wrong order. Ignoring it."

212 + , host->hostname, line, filename, unique_id);

213 errored++;

214 continue;

215 }

@@ -186,11 +218,11 @@ inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filena

218 }

219 else if(unlikely(*pointers[0] == 'U')) {

220 // find the original

189 - for(ae = host->health_log.alarms; ae; ae = ae->next) {

221 + for(ae = host->health_log.alarms; ae ; ae = ae->next) {

222 if(unlikely(unique_id == ae->unique_id)) {

223 if(unlikely(*pointers[0] == 'A')) {

224 error("HEALTH [%s]: line %zu of file '%s' adds duplicate alarm log entry %u. Using the later."

193 - , host->hostname, line, filename, unique_id);

225 + , host->hostname, line, filename, unique_id);

226 *pointers[0] = 'U';

227 duplicate++;

228 }

@@ -270,6 +302,8 @@ inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filena

302 ae->new_value = str2l(pointers[25]);

303 ae->old_value = str2l(pointers[26]);

304

305 + ae->last_repeat = last_repeat;

306 +

307 char value_string[100 + 1];

308 freez(ae->old_value_string);

309 freez(ae->new_value_string);

@@ -339,7 +373,7 @@ inline void health_alarm_log_load(RRDHOST *host) {

373 // ----------------------------------------------------------------------------

374 // health alarm log management

375

342 -inline void health_alarm_log(

376 +inline ALARM_ENTRY* health_create_alarm_entry(

377 RRDHOST *host,

378 uint32_t alarm_id,

379 uint32_t alarm_event_id,

@@ -398,9 +432,24 @@ inline void health_alarm_log(

432 ae->delay_up_to_timestamp = when + delay;

433 ae->flags |= flags;

434

435 + ae->last_repeat = 0;

436 +

437 if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)

438 ae->non_clear_duration += ae->duration;

439

440 + return ae;

441 +}

442 +

443 +inline void health_alarm_log(

444 + RRDHOST *host,

445 + ALARM_ENTRY *ae

446 +) {

447 + debug(D_HEALTH, "Health adding alarm log entry with id: %u", ae->unique_id);

448 +

449 + if(unlikely(alarm_entry_isrepeating(host, ae))) {

450 + error("Repeating alarms cannot be added to host's alarm log entries. It seems somewhere in the logic, API is being misused. Alarm id: %u", ae->alarm_id);

451 + return;

452 + }

453 // link it

454 netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock);

455 ae->next = host->health_log.alarms;

libnetdata/config/appconfig.c

+83

@@ -411,6 +411,27 @@ int appconfig_set_boolean(struct config *root, const char *section, const char *

411 return value;

412 }

413

414 +int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value)

415 +{

416 + int result = 0;

417 + const char *s;

418 +

419 + s = appconfig_get(root, section, name, value);

420 + if(!s) goto fallback;

421 +

422 + if(!config_parse_duration(s, &result)) {

423 + error("config option '[%s].%s = %s' is configured with an valid duration", section, name, s);

424 + goto fallback;

425 + }

426 +

427 + return result;

428 +

429 + fallback:

430 + if(!config_parse_duration(value, &result))

431 + error("INTERNAL ERROR: default duration supplied for option '[%s].%s = %s' is not a valid duration", section, name, value);

432 +

433 + return result;

434 +}

435

436 // ----------------------------------------------------------------------------

437 // config load/save

@@ -586,3 +607,65 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)

607 appconfig_unlock(root);

608 }

609 }

610 +

611 +/**

612 + * Parse Duration

613 + *

614 + * Parse the string setting the result

615 + *

616 + * @param string the timestamp string

617 + * @param result the output variable

618 + *

619 + * @return It returns 1 on success and 0 otherwise

620 + */

621 +int config_parse_duration(const char* string, int* result) {

622 + while(*string && isspace(*string)) string++;

623 +

624 + if(unlikely(!*string)) goto fallback;

625 +

626 + if(*string == 'n' && !strcmp(string, "never")) {

627 + // this is a valid option

628 + *result = 0;

629 + return 1;

630 + }

631 +

632 + // make sure it is a number

633 + if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback;

634 +

635 + char *e = NULL;

636 + calculated_number n = str2ld(string, &e);

637 + if(e && *e) {

638 + switch (*e) {

639 + case 'Y':

640 + *result = (int) (n * 31536000);

641 + break;

642 + case 'M':

643 + *result = (int) (n * 2592000);

644 + break;

645 + case 'w':

646 + *result = (int) (n * 604800);

647 + break;

648 + case 'd':

649 + *result = (int) (n * 86400);

650 + break;

651 + case 'h':

652 + *result = (int) (n * 3600);

653 + break;

654 + case 'm':

655 + *result = (int) (n * 60);

656 + break;

657 + case 's':

658 + default:

659 + *result = (int) (n);

660 + break;

661 + }

662 + }

663 + else

664 + *result = (int)(n);

665 +

666 + return 1;

667 +

668 + fallback:

669 + *result = 0;

670 + return 0;

671 +}

libnetdata/config/appconfig.h

@@ -119,6 +119,7 @@ extern long long appconfig_get_number(struct config *root, const char *section,

119 extern LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value);

120 extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value);

121 extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value);

122 +extern int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value);

123

124 extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value);

125 extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value);

@@ -133,4 +134,6 @@ extern void appconfig_generate(struct config *root, BUFFER *wb, int only_changed

134

135 extern int appconfig_section_compare(void *a, void *b);

136

137 +extern int config_parse_duration(const char* string, int* result);

138 +

139 #endif /* NETDATA_CONFIG_H */

web/api/netdata-swagger.json

@@ -1305,6 +1305,14 @@

1305 "crit_parsed": {

1306 "type": "string"

1307 },

1308 + "warn_repeat_every": {

1309 + "type": "integer",

1310 + "format": "int32"

1311 + },

1312 + "crit_repeat_every": {

1313 + "type": "integer",

1314 + "format": "int32"

1315 + },

1316 "green": {

1317 "type": "string",

1318 "format": "nullable"

web/api/netdata-swagger.yaml

@@ -892,6 +892,12 @@ definitions:

892 type: string

893 crit_parsed:

894 type: string

895 + warn_repeat_every:

896 + type: integer

897 + format: int32

898 + crit_repeat_every:

899 + type: integer

900 + format: int32

901 green:

902 type: string

903 format: nullable

web/gui/main.js

@@ -2100,6 +2100,14 @@ function alarmsUpdateModal() {

2100 + ((chart.red !== null) ? ('<tr><td width="10%" style="text-align:right">red threshold</td><td><code>' + chart.red + ' ' + units + '</code></td></tr>') : '');

2101 }

2102

2103 + if (alarm.warn_repeat_every > 0) {

2104 + html += '<tr><td width="10%" style="text-align:right">repeat warning</td><td>' + NETDATA.seconds4human(alarm.warn_repeat_every) + '</td></tr>';

2105 + }

2106 +

2107 + if (alarm.crit_repeat_every > 0) {

2108 + html += '<tr><td width="10%" style="text-align:right">repeat critical</td><td>' + NETDATA.seconds4human(alarm.crit_repeat_every) + '</td></tr>';

2109 + }

2110 +

2111 var delay = '';

2112 if ((alarm.delay_up_duration > 0 || alarm.delay_down_duration > 0) && alarm.delay_multiplier !== 0 && alarm.delay_max_duration > 0) {

2113 if (alarm.delay_up_duration === alarm.delay_down_duration) {