detect too big updates correctly
Costa Tsaousis (ktsaou) committed
Mar 19, 2015 at 11:51 UTC
b41f0b385b1b614e63f57e22bb47f304ec5662b8
2 files changed
+34
-21
src/log.h
+1
@@ -21,6 +21,7 @@
21
#define D_CHECKS 0x00004000
22
#define D_NFACCT_LOOP 0x00008000
23
#define D_PROCFILE 0x00010000
24
+#define D_RRD_CALLS 0x00020000
25
26
#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)
27
//#define DEBUG 0xffffffff
src/rrd.c
+33
-21
@@ -210,7 +210,7 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
210
unsigned long size = sizeof(RRD_STATS);
211
char *cache_dir = rrd_stats_cache_dir(fullid);
212
213
- debug(D_RRD_STATS, "Creating RRD_STATS for '%s.%s'.", type, id);
213
+ debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
214
215
snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
216
if(memory_mode != NETDATA_MEMORY_MODE_RAM) st = (RRD_STATS *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
@@ -236,6 +236,11 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
236
error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
237
bzero(st, size);
238
}
239
+ else if((time(NULL) - st->last_updated.tv_sec) > update_every * entries) {
240
+ errno = 0;
241
+ error("File %s is too old. Clearing it.", fullfilename);
242
+ bzero(st, size);
243
+ }
244
}
245
246
if(st) {
@@ -312,7 +317,7 @@ RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char
317
RRD_DIMENSION *rd = NULL;
318
unsigned long size = sizeof(RRD_DIMENSION) + (st->entries * sizeof(storage_number));
319
315
- debug(D_RRD_STATS, "Adding dimension '%s/%s'.", st->id, id);
320
+ debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
321
322
rrd_stats_strncpy_name(filename, id, FILENAME_MAX);
323
snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
@@ -429,18 +434,18 @@ void rrd_stats_dimension_free(RRD_DIMENSION *rd)
434
if(rd->next) rrd_stats_dimension_free(rd->next);
435
// free(rd->annotations);
436
if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
432
- debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
437
+ debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
438
savememory(rd->cache_file, rd, rd->memsize);
439
435
- debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
440
+ debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
441
munmap(rd, rd->memsize);
442
}
443
else if(rd->mapped == NETDATA_MEMORY_MODE_MAP) {
439
- debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
444
+ debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
445
munmap(rd, rd->memsize);
446
}
447
else {
443
- debug(D_RRD_STATS, "Removing dimension '%s'.", rd->name);
448
+ debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
449
free(rd);
450
}
451
}
@@ -457,14 +462,14 @@ void rrd_stats_free_all(void)
462
st->dimensions = NULL;
463
464
if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
460
- debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
465
+ debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
466
savememory(st->cache_file, st, st->memsize);
467
463
- debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
468
+ debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
469
munmap(st, st->memsize);
470
}
471
else if(st->mapped == NETDATA_MEMORY_MODE_MAP) {
467
- debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
472
+ debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
473
munmap(st, st->memsize);
474
}
475
else
@@ -487,13 +492,13 @@ void rrd_stats_save_all(void)
492
pthread_rwlock_wrlock(&st->rwlock);
493
494
if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
490
- debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
495
+ debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
496
savememory(st->cache_file, st, st->memsize);
497
}
498
499
for(rd = st->dimensions; rd ; rd = rd->next) {
500
if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
496
- debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
501
+ debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
502
savememory(rd->cache_file, rd, rd->memsize);
503
}
504
}
@@ -506,7 +511,7 @@ void rrd_stats_save_all(void)
511
512
RRD_STATS *rrd_stats_find(const char *id)
513
{
509
- debug(D_RRD_STATS, "rrd_stats_find() for chart %s", id);
514
+ debug(D_RRD_CALLS, "rrd_stats_find() for chart %s", id);
515
516
unsigned long hash = simple_hash(id);
517
@@ -523,7 +528,7 @@ RRD_STATS *rrd_stats_find(const char *id)
528
529
RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
530
{
526
- debug(D_RRD_STATS, "rrd_stats_find_bytype() for chart %s.%s", type, id);
531
+ debug(D_RRD_CALLS, "rrd_stats_find_bytype() for chart %s.%s", type, id);
532
533
char buf[RRD_STATS_NAME_MAX + 1];
534
@@ -539,7 +544,7 @@ RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
544
545
RRD_STATS *rrd_stats_find_byname(const char *name)
546
{
542
- debug(D_RRD_STATS, "rrd_stats_find_byname() for chart %s", name);
547
+ debug(D_RRD_CALLS, "rrd_stats_find_byname() for chart %s", name);
548
549
char b[CONFIG_MAX_VALUE + 1];
550
@@ -558,7 +563,7 @@ RRD_STATS *rrd_stats_find_byname(const char *name)
563
564
RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
565
{
561
- debug(D_RRD_STATS, "rrd_stats_dimension_find() for chart %s, dimension %s", st->name, id);
566
+ debug(D_RRD_CALLS, "rrd_stats_dimension_find() for chart %s, dimension %s", st->name, id);
567
568
unsigned long hash = simple_hash(id);
569
@@ -574,7 +579,7 @@ RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
579
580
int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
581
{
577
- debug(D_RRD_STATS, "rrd_stats_dimension_hide() for chart %s, dimension %s", st->name, id);
582
+ debug(D_RRD_CALLS, "rrd_stats_dimension_hide() for chart %s, dimension %s", st->name, id);
583
584
RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
585
if(!rd) {
@@ -588,7 +593,7 @@ int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
593
594
void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
595
{
591
- debug(D_RRD_STATS, "rrd_stats_dimension_set() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
596
+ debug(D_RRD_CALLS, "rrd_stats_dimension_set() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
597
598
gettimeofday(&rd->last_collected_time, NULL);
599
rd->collected_value = value;
@@ -608,7 +613,7 @@ int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number valu
613
614
void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
615
{
611
- debug(D_RRD_STATS, "rrd_stats_next() for chart %s with microseconds %llu", st->name, microseconds);
616
+ debug(D_RRD_CALLS, "rrd_stats_next() for chart %s with microseconds %llu", st->name, microseconds);
617
618
if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
619
st->usec_since_last_update = microseconds;
@@ -634,7 +639,7 @@ void rrd_stats_next_plugins(RRD_STATS *st)
639
640
unsigned long long rrd_stats_done(RRD_STATS *st)
641
{
637
- debug(D_RRD_STATS, "rrd_stats_done() for chart %s", st->name);
642
+ debug(D_RRD_CALLS, "rrd_stats_done() for chart %s", st->name);
643
644
RRD_DIMENSION *rd, *last;
645
int oldstate;
@@ -645,8 +650,8 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
650
// a read lock is OK here
651
pthread_rwlock_rdlock(&st->rwlock);
652
648
- if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL || usecdiff(&st->last_collected_time, &st->last_updated) > st->entries * st->update_every * 1000000ULL) {
649
- info("History of chart %s too old. Reseting chart.", st->name);
653
+ if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL) {
654
+ info("Chart chart %s took too long to be updated (%0.3Lf secs). Reseting chart history.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
655
rrd_stats_reset(st);
656
st->usec_since_last_update = st->update_every * 1000000ULL;
657
}
@@ -668,6 +673,13 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
673
if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf)", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
674
}
675
676
+ if(usecdiff(&st->last_collected_time, &st->last_updated) > st->entries * st->update_every * 1000000ULL) {
677
+ info("History of chart %s too old (last updated at %u, last collected at %u). Reseting chart.", st->name, st->last_updated.tv_sec, st->last_collected_time.tv_sec);
678
+ rrd_stats_reset(st);
679
+ st->usec_since_last_update = st->update_every * 1000000ULL;
680
+ return(st->usec_since_last_update);
681
+ }
682
+
683
unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
684
unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
685
unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;