30% less memory for 1 hour of data; now it interpolates values at exactly the second bounday, thus eliminating the need for an addition time-difference value per row; added charts to check the precision of netdata and the time-difference of charts to the current time
Costa Tsaousis (ktsaou) committed
May 16, 2014 at 03:42 UTC
bb754ba2ee4599c1c89c082d9158d31f688e67ef
1 file changed
+309
-238
netdata.c
+309
-238
@@ -973,10 +973,10 @@ void *mymmap(const char *filename, unsigned long size)
973
974
#define RRD_STATS_NAME_MAX 1024
975
976
-//typedef long double calculated_number;
977
-//#define CALCULATED_NUMBER_FORMAT "%0.1Lf"
978
-typedef long long calculated_number;
979
-#define CALCULATED_NUMBER_FORMAT "%lld"
976
+typedef long double calculated_number;
977
+#define CALCULATED_NUMBER_FORMAT "%0.3Lf"
978
+//typedef long long calculated_number;
979
+//#define CALCULATED_NUMBER_FORMAT "%lld"
980
981
typedef long long collected_number;
982
#define COLLECTED_NUMBER_FORMAT "%lld"
@@ -988,7 +988,7 @@ typedef int32_t storage_number;
988
typedef uint32_t ustorage_number;
989
#define STORAGE_NUMBER_FORMAT "%d"
990
991
-#define RRD_STATS_MAGIC "NETDATA CACHE STATS FILE V005"
991
+#define RRD_STATS_MAGIC "NETDATA CACHE STATS FILE V006"
992
#define RRD_DIMENSION_MAGIC "NETDATA CACHE DIMENSION FILE V005"
993
994
struct rrd_dimension {
@@ -1011,7 +1011,7 @@ struct rrd_dimension {
1011
long multiplier;
1012
long divisor;
1013
1014
- struct timeval last_updated; // when was this dimension last updated
1014
+ struct timeval last_collected; // when was this dimension last updated
1015
1016
calculated_number calculated_value;
1017
calculated_number last_calculated_value;
@@ -1039,7 +1039,7 @@ struct rrd_stats {
1039
1040
pthread_rwlock_t rwlock;
1041
unsigned long counter; // the number of times we added values to this rrd
1042
- unsigned long counter_since_reload; // the number of times we added values to this rrd
1042
+ unsigned long counter_done; // the number of times we added values to this rrd
1043
1044
int mapped; // if set to 1, this is memory mapped
1045
unsigned long memsize; // how much mem we have allocated for this (without dimensions)
@@ -1057,7 +1057,7 @@ struct rrd_stats {
1057
int update_every; // every how many seconds is this updated?
1058
unsigned long long first_entry_t; // the timestamp (in microseconds) of the oldest entry in the db
1059
struct timeval last_updated; // when this data set was last updated
1060
- struct timeval next_update;
1060
+ struct timeval last_collected;
1061
unsigned long long usec_since_last_update;
1062
1063
total_number absolute_total;
@@ -1072,8 +1072,6 @@ struct rrd_stats {
1072
RRD_DIMENSION *dimensions; // the actual data for every dimension
1073
1074
struct rrd_stats *next; // linking of rrd stats
1075
-
1076
- uint32_t timediff[]; // the time in microseconds between each data entry - HAS TO BE LAST
1075
};
1076
typedef struct rrd_stats RRD_STATS;
1077
@@ -1151,7 +1149,7 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
1149
int enabled = config_get_boolean(fullid, "enabled", 1);
1150
if(!enabled) entries = 5;
1151
1154
- unsigned long size = sizeof(RRD_STATS) + (entries * sizeof(uint32_t));
1152
+ unsigned long size = sizeof(RRD_STATS);
1153
char *cache_dir = rrd_stats_cache_dir(fullid);
1154
1155
debug(D_RRD_STATS, "Creating RRD_STATS for '%s.%s'.", type, id);
@@ -1229,10 +1227,9 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name,
1227
st->enabled = enabled;
1228
1229
st->debug = 0;
1232
- st->counter_since_reload = 0;
1230
1234
- // initialize the next update
1235
- if(!st->next_update.tv_sec) gettimeofday(&st->next_update, NULL);
1231
+ st->last_collected.tv_sec = 0;
1232
+ st->last_collected.tv_usec = 0;
1233
1234
pthread_rwlock_init(&st->rwlock, NULL);
1235
pthread_rwlock_wrlock(&root_rwlock);
@@ -1462,8 +1459,10 @@ int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
1459
1460
void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
1461
{
1465
- rd->last_updated.tv_sec = st->last_updated.tv_sec;
1466
- rd->last_updated.tv_usec = st->last_updated.tv_usec;
1462
+ if(!st->last_collected.tv_sec) gettimeofday(&st->last_collected, NULL);
1463
+
1464
+ rd->last_collected.tv_sec = st->last_collected.tv_sec;
1465
+ rd->last_collected.tv_usec = st->last_collected.tv_usec;
1466
1467
rd->collected_value = value;
1468
}
@@ -1480,54 +1479,61 @@ int rrd_stats_dimension_set(RRD_STATS *st, char *id, collected_number value)
1479
return 0;
1480
}
1481
1483
-void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
1482
+void rrd_stats_next_internal(RRD_STATS *st)
1483
{
1484
// a read lock is OK here
1485
pthread_rwlock_rdlock(&st->rwlock);
1486
1487
RRD_DIMENSION *rd;
1488
for( rd = st->dimensions; rd ; rd = rd->next ) {
1490
- rd->last_calculated_value = rd->calculated_value;
1491
- rd->last_collected_value = rd->collected_value;
1492
-
1493
- rd->calculated_value = 0;
1489
+ rd->last_collected_value = rd->collected_value;
1490
rd->collected_value = 0;
1491
}
1492
1497
- st->last_absolute_total = st->absolute_total;
1498
- st->absolute_total = 0;
1499
- st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
1493
+ pthread_rwlock_unlock(&st->rwlock);
1494
+}
1495
1501
- st->usec_since_last_update = microseconds;
1502
- if(!st->last_updated.tv_sec) gettimeofday(&st->next_update, NULL);
1503
- else {
1504
- unsigned long long usec = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
1505
- usec += microseconds;
1496
+void rrd_stats_next_timeval(RRD_STATS *st, struct timeval *now)
1497
+{
1498
+ if(!st->last_collected.tv_sec) {
1499
+ gettimeofday(&st->last_collected, NULL);
1500
+ unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec - st->update_every * 1000000ULL;
1501
+ st->last_collected.tv_sec = ut / 1000000ULL;
1502
+ st->last_collected.tv_usec = ut % 1000000ULL;
1503
+ }
1504
1507
- st->next_update.tv_sec = usec / 1000000ULL;
1508
- st->next_update.tv_usec = usec % 1000000ULL;
1505
+ st->usec_since_last_update = usecdiff(now, &st->last_collected);
1506
+ st->last_collected.tv_sec = now->tv_sec;
1507
+ st->last_collected.tv_usec = now->tv_usec;
1508
1510
- }
1509
+ rrd_stats_next_internal(st);
1510
+}
1511
1512
- pthread_rwlock_unlock(&st->rwlock);
1512
+void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
1513
+{
1514
+ if(!st->last_collected.tv_sec) {
1515
+ gettimeofday(&st->last_collected, NULL);
1516
+ unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec;
1517
+ st->last_collected.tv_sec = ut / 1000000ULL;
1518
+ st->last_collected.tv_usec = ut % 1000000ULL;
1519
+ }
1520
+ else {
1521
+ unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec + microseconds;
1522
+ st->last_collected.tv_sec = ut / 1000000ULL;
1523
+ st->last_collected.tv_usec = ut % 1000000ULL;
1524
+ }
1525
+ st->usec_since_last_update = microseconds;
1526
1514
- return;
1527
+ rrd_stats_next_internal(st);
1528
}
1529
1530
void rrd_stats_next(RRD_STATS *st)
1531
{
1519
- if(st->last_updated.tv_sec) {
1532
+ if(st->last_collected.tv_sec) {
1533
struct timeval now;
1534
gettimeofday(&now, NULL);
1522
- unsigned long long microseconds = usecdiff(&now, &st->last_updated);
1523
-
1524
- unsigned long long min_microseconds = st->update_every * 1000000ULL / 2ULL;
1525
- if(microseconds < min_microseconds) {
1526
- debug(D_RRD_STATS, "Chart %s is being updated too early (after %llu, expected %llu microseconds). Assuming default.", st->name, microseconds, min_microseconds);
1527
- microseconds = st->update_every * 1000000ULL;
1528
- }
1529
-
1530
- rrd_stats_next_usec(st, microseconds);
1535
+
1536
+ rrd_stats_next_timeval(st, &now);
1537
}
1538
else
1539
rrd_stats_next_usec(st, st->update_every * 1000000ULL);
@@ -1545,209 +1551,230 @@ unsigned long long rrd_stats_done(RRD_STATS *st)
1551
// a read lock is OK here
1552
pthread_rwlock_rdlock(&st->rwlock);
1553
1554
+ if(!st->last_updated.tv_sec) {
1555
+ unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec - st->usec_since_last_update;
1556
+ st->last_updated.tv_sec = ut / 1000000ULL;
1557
+ st->last_updated.tv_usec = ut % 1000000ULL;
1558
+ }
1559
+
1560
+ unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
1561
+ unsigned long long now_ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec + st->usec_since_last_update;
1562
+ unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
1563
+
1564
+ st->counter_done++;
1565
+ if(st->counter_done == 1 || now_ut < next_ut) {
1566
+ if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
1567
+ // we don't have any usable data yet
1568
+ pthread_rwlock_unlock(&st->rwlock);
1569
+ return(st->usec_since_last_update);
1570
+ }
1571
+
1572
+ if(st->debug) debug(D_RRD_STATS, "microseconds since last update: %llu", st->usec_since_last_update);
1573
+
1574
// calculate totals and count the dimensions
1549
- st->absolute_total = 0;
1575
int dimensions;
1576
+ st->last_absolute_total = st->absolute_total;
1577
+ st->absolute_total = 0;
1578
for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
1579
st->absolute_total += rd->collected_value;
1580
1554
- // if this is the second+ value we collect
1555
- if(st->counter) {
1556
- if(st->debug) debug(D_RRD_STATS, "microseconds since last update: %llu", st->usec_since_last_update);
1581
+ // process all dimensions to calculate its values
1582
+ for( rd = st->dimensions ; rd ; rd = rd->next ) {
1583
+ switch(rd->algorithm) {
1584
+ case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
1585
+ // the percentage of the current increment
1586
+ // over the increment of all dimensions together
1587
+ if(st->absolute_total == st->last_absolute_total) rd->calculated_value = 0;
1588
+ else rd->calculated_value =
1589
+ (calculated_number)100
1590
+ * (calculated_number)(rd->collected_value - rd->last_collected_value)
1591
+ / (calculated_number)(st->absolute_total - st->last_absolute_total);
1592
+
1593
+ if(st->debug)
1594
+ debug(D_RRD_STATS, "%s/%s: CALC "
1595
+ CALCULATED_NUMBER_FORMAT " = 100"
1596
+ " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1597
+ " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1598
+ , st->id, rd->name
1599
+ , rd->calculated_value
1600
+ , rd->collected_value, rd->last_collected_value
1601
+ , st->absolute_total, st->last_absolute_total
1602
+ );
1603
+ break;
1604
1558
- // x 10
1559
- // in all calculations we multiply with 10 to allow
1560
- // for all 1 decimal point at the chart
1605
+ case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
1606
+ if(!st->absolute_total) rd->calculated_value = 0;
1607
+ else
1608
+ // the percentage of the current value
1609
+ // over the total of all dimensions
1610
+ rd->calculated_value =
1611
+ (calculated_number)100
1612
+ * (calculated_number)rd->collected_value
1613
+ / (calculated_number)st->absolute_total;
1614
+
1615
+ if(st->debug)
1616
+ debug(D_RRD_STATS, "%s/%s: CALC "
1617
+ CALCULATED_NUMBER_FORMAT " = 100"
1618
+ " * " COLLECTED_NUMBER_FORMAT
1619
+ " / " COLLECTED_NUMBER_FORMAT
1620
+ , st->id, rd->name
1621
+ , rd->calculated_value
1622
+ , rd->collected_value
1623
+ , st->absolute_total
1624
+ );
1625
+ break;
1626
1562
- // process all dimensions to calculate its values
1563
- for( rd = st->dimensions ; rd ; rd = rd->next ) {
1564
- switch(rd->algorithm) {
1565
- case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
1566
- // the percentage of the current increment
1567
- // over the increment of all dimensions together
1568
- if(st->absolute_total == st->last_absolute_total) rd->calculated_value = 0;
1569
- else rd->calculated_value =
1570
- (calculated_number)10
1571
- * (calculated_number)100
1572
- * (calculated_number)(rd->collected_value - rd->last_collected_value)
1573
- / (calculated_number)(st->absolute_total - st->last_absolute_total);
1574
-
1575
- if(st->debug)
1576
- debug(D_RRD_STATS, "%s/%s: CALC "
1577
- CALCULATED_NUMBER_FORMAT " = 10 * 100"
1578
- " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1579
- " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1580
- , st->id, rd->name
1581
- , rd->calculated_value
1582
- , rd->collected_value, rd->last_collected_value
1583
- , st->absolute_total, st->last_absolute_total
1584
- );
1585
- break;
1627
+ case RRD_DIMENSION_INCREMENTAL:
1628
+ // we need the incremental calculation to produce per second results
1629
+ // so, we multiply with 1.000.000 and divide by the microseconds passed since
1630
+ // the last entry
1631
+
1632
+ // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
1633
+ // to reset the calculation (it will give zero as the calculation for this second)
1634
+ if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
1635
+
1636
+ rd->calculated_value =
1637
+ (calculated_number)1000000
1638
+ * (calculated_number)(rd->collected_value - rd->last_collected_value)
1639
+ / (calculated_number)st->usec_since_last_update;
1640
+
1641
+ if(st->debug)
1642
+ debug(D_RRD_STATS, "%s/%s: CALC "
1643
+ CALCULATED_NUMBER_FORMAT " = 1000000"
1644
+ " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1645
+ " / %llu"
1646
+ , st->id, rd->name
1647
+ , rd->calculated_value
1648
+ , rd->collected_value, rd->last_collected_value
1649
+ , st->usec_since_last_update
1650
+ );
1651
+ break;
1652
1587
- case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
1588
- if(!st->absolute_total) rd->calculated_value = 0;
1589
- else
1590
- // the percentage of the current value
1591
- // over the total of all dimensions
1592
- rd->calculated_value =
1593
- (calculated_number)10
1594
- * (calculated_number)100
1595
- * (calculated_number)rd->collected_value
1596
- / (calculated_number)st->absolute_total;
1597
-
1598
- if(st->debug)
1599
- debug(D_RRD_STATS, "%s/%s: CALC "
1600
- CALCULATED_NUMBER_FORMAT " = 10 * 100"
1601
- " * " COLLECTED_NUMBER_FORMAT
1602
- " / " COLLECTED_NUMBER_FORMAT
1603
- , st->id, rd->name
1604
- , rd->calculated_value
1605
- , rd->collected_value
1606
- , st->absolute_total
1607
- );
1608
- break;
1653
+ case RRD_DIMENSION_ABSOLUTE:
1654
+ rd->calculated_value = (calculated_number)rd->collected_value;
1655
1610
- case RRD_DIMENSION_INCREMENTAL:
1611
- // we need the incremental calculation to produce per second results
1612
- // so, we multiply with 1.000.000 and divide by the microseconds passed since
1613
- // the last entry
1614
-
1615
- // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
1616
- // to reset the calculation (it will give zero as the calculation for this second)
1617
- if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
1618
-
1619
- rd->calculated_value =
1620
- (calculated_number)10
1621
- * (calculated_number)1000000
1622
- * (calculated_number)(rd->collected_value - rd->last_collected_value)
1623
- / (calculated_number)st->usec_since_last_update;
1624
-
1625
- if(st->debug)
1626
- debug(D_RRD_STATS, "%s/%s: CALC "
1627
- CALCULATED_NUMBER_FORMAT " = 10 * 1000000"
1628
- " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1629
- " / %llu"
1630
- , st->id, rd->name
1631
- , rd->calculated_value
1632
- , rd->collected_value, rd->last_collected_value
1633
- , st->usec_since_last_update
1634
- );
1635
- break;
1656
+ if(st->debug)
1657
+ debug(D_RRD_STATS, "%s/%s: CALC "
1658
+ CALCULATED_NUMBER_FORMAT " = "
1659
+ COLLECTED_NUMBER_FORMAT
1660
+ , st->id, rd->name
1661
+ , rd->calculated_value
1662
+ , rd->collected_value
1663
+ );
1664
+ break;
1665
1637
- case RRD_DIMENSION_ABSOLUTE:
1638
- rd->calculated_value =
1639
- (calculated_number)10
1640
- * (calculated_number)rd->collected_value;
1641
-
1642
- if(st->debug)
1643
- debug(D_RRD_STATS, "%s/%s: CALC "
1644
- CALCULATED_NUMBER_FORMAT " = 10"
1645
- " * " COLLECTED_NUMBER_FORMAT
1646
- , st->id, rd->name
1647
- , rd->calculated_value
1648
- , rd->collected_value
1649
- );
1650
- break;
1666
+ default:
1667
+ // make the default zero, to make sure
1668
+ // it gets noticed when we add new types
1669
+ rd->calculated_value = 0;
1670
+
1671
+ if(st->debug)
1672
+ debug(D_RRD_STATS, "%s/%s: CALC "
1673
+ CALCULATED_NUMBER_FORMAT " = 0"
1674
+ , st->id, rd->name
1675
+ , rd->calculated_value
1676
+ );
1677
+ break;
1678
+ }
1679
+ }
1680
1652
- default:
1653
- // make the default zero, to make sure
1654
- // it gets noticed when we add new types
1655
- rd->calculated_value = 0;
1656
-
1657
- if(st->debug)
1658
- debug(D_RRD_STATS, "%s/%s: CALC "
1659
- CALCULATED_NUMBER_FORMAT " = 0"
1660
- , st->id, rd->name
1661
- , rd->calculated_value
1662
- );
1663
- break;
1664
- }
1681
+ for( ; next_ut < now_ut ; next_ut += st->update_every * 1000000ULL ) {
1682
+ unsigned long long np = next_ut - last_ut;
1683
+
1684
+ st->last_updated.tv_sec = next_ut / 1000000ULL;
1685
+ st->last_updated.tv_usec = 0;
1686
1666
- if(!st->counter_since_reload) rd->calculated_value = 0;
1687
+ for( rd = st->dimensions ; rd ; rd = rd->next ) {
1688
+ rd->calculated_value = (calculated_number)
1689
+ ( ( (rd->calculated_value - rd->last_calculated_value)
1690
+ * (calculated_number)np
1691
+ / (calculated_number)(now_ut - last_ut)
1692
+ )
1693
+ + rd->last_calculated_value
1694
+ );
1695
1668
- // store the calculated value
1696
rd->values[st->current_entry] = (storage_number)
1697
( rd->calculated_value
1698
+ * (calculated_number)10
1699
* (calculated_number)rd->multiplier
1700
/ (calculated_number)rd->divisor
1701
);
1674
-
1702
+
1703
if(st->debug)
1676
- debug(D_RRD_STATS, "%s/%s STORE "
1677
- STORAGE_NUMBER_FORMAT " = "
1678
- CALCULATED_NUMBER_FORMAT
1704
+ debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
1705
+ STORAGE_NUMBER_FORMAT " = ((("
1706
+ "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
1707
+ " * %llu"
1708
+ " / %llu) + " CALCULATED_NUMBER_FORMAT
1709
+ " * 10 "
1710
" * %ld"
1711
" / %ld"
1712
, st->id, rd->name
1713
+ , st->current_entry
1714
, rd->values[st->current_entry]
1683
- , rd->calculated_value
1715
+ , rd->calculated_value, rd->last_calculated_value
1716
+ , np
1717
+ , (now_ut - last_ut), rd->last_calculated_value
1718
, rd->multiplier
1719
, rd->divisor
1720
);
1687
- }
1721
1689
- if(!st->first_entry_t) {
1690
- // this is the first entry in the database
1691
- st->first_entry_t = st->next_update.tv_sec * 1000000ULL + st->next_update.tv_usec;
1722
+ rd->last_calculated_value = rd->calculated_value;
1723
}
1693
- else {
1694
- if(st->counter > (unsigned long long)st->entries) {
1695
- // the db is overwriting values
1696
- // add the value we will overwrite
1697
- st->first_entry_t += st->timediff[st->current_entry];
1698
- }
1724
+
1725
+ if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
1726
+ // the db is overwriting values
1727
+ // add the value we will overwrite
1728
+ st->first_entry_t += st->update_every * 1000000ULL;
1729
}
1700
- // store the time difference to the last entry
1701
- st->timediff[st->current_entry] = st->usec_since_last_update;
1730
+
1731
+ st->counter++;
1732
+ st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
1733
+ if(!st->first_entry_t) st->first_entry_t = next_ut;
1734
+ last_ut = next_ut;
1735
+ }
1736
1737
+ // ALL DONE ABOUT THE DATA UPDATE
1738
+ // --------------------------------------------------------------------
1739
1704
- // ALL DONE ABOUT THE DATA UPDATE
1705
- // --------------------------------------------------------------------
1740
1741
+ // find if there are any obsolete dimensions (not updated recently)
1742
+ for( rd = st->dimensions; rd ; rd = rd->next )
1743
+ if((rd->last_collected.tv_sec + (10 * st->update_every)) < st->last_collected.tv_sec)
1744
+ break;
1745
1708
- // find if there are any obsolete dimensions (not updated recently)
1709
- for( rd = st->dimensions; rd ; rd = rd->next )
1710
- if((rd->last_updated.tv_sec + (10 * st->update_every)) < st->last_updated.tv_sec)
1711
- break;
1746
+ if(rd) {
1747
+ // there is dimension to free
1748
+ // upgrade our read lock to a write lock
1749
+ pthread_rwlock_unlock(&st->rwlock);
1750
+ pthread_rwlock_wrlock(&st->rwlock);
1751
1713
- if(rd) {
1714
- // there is dimension to free
1715
- // upgrade our read lock to a write lock
1716
- pthread_rwlock_unlock(&st->rwlock);
1717
- pthread_rwlock_wrlock(&st->rwlock);
1718
-
1719
- for( rd = st->dimensions, last = NULL ; rd ; ) {
1720
- if((rd->last_updated.tv_sec + (10 * st->update_every)) < st->last_updated.tv_sec) { // remove it only it is not updated in 10 seconds
1721
- debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
1722
-
1723
- if(!last) {
1724
- st->dimensions = rd->next;
1725
- rd->next = NULL;
1726
- rrd_stats_dimension_free(rd);
1727
- rd = st->dimensions;
1728
- continue;
1729
- }
1730
- else {
1731
- last->next = rd->next;
1732
- rd->next = NULL;
1733
- rrd_stats_dimension_free(rd);
1734
- rd = last->next;
1735
- continue;
1736
- }
1737
- }
1752
+ for( rd = st->dimensions, last = NULL ; rd ; ) {
1753
+ if((rd->last_collected.tv_sec + (10 * st->update_every)) < st->last_collected.tv_sec) { // remove it only it is not updated in 10 seconds
1754
+ debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
1755
1739
- last = rd;
1740
- rd = rd->next;
1756
+ if(!last) {
1757
+ st->dimensions = rd->next;
1758
+ rd->next = NULL;
1759
+ rrd_stats_dimension_free(rd);
1760
+ rd = st->dimensions;
1761
+ continue;
1762
+ }
1763
+ else {
1764
+ last->next = rd->next;
1765
+ rd->next = NULL;
1766
+ rrd_stats_dimension_free(rd);
1767
+ rd = last->next;
1768
+ continue;
1769
+ }
1770
}
1771
1743
- if(!st->dimensions) st->enabled = 0;
1772
+ last = rd;
1773
+ rd = rd->next;
1774
}
1745
- }
1775
1747
- st->last_updated.tv_sec = st->next_update.tv_sec;
1748
- st->last_updated.tv_usec = st->next_update.tv_usec;
1749
- st->counter++;
1750
- st->counter_since_reload++;
1776
+ if(!st->dimensions) st->enabled = 0;
1777
+ }
1778
1779
pthread_rwlock_unlock(&st->rwlock);
1780
return(st->usec_since_last_update);
@@ -1820,7 +1847,7 @@ void web_buffer_rrd_value(struct web_buffer *wb, storage_number value)
1847
}
1848
1849
// generate a javascript date, the fastest possible way...
1823
-void web_buffer_jsdate(struct web_buffer *wb, int year, int month, int day, int hours, int minutes, int seconds, int milliseconds)
1850
+void web_buffer_jsdate(struct web_buffer *wb, int year, int month, int day, int hours, int minutes, int seconds)
1851
{
1852
// 10 20 30 = 35
1853
// 01234567890123456789012345678901234
@@ -1859,15 +1886,10 @@ void web_buffer_jsdate(struct web_buffer *wb, int year, int month, int day, int
1886
b[26]=' ';
1887
b[27]= 48 + seconds / 10;
1888
b[28]= 48 + seconds % 10;
1862
- b[29]=',';
1863
- b[30]=' ';
1864
- b[31]= 48 + milliseconds / 100; milliseconds -= (milliseconds / 100) * 100;
1865
- b[32]= 48 + milliseconds / 10;
1866
- b[33]= 48 + milliseconds % 10;
1867
- b[34]=')';
1868
- b[35]='\0';
1869
-
1870
- wb->bytes += 35;
1889
+ b[29]=')';
1890
+ b[30]='\0';
1891
+
1892
+ wb->bytes += 30;
1893
}
1894
1895
struct web_buffer *web_buffer_create(long size)
@@ -2126,7 +2148,7 @@ unsigned long rrd_stats_one_json(RRD_STATS *st, char *options, struct web_buffer
2148
, algorithm_name(rd->algorithm)
2149
, rd->multiplier
2150
, rd->divisor
2129
- , rd->last_updated.tv_sec
2151
+ , rd->last_collected.tv_sec
2152
, rd->collected_value
2153
, rd->calculated_value
2154
, rd->last_collected_value
@@ -2223,17 +2245,17 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
2245
if(group < 1) group = 1;
2246
2247
// make sure current_entry is within limits
2226
- long current_entry = st->current_entry;
2248
+ long current_entry = (long)st->current_entry - (long)1;
2249
if(current_entry < 0) current_entry = 0;
2250
else if(current_entry >= st->entries) current_entry = st->entries - 1;
2251
2252
// find the oldest entry of the round-robin
2231
- long max_entries_init = (st->counter <= (unsigned long)st->entries) ? st->counter - 1 : (unsigned long)st->entries;
2253
+ long max_entries_init = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
2254
2255
if(before == 0) before = st->last_updated.tv_sec;
2256
if(after == 0) after = rrd_stats_first_entry_t(st);
2257
2236
- unsigned long long time_usec_init = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
2258
+ time_t time_init = st->last_updated.tv_sec;
2259
2260
// ---
2261
@@ -2345,21 +2367,17 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
2367
// the minimum line length we expect
2368
int line_size = 4096 + (dimensions * 200);
2369
2348
- unsigned long long time_usec = time_usec_init;
2370
+ time_t now = time_init;
2371
long max_entries = max_entries_init;
2372
2373
long t;
2374
2375
long count = 0, printed = 0, group_count = 0;
2376
last_timestamp = 0;
2355
- for(t = current_entry; max_entries ; time_usec -= st->timediff[t], t--, max_entries--) {
2377
+ for(t = current_entry; max_entries ; now--, t--, max_entries--) {
2378
if(t < 0) t = st->entries - 1;
2379
2358
- if(st->debug)
2359
- if(st->timediff[t] == 0) debug(D_RRD_STATS, "WARNING: %s time diff is zero at position %ld", st->id, t);
2360
-
2380
int print_this = 0;
2362
- time_t now = time_usec / 1000000ULL;
2381
2382
if(st->debug) {
2383
debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
@@ -2405,7 +2423,7 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
2423
2424
if(printed) web_buffer_strcpy(wb, "]},\n");
2425
web_buffer_strcpy(wb, pre_date);
2408
- web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec, (int)((time_usec % 1000000ULL) / 1000));
2426
+ web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
2427
web_buffer_strcpy(wb, post_date);
2428
2429
print_this = 1;
@@ -2463,6 +2481,8 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int
2481
if(only_non_zero && max_loop > 1) {
2482
int changed = 0;
2483
for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2484
+ group_values[c] = 0;
2485
+
2486
if(!print_hidden[c] && !found_non_zero[c]) {
2487
changed = 1;
2488
print_hidden[c] = 1;
@@ -6051,7 +6071,28 @@ void *cpuidlejitter_main(void *ptr)
6071
sleep_ms = CPU_IDLEJITTER_SLEEP_TIME_MS;
6072
}
6073
6074
+ unsigned long long total_susec = 0, loop_usec = 0;
6075
struct timeval before, after;
6076
+ struct timeval loop1, loop2;
6077
+
6078
+ gettimeofday(&loop1, NULL);
6079
+
6080
+ RRD_STATS *jitter, *check1, *check2, *check3;
6081
+
6082
+ jitter = rrd_stats_create("system", "idlejitter", NULL, "cpu", "CPU Idle Jitter", "microseconds lost/s", 9999, update_every, CHART_TYPE_LINE);
6083
+ rrd_stats_dimension_add(jitter, "jitter", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6084
+
6085
+ check1 = rrd_stats_create("netdata", "check1", NULL, "netdata", "Caller gives microseconds", "a million !", 99999, update_every, CHART_TYPE_LINE);
6086
+ rrd_stats_dimension_add(check1, "absolute", NULL, -1, 1, RRD_DIMENSION_ABSOLUTE);
6087
+ rrd_stats_dimension_add(check1, "incremental", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
6088
+
6089
+ check2 = rrd_stats_create("netdata", "check2", NULL, "netdata", "Netdata calcs microseconds", "a million !", 99999, update_every, CHART_TYPE_LINE);
6090
+ rrd_stats_dimension_add(check2, "absolute", NULL, -1, 1, RRD_DIMENSION_ABSOLUTE);
6091
+ rrd_stats_dimension_add(check2, "incremental", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
6092
+
6093
+ check3 = rrd_stats_create("netdata", "checkdt", NULL, "netdata", "Clock difference", "microseconds diff", 99999, update_every, CHART_TYPE_LINE);
6094
+ rrd_stats_dimension_add(check3, "caller", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6095
+ rrd_stats_dimension_add(check3, "netdata", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6096
6097
while(1) {
6098
unsigned long long usec = 0, susec = 0;
@@ -6068,16 +6109,46 @@ void *cpuidlejitter_main(void *ptr)
6109
}
6110
usec -= (sleep_ms * 1000);
6111
6071
- RRD_STATS *st = rrd_stats_find("system.idlejitter");
6072
- if(!st) {
6073
- st = rrd_stats_create("system", "idlejitter", NULL, "cpu", "CPU Idle Jitter", "microseconds lost/s", 9999, update_every, CHART_TYPE_LINE);
6112
+ // --------------------------------------------------------------------
6113
+ // Calculate loop time
6114
6075
- rrd_stats_dimension_add(st, "jitter", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6076
- }
6077
- else rrd_stats_next_usec(st, susec);
6115
+ gettimeofday(&loop2, NULL);
6116
+ loop_usec = usecdiff(&loop2, &loop1);
6117
+ loop1.tv_sec = loop2.tv_sec;
6118
+ loop1.tv_usec = loop2.tv_usec;
6119
+ total_susec += loop_usec;
6120
6079
- rrd_stats_dimension_set(st, "jitter", usec);
6080
- rrd_stats_done(st);
6121
+ // --------------------------------------------------------------------
6122
+ // idle jitter
6123
+
6124
+ if(jitter->counter_done) rrd_stats_next_usec(jitter, susec);
6125
+ rrd_stats_dimension_set(jitter, "jitter", loop_usec);
6126
+ rrd_stats_done(jitter);
6127
+
6128
+ // --------------------------------------------------------------------
6129
+ // check chart 1
6130
+
6131
+ if(check1->counter_done) rrd_stats_next_usec(check1, loop_usec);
6132
+ rrd_stats_dimension_set(check1, "absolute", 1000000);
6133
+ rrd_stats_dimension_set(check1, "incremental", total_susec);
6134
+ rrd_stats_done(check1);
6135
+
6136
+ // --------------------------------------------------------------------
6137
+ // check chart 2
6138
+
6139
+ if(check2->counter_done) rrd_stats_next(check2);
6140
+ rrd_stats_dimension_set(check2, "absolute", 1000000);
6141
+ rrd_stats_dimension_set(check2, "incremental", total_susec);
6142
+ rrd_stats_done(check2);
6143
+
6144
+ // --------------------------------------------------------------------
6145
+ // check chart 3
6146
+
6147
+ if(check3->counter_done) rrd_stats_next_usec(check3, susec);
6148
+ gettimeofday(&loop2, NULL);
6149
+ rrd_stats_dimension_set(check3, "caller", (long long)usecdiff(&loop2, &check1->last_collected));
6150
+ rrd_stats_dimension_set(check3, "netdata", (long long)usecdiff(&loop2, &check2->last_collected));
6151
+ rrd_stats_done(check3);
6152
}
6153
6154
return NULL;
@@ -6220,7 +6291,7 @@ void *pluginsd_worker_thread(void *arg)
6291
break;
6292
}
6293
6223
- if(st->counter) {
6294
+ if(st->counter_done) {
6295
unsigned long long microseconds = 0;
6296
if(microseconds_txt && *microseconds_txt) microseconds = strtoull(microseconds_txt, NULL, 10);
6297
if(microseconds) rrd_stats_next_usec(st, microseconds);