added API parameter gpoints=N to allow emulating different data collection periods
Costa Tsaousis (ktsaou) committed
Jan 27, 2018 at 02:16 UTC
4ac431fedbbdbf0ed05d20182a84aa58c33c63ca
6 files changed
+90
-43
src/health.c
+1
@@ -428,6 +428,7 @@ void *health_main(void *ptr) {
428
, rc->after
429
, rc->before
430
, rc->group
431
+ , 0
432
, rc->options
433
, &rc->db_after
434
, &rc->db_before
src/rrd2json.c
+63
-38
@@ -1413,7 +1413,7 @@ static RRDR *rrdr_create(RRDSET *st, long n)
1413
return r;
1414
}
1415
1416
-RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, int aligned)
1416
+RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, long group_points, int aligned)
1417
{
1418
int debug = rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?1:0;
1419
int absolute_period_requested = -1;
@@ -1476,20 +1476,30 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
1476
if(duration <= 0 || available_points <= 0)
1477
return rrdr_create(st, 1);
1478
1479
- // check the wanted points
1480
- if(points < 0) points = -points;
1481
- if(points > available_points) points = available_points;
1482
- if(points == 0) points = available_points;
1479
+ // check the number of wanted points in the result
1480
+ if(unlikely(points < 0)) points = -points;
1481
+ if(unlikely(points > available_points)) points = available_points;
1482
+ if(unlikely(points == 0)) points = available_points;
1483
1484
- // calculate proper grouping of source data
1484
+ // calculate the desired grouping of source data points
1485
long group = available_points / points;
1486
- if(group <= 0) group = 1;
1486
+ if(unlikely(group <= 0)) group = 1;
1487
+ if(unlikely(available_points % points > points / 2)) group++; // rounding to the closest integer
1488
1488
- // round group to the closest integer
1489
- if(available_points % points > points / 2) group++;
1489
+ // group_points enforces a certain grouping multiple
1490
+ calculated_number group_sum_divisor = 1.0;
1491
+ if(likely(group_points < 1)) group_points = 1;
1492
+ else {
1493
+ if (unlikely(group_points > available_points)) group_points = available_points; // group_points is above all the available points
1494
+ if (unlikely(group < group_points)) group = group_points; // do not allow grouping below the desired one
1495
+ if (unlikely(group > group_points && (group % group_points) > 0)) // make sure group is multiple of group_points
1496
+ group += group_points - (group % group_points);
1497
+
1498
+ group_sum_divisor = group / group_points;
1499
+ }
1500
1491
- time_t after_new = (aligned) ? (after - (after % (group * st->update_every))) : after;
1492
- time_t before_new = (aligned) ? (before - (before % (group * st->update_every))) : before;
1501
+ time_t after_new = after - (after % ( ((aligned)?group:group_points) * st->update_every ));
1502
+ time_t before_new = before - (before % ( ((aligned)?group:group_points) * st->update_every ));
1503
long points_new = (before_new - after_new) / st->update_every / group;
1504
1505
// find the starting and ending slots in our round robin db
@@ -1497,27 +1507,35 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
1507
stop_at_slot = rrdset_time2slot(st, after_new);
1508
1509
#ifdef NETDATA_INTERNAL_CHECKS
1500
- if(after_new < first_entry_t) {
1501
- error("after_new %u is too small, minimum %u", (uint32_t)after_new, (uint32_t)first_entry_t);
1502
- }
1503
- if(after_new > last_entry_t) {
1504
- error("after_new %u is too big, maximum %u", (uint32_t)after_new, (uint32_t)last_entry_t);
1505
- }
1506
- if(before_new < first_entry_t) {
1507
- error("before_new %u is too small, minimum %u", (uint32_t)before_new, (uint32_t)first_entry_t);
1508
- }
1509
- if(before_new > last_entry_t) {
1510
- error("before_new %u is too big, maximum %u", (uint32_t)before_new, (uint32_t)last_entry_t);
1511
- }
1512
- if(start_at_slot < 0 || start_at_slot >= st->entries) {
1513
- error("start_at_slot is invalid %ld, expected 0 to %ld", start_at_slot, st->entries - 1);
1514
- }
1515
- if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1516
- error("stop_at_slot is invalid %ld, expected 0 to %ld", stop_at_slot, st->entries - 1);
1517
- }
1518
- if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1519
- error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1520
- }
1510
+ if(after_new < first_entry_t)
1511
+ error("INTERNAL CHECK: after_new %u is too small, minimum %u", (uint32_t)after_new, (uint32_t)first_entry_t);
1512
+
1513
+ if(after_new > last_entry_t)
1514
+ error("INTERNAL CHECK: after_new %u is too big, maximum %u", (uint32_t)after_new, (uint32_t)last_entry_t);
1515
+
1516
+ if(before_new < first_entry_t)
1517
+ error("INTERNAL CHECK: before_new %u is too small, minimum %u", (uint32_t)before_new, (uint32_t)first_entry_t);
1518
+
1519
+ if(before_new > last_entry_t)
1520
+ error("INTERNAL CHECK: before_new %u is too big, maximum %u", (uint32_t)before_new, (uint32_t)last_entry_t);
1521
+
1522
+ if(start_at_slot < 0 || start_at_slot >= st->entries)
1523
+ error("INTERNAL CHECK: start_at_slot is invalid %ld, expected 0 to %ld", start_at_slot, st->entries - 1);
1524
+
1525
+ if(stop_at_slot < 0 || stop_at_slot >= st->entries)
1526
+ error("INTERNAL CHECK: stop_at_slot is invalid %ld, expected 0 to %ld", stop_at_slot, st->entries - 1);
1527
+
1528
+ if(points_new > (before_new - after_new) / group / st->update_every + 1)
1529
+ error("INTERNAL CHECK: points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1530
+
1531
+ if(group < group_points)
1532
+ error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, group_points);
1533
+
1534
+ if(group > group_points && group % group_points)
1535
+ error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, group_points);
1536
+
1537
+ if(group_points > 1 && group_points * group_sum_divisor != group)
1538
+ error("INTERNAL CHECK: group_sum_divisor " CALCULATED_NUMBER_FORMAT " * group_points %ld, is not equal to group %ld", group_sum_divisor, group_points, group);
1539
#endif
1540
1541
//info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
@@ -1542,13 +1560,13 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
1560
RRDR *r = rrdr_create(st, points);
1561
if(!r) {
1562
#ifdef NETDATA_INTERNAL_CHECKS
1545
- error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1563
+ error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1564
#endif
1565
return NULL;
1566
}
1567
if(!r->d) {
1568
#ifdef NETDATA_INTERNAL_CHECKS
1551
- error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1569
+ error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1570
#endif
1571
return r;
1572
}
@@ -1565,7 +1583,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
1583
// -------------------------------------------------------------------------
1584
// checks for debugging
1585
1568
- if(debug) debug(D_RRD_STATS, "INFO %s first_t: %u, last_t: %u, all_duration: %u, after: %u, before: %u, duration: %u, points: %ld, group: %ld"
1586
+ if(debug) debug(D_RRD_STATS, "INFO %s first_t: %u, last_t: %u, all_duration: %u, after: %u, before: %u, duration: %u, points: %ld, group: %ld, group_points: %ld"
1587
, st->id
1588
, (uint32_t)first_entry_t
1589
, (uint32_t)last_entry_t
@@ -1575,6 +1593,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
1593
, (uint32_t)duration
1594
, points
1595
, group
1596
+ , group_points
1597
);
1598
1599
@@ -1746,7 +1765,11 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
1765
default:
1766
case GROUP_AVERAGE:
1767
case GROUP_UNDEFINED:
1749
- cn[c] = group_values[c] / group_counts[c];
1768
+ if(unlikely(group_points != 1))
1769
+ cn[c] = group_values[c] / group_sum_divisor;
1770
+ else
1771
+ cn[c] = group_values[c] / group_counts[c];
1772
+
1773
group_values[c] = 0;
1774
break;
1775
}
@@ -1781,12 +1804,13 @@ int rrdset2value_api_v1(
1804
, long long after
1805
, long long before
1806
, int group_method
1807
+ , long group_points
1808
, uint32_t options
1809
, time_t *db_after
1810
, time_t *db_before
1811
, int *value_is_null
1812
) {
1789
- RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1813
+ RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_points, !(options & RRDR_OPTION_NOT_ALIGNED));
1814
if(!r) {
1815
if(value_is_null) *value_is_null = 1;
1816
return 500;
@@ -1831,12 +1855,13 @@ int rrdset2anything_api_v1(
1855
, long long after
1856
, long long before
1857
, int group_method
1858
+ , long group_points
1859
, uint32_t options
1860
, time_t *latest_timestamp
1861
) {
1862
st->last_accessed_time = now_realtime_sec();
1863
1839
- RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1864
+ RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_points, !(options & RRDR_OPTION_NOT_ALIGNED));
1865
if(!r) {
1866
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1867
return 500;
src/rrd2json.h
+2
-2
@@ -72,11 +72,11 @@ extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb);
72
extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb);
73
74
extern int rrdset2anything_api_v1(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points
75
- , long long after, long long before, int group_method, uint32_t options
75
+ , long long after, long long before, int group_method, long group_points, uint32_t options
76
, time_t *latest_timestamp);
77
78
extern int rrdset2value_api_v1(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points
79
- , long long after, long long before, int group_method, uint32_t options
79
+ , long long after, long long before, int group_method, long group_points, uint32_t options
80
, time_t *db_after, time_t *db_before, int *value_is_null);
81
82
#endif /* NETDATA_RRD2JSON_H */
src/web_api_v1.c
+6
-3
@@ -558,7 +558,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
558
// if the collected value is too old, don't calculate its value
559
if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above)))
560
ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL
561
- , points, after, before, group, options, NULL, &latest_timestamp, &value_is_null);
561
+ , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null);
562
563
// if the value cannot be calculated, show empty badge
564
if (ret != 200) {
@@ -611,6 +611,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
611
char *chart = NULL
612
, *before_str = NULL
613
, *after_str = NULL
614
+ , *group_points_str = NULL
615
, *points_str = NULL;
616
617
int group = GROUP_AVERAGE;
@@ -639,6 +640,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
640
else if(!strcmp(name, "after")) after_str = value;
641
else if(!strcmp(name, "before")) before_str = value;
642
else if(!strcmp(name, "points")) points_str = value;
643
+ else if(!strcmp(name, "gpoints")) group_points_str = value;
644
else if(!strcmp(name, "group")) {
645
group = web_client_api_request_v1_data_group(value, GROUP_AVERAGE);
646
}
@@ -705,6 +707,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
707
long long before = (before_str && *before_str)?str2l(before_str):0;
708
long long after = (after_str && *after_str) ?str2l(after_str):0;
709
int points = (points_str && *points_str)?str2i(points_str):0;
710
+ long group_points = (group_points_str && *group_points_str)?str2l(group_points_str):0;
711
712
debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', format '%u', options '0x%08x'"
713
, w->id
@@ -743,8 +746,8 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
746
buffer_strcat(w->response.data, "(");
747
}
748
746
- ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, points, after, before, group, options
747
- , &last_timestamp_in_data);
749
+ ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, points, after, before, group, group_points
750
+ , options, &last_timestamp_in_data);
751
752
if(format == DATASOURCE_DATATABLE_JSONP) {
753
if(google_timestamp < last_timestamp_in_data)
web/netdata-swagger.json
+10
@@ -132,6 +132,16 @@
132
"default": "average",
133
"allowEmptyValue": false
134
},
135
+ {
136
+ "name": "gpoints",
137
+ "in": "query",
138
+ "description": "The grouping number of points. This is used in conjunction with group=average to change the units of metrics (ie when the data is per-second, setting gpoints=60 will turn them to per-minute).",
139
+ "required": false,
140
+ "type": "number",
141
+ "format": "integer",
142
+ "allowEmptyValue": false,
143
+ "default": 0
144
+ },
145
{
146
"name": "format",
147
"in": "query",
web/netdata-swagger.yaml
+8
@@ -95,6 +95,14 @@ paths:
95
enum: [ 'min', 'max', 'average', 'sum', 'incremental-sum' ]
96
default: 'average'
97
allowEmptyValue: false
98
+ - name: gpoints
99
+ in: query
100
+ description: 'The grouping number of points. This is used in conjunction with group=average to change the units of metrics (ie when the data is per-second, setting gpoints=60 will turn them to per-minute).'
101
+ required: false
102
+ type: number
103
+ format: integer
104
+ allowEmptyValue: false
105
+ default: 0
106
- name: format
107
in: query
108
description: 'The format of the data to be returned.'