@cryptotaxi247 / netdata-1 / commits / b32ca4431

Query Engine multi-granularity support (and MC improvements) (#13155)

* set grouping functions * storage engine should check the validity of timestamps, not the query engine * calculate and store in RRDR anomaly rates for every query * anomaly rate used by volume metric correlations * mc volume should use absolute data, to avoid cancelling effect * return anomaly-rates in jasonwrap with jw-anomaly-rates option to data queries * dont return null on anomaly rates * allow passing group query options from the URL * added countif to the query engine and used it in metric correlations * fix configure * fix countif and anomaly rate percentages * added group_options to metric correlations; updated swagger * added newline at the end of yaml file * always check the time the highlighted window was above/below the highlighted window * properly track time in memory queries * error for internal checks only * moved pack_storage_number() into the storage engines * moved unpack_storage_number() inside the storage engines * remove old comment * pass unit tests * properly detect zero or subnormal values in pack_storage_number() * fill nulls before the value, not after * make sure math.h is included * workaround for isfinite() * fix for isfinite() * faster isfinite() alternative * fix for faster isfinite() alternative * next_metric() now returns end_time too * variable step implemented in a generic way * remove left-over variables * ensure we always complete the wanted number of points * fixes * ensure no infinite loop * mc-volume-improvements: Add information about invalid condition * points should have a duration in the past * removed unneeded info() line * Fix unit tests for exporting engine * new_point should only be checked when it is fetched from the db; better comment about the premature breaking of the main query loop Co-authored-by: Thiago Marques <thiagoftsm@gmail.com> Co-authored-by: Vladimir Kobal <vlad@prokk.net>

Costa Tsaousis committed Jun 22, 2022 at 11:19 UTC b32ca44319e35eb38e5858730f2ea44ea2268926

60 files changed +1065 -1007

CMakeLists.txt

@@ -727,6 +727,8 @@ set(API_PLUGIN_FILES

727 web/api/queries/query.h

728 web/api/queries/average/average.c

729 web/api/queries/average/average.h

730 + web/api/queries/countif/countif.c

731 + web/api/queries/countif/countif.h

732 web/api/queries/incremental_sum/incremental_sum.c

733 web/api/queries/incremental_sum/incremental_sum.h

734 web/api/queries/max/max.c

Makefile.am

@@ -511,6 +511,8 @@ API_PLUGIN_FILES = \

511 web/api/exporters/shell/allmetrics_shell.h \

512 web/api/queries/average/average.c \

513 web/api/queries/average/average.h \

514 + web/api/queries/countif/countif.c \

515 + web/api/queries/countif/countif.h \

516 web/api/queries/des/des.c \

517 web/api/queries/des/des.h \

518 web/api/queries/incremental_sum/incremental_sum.c \

configure.ac

@@ -363,6 +363,8 @@ else

363 fi

364 AC_MSG_RESULT([${with_math}])

365

366 +AC_CHECK_FUNCS([isfinite])

367 +AC_CHECK_FUNCS([finite])

368

369 # -----------------------------------------------------------------------------

370 # libuv multi-platform support library with a focus on asynchronous I/O

@@ -1785,6 +1787,7 @@ AC_CONFIG_FILES([

1787 web/api/formatters/value/Makefile

1788 web/api/queries/Makefile

1789 web/api/queries/average/Makefile

1790 + web/api/queries/countif/Makefile

1791 web/api/queries/des/Makefile

1792 web/api/queries/incremental_sum/Makefile

1793 web/api/queries/max/Makefile

daemon/unit_test.c

+81 -33

@@ -1242,6 +1242,8 @@ int run_test(struct test *test)

1242 }

1243

1244 static int test_variable_renames(void) {

1245 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1246 +

1247 fprintf(stderr, "Creating chart\n");

1248 RRDSET *st = rrdset_create_localhost("chart", "ID", NULL, "family", "context", "Unit Testing", "a value", "unittest", NULL, 1, 1, RRDSET_TYPE_LINE);

1249 fprintf(stderr, "Created chart with id '%s', name '%s'\n", st->id, st->name);

@@ -1326,6 +1328,7 @@ int check_strdupz_path_subpath() {

1328

1329 int run_all_mockup_tests(void)

1330 {

1331 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1332 if(check_strdupz_path_subpath())

1333 return 1;

1334

@@ -1399,6 +1402,7 @@ int run_all_mockup_tests(void)

1402

1403 int unit_test(long delay, long shift)

1404 {

1405 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1406 static int repeat = 0;

1407 repeat++;

1408

@@ -1501,6 +1505,7 @@ int unit_test(long delay, long shift)

1505 }

1506

1507 int test_sqlite(void) {

1508 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1509 sqlite3 *db_meta;

1510 fprintf(stderr, "Testing SQLIte\n");

1511

@@ -1590,6 +1595,7 @@ static const int QUERY_BATCH = 4096;

1595 static void test_dbengine_create_charts(RRDHOST *host, RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS],

1596 int update_every)

1597 {

1598 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1599 int i, j;

1600 char name[101];

1601

@@ -1637,6 +1643,7 @@ static void test_dbengine_create_charts(RRDHOST *host, RRDSET *st[CHARTS], RRDDI

1643 static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS],

1644 int current_region, time_t time_start)

1645 {

1646 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1647 time_t time_now;

1648 int i, j, c, update_every;

1649 collected_number next;

@@ -1672,13 +1679,15 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS

1679 static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS],

1680 int current_region, time_t time_start)

1681 {

1682 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1683 uint8_t same;

1684 time_t time_now, time_retrieved;

1685 int i, j, k, c, errors, update_every;

1686 collected_number last;

1687 calculated_number value, expected;

1680 - storage_number n;

1688 + SN_FLAGS nflags;

1689 struct rrddim_query_handle handle;

1690 + size_t value_errors = 0, time_errors = 0;

1691

1692 update_every = REGION_UPDATE_EVERY[current_region];

1693 errors = 0;

@@ -1694,19 +1703,23 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI

1703 j * REGION_POINTS[current_region] + c + k;

1704 expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));

1705

1697 - n = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved);

1698 - value = unpack_storage_number(n);

1706 + time_t end_time;

1707 + value = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);

1708

1709 same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;

1710 if(!same) {

1702 - fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "

1711 + if(!value_errors)

1712 + fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "

1713 CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n",

1714 st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, expected, value);

1715 + value_errors++;

1716 errors++;

1717 }

1707 - if(time_retrieved != time_now + k * update_every) {

1708 - fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n",

1718 + if(end_time != time_now + k * update_every) {

1719 + if(!time_errors)

1720 + fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n",

1721 st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, (unsigned long)time_retrieved);

1722 + time_errors++;

1723 errors++;

1724 }

1725 }

@@ -1714,6 +1727,13 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI

1727 }

1728 }

1729 }

1730 +

1731 + if(value_errors)

1732 + fprintf(stderr, "%zu value errors encountered\n", value_errors);

1733 +

1734 + if(time_errors)

1735 + fprintf(stderr, "%zu time errors encountered\n", time_errors);

1736 +

1737 return errors;

1738 }

1739

@@ -1721,27 +1741,28 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI

1741 static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS],

1742 int current_region, time_t time_start, time_t time_end)

1743 {

1744 + int update_every = REGION_UPDATE_EVERY[current_region];

1745 + fprintf(stderr, "%s() running on region %d, start time %ld, end time %ld, update every %d...\n", __FUNCTION__, current_region, time_start, time_end, update_every);

1746 uint8_t same;

1747 time_t time_now, time_retrieved;

1726 - int i, j, errors, update_every;

1748 + int i, j, errors, value_errors = 0, time_errors = 0;

1749 long c;

1750 collected_number last;

1751 calculated_number value, expected;

1752

1753 errors = 0;

1732 - update_every = REGION_UPDATE_EVERY[current_region];

1754 long points = (time_end - time_start) / update_every;

1755 for (i = 0 ; i < CHARTS ; ++i) {

1756 ONEWAYALLOC *owa = onewayalloc_create(0);

1736 - RRDR *r = rrd2rrdr(owa, st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);

1757 + RRDR *r = rrd2rrdr(owa, st[i], points, time_start, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, NULL, 0);

1758 if (!r) {

1738 - fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);

1759 + fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", st[i]->name, current_region);

1760 return ++errors;

1761 } else {

1762 assert(r->st == st[i]);

1763 for (c = 0; c != rrdr_rows(r) ; ++c) {

1764 RRDDIM *d;

1744 - time_now = time_start + (c + 2) * update_every;

1765 + time_now = time_start + (c + 1) * update_every;

1766 time_retrieved = r->t[c];

1767

1768 // for each dimension

@@ -1750,20 +1771,22 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]

1771 value = cn[j];

1772 assert(rd[i][j] == d);

1773

1753 - last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c + 1;

1774 + last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c;

1775 expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));

1776

1777 same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;

1778 if(!same) {

1758 - fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "

1779 + if(value_errors < 10)

1780 + fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "

1781 CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n",

1782 st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);

1761 - errors++;

1783 + value_errors++;

1784 }

1785 if(time_retrieved != time_now) {

1764 - fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",

1786 + if(!time_errors)

1787 + fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",

1788 st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved);

1766 - errors++;

1789 + time_errors++;

1790 }

1791 }

1792 }

@@ -1771,12 +1794,20 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]

1794 }

1795 onewayalloc_destroy(owa);

1796 }

1774 - return errors;

1797 +

1798 + if(value_errors)

1799 + fprintf(stderr, "%d value errors encountered\n", value_errors);

1800 +

1801 + if(time_errors)

1802 + fprintf(stderr, "%d time errors encountered\n", time_errors);

1803 +

1804 + return errors + value_errors + time_errors;

1805 }

1806

1807 int test_dbengine(void)

1808 {

1779 - int i, j, errors, update_every, current_region;

1809 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1810 + int i, j, errors, value_errors = 0, time_errors = 0, update_every, current_region;

1811 RRDHOST *host = NULL;

1812 RRDSET *st[CHARTS];

1813 RRDDIM *rd[CHARTS][DIMS];

@@ -1854,7 +1885,7 @@ int test_dbengine(void)

1885 long point_offset = (time_start[current_region] - time_start[0]) / update_every;

1886 for (i = 0 ; i < CHARTS ; ++i) {

1887 ONEWAYALLOC *owa = onewayalloc_create(0);

1857 - RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);

1888 + RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, NULL, 0);

1889 if (!r) {

1890 fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);

1891 ++errors;

@@ -1879,15 +1910,17 @@ int test_dbengine(void)

1910

1911 uint8_t same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;

1912 if(!same) {

1882 - fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "

1913 + if(!value_errors)

1914 + fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "

1915 CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n",

1916 st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);

1885 - errors++;

1917 + value_errors++;

1918 }

1919 if(time_retrieved != time_now) {

1888 - fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",

1920 + if(!time_errors)

1921 + fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",

1922 st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved);

1890 - errors++;

1923 + time_errors++;

1924 }

1925 }

1926 }

@@ -1902,7 +1935,7 @@ error_out:

1935 rrdeng_exit(host->rrdeng_ctx);

1936 rrd_unlock();

1937

1905 - return errors;

1938 + return errors + value_errors + time_errors;

1939 }

1940

1941 struct dbengine_chart_thread {

@@ -1937,6 +1970,7 @@ collected_number generate_dbengine_chart_value(int chart_i, int dim_i, time_t ti

1970

1971 static void generate_dbengine_chart(void *arg)

1972 {

1973 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

1974 struct dbengine_chart_thread *thread_info = (struct dbengine_chart_thread *)arg;

1975 RRDHOST *host = thread_info->host;

1976 char *chartname = thread_info->chartname;

@@ -1989,6 +2023,7 @@ static void generate_dbengine_chart(void *arg)

2023

2024 void generate_dbengine_dataset(unsigned history_seconds)

2025 {

2026 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

2027 const int DSET_CHARTS = 16;

2028 const int DSET_DIMS = 128;

2029 const uint64_t EXPECTED_COMPRESSION_RATIO = 20;

@@ -2063,6 +2098,7 @@ struct dbengine_query_thread {

2098

2099 static void query_dbengine_chart(void *arg)

2100 {

2101 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

2102 struct dbengine_query_thread *thread_info = (struct dbengine_query_thread *)arg;

2103 const int DSET_CHARTS = thread_info->dset_charts;

2104 const int DSET_DIMS = thread_info->dset_dims;

@@ -2074,8 +2110,9 @@ static void query_dbengine_chart(void *arg)

2110 time_t time_now, time_retrieved;

2111 collected_number generatedv;

2112 calculated_number value, expected;

2077 - storage_number n;

2113 + SN_FLAGS nflags;

2114 struct rrddim_query_handle handle;

2115 + size_t value_errors = 0, time_errors = 0;

2116

2117 do {

2118 // pick a chart and dimension

@@ -2116,8 +2153,9 @@ static void query_dbengine_chart(void *arg)

2153 }

2154 break;

2155 }

2119 - n = rd->state->query_ops.next_metric(&handle, &time_retrieved);

2120 - if (SN_EMPTY_SLOT == n) {

2156 + time_t end_time;

2157 + value = rd->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);

2158 + if (!calculated_number_isnumber(value)) {

2159 if (!thread_info->delete_old_data) { /* data validation only when we don't delete */

2160 fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value "

2161 CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n",

@@ -2127,34 +2165,44 @@ static void query_dbengine_chart(void *arg)

2165 break;

2166 }

2167 ++thread_info->queried_metrics_nr;

2130 - value = unpack_storage_number(n);

2168

2169 same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;

2170 if (!same) {

2171 if (!thread_info->delete_old_data) { /* data validation only when we don't delete */

2135 - fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value "

2172 + if(!value_errors)

2173 + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value "

2174 CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT

2175 ", ### E R R O R ###\n",

2176 st->name, rd->name, (unsigned long) time_now, expected, value);

2139 - ++thread_info->errors;

2177 + value_errors++;

2178 + thread_info->errors++;

2179 }

2180 }

2142 - if (time_retrieved != time_now) {

2181 + if (end_time != time_now) {

2182 if (!thread_info->delete_old_data) { /* data validation only when we don't delete */

2144 - fprintf(stderr,

2183 + if(!time_errors)

2184 + fprintf(stderr,

2185 " DB-engine stresstest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n",

2186 st->name, rd->name, (unsigned long) time_now, (unsigned long) time_retrieved);

2147 - ++thread_info->errors;

2187 + time_errors++;

2188 + thread_info->errors++;

2189 }

2190 }

2191 }

2192 rd->state->query_ops.finalize(&handle);

2193 } while(!thread_info->done);

2194 +

2195 + if(value_errors)

2196 + fprintf(stderr, "%zu value errors encountered\n", value_errors);

2197 +

2198 + if(time_errors)

2199 + fprintf(stderr, "%zu time errors encountered\n", time_errors);

2200 }

2201

2202 void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsigned QUERY_THREADS,

2203 unsigned RAMP_UP_SECONDS, unsigned PAGE_CACHE_MB, unsigned DISK_SPACE_MB)

2204 {

2205 + fprintf(stderr, "%s() running...\n", __FUNCTION__ );

2206 const unsigned DSET_DIMS = 128;

2207 const uint64_t EXPECTED_COMPRESSION_RATIO = 20;

2208 const unsigned HISTORY_SECONDS = 3600 * 24 * 365 * 50; /* 50 year of history */

database/engine/rrdengineapi.c

+33 -15

@@ -193,8 +193,10 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)

193 handle->descr = NULL;

194 }

195

196 -void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number number)

196 +void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, calculated_number n, SN_FLAGS flags)

197 {

198 + storage_number number = pack_storage_number(n, flags);

199 +

200 struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)rd->state->handle;

201 struct rrdengine_instance *ctx;

202 struct page_cache *pg_cache;

@@ -509,6 +511,8 @@ unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t e

511 */

512 void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle, time_t start_time, time_t end_time)

513 {

514 + // fprintf(stderr, "%s: %s/%s start time %ld, end time %ld\n", __FUNCTION__ , rd->rrdset->name, rd->name, start_time, end_time);

515 +

516 struct rrdeng_query_handle *handle;

517 struct rrdengine_instance *ctx;

518 unsigned pages_nr;

@@ -520,6 +524,8 @@ void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_hand

524 handle = callocz(1, sizeof(struct rrdeng_query_handle));

525 handle->next_page_time = start_time;

526 handle->now = start_time;

527 + handle->dt = rd->update_every * USEC_PER_SEC;

528 + handle->dt_sec = rd->update_every;

529 handle->position = 0;

530 handle->ctx = ctx;

531 handle->descr = NULL;

@@ -527,7 +533,7 @@ void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_hand

533 pages_nr = pg_cache_preload(ctx, rd->state->rrdeng_uuid, start_time * USEC_PER_SEC, end_time * USEC_PER_SEC,

534 NULL, &handle->page_index);

535 if (unlikely(NULL == handle->page_index || 0 == pages_nr))

530 - /* there are no metrics to load */

536 + // there are no metrics to load

537 handle->next_page_time = INVALID_TIME;

538 }

539

@@ -585,40 +591,50 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {

591 usec_t entries = handle->entries = page_length / sizeof(storage_number);

592 if (likely(entries > 1))

593 handle->dt = (page_end_time - descr->start_time) / (entries - 1);

588 - else

589 - handle->dt = 0;

594

591 - handle->dt_sec = handle->dt / USEC_PER_SEC;

595 + handle->dt_sec = (time_t)(handle->dt / USEC_PER_SEC);

596 handle->position = position;

597

598 return 0;

599 }

600

597 -/* Returns the metric and sets its timestamp into current_time */

598 -storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *current_time) {

601 +// Returns the metric and sets its timestamp into current_time

602 +// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)

603 +// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES

604 +calculated_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {

605 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;

606

601 - if (unlikely(INVALID_TIME == handle->next_page_time))

602 - return SN_EMPTY_SLOT;

603 -

607 struct rrdeng_page_descr *descr = handle->descr;

608 unsigned position = handle->position + 1;

609 time_t now = handle->now + handle->dt_sec;

610

611 + if (unlikely(INVALID_TIME == handle->next_page_time)) {

612 + handle->next_page_time = INVALID_TIME;

613 + handle->now = now;

614 + *start_time = now - handle->dt_sec;

615 + *end_time = now;

616 + *flags = SN_EMPTY_SLOT;

617 + return NAN;

618 + }

619 +

620 if (unlikely(!descr || position >= handle->entries)) {

621 // We need to get a new page

622 if(rrdeng_load_page_next(rrdimm_handle)) {

623 // next calls will not load any more metrics

624 handle->next_page_time = INVALID_TIME;

613 - return SN_EMPTY_SLOT;

625 + handle->now = now;

626 + *start_time = now - handle->dt_sec;

627 + *end_time = now;

628 + *flags = SN_EMPTY_SLOT;

629 + return NAN;

630 }

631

632 descr = handle->descr;

633 position = handle->position;

618 - now = (descr->start_time + position * handle->dt) / USEC_PER_SEC;

634 + now = (time_t)((descr->start_time + position * handle->dt) / USEC_PER_SEC);

635 }

636

621 - storage_number ret = handle->page[position];

637 + storage_number n = handle->page[position];

638 handle->position = position;

639 handle->now = now;

640

@@ -627,8 +643,10 @@ storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle

643 handle->next_page_time = INVALID_TIME;

644 }

645

630 - *current_time = now;

631 - return ret;

646 + *flags = n & SN_ALL_FLAGS;

647 + *start_time = now - handle->dt_sec;

648 + *end_time = now;

649 + return unpack_storage_number(n);

650 }

651

652 int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle)

database/engine/rrdengineapi.h

+2 -2

@@ -40,14 +40,14 @@ extern void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN +

40 extern void rrdeng_metric_init(RRDDIM *rd);

41 extern void rrdeng_store_metric_init(RRDDIM *rd);

42 extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd);

43 -extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number number);

43 +extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags);

44 extern int rrdeng_store_metric_finalize(RRDDIM *rd);

45 extern unsigned

46 rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,

47 struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);

48 extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle,

49 time_t start_time, time_t end_time);

50 -extern storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *current_time);

50 +extern calculated_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);

51 extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle);

52 extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);

53 extern time_t rrdeng_metric_latest_time(RRDDIM *rd);

database/metric_correlations.c

+94 -35

@@ -9,6 +9,7 @@ int metric_correlations_version = 1;

9 METRIC_CORRELATIONS_METHOD default_metric_correlations_method = METRIC_CORRELATIONS_KS2;

11 typedef struct mc_stats {

12 + calculated_number max_base_high_ratio;

13 size_t db_points;

14 size_t result_points;

15 size_t db_queries;

@@ -46,7 +47,13 @@ const char *mc_method_to_string(METRIC_CORRELATIONS_METHOD method) {

47 // ----------------------------------------------------------------------------

48 // The results per dimension are aggregated into a dictionary

50 +typedef enum {

51 + RESULT_IS_BASE_HIGH_RATIO = (1 << 0),

52 + RESULT_IS_PERCENTAGE_OF_TIME = (1 << 1),

53 +} RESULT_FLAGS;

54 +

55 struct register_result {

56 + RESULT_FLAGS flags;

57 RRDSET *st;

58 const char *chart_id;

59 const char *context;

@@ -86,15 +93,26 @@ static void register_result_destroy(DICTIONARY *results) {

93 dictionary_destroy(results);

94 }

89 -static void register_result(DICTIONARY *results, RRDSET *st, RRDDIM *d, calculated_number value) {

96 +static void register_result(DICTIONARY *results, RRDSET *st, RRDDIM *d, calculated_number value, RESULT_FLAGS flags, MC_STATS *stats) {

97 if(!calculated_number_isnumber(value)) return;

99 + // make it positive

100 + calculated_number v = calculated_number_fabs(value);

101 +

102 + // no need to store zero scored values

103 + if(v == 0.0) return;

104 +

105 + // keep track of the max of the baseline / highlight ratio

106 + if(flags & RESULT_IS_BASE_HIGH_RATIO && v > stats->max_base_high_ratio)

107 + stats->max_base_high_ratio = v;

108 +

109 struct register_result t = {

110 + .flags = flags,

111 .st = st,

112 .chart_id = st->id,

113 .context = st->context,

114 .dim_name = d->name,

97 - .value = value

115 + .value = v

116 };

117

118 char buf[5000 + 1];

@@ -361,7 +379,8 @@ static double kstwo(calculated_number baseline[], int baseline_points, calculate

379 static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,

380 long long baseline_after, long long baseline_before,

381 long long after, long long before,

364 - long long points, RRDR_OPTIONS options, RRDR_GROUPING group,

382 + long long points, RRDR_OPTIONS options,

383 + RRDR_GROUPING group, const char *group_options,

384 uint32_t shifts, int timeout, MC_STATS *stats) {

385 long group_time = 0;

386 struct context_param *context_param_list = NULL;

@@ -377,7 +396,8 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,

396 ONEWAYALLOC *owa = onewayalloc_create(0);

397 high_rrdr = rrd2rrdr(owa, st, points,

398 after, before, group,

380 - group_time, options, NULL, context_param_list, timeout);

399 + group_time, options, NULL, context_param_list, group_options,

400 + timeout);

401 if(!high_rrdr) {

402 info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name);

403 goto cleanup;

@@ -402,7 +422,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,

422 stats->db_queries++;

423 base_rrdr = rrd2rrdr(owa, st,high_points << shifts,

424 baseline_after, baseline_before, group,

405 - group_time, options, NULL, context_param_list,

425 + group_time, options, NULL, context_param_list, group_options,

426 (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)));

427 if(!base_rrdr) {

428 info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name);

@@ -477,7 +497,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,

497

498 // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated

499 // so we flip the result of kstwo()

480 - register_result(results, base_rrdr->st, d, 1.0 - prob);

500 + register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats);

501 }

502 }

503

@@ -494,8 +514,9 @@ cleanup:

514 static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,

515 long long baseline_after, long long baseline_before,

516 long long after, long long before,

497 - RRDR_OPTIONS options, RRDR_GROUPING group, int timeout, MC_STATS *stats) {

498 - options |= RRDR_OPTION_MATCH_IDS;

517 + RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,

518 + int timeout, MC_STATS *stats) {

519 + options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE;

520 long group_time = 0;

521

522 int correlated_dimensions = 0;

@@ -515,47 +536,80 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,

536 // since the query engine checks for a timeout between

537 // dimensions, and we query a single dimension at a time.

538

539 + stats->db_queries++;

540 + calculated_number baseline_average = NAN;

541 + uint8_t base_anomaly_rate = 0;

542 + value_is_null = 1;

543 + ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,

544 + baseline_after, baseline_before,

545 + group, group_options, group_time, options,

546 + NULL, NULL,

547 + &stats->db_points, &stats->result_points,

548 + &value_is_null, &base_anomaly_rate, 0);

549 +

550 + if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(baseline_average)) {

551 + // this means no data for the baseline window, but we may have data for the highlighted one - assume zero

552 + baseline_average = 0.0;

553 + }

554 +

555 stats->db_queries++;

556 calculated_number highlight_average = NAN;

557 + uint8_t high_anomaly_rate = 0;

558 value_is_null = 1;

559 ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1,

560 after, before,

523 - group, group_time, options,

561 + group, group_options, group_time, options,

562 NULL, NULL,

563 &stats->db_points, &stats->result_points,

526 - &value_is_null, 0);

564 + &value_is_null, &high_anomaly_rate, 0);

565

566 if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(highlight_average)) {

529 - // error("Metric correlations: cannot query highlight duration of dimension '%s' of chart '%s', %d %s %s %s", st->name, d->name, ret, (ret != HTTP_RESP_OK)?"response failed":"", (value_is_null)?"value is null":"", (!calculated_number_isnumber(highlight_average))?"result is NAN":"");

567 // this means no data for the highlighted duration - so skip it

568 continue;

569 }

570

571 + if(baseline_average == highlight_average) {

572 + // they are the same - let's move on

573 + continue;

574 + }

575 +

576 stats->db_queries++;

535 - calculated_number baseline_average = NAN;

577 + calculated_number highlight_countif = NAN;

578 value_is_null = 1;

537 - ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,

538 - baseline_after, baseline_before,

539 - group, group_time, options,

579 +

580 + char highlighted_countif_options[50 + 1];

581 + snprintfz(highlighted_countif_options, 50, "%s" CALCULATED_NUMBER_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);

582 +

583 + ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1,

584 + after, before,

585 + RRDR_GROUPING_COUNTIF,highlighted_countif_options,

586 + group_time, options,

587 NULL, NULL,

588 &stats->db_points, &stats->result_points,

542 - &value_is_null, 0);

589 + &value_is_null, NULL, 0);

590

544 - if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(baseline_average)) {

545 - // error("Metric correlations: cannot query baseline duration of dimension '%s' of chart '%s', %d %s %s %s", st->name, d->name, ret, (ret != HTTP_RESP_OK)?"response failed":"", (value_is_null)?"value is null":"", (!calculated_number_isnumber(baseline_average))?"result is NAN":"");

546 - // continue;

547 - // this means no data for the baseline window, but we have data for the highlighted one - assume zero

548 - baseline_average = 0.0;

591 + if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(highlight_countif)) {

592 + info("MC: highlighted countif query failed, but highlighted average worked - strange...");

593 + continue;

594 }

595

551 - calculated_number pcent = NAN;

552 - if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0))

553 - pcent = (highlight_average - baseline_average) / baseline_average;

596 + // this represents the percentage of time

597 + // the highlighted window was above/below the baseline window

598 + // (above or below depending on their averages)

599 + highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0

600

555 - else if(isgreater(highlight_average, 0.0) || isless(highlight_average, 0.0))

556 - pcent = highlight_average;

601 + RESULT_FLAGS flags;

602 + calculated_number pcent = NAN;

603 + if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) {

604 + flags = RESULT_IS_BASE_HIGH_RATIO;

605 + pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif;

606 + }

607 + else {

608 + flags = RESULT_IS_PERCENTAGE_OF_TIME;

609 + pcent = highlight_countif;

610 + }

611

558 - register_result(results, st, d, pcent);

612 + register_result(results, st, d, pcent, flags, stats);

613 }

614

615 return correlated_dimensions;

@@ -590,18 +644,23 @@ static inline int binary_search_bigger_than_calculated_number(const calculated_n

644 // ----------------------------------------------------------------------------

645 // spread the results evenly according to their value

646

593 -static size_t spread_results_evenly(DICTIONARY *results) {

647 +static size_t spread_results_evenly(DICTIONARY *results, MC_STATS *stats) {

648 struct register_result *t;

649

650 // count the dimensions

651 size_t dimensions = dictionary_stats_entries(results);

652 if(!dimensions) return 0;

653

654 + if(stats->max_base_high_ratio == 0.0)

655 + stats->max_base_high_ratio = 1.0;

656 +

657 // create an array of the right size and copy all the values in it

658 calculated_number slots[dimensions];

659 dimensions = 0;

660 dfe_start_read(results, t) {

604 - t->value = calculated_number_fabs(t->value);

661 + if(t->flags & (RESULT_IS_PERCENTAGE_OF_TIME))

662 + t->value = t->value * stats->max_base_high_ratio;

663 +

664 slots[dimensions++] = t->value;

665 }

666 dfe_done(t);

@@ -639,7 +698,8 @@ static size_t spread_results_evenly(DICTIONARY *results) {

698 // ----------------------------------------------------------------------------

699 // The main function

700

642 -int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD method, RRDR_GROUPING group,

701 +int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD method,

702 + RRDR_GROUPING group, const char *group_options,

703 long long baseline_after, long long baseline_before,

704 long long after, long long before,

705 long long points, RRDR_OPTIONS options, int timeout) {

@@ -727,8 +787,7 @@ int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD me

787 while((points << shifts) > MAX_POINTS)

788 points = points >> 1;

789

730 - if(points < 100) {

731 - // error = "cannot comply to at least 100 points";

790 + if(points < 15) {

791 resp = HTTP_RESP_BAD_REQUEST;

792 goto cleanup;

793 }

@@ -769,7 +828,7 @@ int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD me

828 correlated_dimensions += rrdset_metric_correlations_volume(st, results,

829 baseline_after, baseline_before,

830 after, before,

772 - options, group,

831 + options, group, group_options,

832 (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),

833 &stats);

834 break;

@@ -779,7 +838,7 @@ int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD me

838 correlated_dimensions += rrdset_metric_correlations_ks2(st, results,

839 baseline_after, baseline_before,

840 after, before,

782 - points, options, group, shifts,

841 + points, options, group, group_options, shifts,

842 (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),

843 &stats);

844 break;

@@ -790,7 +849,7 @@ int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD me

849 dfe_done(ptr);

850

851 if(!(options & RRDR_OPTION_RETURN_RAW))

793 - spread_results_evenly(results);

852 + spread_results_evenly(results, &stats);

853

854 usec_t ended_usec = now_realtime_usec();

855

database/metric_correlations.h

+2 -1

@@ -14,7 +14,8 @@ extern int enable_metric_correlations;

14 extern int metric_correlations_version;

15 extern METRIC_CORRELATIONS_METHOD default_metric_correlations_method;

17 -extern int metric_correlations (RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD method, RRDR_GROUPING group,

17 +extern int metric_correlations (RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD method,

18 + RRDR_GROUPING group, const char *group_options,

19 long long baseline_after, long long baseline_before,

20 long long after, long long before,

21 long long points, RRDR_OPTIONS options, int timeout);

database/ram/rrddim_mem.c

+42 -12

@@ -9,9 +9,9 @@ void rrddim_collect_init(RRDDIM *rd) {

9 rd->values[rd->rrdset->current_entry] = SN_EMPTY_SLOT;

10 rd->state->handle = calloc(1, sizeof(struct mem_collect_handle));

11 }

12 -void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, storage_number number) {

12 +void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags) {

13 (void)point_in_time;

14 - rd->values[rd->rrdset->current_entry] = number;

14 + rd->values[rd->rrdset->current_entry] = pack_storage_number(number, flags);

15 }

16 int rrddim_collect_finalize(RRDDIM *rd) {

17 free((struct mem_collect_handle*)rd->state->handle);

@@ -28,33 +28,63 @@ void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t st

28 struct mem_query_handle* h = calloc(1, sizeof(struct mem_query_handle));

29 h->slot = rrdset_time2slot(rd->rrdset, start_time);

30 h->last_slot = rrdset_time2slot(rd->rrdset, end_time);

31 - h->finished = 0;

31 + h->dt = rd->update_every;

32 +

33 + h->next_timestamp = start_time;

34 + h->slot_timestamp = rrdset_slot2time(rd->rrdset, h->slot);

35 + h->last_timestamp = rrdset_slot2time(rd->rrdset, h->last_slot);

36 +

37 + // info("QUERY: start %ld, end %ld, next %ld, first %ld, last %ld", start_time, end_time, h->next_timestamp, h->slot_timestamp, h->last_timestamp);

38 +

39 handle->handle = (STORAGE_QUERY_HANDLE *)h;

40 }

35 -storage_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time) {

42 +// Returns the metric and sets its timestamp into current_time

43 +// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)

44 +// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES

45 +calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {

46 RRDDIM *rd = handle->rd;

47 struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;

38 - long entries = rd->rrdset->entries;

39 - long slot = h->slot;

48 + size_t entries = rd->rrdset->entries;

49 + size_t slot = h->slot;

41 - (void)current_time;

42 - if (unlikely(h->slot == h->last_slot))

43 - h->finished = 1;

44 - storage_number n = rd->values[slot++];

51 + time_t this_timestamp = h->next_timestamp;

52 + h->next_timestamp += h->dt;

53 +

54 + // set this timestamp for our caller

55 + *start_time = this_timestamp - h->dt;

56 + *end_time = this_timestamp;

58 + if(unlikely(this_timestamp < h->slot_timestamp)) {

59 + *flags = SN_EMPTY_SLOT;

60 + return NAN;

61 + }

62 +

63 + if(unlikely(this_timestamp > h->last_timestamp)) {

64 + *flags = SN_EMPTY_SLOT;

65 + return NAN;

66 + }

67 +

68 + storage_number n = rd->values[slot++];

69 if(unlikely(slot >= entries)) slot = 0;

70 +

71 h->slot = slot;

72 + h->slot_timestamp += h->dt;

49 - return n;

74 + *flags = (n & SN_ALL_FLAGS);

75 + return unpack_storage_number(n);

76 }

78 int rrddim_query_is_finished(struct rrddim_query_handle *handle) {

79 struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;

54 - return h->finished;

80 + return (h->next_timestamp > handle->end_time);

81 }

83 void rrddim_query_finalize(struct rrddim_query_handle *handle) {

84 +#ifdef NETDATA_INTERNAL_CHECKS

85 + if(!rrddim_query_is_finished(handle))

86 + error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", handle->rd->rrdset->id, handle->rd->name);

87 +#endif

88 freez(handle->handle);

89 }

database/ram/rrddim_mem.h

+9 -5

@@ -9,18 +9,22 @@ struct mem_collect_handle {

9 long slot;

10 long entries;

11 };

12 +

13 struct mem_query_handle {

13 - long slot;

14 - long last_slot;

15 - uint8_t finished;

14 + time_t dt;

15 + time_t next_timestamp;

16 + time_t last_timestamp;

17 + time_t slot_timestamp;

18 + size_t slot;

19 + size_t last_slot;

20 };

22 extern void rrddim_collect_init(RRDDIM *rd);

19 -extern void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, storage_number number);

23 +extern void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags);

24 extern int rrddim_collect_finalize(RRDDIM *rd);

26 extern void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);

23 -extern storage_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time);

27 +extern calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);

28 extern int rrddim_query_is_finished(struct rrddim_query_handle *handle);

29 extern void rrddim_query_finalize(struct rrddim_query_handle *handle);

30 extern time_t rrddim_query_latest_time(RRDDIM *rd);

database/rrd.h

+9 -11

@@ -332,7 +332,7 @@ struct rrddim_collect_ops {

332 void (*init)(RRDDIM *rd);

333

334 // run this to store each metric into the database

335 - void (*store_metric)(RRDDIM *rd, usec_t point_in_time, storage_number number);

335 + void (*store_metric)(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags);

336

337 // an finalization function to run after collection is over

338 // returns 1 if it's safe to delete the dimension

@@ -345,7 +345,7 @@ struct rrddim_query_ops {

345 void (*init)(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);

346

347 // run this to load each metric number from the database

348 - storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time);

348 + calculated_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags);

349

350 // run this to test if the series of next_metric() database queries is finished

351 int (*is_finished)(struct rrddim_query_handle *handle);

@@ -1184,10 +1184,10 @@ static inline size_t rrdset_time2slot(RRDSET *st, time_t t) {

1184 ret = rrdset_first_slot(st);

1185 }

1186 else {

1187 - if(rrdset_last_slot(st) >= ((last_entry_t - t) / (size_t)(st->update_every)))

1188 - ret = rrdset_last_slot(st) - ((last_entry_t - t) / (size_t)(st->update_every));

1187 + if(rrdset_last_slot(st) >= (size_t)((last_entry_t - t) / st->update_every))

1188 + ret = rrdset_last_slot(st) - ((last_entry_t - t) / st->update_every);

1189 else

1190 - ret = rrdset_last_slot(st) - ((last_entry_t - t) / (size_t)(st->update_every)) + (unsigned long)st->entries;

1190 + ret = rrdset_last_slot(st) - ((last_entry_t - t) / st->update_every) + st->entries;

1191 }

1192 }

1193

@@ -1211,12 +1211,10 @@ static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {

1211 slot = (size_t)st->entries - 1;

1212 }

1213

1214 - if(slot > rrdset_last_slot(st)) {

1215 - ret = last_entry_t - (size_t)st->update_every * (rrdset_last_slot(st) - slot + (size_t)st->entries);

1216 - }

1217 - else {

1218 - ret = last_entry_t - (size_t)st->update_every;

1219 - }

1214 + if(slot > rrdset_last_slot(st))

1215 + ret = last_entry_t - (time_t)(st->update_every * (rrdset_last_slot(st) - slot + (size_t)st->entries));

1216 + else

1217 + ret = last_entry_t - (time_t)(st->update_every * (rrdset_last_slot(st) - slot));

1218

1219 if(unlikely(ret < first_entry_t)) {

1220 error("INTERNAL ERROR: rrdset_slot2time() on %s returns time too far in the past", st->name);

database/rrdset.c

+5 -46

@@ -1130,7 +1130,7 @@ static inline size_t rrdset_done_interpolate(

1130 size_t counter = st->counter;

1131 long current_entry = st->current_entry;

1132

1133 - uint32_t storage_flags = SN_DEFAULT_FLAGS;

1133 + SN_FLAGS storage_flags = SN_DEFAULT_FLAGS;

1134

1135 if (has_reset_value)

1136 storage_flags |= SN_EXISTS_RESET;

@@ -1234,8 +1234,7 @@ static inline size_t rrdset_done_interpolate(

1234 if(unlikely(!store_this_entry)) {

1235 (void) ml_is_anomalous(rd, 0, false);

1236

1237 - rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT);

1238 -// rd->values[current_entry] = SN_EMPTY_SLOT;

1237 + rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT);

1238 continue;

1239 }

1240

@@ -1247,18 +1246,8 @@ static inline size_t rrdset_done_interpolate(

1246 dim_storage_flags &= ~ ((uint32_t) SN_ANOMALY_BIT);

1247 }

1248

1250 - rd->state->collect_ops.store_metric(rd, next_store_ut, pack_storage_number(new_value, dim_storage_flags));

1251 -// rd->values[current_entry] = pack_storage_number(new_value, storage_flags );

1249 + rd->state->collect_ops.store_metric(rd, next_store_ut, new_value, dim_storage_flags);

1250 rd->last_stored_value = new_value;

1253 -

1254 - #ifdef NETDATA_INTERNAL_CHECKS

1255 - rrdset_debug(st, "%s: STORE[%ld] "

1256 - CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT

1257 - , rd->name

1258 - , current_entry

1259 - , unpack_storage_number(rd->values[current_entry]), new_value

1260 - );

1261 - #endif

1251 }

1252 else {

1253 (void) ml_is_anomalous(rd, 0, false);

@@ -1270,42 +1259,11 @@ static inline size_t rrdset_done_interpolate(

1259 );

1260 #endif

1261

1273 -// rd->values[current_entry] = SN_EMPTY_SLOT;

1274 - rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT);

1262 + rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT);

1263 rd->last_stored_value = NAN;

1264 }

1265

1266 stored_entries++;

1279 -

1280 - #ifdef NETDATA_INTERNAL_CHECKS

1281 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {

1282 - calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor;

1283 - calculated_number t2 = unpack_storage_number(rd->values[current_entry]);

1284 -

1285 - calculated_number accuracy = accuracy_loss(t1, t2);

1286 - debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"

1287 - , st->id, rd->name

1288 - , current_entry

1289 - , t2

1290 - , t1

1291 - , accuracy

1292 - , (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : ""

1293 - );

1294 -

1295 - rd->collected_volume += t1;

1296 - rd->stored_volume += t2;

1297 -

1298 - accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume);

1299 - debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s"

1300 - , st->id, rd->name

1301 - , current_entry

1302 - , rd->stored_volume

1303 - , rd->collected_volume

1304 - , accuracy

1305 - , (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : ""

1306 - );

1307 - }

1308 - #endif

1267 }

1268 // reset the storage flags for the next point, if any;

1269 storage_flags = SN_DEFAULT_FLAGS;

@@ -1503,6 +1461,7 @@ void rrdset_done(RRDSET *st) {

1461 // and we have collected metrics for this chart in the past (st->counter != 0)

1462 // fill the gap (the chart has been just loaded from disk)

1463 if(unlikely(st->counter) && st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {

1464 + // TODO this should be inside the storage engine

1465 rrdset_done_fill_the_gap(st);

1466 last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;

1467 next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;

exporting/process_data.c

+5 -5

@@ -81,7 +81,6 @@ calculated_number exporting_calculate_value_from_stored_data(

81 time_t last_t = rd->state->query_ops.latest_time(rd);

82 time_t update_every = st->update_every;

83 struct rrddim_query_handle handle;

84 - storage_number n;

85 // step back a little, to make sure we have complete data collection

86 // for all metrics

@@ -125,17 +124,18 @@ calculated_number exporting_calculate_value_from_stored_data(

124

125 size_t counter = 0;

126 calculated_number sum = 0;

127 + calculated_number value;

128

129 for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {

130 - time_t curr_t;

131 - n = rd->state->query_ops.next_metric(&handle, &curr_t);

130 + time_t curr_t, end_t;

131 + SN_FLAGS flags;

132 + value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags);

133

133 - if (unlikely(!does_storage_number_exist(n))) {

134 + if (unlikely(!calculated_number_isnumber(value))) {

135 // not collected

136 continue;

137 }

138

138 - calculated_number value = unpack_storage_number(n);

139 sum += value;

140

141 counter++;

exporting/tests/netdata_doubles.c

+6 -3

@@ -230,13 +230,16 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)

230 return mock_type(int);

231 }

232

233 -storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time)

233 +calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)

234 {

235 (void)handle;

236 - (void)current_time;

236 + (void)start_time;

237 + (void)end_time;

238 + (void) flags;

239 +

240

241 function_called();

239 - return mock_type(storage_number);

242 + return mock_type(calculated_number);

243 }

244

245 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)

exporting/tests/test_exporting_engine.c

+3 -3

@@ -307,19 +307,19 @@ static void test_exporting_calculate_value_from_stored_data(void **state)

307 expect_function_call(__mock_rrddim_query_is_finished);

308 will_return(__mock_rrddim_query_is_finished, 0);

309 expect_function_call(__mock_rrddim_query_next_metric);

310 - will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_DEFAULT_FLAGS));

310 + will_return(__mock_rrddim_query_next_metric, 27);

311

312 expect_function_call(__mock_rrddim_query_is_finished);

313 will_return(__mock_rrddim_query_is_finished, 0);

314 expect_function_call(__mock_rrddim_query_next_metric);

315 - will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_DEFAULT_FLAGS));

315 + will_return(__mock_rrddim_query_next_metric, 45);

316

317 expect_function_call(__mock_rrddim_query_is_finished);

318 will_return(__mock_rrddim_query_is_finished, 1);

319

320 expect_function_call(__mock_rrddim_query_finalize);

321

322 - assert_int_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, &timestamp), 36);

322 + assert_float_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, &timestamp), 36, 0.1);

323 }

324

325 static void test_prepare_buffers(void **state)

exporting/tests/test_exporting_engine.h

+1 -1

@@ -61,7 +61,7 @@ time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);

61 time_t __mock_rrddim_query_latest_time(RRDDIM *rd);

62 void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);

63 int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);

64 -storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time);

64 +calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);

65 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);

67 // -----------------------------------------------------------------------

health/health.c

+2 -2

@@ -856,11 +856,11 @@ void *health_main(void *ptr) {

856 int value_is_null = 0;

857

858 int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1,

859 - rc->after, rc->before, rc->group,

859 + rc->after, rc->before, rc->group, NULL,

860 0, rc->options,

861 &rc->db_after,&rc->db_before,

862 NULL, NULL,

863 - &value_is_null, 0);

863 + &value_is_null, NULL, 0);

864

865 if (unlikely(ret != 200)) {

866 // database lookup failed

libnetdata/storage_number/storage_number.c

+6 -8

@@ -2,12 +2,7 @@

3 #include "../libnetdata.h"

5 -#define get_storage_number_flags(value) \

6 - ((((storage_number)(value)) & (1 << 24)) | \

7 - (((storage_number)(value)) & (1 << 25)) | \

8 - (((storage_number)(value)) & (1 << 26)))

9 -

10 -storage_number pack_storage_number(calculated_number value, uint32_t flags) {

5 +storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {

6 // bit 32 = sign 0:positive, 1:negative

7 // bit 31 = 0:divide, 1:multiply

8 // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)

@@ -16,8 +11,11 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) {

11 // bit 25 SN_ANOMALY_BIT = 0: anomalous, 1: not anomalous

12 // bit 24 to bit 1 = the value

19 - storage_number r = get_storage_number_flags(flags);

20 - if(!value)

14 + storage_number r = flags & SN_ALL_FLAGS;

15 +

16 + // The isnormal() macro shall determine whether its argument value

17 + // is normal (neither zero, subnormal, infinite, nor NaN).

18 + if(unlikely(!isnormal(value)))

19 goto RET_SN;

21 int m = 0;

libnetdata/storage_number/storage_number.h

+21 -8

@@ -3,6 +3,7 @@

3 #ifndef NETDATA_STORAGE_NUMBER_H

4 #define NETDATA_STORAGE_NUMBER_H 1

6 +#include <math.h>

7 #include "../libnetdata.h"

9 #ifdef NETDATA_WITHOUT_LONG_DOUBLE

@@ -59,18 +60,29 @@ typedef long double collected_number;

61 #define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon)

62 -#define calculated_number_isnumber(a) (!(fpclassify(a) & (FP_NAN|FP_INFINITE)))

63 +#if defined(HAVE_ISFINITE) || defined(isfinite)

64 +// The isfinite() macro shall determine whether its argument has a

65 +// finite value (zero, subnormal, or normal, and not infinite or NaN).

66 +#define calculated_number_isnumber(a) (isfinite(a))

67 +#elif defined(HAVE_FINITE) || defined(finite)

68 +#define calculated_number_isnumber(a) (finite(a))

69 +#else

70 +#define calculated_number_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE)

71 +#endif

73 typedef uint32_t storage_number;

74 #define STORAGE_NUMBER_FORMAT "%u"

67 -#define SN_ANOMALY_BIT (1 << 24) // the anomaly bit of the value

68 -#define SN_EXISTS_RESET (1 << 25) // the value has been overflown

69 -#define SN_EXISTS_100 (1 << 26) // very large value (multiplier is 100 instead of 10)

76 +typedef enum {

77 + SN_ANOMALY_BIT = (1 << 24), // the anomaly bit of the value

78 + SN_EXISTS_RESET = (1 << 25), // the value has been overflown

79 + SN_EXISTS_100 = (1 << 26) // very large value (multiplier is 100 instead of 10)

80 +} SN_FLAGS;

71 -#define SN_DEFAULT_FLAGS SN_ANOMALY_BIT

82 +#define SN_ALL_FLAGS (SN_ANOMALY_BIT|SN_EXISTS_RESET|SN_EXISTS_100)

84 #define SN_EMPTY_SLOT 0x00000000

85 +#define SN_DEFAULT_FLAGS SN_ANOMALY_BIT

87 // When the calculated number is zero and the value is anomalous (ie. it's bit

88 // is zero) we want to return a storage_number representation that is

@@ -83,7 +95,7 @@ typedef uint32_t storage_number;

95 #define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT)

96 #define did_storage_number_reset(value) ((((storage_number) (value)) & SN_EXISTS_RESET) != 0)

86 -storage_number pack_storage_number(calculated_number value, uint32_t flags);

98 +storage_number pack_storage_number(calculated_number value, SN_FLAGS flags);

99 static inline calculated_number unpack_storage_number(storage_number value) __attribute__((const));

100

101 int print_calculated_number(char *str, calculated_number value);

@@ -106,7 +118,8 @@ int print_calculated_number(char *str, calculated_number value);

118 static inline calculated_number unpack_storage_number(storage_number value) {

119 extern calculated_number unpack_storage_number_lut10x[4 * 8];

120

109 - if(!value) return 0;

121 + if(unlikely(value == SN_EMPTY_SLOT))

122 + return NAN;

123

124 int sign = 1, exp = 0;

125 int factor = 0;

@@ -127,7 +140,7 @@ static inline calculated_number unpack_storage_number(storage_number value) {

140 // bit 25 SN_ANOMALY_BIT

141

142 // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)

130 - int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27;

143 + int mul = (int)((value & ((1<<29)|(1<<28)|(1<<27))) >> 27);

144

145 // bit 24 to bit 1 = the value, so remove all other bits

146 value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));

ml/Dimension.cc

+3 -52

@@ -6,55 +6,6 @@

7 using namespace ml;

9 -/*

10 - * Copy of the unpack_storage_number which allows us to convert

11 - * a storage_number to double.

12 - */

13 -static CalculatedNumber unpack_storage_number_dbl(storage_number value) {

14 - if(!value)

15 - return 0;

16 -

17 - int sign = 0, exp = 0;

18 - int factor = 10;

19 -

20 - // bit 32 = 0:positive, 1:negative

21 - if(unlikely(value & (1 << 31)))

22 - sign = 1;

23 -

24 - // bit 31 = 0:divide, 1:multiply

25 - if(unlikely(value & (1 << 30)))

26 - exp = 1;

27 -

28 - // bit 27 SN_EXISTS_100

29 - if(unlikely(value & (1 << 26)))

30 - factor = 100;

31 -

32 - // bit 26 SN_EXISTS_RESET

33 - // bit 25 SN_ANOMALY_BIT

34 -

35 - // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)

36 - int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27;

37 -

38 - // bit 24 to bit 1 = the value, so remove all other bits

39 - value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));

40 -

41 - CalculatedNumber CN = value;

42 -

43 - if(exp) {

44 - for(; mul; mul--)

45 - CN *= factor;

46 - }

47 - else {

48 - for( ; mul ; mul--)

49 - CN /= 10;

50 - }

51 -

52 - if(sign)

53 - CN = -CN;

54 -

55 - return CN;

56 -}

57 -

9 std::pair<CalculatedNumber *, size_t>

10 TrainableDimension::getCalculatedNumbers() {

11 size_t MinN = Cfg.MinTrainSamples;

@@ -89,10 +40,10 @@ TrainableDimension::getCalculatedNumbers() {

40 break;

42 auto P = Q.nextMetric();

92 - storage_number SN = P.second;

43 + CalculatedNumber Value = P.second;

94 - if (does_storage_number_exist(SN)) {

95 - CNs[Idx] = unpack_storage_number_dbl(SN);

45 + if (calculated_number_isnumber(Value)) {

46 + CNs[Idx] = Value;

47 LastValue = CNs[Idx];

48 CollectedValues++;

49 } else

ml/Query.h

+5 -4

@@ -27,10 +27,11 @@ public:

27 return Ops->is_finished(&Handle);

28 }

30 - std::pair<time_t, storage_number> nextMetric() {

31 - time_t CurrT;

32 - storage_number SN = Ops->next_metric(&Handle, &CurrT);

33 - return { CurrT, SN };

30 + std::pair<time_t, CalculatedNumber> nextMetric() {

31 + time_t CurrT, EndT;

32 + SN_FLAGS Flags;

33 + auto Value = (CalculatedNumber)Ops->next_metric(&Handle, &CurrT, &EndT, &Flags);

34 + return { CurrT, Value };

35 }

37 ~Query() {

web/api/badges/web_buffer_svg.c

+5 -3

@@ -884,7 +884,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u

884 , *fixed_width_lbl_str = NULL

885 , *fixed_width_val_str = NULL

886 , *text_color_lbl_str = NULL

887 - , *text_color_val_str = NULL;

887 + , *text_color_val_str = NULL

888 + , *group_options = NULL;

889

890 int group = RRDR_GROUPING_AVERAGE;

891 uint32_t options = 0x00000000;

@@ -913,6 +914,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u

914 else if(!strcmp(name, "after")) after_str = value;

915 else if(!strcmp(name, "before")) before_str = value;

916 else if(!strcmp(name, "points")) points_str = value;

917 + else if(!strcmp(name, "group_options")) group_options = value;

918 else if(!strcmp(name, "group")) {

919 group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE);

920 }

@@ -1103,10 +1105,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u

1105 if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above)))

1106 ret = rrdset2value_api_v1(st, w->response.data, &n,

1107 (dimensions) ? buffer_tostring(dimensions) : NULL,

1106 - points, after, before, group, 0, options,

1108 + points, after, before, group, group_options, 0, options,

1109 NULL, &latest_timestamp,

1110 NULL, NULL,

1109 - &value_is_null, 0);

1111 + &value_is_null, NULL, 0);

1112

1113 // if the value cannot be calculated, show empty badge

1114 if (ret != HTTP_RESP_OK) {

web/api/formatters/json/json.c

+12 -3

@@ -162,6 +162,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct

162 for(i = start; i != end ;i += step) {

163 calculated_number *cn = &r->v[ i * r->d ];

164 RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];

165 + uint8_t *ar = &r->ar[ i * r->d ];

166

167 time_t now = r->t[i];

168

@@ -222,7 +223,11 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct

223 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {

224 total = 0;

225 for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {

225 - calculated_number n = cn[c];

226 + calculated_number n;

227 + if(unlikely(options & RRDR_OPTION_INTERNAL_AR))

228 + n = (calculated_number)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200

229 + else

230 + n = cn[c];

231

232 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

233 n = -n;

@@ -239,14 +244,18 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct

244 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;

245 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

246

242 - calculated_number n = cn[c];

247 + calculated_number n;

248 + if(unlikely(options & RRDR_OPTION_INTERNAL_AR))

249 + n = (calculated_number)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200

250 + else

251 + n = cn[c];

252

253 buffer_fast_strcat(wb, pre_value, pre_value_len);

254

255 if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))

256 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);

257

249 - if(co[c] & RRDR_VALUE_EMPTY) {

258 + if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) {

259 if(unlikely(options & RRDR_OPTION_NULL2ZERO))

260 buffer_fast_strcat(wb, "0", 1);

261 else

web/api/formatters/json_wrapper.c

+21

@@ -356,6 +356,27 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS

356 //info("JSONWRAPPER(): %s: END", r->st->id);

357 }

358

359 +void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) {

360 + (void)r;

361 + (void)format;

362 +

363 + char kq[2] = "", // key quote

364 + sq[2] = ""; // string quote

365 +

366 + if( options & RRDR_OPTION_GOOGLE_JSON ) {

367 + kq[0] = '\0';

368 + sq[0] = '\'';

369 + }

370 + else {

371 + kq[0] = '"';

372 + sq[0] = '"';

373 + }

374 +

375 + if(string_value) buffer_strcat(wb, sq);

376 +

377 + buffer_sprintf(wb, ",\n %sanomaly_rates%s: ", kq, kq);

378 +}

379 +

380 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) {

381 (void)format;

382

web/api/formatters/json_wrapper.h

@@ -9,6 +9,7 @@

10 extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,

11 RRDR_GROUPING group_method, QUERY_PARAMS *query_params);

12 +extern void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);

13 extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);

15 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H

web/api/formatters/rrd2json.c

+14 -3

@@ -158,6 +158,7 @@ int rrdset2value_api_v1(

158 , long long after

159 , long long before

160 , int group_method

161 + , const char *group_options

162 , long group_time

163 , uint32_t options

164 , time_t *db_after

@@ -165,13 +166,16 @@ int rrdset2value_api_v1(

166 , size_t *db_points_read

167 , size_t *result_points_generated

168 , int *value_is_null

169 + , uint8_t *anomaly_rate

170 , int timeout

171 ) {

172 int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;

173

174 ONEWAYALLOC *owa = onewayalloc_create(0);

175

174 - RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout);

176 + RRDR *r = rrd2rrdr(owa, st, points, after, before,

177 + group_method, group_time, options, dimensions, NULL,

178 + group_options, timeout);

179

180 if(!r) {

181 if(value_is_null) *value_is_null = 1;

@@ -205,7 +209,7 @@ int rrdset2value_api_v1(

209 if(db_before) *db_before = r->before;

210

211 long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;

208 - *n = rrdr2value(r, i, options, value_is_null, NULL);

212 + *n = rrdr2value(r, i, options, value_is_null, anomaly_rate, NULL);

213 ret = HTTP_RESP_OK;

214

215 cleanup:

@@ -224,6 +228,7 @@ int rrdset2anything_api_v1(

228 , long long after

229 , long long before

230 , int group_method

231 + , const char *group_options

232 , long group_time

233 , uint32_t options

234 , time_t *latest_timestamp

@@ -244,6 +249,7 @@ int rrdset2anything_api_v1(

249 options,

250 dimensions ? buffer_tostring(dimensions) : NULL,

251 query_params->context_param_list,

252 + group_options,

253 query_params->timeout);

254 if(!r) {

255 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");

@@ -425,8 +431,13 @@ int rrdset2anything_api_v1(

431

432 rrdr2json(r, wb, options, 0, query_params->context_param_list);

433

428 - if(options & RRDR_OPTION_JSON_WRAP)

434 + if(options & RRDR_OPTION_JSON_WRAP) {

435 + if(options & RRDR_OPTION_RETURN_JWAR) {

436 + rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0);

437 + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0, query_params->context_param_list);

438 + }

439 rrdr_json_wrapper_end(r, wb, format, options, 0);

440 + }

441 break;

442 }

443

web/api/formatters/rrd2json.h

@@ -74,6 +74,7 @@ extern int rrdset2anything_api_v1(

74 , long long after

75 , long long before

76 , int group_method

77 + , const char *group_options

78 , long group_time

79 , uint32_t options

80 , time_t *latest_timestamp

@@ -88,6 +89,7 @@ extern int rrdset2value_api_v1(

89 , long long after

90 , long long before

91 , int group_method

92 + , const char *group_options

93 , long group_time

94 , uint32_t options

95 , time_t *db_after

@@ -95,6 +97,7 @@ extern int rrdset2value_api_v1(

97 , size_t *db_points_read

98 , size_t *result_points_generated

99 , int *value_is_null

100 + , uint8_t *anomaly_rate

101 , int timeout

102 );

103

web/api/formatters/ssv/ssv.c

+1 -1

@@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con

17 // for each line in the array

18 for(i = start; i != end ;i += step) {

19 int all_values_are_null = 0;

20 - calculated_number v = rrdr2value(r, i, options, &all_values_are_null, temp_rd);

20 + calculated_number v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd);

22 if(likely(i != start)) {

23 if(r->min > v) r->min = v;

web/api/formatters/value/value.c

+11 -1

@@ -3,7 +3,7 @@

3 #include "value.h"

6 -inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, RRDDIM *temp_rd) {

6 +inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) {

7 if (r->st_needs_lock)

8 rrdset_check_rdlock(r->st);

@@ -12,11 +12,14 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *

13 calculated_number *cn = &r->v[ i * r->d ];

14 RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];

15 + uint8_t *ar = &r->ar[ i * r->d ];

17 calculated_number sum = 0, min = 0, max = 0, v;

18 int all_null = 1, init = 1;

20 calculated_number total = 1;

21 + size_t total_anomaly_rate = 0;

22 +

23 int set_min_max = 0;

24 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {

25 total = 0;

@@ -74,6 +77,13 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *

78 if(n < min) min = n;

79 if(n > max) max = n;

80 +

81 + total_anomaly_rate += ar[c];

82 + }

83 +

84 + if(anomaly_rate) {

85 + if(!r->d) *anomaly_rate = 0;

86 + else *anomaly_rate = total_anomaly_rate / r->d;

87 }

89 if(unlikely(all_null)) {

web/api/formatters/value/value.h

+1 -1

@@ -5,6 +5,6 @@

6 #include "../rrd2json.h"

8 -extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, RRDDIM *temp_rd);

8 +extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd);

10 #endif //NETDATA_API_FORMATTER_VALUE_H

web/api/netdata-swagger.json

+122 -7

@@ -236,11 +236,25 @@

236 "median",

237 "stddev",

238 "sum",

239 - "incremental-sum"

239 + "incremental-sum",

240 + "ses",

241 + "des",

242 + "cv",

243 + "countif"

244 ],

245 "default": "average"

246 }

247 },

248 + {

249 + "name": "group_options",

250 + "in": "query",

251 + "description": "When the group function supports additional parameters, this field can be used to pass them to it. Currently only \"countif\" supports this.",

252 + "required": false,

253 + "allowEmptyValue": false,

254 + "schema": {

255 + "type": "string"

256 + }

257 + },

258 {

259 "name": "gtime",

260 "in": "query",

@@ -1161,7 +1175,7 @@

1175 }

1176 },

1177 {

1164 - "name": "highlight_after",

1178 + "name": "after",

1179 "in": "query",

1180 "description": "This parameter can either be an absolute timestamp specifying the starting point of highlighted window, or a relative number of seconds (negative, relative to parameter highlight_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",

1181 "required": false,

@@ -1173,7 +1187,7 @@

1187 }

1188 },

1189 {

1176 - "name": "highlight_before",

1190 + "name": "before",

1191 "in": "query",

1192 "description": "This parameter can either be an absolute timestamp specifying the ending point of the highlighted window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",

1193 "required": false,

@@ -1252,6 +1266,40 @@

1266 "unaligned"

1267 ]

1268 }

1269 + },

1270 + {

1271 + "name": "group",

1272 + "in": "query",

1273 + "description": "The grouping method. If multiple collected values are to be grouped in order to return fewer points, this parameters defines the method of grouping. methods supported \"min\", \"max\", \"average\", \"sum\", \"incremental-sum\". \"max\" is actually calculated on the absolute value collected (so it works for both positive and negative dimensions to return the most extreme value in either direction).",

1274 + "required": true,

1275 + "allowEmptyValue": false,

1276 + "schema": {

1277 + "type": "string",

1278 + "enum": [

1279 + "min",

1280 + "max",

1281 + "average",

1282 + "median",

1283 + "stddev",

1284 + "sum",

1285 + "incremental-sum",

1286 + "ses",

1287 + "des",

1288 + "cv",

1289 + "countif"

1290 + ],

1291 + "default": "average"

1292 + }

1293 + },

1294 + {

1295 + "name": "group_options",

1296 + "in": "query",

1297 + "description": "When the group function supports additional parameters, this field can be used to pass them to it. Currently only \"countif\" supports this.",

1298 + "required": false,

1299 + "allowEmptyValue": false,

1300 + "schema": {

1301 + "type": "string"

1302 + }

1303 }

1304 ],

1305 "responses": {

@@ -2325,6 +2373,77 @@

2373 "metric_correlations": {

2374 "type": "object",

2375 "properties": {

2376 + "after": {

2377 + "description": "the start time of the highlighted window",

2378 + "type": "integer"

2379 + },

2380 + "before": {

2381 + "description": "the end time of the highlighted window",

2382 + "type": "integer"

2383 + },

2384 + "duration": {

2385 + "description": "the duration of the highlighted window",

2386 + "type": "integer"

2387 + },

2388 + "points": {

2389 + "description": "the points of the highlighted window",

2390 + "type": "integer"

2391 + },

2392 + "baseline_after": {

2393 + "description": "the start time of the baseline window",

2394 + "type": "integer"

2395 + },

2396 + "baseline_before": {

2397 + "description": "the end time of the baseline window",

2398 + "type": "integer"

2399 + },

2400 + "baseline_duration": {

2401 + "description": "the duration of the baseline window",

2402 + "type": "integer"

2403 + },

2404 + "baseline_points": {

2405 + "description": "the points of the baseline window",

2406 + "type": "integer"

2407 + },

2408 + "group": {

2409 + "description": "the grouping method across time",

2410 + "type": "string"

2411 + },

2412 + "method": {

2413 + "description": "the correlation method used",

2414 + "type": "string"

2415 + },

2416 + "options": {

2417 + "description": "a comma separated list of the query options set",

2418 + "type": "string"

2419 + },

2420 + "correlated_dimensions": {

2421 + "description": "the number of dimensions returned in the result"

2422 + },

2423 + "total_dimensions_count": {

2424 + "description": "the total number of dimensions evaluated",

2425 + "type": "integer"

2426 + },

2427 + "statistics": {

2428 + "type": "object",

2429 + "properties": {

2430 + "query_time_ms": {

2431 + "type": "number"

2432 + },

2433 + "db_queries": {

2434 + "type": "integer"

2435 + },

2436 + "db_points_read": {

2437 + "type": "integer"

2438 + },

2439 + "query_result_points": {

2440 + "type": "integer"

2441 + },

2442 + "binary_searches": {

2443 + "type": "integer"

2444 + }

2445 + }

2446 + },

2447 "correlated_charts": {

2448 "type": "object",

2449 "description": "An object containing chart objects with their metrics correlations.",

@@ -2368,10 +2487,6 @@

2487 }

2488 }

2489 }

2371 - },

2372 - "total_dimensions_count": {

2373 - "description": "The number of dimensions correlated",

2374 - "type": "integer"

2490 }

2491 }

2492 }

web/api/netdata-swagger.yaml

+99 -6

@@ -208,7 +208,19 @@ paths:

208 - stddev

209 - sum

210 - incremental-sum

211 + - ses

212 + - des

213 + - cv

214 + - countif

215 default: average

216 + - name: group_options

217 + in: query

218 + description: When the group function supports additional parameters, this field

219 + can be used to pass them to it. Currently only "countif" supports this.

220 + required: false

221 + allowEmptyValue: false

222 + schema:

223 + type: string

224 - name: gtime

225 in: query

226 description: The grouping number of seconds. This is used in conjunction with

@@ -951,7 +963,7 @@ paths:

963 type: number

964 format: integer

965 default: -60

954 - - name: highlight_after

966 + - name: after

967 in: query

968 description: This parameter can either be an absolute timestamp specifying the

969 starting point of highlighted window, or a relative number of

@@ -963,7 +975,7 @@ paths:

975 type: number

976 format: integer

977 default: -60

966 - - name: highlight_before

978 + - name: before

979 in: query

980 description: This parameter can either be an absolute timestamp specifying the

981 ending point of the highlighted window, or a relative number of

@@ -995,7 +1007,7 @@ paths:

1007 enum:

1008 - ks2

1009 - volume

998 - default: volume

1010 + default: ks2

1011 - name: timeout

1012 in: query

1013 description: Cancel the query if to takes more that this amount of milliseconds.

@@ -1031,6 +1043,39 @@ paths:

1043 - allow_past

1044 - nonzero

1045 - unaligned

1046 + - name: group

1047 + in: query

1048 + description: The grouping method. If multiple collected values are to be grouped

1049 + in order to return fewer points, this parameters defines the method

1050 + of grouping. methods supported "min", "max", "average", "sum",

1051 + "incremental-sum". "max" is actually calculated on the absolute

1052 + value collected (so it works for both positive and negative

1053 + dimensions to return the most extreme value in either direction).

1054 + required: true

1055 + allowEmptyValue: false

1056 + schema:

1057 + type: string

1058 + enum:

1059 + - min

1060 + - max

1061 + - average

1062 + - median

1063 + - stddev

1064 + - sum

1065 + - incremental-sum

1066 + - ses

1067 + - des

1068 + - cv

1069 + - countif

1070 + default: average

1071 + - name: group_options

1072 + in: query

1073 + description: When the group function supports additional parameters, this field

1074 + can be used to pass them to it. Currently only "countif" supports this.

1075 + required: false

1076 + allowEmptyValue: false

1077 + schema:

1078 + type: string

1079 responses:

1080 "200":

1081 description: JSON object with weights for each chart and dimension.

@@ -1833,6 +1878,57 @@ components:

1878 metric_correlations:

1879 type: object

1880 properties:

1881 + after:

1882 + description: the start time of the highlighted window

1883 + type: integer

1884 + before:

1885 + description: the end time of the highlighted window

1886 + type: integer

1887 + duration:

1888 + description: the duration of the highlighted window

1889 + type: integer

1890 + points:

1891 + description: the points of the highlighted window

1892 + type: integer

1893 + baseline_after:

1894 + description: the start time of the baseline window

1895 + type: integer

1896 + baseline_before:

1897 + description: the end time of the baseline window

1898 + type: integer

1899 + baseline_duration:

1900 + description: the duration of the baseline window

1901 + type: integer

1902 + baseline_points:

1903 + description: the points of the baseline window

1904 + type: integer

1905 + group:

1906 + description: the grouping method across time

1907 + type: string

1908 + method:

1909 + description: the correlation method used

1910 + type: string

1911 + options:

1912 + description: a comma separated list of the query options set

1913 + type: string

1914 + correlated_dimensions:

1915 + description: the number of dimensions returned in the result

1916 + total_dimensions_count:

1917 + description: the total number of dimensions evaluated

1918 + type: integer

1919 + statistics:

1920 + type: object

1921 + properties:

1922 + query_time_ms:

1923 + type: number

1924 + db_queries:

1925 + type: integer

1926 + db_points_read:

1927 + type: integer

1928 + query_result_points:

1929 + type: integer

1930 + binary_searches:

1931 + type: integer

1932 correlated_charts:

1933 type: object

1934 description: An object containing chart objects with their metrics correlations.

@@ -1861,6 +1957,3 @@ components:

1957 type: number

1958 dimension2-name:

1959 type: number

1864 - total_dimensions_count:

1865 - description: The number of dimensions correlated

1866 - type: integer

web/api/queries/Makefile.am

@@ -5,6 +5,7 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

6 SUBDIRS = \

7 average \

8 + countif \

9 des \

10 incremental_sum \

11 max \

web/api/queries/average/average.c

+1 -1

@@ -10,7 +10,7 @@ struct grouping_average {

10 size_t count;

11 };

13 -void grouping_create_average(RRDR *r) {

13 +void grouping_create_average(RRDR *r, const char *options __maybe_unused) {

14 r->internal.grouping_data = callocz(1, sizeof(struct grouping_average));

15 }

web/api/queries/average/average.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "../query.h"

7 #include "../rrdr.h"

9 -extern void grouping_create_average(RRDR *r);

9 +extern void grouping_create_average(RRDR *r, const char *options __maybe_unused);

10 extern void grouping_reset_average(RRDR *r);

11 extern void grouping_free_average(RRDR *r);

12 extern void grouping_add_average(RRDR *r, calculated_number value);

web/api/queries/countif/Makefile.am new

@@ -0,0 +1,8 @@

1 +# SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +AUTOMAKE_OPTIONS = subdir-objects

4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

5 +

6 +dist_noinst_DATA = \

7 + README.md \

8 + $(NULL)

web/api/queries/countif/README.md new

+36

@@ -0,0 +1,36 @@

1 +<!--

2 +title: "CountIf"

3 +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/countif/README.md

4 +-->

5 +

6 +# CountIf

7 +

8 +> This query is available as `countif`.

9 +

10 +CountIf returns the percentage of points in the database that satisfy the condition supplied.

11 +

12 +The following conditions are available:

13 +

14 +- `!` or `!=` or `<>`, different than

15 +- `=` or `:`, equal to

16 +- `>`, greater than

17 +- `<`, less than

18 +- `>=`, greater or equal to

19 +- `<=`, less or equal to

20 +

21 +The target number and the desired condition can be set using the `group_options` query parameter, as a string, like in these examples:

22 +

23 +- `!0`, to match any number except zero.

24 +- `>=-3` to match any number bigger or equal to -3.

25 +

26 +. When an invalid condition is given, the web server can deliver a not accurate response.

27 +

28 +## how to use

29 +

30 +This query cannot be used in alarms.

31 +

32 +`countif` changes the units of charts. The result of the calculation is always from zero to 1, expressing the percentage of database points that matched the condition.

33 +

34 +In APIs and badges can be used like this: `&group=countif&group_options=>10` in the URL.

35 +

36 +

web/api/queries/countif/countif.c new

+136

@@ -0,0 +1,136 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "countif.h"

4 +

5 +// ----------------------------------------------------------------------------

6 +// countif

7 +

8 +struct grouping_countif {

9 + size_t (*comparison)(calculated_number, calculated_number);

10 + calculated_number target;

11 + size_t count;

12 + size_t matched;

13 +};

14 +

15 +static size_t countif_equal(calculated_number v, calculated_number target) {

16 + return (v == target);

17 +}

18 +

19 +static size_t countif_notequal(calculated_number v, calculated_number target) {

20 + return (v != target);

21 +}

22 +

23 +static size_t countif_less(calculated_number v, calculated_number target) {

24 + return (v < target);

25 +}

26 +

27 +static size_t countif_lessequal(calculated_number v, calculated_number target) {

28 + return (v <= target);

29 +}

30 +

31 +static size_t countif_greater(calculated_number v, calculated_number target) {

32 + return (v > target);

33 +}

34 +

35 +static size_t countif_greaterequal(calculated_number v, calculated_number target) {

36 + return (v >= target);

37 +}

38 +

39 +void grouping_create_countif(RRDR *r, const char *options __maybe_unused) {

40 + struct grouping_countif *g = callocz(1, sizeof(struct grouping_countif));

41 + r->internal.grouping_data = g;

42 +

43 + if(options && *options) {

44 + // skip any leading spaces

45 + while(isspace(*options)) options++;

46 +

47 + // find the comparison function

48 + switch(*options) {

49 + case '!':

50 + options++;

51 + if(*options != '=' && *options != ':')

52 + options--;

53 + g->comparison = countif_notequal;

54 + break;

55 +

56 + case '>':

57 + options++;

58 + if(*options == '=' || *options == ':') {

59 + g->comparison = countif_greaterequal;

60 + }

61 + else {

62 + options--;

63 + g->comparison = countif_greater;

64 + }

65 + break;

66 +

67 + case '<':

68 + options++;

69 + if(*options == '>') {

70 + g->comparison = countif_notequal;

71 + }

72 + else if(*options == '=' || *options == ':') {

73 + g->comparison = countif_lessequal;

74 + }

75 + else {

76 + options--;

77 + g->comparison = countif_less;

78 + }

79 + break;

80 +

81 + default:

82 + case '=':

83 + case ':':

84 + g->comparison = countif_equal;

85 + break;

86 + }

87 + if(*options) options++;

88 +

89 + // skip everything up to the first digit

90 + while(isspace(*options)) options++;

91 +

92 + g->target = str2ld(options, NULL);

93 + }

94 + else {

95 + g->target = 0.0;

96 + g->comparison = countif_equal;

97 + }

98 +}

99 +

100 +// resets when switches dimensions

101 +// so, clear everything to restart

102 +void grouping_reset_countif(RRDR *r) {

103 + struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;

104 + g->matched = 0;

105 + g->count = 0;

106 +}

107 +

108 +void grouping_free_countif(RRDR *r) {

109 + freez(r->internal.grouping_data);

110 + r->internal.grouping_data = NULL;

111 +}

112 +

113 +void grouping_add_countif(RRDR *r, calculated_number value) {

114 + struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;

115 + g->matched += g->comparison(value, g->target);

116 + g->count++;

117 +}

118 +

119 +calculated_number grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {

120 + struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;

121 +

122 + calculated_number value;

123 +

124 + if(unlikely(!g->count)) {

125 + value = 0.0;

126 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;

127 + }

128 + else {

129 + value = (calculated_number)g->matched * 100 / (calculated_number)g->count;

130 + }

131 +

132 + g->matched = 0;

133 + g->count = 0;

134 +

135 + return value;

136 +}

web/api/queries/countif/countif.h new

+15

@@ -0,0 +1,15 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_API_QUERY_COUNTIF_H

4 +#define NETDATA_API_QUERY_COUNTIF_H

5 +

6 +#include "../query.h"

7 +#include "../rrdr.h"

8 +

9 +extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused);

10 +extern void grouping_reset_countif(RRDR *r);

11 +extern void grouping_free_countif(RRDR *r);

12 +extern void grouping_add_countif(RRDR *r, calculated_number value);

13 +extern calculated_number grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);

14 +

15 +#endif //NETDATA_API_QUERY_COUNTIF_H

web/api/queries/des/des.c

+1 -1

@@ -69,7 +69,7 @@ static inline void set_beta(RRDR *r, struct grouping_des *g) {

69 //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta);

70 }

72 -void grouping_create_des(RRDR *r) {

72 +void grouping_create_des(RRDR *r, const char *options __maybe_unused) {

73 struct grouping_des *g = (struct grouping_des *)mallocz(sizeof(struct grouping_des));

74 set_alpha(r, g);

75 set_beta(r, g);

web/api/queries/des/des.h

+1 -1

@@ -8,7 +8,7 @@

9 extern void grouping_init_des(void);

11 -extern void grouping_create_des(RRDR *r);

11 +extern void grouping_create_des(RRDR *r, const char *options __maybe_unused);

12 extern void grouping_reset_des(RRDR *r);

13 extern void grouping_free_des(RRDR *r);

14 extern void grouping_add_des(RRDR *r, calculated_number value);

web/api/queries/incremental_sum/incremental_sum.c

+1 -1

@@ -11,7 +11,7 @@ struct grouping_incremental_sum {

11 size_t count;

12 };

14 -void grouping_create_incremental_sum(RRDR *r) {

14 +void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused) {

15 r->internal.grouping_data = callocz(1, sizeof(struct grouping_incremental_sum));

16 }

web/api/queries/incremental_sum/incremental_sum.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "../query.h"

7 #include "../rrdr.h"

9 -extern void grouping_create_incremental_sum(RRDR *r);

9 +extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);

10 extern void grouping_reset_incremental_sum(RRDR *r);

11 extern void grouping_free_incremental_sum(RRDR *r);

12 extern void grouping_add_incremental_sum(RRDR *r, calculated_number value);

web/api/queries/max/max.c

+1 -1

@@ -10,7 +10,7 @@ struct grouping_max {

10 size_t count;

11 };

13 -void grouping_create_max(RRDR *r) {

13 +void grouping_create_max(RRDR *r, const char *options __maybe_unused) {

14 r->internal.grouping_data = callocz(1, sizeof(struct grouping_max));

15 }

web/api/queries/max/max.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "../query.h"

7 #include "../rrdr.h"

9 -extern void grouping_create_max(RRDR *r);

9 +extern void grouping_create_max(RRDR *r, const char *options __maybe_unused);

10 extern void grouping_reset_max(RRDR *r);

11 extern void grouping_free_max(RRDR *r);

12 extern void grouping_add_max(RRDR *r, calculated_number value);

web/api/queries/median/median.c

+1 -1

@@ -13,7 +13,7 @@ struct grouping_median {

13 LONG_DOUBLE series[];

14 };

16 -void grouping_create_median(RRDR *r) {

16 +void grouping_create_median(RRDR *r, const char *options __maybe_unused) {

17 long entries = r->group;

18 if(entries < 0) entries = 0;

web/api/queries/median/median.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "../query.h"

7 #include "../rrdr.h"

9 -extern void grouping_create_median(RRDR *r);

9 +extern void grouping_create_median(RRDR *r, const char *options __maybe_unused);

10 extern void grouping_reset_median(RRDR *r);

11 extern void grouping_free_median(RRDR *r);

12 extern void grouping_add_median(RRDR *r, calculated_number value);

web/api/queries/min/min.c

+1 -1

@@ -10,7 +10,7 @@ struct grouping_min {

10 size_t count;

11 };

13 -void grouping_create_min(RRDR *r) {

13 +void grouping_create_min(RRDR *r, const char *options __maybe_unused) {

14 r->internal.grouping_data = callocz(1, sizeof(struct grouping_min));

15 }

web/api/queries/min/min.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "../query.h"

7 #include "../rrdr.h"

9 -extern void grouping_create_min(RRDR *r);

9 +extern void grouping_create_min(RRDR *r, const char *options __maybe_unused);

10 extern void grouping_reset_min(RRDR *r);

11 extern void grouping_free_min(RRDR *r);

12 extern void grouping_add_min(RRDR *r, calculated_number value);

web/api/queries/query.c

+180 -685

@@ -6,6 +6,7 @@

6 #include "database/ram/rrddim_mem.h"

8 #include "average/average.h"

9 +#include "countif/countif.h"

10 #include "incremental_sum/incremental_sum.h"

11 #include "max/max.h"

12 #include "median/median.h"

@@ -28,7 +29,7 @@ static struct {

30 // Allocate all required structures for a query.

31 // This is called once for each netdata query.

31 - void (*create)(struct rrdresult *r);

32 + void (*create)(struct rrdresult *r, const char *options);

34 // Cleanup collected values, but don't destroy the structures.

35 // This is called when the query engine switches dimensions,

@@ -233,6 +234,17 @@ static struct {

234 .flush = grouping_flush_des

235 },

236

237 + {.name = "countif",

238 + .hash = 0,

239 + .value = RRDR_GROUPING_COUNTIF,

240 + .init = NULL,

241 + .create= grouping_create_countif,

242 + .reset = grouping_reset_countif,

243 + .free = grouping_free_countif,

244 + .add = grouping_add_countif,

245 + .flush = grouping_flush_countif

246 + },

247 +

248 // terminator

249 {.name = NULL,

250 .hash = 0,

@@ -290,6 +302,31 @@ const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group)

302 return "unknown";

303 }

304

305 +static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) {

306 + int i, found = 0;

307 + for(i = 0; !found && api_v1_data_groups[i].name ;i++) {

308 + if(api_v1_data_groups[i].value == group_method) {

309 + r->internal.grouping_create= api_v1_data_groups[i].create;

310 + r->internal.grouping_reset = api_v1_data_groups[i].reset;

311 + r->internal.grouping_free = api_v1_data_groups[i].free;

312 + r->internal.grouping_add = api_v1_data_groups[i].add;

313 + r->internal.grouping_flush = api_v1_data_groups[i].flush;

314 + found = 1;

315 + }

316 + }

317 + if(!found) {

318 + errno = 0;

319 +#ifdef NETDATA_INTERNAL_CHECKS

320 + error("INTERNAL ERROR: grouping method %u not found. Using 'average'", (unsigned int)group_method);

321 +#endif

322 + r->internal.grouping_create= grouping_create_average;

323 + r->internal.grouping_reset = grouping_reset_average;

324 + r->internal.grouping_free = grouping_free_average;

325 + r->internal.grouping_add = grouping_add_average;

326 + r->internal.grouping_flush = grouping_flush_average;

327 + }

328 +}

329 +

330 // ----------------------------------------------------------------------------

331

332 static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,

@@ -393,155 +430,7 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) {

430 // ----------------------------------------------------------------------------

431 // fill RRDR for a single dimension

432

396 -static inline void do_dimension_variablestep(

397 - RRDR *r

398 - , long points_wanted

399 - , RRDDIM *rd

400 - , long dim_id_in_rrdr

401 - , time_t after_wanted

402 - , time_t before_wanted

403 - , uint32_t options

404 -){

405 -// RRDSET *st = r->st;

406 -

407 - time_t

408 - now = after_wanted,

409 - dt = r->update_every,

410 - max_date = 0,

411 - min_date = 0;

412 -

413 - long

414 -// group_size = r->group,

415 - points_added = 0,

416 - values_in_group = 0,

417 - values_in_group_non_zero = 0,

418 - rrdr_line = -1;

419 -

420 - RRDR_VALUE_FLAGS

421 - group_value_flags = RRDR_VALUE_NOTHING;

422 -

423 - struct rrddim_query_handle handle;

424 -

425 - calculated_number min = r->min, max = r->max;

426 - size_t db_points_read = 0;

427 - time_t db_now = now;

428 - storage_number n_curr, n_prev = SN_EMPTY_SLOT;

429 - calculated_number value;

430 -

431 - for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) {

432 - // make sure we return data in the proper time range

433 - if (unlikely(now > before_wanted)) {

434 -#ifdef NETDATA_INTERNAL_CHECKS

435 - r->internal.log = "stopped, because attempted to access the db after 'wanted before'";

436 -#endif

437 - break;

438 - }

439 - if (unlikely(now < after_wanted)) {

440 -#ifdef NETDATA_INTERNAL_CHECKS

441 - r->internal.log = "skipped, because attempted to access the db before 'wanted after'";

442 -#endif

443 - continue;

444 - }

445 -

446 - while (now >= db_now && (!rd->state->query_ops.is_finished(&handle) ||

447 - does_storage_number_exist(n_prev))) {

448 - value = NAN;

449 - if (does_storage_number_exist(n_prev)) {

450 - // use the previously read database value

451 - n_curr = n_prev;

452 - } else {

453 - // read the value from the database

454 - n_curr = rd->state->query_ops.next_metric(&handle, &db_now);

455 - }

456 - n_prev = SN_EMPTY_SLOT;

457 - // db_now has a different value than above

458 - if (likely(now >= db_now)) {

459 - if (likely(does_storage_number_exist(n_curr))) {

460 - if (options & RRDR_OPTION_ANOMALY_BIT)

461 - value = (n_curr & SN_ANOMALY_BIT) ? 0.0 : 100.0;

462 - else

463 - value = unpack_storage_number(n_curr);

464 -

465 - if (likely(value != 0.0))

466 - values_in_group_non_zero++;

467 -

468 - if (unlikely(did_storage_number_reset(n_curr)))

469 - group_value_flags |= RRDR_VALUE_RESET;

470 - }

471 - } else {

472 - // We must postpone processing the value and fill the result with gaps instead

473 - if (likely(does_storage_number_exist(n_curr))) {

474 - n_prev = n_curr;

475 - }

476 - }

477 - // add this value to grouping

478 - if(likely(!isnan(value)))

479 - r->internal.grouping_add(r, value);

480 -

481 - values_in_group++;

482 - db_points_read++;

483 - }

484 -

485 - if (0 == values_in_group) {

486 - // add NAN to grouping

487 - r->internal.grouping_add(r, NAN);

488 - }

489 -

490 - rrdr_line = rrdr_line_init(r, now, rrdr_line);

491 -

492 - if(unlikely(!min_date)) min_date = now;

493 - max_date = now;

494 -

495 - // find the place to store our values

496 - RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr];

497 -

498 - // update the dimension options

499 - if(likely(values_in_group_non_zero))

500 - r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;

501 -

502 - // store the specific point options

503 - *rrdr_value_options_ptr = group_value_flags;

504 -

505 - // store the value

506 - value = r->internal.grouping_flush(r, rrdr_value_options_ptr);

507 - r->v[rrdr_line * r->d + dim_id_in_rrdr] = value;

508 -

509 - if(likely(points_added || dim_id_in_rrdr)) {

510 - // find the min/max across all dimensions

511 -

512 - if(unlikely(value < min)) min = value;

513 - if(unlikely(value > max)) max = value;

514 -

515 - }

516 - else {

517 - // runs only when dim_id_in_rrdr == 0 && points_added == 0

518 - // so, on the first point added for the query.

519 - min = max = value;

520 - }

521 -

522 - points_added++;

523 - values_in_group = 0;

524 - group_value_flags = RRDR_VALUE_NOTHING;

525 - values_in_group_non_zero = 0;

526 - }

527 - rd->state->query_ops.finalize(&handle);

528 -

529 - r->internal.db_points_read += db_points_read;

530 - r->internal.result_points_generated += points_added;

531 -

532 - r->min = min;

533 - r->max = max;

534 - r->before = max_date;

535 - r->after = min_date - (r->group - 1) * dt;

536 - rrdr_done(r, rrdr_line);

537 -

538 - #ifdef NETDATA_INTERNAL_CHECKS

539 - if(unlikely(r->rows != points_added))

540 - error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows);

541 - #endif

542 -}

543 -

544 -static inline void do_dimension_fixedstep(

433 +static inline void rrd2rrdr_do_dimension(

434 RRDR *r

435 , long points_wanted

436 , RRDDIM *rd

@@ -555,117 +444,152 @@ static inline void do_dimension_fixedstep(

444 max_date = 0,

445 min_date = 0;

446

558 - long group_size = r->group,

559 - points_added = 0,

560 - values_in_group = 0,

561 - values_in_group_non_zero = 0,

447 + long group_points_wanted = r->group,

448 + points_added = 0, group_points_added = 0, group_points_non_zero = 0,

449 rrdr_line = -1;

450

451 + size_t group_anomaly_rate = 0;

452 +

453 RRDR_VALUE_FLAGS group_value_flags = RRDR_VALUE_NOTHING;

454

455 struct rrddim_query_handle handle;

456

457 calculated_number min = r->min, max = r->max;

458 size_t db_points_read = 0;

570 - time_t db_now = now;

571 - time_t first_time_t = rrddim_first_entry_t(rd);

459

460 // cache the function pointers we need in the loop

574 - storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time) = rd->state->query_ops.next_metric;

461 + calculated_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags) = rd->state->query_ops.next_metric;

462 void (*grouping_add)(struct rrdresult *r, calculated_number value) = r->internal.grouping_add;

463 calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) = r->internal.grouping_flush;

577 - RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;

464 +

465 + calculated_number last_point_value;

466 + SN_FLAGS last_point_flags;

467 + time_t last_point_start_time;

468 + time_t last_point_end_time;

469 + size_t last_point_anomaly;

470 +

471 + calculated_number new_point_value = NAN;

472 + SN_FLAGS new_point_flags = SN_EMPTY_SLOT;

473 + time_t new_point_start_time = 0;

474 + time_t new_point_end_time = 0;

475 + size_t new_point_anomaly = 0;

476

477 for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) {

580 - // make sure we return data in the proper time range

581 - if(unlikely(now > before_wanted)) {

582 -#ifdef NETDATA_INTERNAL_CHECKS

583 - r->internal.log = "stopped, because attempted to access the db after 'wanted before'";

584 -#endif

478 +

479 + // TODO - should be removed when before and after are always respected

480 + // independently of the databaase first and last time and points_wanted

481 + // is set to a sane number for the user to get the timeframe wanted.

482 + // Without the above, this check is needed to stop the loop when the

483 + // points_wanted is set to an unreasonably high number for the duration

484 + // of the query.

485 + if(unlikely(now > before_wanted))

486 break;

586 - }

487

588 - if(unlikely(now < after_wanted)) {

589 -#ifdef NETDATA_INTERNAL_CHECKS

590 - r->internal.log = "skipped, because attempted to access the db before 'wanted after'";

591 -#endif

592 - continue;

593 - }

488 + // save the old point, in case we need it

489 + last_point_value = new_point_value;

490 + last_point_flags = new_point_flags;

491 + last_point_anomaly = new_point_anomaly;

492 + last_point_start_time = new_point_start_time;

493 + last_point_end_time = new_point_end_time;

494

595 - // read the value from the database

596 - //storage_number n = rd->values[slot];

495 + if(likely(!rd->state->query_ops.is_finished(&handle))) {

496 + // fetch the new point

497 + new_point_value = next_metric(&handle, &new_point_start_time, &new_point_end_time, &new_point_flags);

498

598 -#ifdef NETDATA_INTERNAL_CHECKS

599 - struct mem_query_handle* mem_handle = (struct mem_query_handle*)handle.handle;

600 - if ((rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) &&

601 - (rrdset_time2slot(r->st, now) != (long unsigned)(mem_handle->slot))) {

602 - error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)mem_handle->slot, rrdset_time2slot(r->st, now));

603 - }

604 -#endif

499 + if(likely(calculated_number_isnumber(new_point_value))) {

500 + new_point_anomaly = (new_point_flags & SN_ANOMALY_BIT) ? 0 : 100;

501

606 - db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it

502 + if(unlikely(options & RRDR_OPTION_ANOMALY_BIT))

503 + new_point_value = (calculated_number)new_point_anomaly;

504 + }

505 + else {

506 + new_point_flags = SN_EMPTY_SLOT;

507 + new_point_value = NAN;

508 + new_point_anomaly = 0;

509 + }

510

608 - storage_number n;

609 - calculated_number value;

511 + if(unlikely(new_point_start_time == new_point_end_time)) {

512 + error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point start time %ld, end time %ld, that are both equal", rd->rrdset->name, rd->name, new_point_start_time, new_point_end_time);

513 + new_point_start_time = new_point_end_time - rd->update_every;

514 + }

515

611 - if (unlikely(rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t)) {

612 - n = SN_EMPTY_SLOT;

613 - value = NAN;

614 - }

615 - else {

616 - // load the metric value

617 - n = next_metric(&handle, &db_now);

618 - db_points_read++;

619 -

620 - // and unpack it

621 - if(likely(does_storage_number_exist(n))) {

622 - if (options & RRDR_OPTION_ANOMALY_BIT)

623 - value = (n & SN_ANOMALY_BIT) ? 0.0 : 100.0;

624 - else

625 - value = unpack_storage_number(n);

516 + if(unlikely(new_point_start_time < last_point_start_time && new_point_end_time < last_point_end_time)) {

517 + error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point start time %ld, end time %ld, before the last point start time %ld, end time %ld", rd->rrdset->name, rd->name, new_point_start_time, new_point_end_time, last_point_start_time, last_point_end_time);

518 + new_point_value = last_point_value;

519 + new_point_flags = last_point_flags;

520 + new_point_start_time = last_point_start_time;

521 + new_point_end_time = last_point_end_time;

522 }

627 - else

628 - value = NAN;

629 - }

523

631 - if(unlikely(db_now > before_wanted)) {

632 -#ifdef NETDATA_INTERNAL_CHECKS

633 - r->internal.log = "stopped, because attempted to access the db after 'wanted before'";

634 -#endif

635 - break;

524 + if(unlikely(new_point_end_time < last_point_end_time)) {

525 + error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point end time %ld, before the last point end time %ld", rd->rrdset->name, rd->name, new_point_end_time, last_point_end_time);

526 + new_point_value = last_point_value;

527 + new_point_flags = last_point_flags;

528 + new_point_start_time = last_point_start_time;

529 + new_point_end_time = last_point_end_time;

530 + }

531 +

532 + if(unlikely(new_point_end_time < now)) {

533 + error("QUERY: INTERNAL BUG: next_metric(%s, %s) returned point %ld to %ld, before now (now = %ld, after_wanted = %ld, before_wanted = %ld, dt = %ld)", rd->rrdset->name, rd->name, new_point_start_time, new_point_end_time, now, after_wanted, before_wanted, dt);

534 + new_point_end_time = now;

535 + }

536 + }

537 + else {

538 + new_point_value = NAN;

539 + new_point_flags = SN_EMPTY_SLOT;

540 + new_point_start_time = last_point_end_time;

541 + new_point_end_time = now;

542 }

543

638 - // this loop exists only to fill nulls

639 - // so, if there is a value already, we use it for the first iteration

640 - // but the following iterations will just fill nulls to the destination

641 - for ( ; now <= db_now ; now += dt, value = NAN, n = SN_EMPTY_SLOT) {

642 - if(likely(does_storage_number_exist(n))) {

643 -

644 -#if defined(NETDATA_INTERNAL_CHECKS) && defined(ENABLE_DBENGINE)

645 - if(now >= db_now) {

646 - struct rrdeng_query_handle *rrd_handle = (struct rrdeng_query_handle *)handle.handle;

647 - if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != rrd_handle->now))

648 - error(

649 - "INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld",

650 - rd->id,

651 - (long)rrd_handle->now,

652 - (long)now);

653 - }

654 -#endif

544 + size_t iterations = 0;

545 + for ( ; now <= new_point_end_time && points_added < points_wanted; now += dt, iterations++) {

546 +

547 + calculated_number current_point_value;

548 + SN_FLAGS current_point_flags;

549 + size_t current_point_anomaly;

550 + //time_t current_point_start_time;

551 + //time_t current_point_end_time;

552 +

553 + if(likely(now > new_point_start_time)) {

554 + // it is time for our NEW point to be used

555 + current_point_value = new_point_value;

556 + current_point_flags = new_point_flags;

557 + current_point_anomaly = new_point_anomaly;

558 + //current_point_start_time = new_point_start_time;

559 + //current_point_end_time = new_point_end_time;

560 + }

561 + else if(likely(now <= last_point_end_time)) {

562 + // our LAST point is still valid

563 + current_point_value = last_point_value;

564 + current_point_flags = last_point_flags;

565 + current_point_anomaly = last_point_anomaly;

566 + //current_point_start_time = last_point_start_time;

567 + //current_point_end_time = last_point_end_time;

568 + }

569 + else {

570 + // a GAP, we don't have a value this time

571 + current_point_value = NAN;

572 + current_point_flags = SN_EMPTY_SLOT;

573 + current_point_anomaly = 0;

574 + //current_point_start_time = now - dt;

575 + //current_point_end_time = now;

576 + }

577

656 - if(likely(value != 0.0))

657 - values_in_group_non_zero++;

578 + if(likely(calculated_number_isnumber(current_point_value))) {

579 + if(likely(current_point_value != 0.0))

580 + group_points_non_zero++;

581

659 - if(unlikely(did_storage_number_reset(n)))

582 + if(unlikely(current_point_flags & SN_EXISTS_RESET))

583 group_value_flags |= RRDR_VALUE_RESET;

584

662 - grouping_add(r, value);

585 + grouping_add(r, current_point_value);

586 }

587

588 // add this value for grouping

666 - values_in_group++;

589 + group_points_added++;

590 + group_anomaly_rate += current_point_anomaly;

591

668 - if(unlikely(values_in_group == group_size)) {

592 + if(unlikely(group_points_added == group_points_wanted)) {

593 rrdr_line = rrdr_line_init(r, now, rrdr_line);

594 size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;

595

@@ -676,7 +600,7 @@ static inline void do_dimension_fixedstep(

600 RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];

601

602 // update the dimension options

679 - if(likely(values_in_group_non_zero))

603 + if(likely(group_points_non_zero))

604 r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;

605

606 // store the specific point options

@@ -686,6 +610,12 @@ static inline void do_dimension_fixedstep(

610 calculated_number group_value = grouping_flush(r, rrdr_value_options_ptr);

611 r->v[rrdr_o_v_index] = group_value;

612

613 + // we only store uint8_t anomaly rates,

614 + // so let's get double precision by storing

615 + // anomaly rates in the range 0 - 200

616 + group_anomaly_rate = (group_anomaly_rate << 1) / group_points_added;

617 + r->ar[rrdr_o_v_index] = (uint8_t)group_anomaly_rate;

618 +

619 if(likely(points_added || dim_id_in_rrdr)) {

620 // find the min/max across all dimensions

621

@@ -700,12 +630,17 @@ static inline void do_dimension_fixedstep(

630 }

631

632 points_added++;

703 - values_in_group = 0;

633 + group_points_added = 0;

634 group_value_flags = RRDR_VALUE_NOTHING;

705 - values_in_group_non_zero = 0;

635 + group_points_non_zero = 0;

636 + group_anomaly_rate = 0;

637 }

638 }

708 - now = db_now;

639 + // the loop above increased "now" by dt,

640 + // but the main loop will increase it,

641 + // so, let's undo the last iteration of this loop

642 + if(iterations)

643 + now -= dt;

644 }

645 rd->state->query_ops.finalize(&handle);

646

@@ -718,10 +653,8 @@ static inline void do_dimension_fixedstep(

653 r->after = min_date - (r->group - 1) * dt;

654 rrdr_done(r, rrdr_line);

655

721 -#ifdef NETDATA_INTERNAL_CHECKS

722 - if(unlikely(r->rows != points_added))

723 - error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows);

724 -#endif

656 + if(unlikely(points_wanted != points_added))

657 + error("QUERY: INTERNAL ERROR: query on %s/%s requested %zu points, but RRDR added %zu.", r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added);

658 }

659

660 // ----------------------------------------------------------------------------

@@ -873,7 +806,7 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before, int up

806 return absolute_period_requested;

807 }

808

876 -static RRDR *rrd2rrdr_fixedstep(

809 +static RRDR *rrd2rrdr_do_chart(

810 ONEWAYALLOC *owa

811 , RRDSET *st

812 , long points_requested

@@ -888,6 +821,7 @@ static RRDR *rrd2rrdr_fixedstep(

821 , time_t last_entry_t

822 , int absolute_period_requested

823 , struct context_param *context_param_list

824 + , const char *group_options

825 , int timeout

826 ) {

827 UNUSED(last_entry_t);

@@ -1073,34 +1007,10 @@ static RRDR *rrd2rrdr_fixedstep(

1007

1008 // -------------------------------------------------------------------------

1009 // assign the processor functions

1076 -

1077 - {

1078 - int i, found = 0;

1079 - for(i = 0; !found && api_v1_data_groups[i].name ;i++) {

1080 - if(api_v1_data_groups[i].value == group_method) {

1081 - r->internal.grouping_create= api_v1_data_groups[i].create;

1082 - r->internal.grouping_reset = api_v1_data_groups[i].reset;

1083 - r->internal.grouping_free = api_v1_data_groups[i].free;

1084 - r->internal.grouping_add = api_v1_data_groups[i].add;

1085 - r->internal.grouping_flush = api_v1_data_groups[i].flush;

1086 - found = 1;

1087 - }

1088 - }

1089 - if(!found) {

1090 - errno = 0;

1091 - #ifdef NETDATA_INTERNAL_CHECKS

1092 - error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name);

1093 - #endif

1094 - r->internal.grouping_create= grouping_create_average;

1095 - r->internal.grouping_reset = grouping_reset_average;

1096 - r->internal.grouping_free = grouping_free_average;

1097 - r->internal.grouping_add = grouping_add_average;

1098 - r->internal.grouping_flush = grouping_flush_average;

1099 - }

1100 - }

1010 + rrdr_set_grouping_function(r, group_method);

1011

1012 // allocate any memory required by the grouping method

1103 - r->internal.grouping_create(r);

1013 + r->internal.grouping_create(r, group_options);

1014

1015

1016 // -------------------------------------------------------------------------

@@ -1137,15 +1047,7 @@ static RRDR *rrd2rrdr_fixedstep(

1047 // reset the grouping for the new dimension

1048 r->internal.grouping_reset(r);

1049

1140 - do_dimension_fixedstep(

1141 - r

1142 - , points_wanted

1143 - , rd

1144 - , c

1145 - , after_wanted

1146 - , before_wanted

1147 - , options

1148 - );

1050 + rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted, options);

1051 if (timeout)

1052 now_realtime_timeval(&query_current_time);

1053

@@ -1237,381 +1139,6 @@ static RRDR *rrd2rrdr_fixedstep(

1139 return r;

1140 }

1141

1240 -#ifdef ENABLE_DBENGINE

1241 -static RRDR *rrd2rrdr_variablestep(

1242 - ONEWAYALLOC *owa

1243 - , RRDSET *st

1244 - , long points_requested

1245 - , long long after_requested

1246 - , long long before_requested

1247 - , RRDR_GROUPING group_method

1248 - , long resampling_time_requested

1249 - , RRDR_OPTIONS options

1250 - , const char *dimensions

1251 - , int update_every

1252 - , time_t first_entry_t

1253 - , time_t last_entry_t

1254 - , int absolute_period_requested

1255 - , struct rrdeng_region_info *region_info_array

1256 - , struct context_param *context_param_list

1257 - , int timeout

1258 -) {

1259 - UNUSED(last_entry_t);

1260 - int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);

1261 -

1262 - // the duration of the chart

1263 - time_t duration = before_requested - after_requested;

1264 - long available_points = duration / update_every;

1265 -

1266 - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;

1267 -

1268 - if(duration <= 0 || available_points <= 0) {

1269 - freez(region_info_array);

1270 - return rrdr_create(owa, st, 1, context_param_list);

1271 - }

1272 -

1273 - // check the number of wanted points in the result

1274 - if(unlikely(points_requested < 0)) points_requested = -points_requested;

1275 - if(unlikely(points_requested > available_points)) points_requested = available_points;

1276 - if(unlikely(points_requested == 0)) points_requested = available_points;

1277 -

1278 - // calculate the desired grouping of source data points

1279 - long group = available_points / points_requested;

1280 - if(unlikely(group <= 0)) group = 1;

1281 - if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer

1282 -

1283 - // resampling_time_requested enforces a certain grouping multiple

1284 - calculated_number resampling_divisor = 1.0;

1285 - long resampling_group = 1;

1286 - if(unlikely(resampling_time_requested > update_every)) {

1287 - if (unlikely(resampling_time_requested > duration)) {

1288 - // group_time is above the available duration

1289 -

1290 - #ifdef NETDATA_INTERNAL_CHECKS

1291 - info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, resampling_time_requested, duration);

1292 - #endif

1293 -

1294 - after_requested = before_requested - resampling_time_requested;

1295 - duration = before_requested - after_requested;

1296 - available_points = duration / update_every;

1297 - group = available_points / points_requested;

1298 - }

1299 -

1300 - // if the duration is not aligned to resampling time

1301 - // extend the duration to the past, to avoid a gap at the chart

1302 - // only when the missing duration is above 1/10th of a point

1303 - if(duration % resampling_time_requested) {

1304 - time_t delta = duration % resampling_time_requested;

1305 - if(delta > resampling_time_requested / 10) {

1306 - after_requested -= resampling_time_requested - delta;

1307 - duration = before_requested - after_requested;

1308 - available_points = duration / update_every;

1309 - group = available_points / points_requested;

1310 - }

1311 - }

1312 -

1313 - // the points we should group to satisfy gtime

1314 - resampling_group = resampling_time_requested / update_every;

1315 - if(unlikely(resampling_time_requested % update_every)) {

1316 - #ifdef NETDATA_INTERNAL_CHECKS

1317 - info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every);

1318 - #endif

1319 -

1320 - resampling_group++;

1321 - }

1322 -

1323 - // adapt group according to resampling_group

1324 - if(unlikely(group < resampling_group)) group = resampling_group; // do not allow grouping below the desired one

1325 - if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group

1326 -

1327 - //resampling_divisor = group / resampling_group;

1328 - resampling_divisor = (calculated_number)(group * update_every) / (calculated_number)resampling_time_requested;

1329 - }

1330 -

1331 - // now that we have group,

1332 - // align the requested timeframe to fit it.

1333 -

1334 - if(aligned) {

1335 - // alignment has been requested, so align the values

1336 - before_requested -= before_requested % (group * update_every);

1337 - after_requested -= after_requested % (group * update_every);

1338 - }

1339 -

1340 - // we align the request on requested_before

1341 - time_t before_wanted = before_requested;

1342 -

1343 - //size_t before_slot = rrdset_time2slot(st, before_wanted);

1344 -

1345 - // we need to estimate the number of points, for having

1346 - // an integer number of values per point

1347 - long points_wanted = (before_wanted - after_requested) / (update_every * group);

1348 -

1349 - time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;

1350 - if(unlikely(after_wanted < first_entry_t)) {

1351 - // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning

1352 - points_wanted = (before_wanted - first_entry_t) / group;

1353 -

1354 - // recalculate after wanted with the new number of points

1355 - after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;

1356 -

1357 - if(unlikely(after_wanted < first_entry_t)) {

1358 - #ifdef NETDATA_INTERNAL_CHECKS

1359 - error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name);

1360 - #endif

1361 -

1362 - after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every );

1363 - }

1364 - }

1365 - //size_t after_slot = rrdset_time2slot(st, after_wanted);

1366 -

1367 - // check if they are reversed

1368 - if(unlikely(after_wanted > before_wanted)) {

1369 - #ifdef NETDATA_INTERNAL_CHECKS

1370 - error("INTERNAL ERROR: rrd2rrdr() on %s, reversed wanted after/before", st->name);

1371 - #endif

1372 - time_t tmp = before_wanted;

1373 - before_wanted = after_wanted;

1374 - after_wanted = tmp;

1375 - }

1376 -

1377 - // recalculate points_wanted using the final time-frame

1378 - points_wanted = (before_wanted - after_wanted) / update_every / group + 1;

1379 - if(unlikely(points_wanted < 0)) {

1380 - #ifdef NETDATA_INTERNAL_CHECKS

1381 - error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted);

1382 - #endif

1383 - points_wanted = 0;

1384 - }

1385 -

1386 -#ifdef NETDATA_INTERNAL_CHECKS

1387 - duration = before_wanted - after_wanted;

1388 -

1389 - if(after_wanted < first_entry_t)

1390 - error("INTERNAL CHECK: after_wanted %u is too small, minimum %u", (uint32_t)after_wanted, (uint32_t)first_entry_t);

1391 -

1392 - if(before_wanted < first_entry_t)

1393 - error("INTERNAL CHECK: before_wanted %u is too small, minimum %u", (uint32_t)before_wanted, (uint32_t)first_entry_t);

1394 -

1395 - if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1)

1396 - error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1);

1397 -

1398 - if(group < resampling_group)

1399 - error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group);

1400 -

1401 - if(group > resampling_group && group % resampling_group)

1402 - error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, resampling_group);

1403 -#endif

1404 -

1405 - // -------------------------------------------------------------------------

1406 - // initialize our result set

1407 - // this also locks the chart for us

1408 -

1409 - RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);

1410 - if(unlikely(!r)) {

1411 - #ifdef NETDATA_INTERNAL_CHECKS

1412 - error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);

1413 - #endif

1414 - freez(region_info_array);

1415 - return NULL;

1416 - }

1417 -

1418 - if(unlikely(!r->d || !points_wanted)) {

1419 - #ifdef NETDATA_INTERNAL_CHECKS

1420 - error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);

1421 - #endif

1422 - freez(region_info_array);

1423 - return r;

1424 - }

1425 -

1426 - r->result_options |= RRDR_RESULT_OPTION_VARIABLE_STEP;

1427 - if(unlikely(absolute_period_requested == 1))

1428 - r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;

1429 - else

1430 - r->result_options |= RRDR_RESULT_OPTION_RELATIVE;

1431 -

1432 - // find how many dimensions we have

1433 - long dimensions_count = r->d;

1434 -

1435 - // -------------------------------------------------------------------------

1436 - // initialize RRDR

1437 -

1438 - r->group = group;

1439 - r->update_every = (int)group * update_every;

1440 - r->before = before_wanted;

1441 - r->after = after_wanted;

1442 - r->internal.points_wanted = points_wanted;

1443 - r->internal.resampling_group = resampling_group;

1444 - r->internal.resampling_divisor = resampling_divisor;

1445 -

1446 -

1447 - // -------------------------------------------------------------------------

1448 - // assign the processor functions

1449 -

1450 - {

1451 - int i, found = 0;

1452 - for(i = 0; !found && api_v1_data_groups[i].name ;i++) {

1453 - if(api_v1_data_groups[i].value == group_method) {

1454 - r->internal.grouping_create= api_v1_data_groups[i].create;

1455 - r->internal.grouping_reset = api_v1_data_groups[i].reset;

1456 - r->internal.grouping_free = api_v1_data_groups[i].free;

1457 - r->internal.grouping_add = api_v1_data_groups[i].add;

1458 - r->internal.grouping_flush = api_v1_data_groups[i].flush;

1459 - found = 1;

1460 - }

1461 - }

1462 - if(!found) {

1463 - errno = 0;

1464 - #ifdef NETDATA_INTERNAL_CHECKS

1465 - error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name);

1466 - #endif

1467 - r->internal.grouping_create= grouping_create_average;

1468 - r->internal.grouping_reset = grouping_reset_average;

1469 - r->internal.grouping_free = grouping_free_average;

1470 - r->internal.grouping_add = grouping_add_average;

1471 - r->internal.grouping_flush = grouping_flush_average;

1472 - }

1473 - }

1474 -

1475 - // allocate any memory required by the grouping method

1476 - r->internal.grouping_create(r);

1477 -

1478 -

1479 - // -------------------------------------------------------------------------

1480 - // disable the not-wanted dimensions

1481 - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))

1482 - rrdset_check_rdlock(st);

1483 -

1484 - if(dimensions)

1485 - rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);

1486 -

1487 -

1488 - // -------------------------------------------------------------------------

1489 - // do the work for each dimension

1490 -

1491 - time_t max_after = 0, min_before = 0;

1492 - long max_rows = 0;

1493 -

1494 - RRDDIM *rd;

1495 - long c, dimensions_used = 0, dimensions_nonzero = 0;

1496 - struct timeval query_start_time;

1497 - struct timeval query_current_time;

1498 - if (timeout)

1499 - now_realtime_timeval(&query_start_time);

1500 - for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {

1501 -

1502 - // if we need a percentage, we need to calculate all dimensions

1503 - if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) {

1504 - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;

1505 - continue;

1506 - }

1507 - r->od[c] |= RRDR_DIMENSION_SELECTED;

1508 -

1509 - // reset the grouping for the new dimension

1510 - r->internal.grouping_reset(r);

1511 -

1512 - do_dimension_variablestep(

1513 - r

1514 - , points_wanted

1515 - , rd

1516 - , c

1517 - , after_wanted

1518 - , before_wanted

1519 - , options

1520 - );

1521 - if (timeout)

1522 - now_realtime_timeval(&query_current_time);

1523 -

1524 - if(r->od[c] & RRDR_DIMENSION_NONZERO)

1525 - dimensions_nonzero++;

1526 -

1527 - // verify all dimensions are aligned

1528 - if(unlikely(!dimensions_used)) {

1529 - min_before = r->before;

1530 - max_after = r->after;

1531 - max_rows = r->rows;

1532 - }

1533 - else {

1534 - if(r->after != max_after) {

1535 - #ifdef NETDATA_INTERNAL_CHECKS

1536 - error("INTERNAL ERROR: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",

1537 - st->name, (size_t)max_after, rd->name, (size_t)r->after);

1538 - #endif

1539 - r->after = (r->after > max_after) ? r->after : max_after;

1540 - }

1541 -

1542 - if(r->before != min_before) {

1543 - #ifdef NETDATA_INTERNAL_CHECKS

1544 - error("INTERNAL ERROR: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",

1545 - st->name, (size_t)min_before, rd->name, (size_t)r->before);

1546 - #endif

1547 - r->before = (r->before < min_before) ? r->before : min_before;

1548 - }

1549 -

1550 - if(r->rows != max_rows) {

1551 - #ifdef NETDATA_INTERNAL_CHECKS

1552 - error("INTERNAL ERROR: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",

1553 - st->name, (size_t)max_rows, rd->name, (size_t)r->rows);

1554 - #endif

1555 - r->rows = (r->rows > max_rows) ? r->rows : max_rows;

1556 - }

1557 - }

1558 -

1559 - dimensions_used++;

1560 - if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {

1561 - log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",

1562 - dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);

1563 - r->result_options |= RRDR_RESULT_OPTION_CANCEL;

1564 - break;

1565 - }

1566 - }

1567 -

1568 - #ifdef NETDATA_INTERNAL_CHECKS

1569 -

1570 - if (dimensions_used) {

1571 - if(r->internal.log)

1572 - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log);

1573 -

1574 - if(r->rows != points_wanted)

1575 - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'");

1576 -

1577 - if(aligned && (r->before % group) != 0)

1578 - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required");

1579 -

1580 - // 'after' should not be aligned, since we start inside the first group

1581 - //if(aligned && (r->after % group) != 0)

1582 - // rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");

1583 -

1584 - if(r->before != before_requested)

1585 - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'");

1586 -

1587 - if(r->before != before_wanted)

1588 - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'");

1589 -

1590 - // reported 'after' varies, depending on group

1591 - if(r->after != after_wanted)

1592 - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'");

1593 - }

1594 - #endif

1595 -

1596 - // free all resources used by the grouping method

1597 - r->internal.grouping_free(r);

1598 -

1599 - // when all the dimensions are zero, we should return all of them

1600 - if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {

1601 - // all the dimensions are zero

1602 - // mark them as NONZERO to send them all

1603 - for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {

1604 - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;

1605 - r->od[c] |= RRDR_DIMENSION_NONZERO;

1606 - }

1607 - }

1608 -

1609 - rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);

1610 - freez(region_info_array);

1611 - return r;

1612 -}

1613 -#endif //#ifdef ENABLE_DBENGINE

1614 -

1142 RRDR *rrd2rrdr(

1143 ONEWAYALLOC *owa

1144 , RRDSET *st

@@ -1623,6 +1150,7 @@ RRDR *rrd2rrdr(

1150 , RRDR_OPTIONS options

1151 , const char *dimensions

1152 , struct context_param *context_param_list

1153 + , const char *group_options

1154 , int timeout

1155 )

1156 {

@@ -1677,42 +1205,9 @@ RRDR *rrd2rrdr(

1205 return NULL;

1206 }

1207

1680 -#ifdef ENABLE_DBENGINE

1681 - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

1682 - struct rrdeng_region_info *region_info_array;

1683 - unsigned regions, max_interval;

1684 -

1685 - /* This call takes the chart read-lock */

1686 - regions = rrdeng_variable_step_boundaries(st, after_requested, before_requested,

1687 - &region_info_array, &max_interval, context_param_list);

1688 - if (1 == regions) {

1689 - if (region_info_array) {

1690 - if (rrd_update_every != region_info_array[0].update_every) {

1691 - rrd_update_every = region_info_array[0].update_every;

1692 - /* recalculate query alignment */

1693 - absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested,

1694 - rrd_update_every, points_requested);

1695 - }

1696 - freez(region_info_array);

1697 - }

1698 - return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method,

1699 - resampling_time_requested, options, dimensions, rrd_update_every,

1700 - first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);

1701 - }

1702 - else {

1703 - if (rrd_update_every != (uint16_t)max_interval) {

1704 - rrd_update_every = (uint16_t) max_interval;

1705 - /* recalculate query alignment */

1706 - absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested,

1707 - rrd_update_every, points_requested);

1708 - }

1709 - return rrd2rrdr_variablestep(owa, st, points_requested, after_requested, before_requested, group_method,

1710 - resampling_time_requested, options, dimensions, rrd_update_every,

1711 - first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout);

1712 - }

1713 - }

1714 -#endif

1715 - return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method,

1716 - resampling_time_requested, options, dimensions,

1717 - rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);

1208 + return rrd2rrdr_do_chart(owa, st, points_requested,

1209 + after_requested, before_requested,

1210 + group_method, resampling_time_requested, options, dimensions, rrd_update_every,

1211 + first_entry_t, last_entry_t, absolute_period_requested,

1212 + context_param_list, group_options, timeout);

1213 }

web/api/queries/query.h

@@ -15,6 +15,7 @@ typedef enum rrdr_grouping {

15 RRDR_GROUPING_CV,

16 RRDR_GROUPING_SES,

17 RRDR_GROUPING_DES,

18 + RRDR_GROUPING_COUNTIF,

19 } RRDR_GROUPING;

21 extern const char *group_method2string(RRDR_GROUPING group);

web/api/queries/rrdr.c

@@ -91,6 +91,7 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {

91 onewayalloc_freez(owa, r->v);

92 onewayalloc_freez(owa, r->o);

93 onewayalloc_freez(owa, r->od);

94 + onewayalloc_freez(owa, r->ar);

95 onewayalloc_freez(owa, r);

96 }

@@ -125,6 +126,7 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_pa

126 r->t = onewayalloc_callocz(owa, (size_t)n, sizeof(time_t));

127 r->v = onewayalloc_mallocz(owa, n * r->d * sizeof(calculated_number));

128 r->o = onewayalloc_mallocz(owa, n * r->d * sizeof(RRDR_VALUE_FLAGS));

129 + r->ar = onewayalloc_mallocz(owa, n * r->d * sizeof(uint8_t));

130 r->od = onewayalloc_mallocz(owa, r->d * sizeof(RRDR_DIMENSION_FLAGS));

131

132 // set the hidden flag on hidden dimensions

web/api/queries/rrdr.h

+7 -2

@@ -26,6 +26,10 @@ typedef enum rrdr_options {

26 RRDR_OPTION_ALLOW_PAST = 0x00020000, // The after parameter can extend in the past before the first entry

27 RRDR_OPTION_ANOMALY_BIT = 0x00040000, // Return the anomaly bit stored in each collected_number

28 RRDR_OPTION_RETURN_RAW = 0x00080000, // Return raw data for aggregating across multiple nodes

29 + RRDR_OPTION_RETURN_JWAR = 0x00100000, // Return anomaly rates in jsonwrap

30 +

31 + // internal ones - not to be exposed to the API

32 + RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate

33 } RRDR_OPTIONS;

35 typedef enum rrdr_value_flag {

@@ -65,6 +69,7 @@ typedef struct rrdresult {

69 time_t *t; // array of n timestamps

70 calculated_number *v; // array n x d values

71 RRDR_VALUE_FLAGS *o; // array n x d options for each value returned

72 + uint8_t *ar; // array n x d of anomaly rates (0 - 200)

74 long group; // how many collected values were grouped for each row

75 int update_every; // what is the suggested update frequency in seconds

@@ -84,7 +89,7 @@ typedef struct rrdresult {

89 long resampling_group;

90 calculated_number resampling_divisor;

87 - void (*grouping_create)(struct rrdresult *r);

92 + void (*grouping_create)(struct rrdresult *r, const char *options);

93 void (*grouping_reset)(struct rrdresult *r);

94 void (*grouping_free)(struct rrdresult *r);

95 void (*grouping_add)(struct rrdresult *r, calculated_number value);

@@ -113,7 +118,7 @@ extern RRDR *rrd2rrdr(

118 ONEWAYALLOC *owa,

119 RRDSET *st, long points_requested, long long after_requested, long long before_requested,

120 RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,

116 - struct context_param *context_param_list, int timeout);

121 + struct context_param *context_param_list, const char *group_options, int timeout);

122

123 extern int rrdr_relative_window_to_absolute(long long *after, long long *before, int update_every, long points);

124

web/api/queries/ses/ses.c

+1 -1

@@ -48,7 +48,7 @@ static inline void set_alpha(RRDR *r, struct grouping_ses *g) {

48 g->alpha_other = 1.0 - g->alpha;

49 }

51 -void grouping_create_ses(RRDR *r) {

51 +void grouping_create_ses(RRDR *r, const char *options __maybe_unused) {

52 struct grouping_ses *g = (struct grouping_ses *)callocz(1, sizeof(struct grouping_ses));

53 set_alpha(r, g);

54 g->level = 0.0;

web/api/queries/ses/ses.h

+1 -1

@@ -8,7 +8,7 @@

9 extern void grouping_init_ses(void);

11 -extern void grouping_create_ses(RRDR *r);

11 +extern void grouping_create_ses(RRDR *r, const char *options __maybe_unused);

12 extern void grouping_reset_ses(RRDR *r);

13 extern void grouping_free_ses(RRDR *r);

14 extern void grouping_add_ses(RRDR *r, calculated_number value);

web/api/queries/stddev/stddev.c

+1 -1

@@ -14,7 +14,7 @@ struct grouping_stddev {

14 calculated_number m_oldM, m_newM, m_oldS, m_newS;

15 };

17 -void grouping_create_stddev(RRDR *r) {

17 +void grouping_create_stddev(RRDR *r, const char *options __maybe_unused) {

18 r->internal.grouping_data = callocz(1, sizeof(struct grouping_stddev));

19 }

web/api/queries/stddev/stddev.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "../query.h"

7 #include "../rrdr.h"

9 -extern void grouping_create_stddev(RRDR *r);

9 +extern void grouping_create_stddev(RRDR *r, const char *options __maybe_unused);

10 extern void grouping_reset_stddev(RRDR *r);

11 extern void grouping_free_stddev(RRDR *r);

12 extern void grouping_add_stddev(RRDR *r, calculated_number value);

web/api/queries/sum/sum.c

+1 -1

@@ -10,7 +10,7 @@ struct grouping_sum {

10 size_t count;

11 };

13 -void grouping_create_sum(RRDR *r) {

13 +void grouping_create_sum(RRDR *r, const char *options __maybe_unused) {

14 r->internal.grouping_data = callocz(1, sizeof(struct grouping_sum));

15 }

web/api/queries/sum/sum.h

+1 -1

@@ -6,7 +6,7 @@

6 #include "../query.h"

7 #include "../rrdr.h"

9 -extern void grouping_create_sum(RRDR *r);

9 +extern void grouping_create_sum(RRDR *r, const char *options __maybe_unused);

10 extern void grouping_reset_sum(RRDR *r);

11 extern void grouping_free_sum(RRDR *r);

12 extern void grouping_add_sum(RRDR *r, calculated_number value);

web/api/web_api_v1.c

+35 -31

@@ -10,35 +10,36 @@ static struct {

10 uint32_t hash;

11 RRDR_OPTIONS value;

12 } api_v1_data_options[] = {

13 - { "nonzero" , 0 , RRDR_OPTION_NONZERO}

14 - , {"flip" , 0 , RRDR_OPTION_REVERSED}

15 - , {"reversed" , 0 , RRDR_OPTION_REVERSED}

16 - , {"reverse" , 0 , RRDR_OPTION_REVERSED}

17 - , {"jsonwrap" , 0 , RRDR_OPTION_JSON_WRAP}

18 - , {"min2max" , 0 , RRDR_OPTION_MIN2MAX}

19 - , {"ms" , 0 , RRDR_OPTION_MILLISECONDS}

20 - , {"milliseconds" , 0 , RRDR_OPTION_MILLISECONDS}

21 - , {"abs" , 0 , RRDR_OPTION_ABSOLUTE}

22 - , {"absolute" , 0 , RRDR_OPTION_ABSOLUTE}

23 - , {"absolute_sum" , 0 , RRDR_OPTION_ABSOLUTE}

24 - , {"absolute-sum" , 0 , RRDR_OPTION_ABSOLUTE}

25 - , {"display_absolute", 0 , RRDR_OPTION_DISPLAY_ABS}

26 - , {"display-absolute", 0 , RRDR_OPTION_DISPLAY_ABS}

27 - , {"seconds" , 0 , RRDR_OPTION_SECONDS}

28 - , {"null2zero" , 0 , RRDR_OPTION_NULL2ZERO}

29 - , {"objectrows" , 0 , RRDR_OPTION_OBJECTSROWS}

30 - , {"google_json" , 0 , RRDR_OPTION_GOOGLE_JSON}

31 - , {"google-json" , 0 , RRDR_OPTION_GOOGLE_JSON}

32 - , {"percentage" , 0 , RRDR_OPTION_PERCENTAGE}

33 - , {"unaligned" , 0 , RRDR_OPTION_NOT_ALIGNED}

34 - , {"match_ids" , 0 , RRDR_OPTION_MATCH_IDS}

35 - , {"match-ids" , 0 , RRDR_OPTION_MATCH_IDS}

36 - , {"match_names" , 0 , RRDR_OPTION_MATCH_NAMES}

37 - , {"match-names" , 0 , RRDR_OPTION_MATCH_NAMES}

38 - , {"showcustomvars" , 0 , RRDR_OPTION_CUSTOM_VARS}

39 - , {"anomaly-bit" , 0 , RRDR_OPTION_ANOMALY_BIT}

40 - , {"raw" , 0 , RRDR_OPTION_RETURN_RAW}

41 - , {NULL , 0 , 0}

13 + { "nonzero" , 0 , RRDR_OPTION_NONZERO}

14 + , {"flip" , 0 , RRDR_OPTION_REVERSED}

15 + , {"reversed" , 0 , RRDR_OPTION_REVERSED}

16 + , {"reverse" , 0 , RRDR_OPTION_REVERSED}

17 + , {"jsonwrap" , 0 , RRDR_OPTION_JSON_WRAP}

18 + , {"min2max" , 0 , RRDR_OPTION_MIN2MAX}

19 + , {"ms" , 0 , RRDR_OPTION_MILLISECONDS}

20 + , {"milliseconds" , 0 , RRDR_OPTION_MILLISECONDS}

21 + , {"abs" , 0 , RRDR_OPTION_ABSOLUTE}

22 + , {"absolute" , 0 , RRDR_OPTION_ABSOLUTE}

23 + , {"absolute_sum" , 0 , RRDR_OPTION_ABSOLUTE}

24 + , {"absolute-sum" , 0 , RRDR_OPTION_ABSOLUTE}

25 + , {"display_absolute" , 0 , RRDR_OPTION_DISPLAY_ABS}

26 + , {"display-absolute" , 0 , RRDR_OPTION_DISPLAY_ABS}

27 + , {"seconds" , 0 , RRDR_OPTION_SECONDS}

28 + , {"null2zero" , 0 , RRDR_OPTION_NULL2ZERO}

29 + , {"objectrows" , 0 , RRDR_OPTION_OBJECTSROWS}

30 + , {"google_json" , 0 , RRDR_OPTION_GOOGLE_JSON}

31 + , {"google-json" , 0 , RRDR_OPTION_GOOGLE_JSON}

32 + , {"percentage" , 0 , RRDR_OPTION_PERCENTAGE}

33 + , {"unaligned" , 0 , RRDR_OPTION_NOT_ALIGNED}

34 + , {"match_ids" , 0 , RRDR_OPTION_MATCH_IDS}

35 + , {"match-ids" , 0 , RRDR_OPTION_MATCH_IDS}

36 + , {"match_names" , 0 , RRDR_OPTION_MATCH_NAMES}

37 + , {"match-names" , 0 , RRDR_OPTION_MATCH_NAMES}

38 + , {"showcustomvars" , 0 , RRDR_OPTION_CUSTOM_VARS}

39 + , {"anomaly-bit" , 0 , RRDR_OPTION_ANOMALY_BIT}

40 + , {"raw" , 0 , RRDR_OPTION_RETURN_RAW}

41 + , {"jw-anomaly-rates" , 0 , RRDR_OPTION_RETURN_JWAR}

42 + , {NULL , 0 , 0}

43 };

45 static struct {

@@ -434,6 +435,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c

435 char *context = NULL;

436 char *chart_label_key = NULL;

437 char *chart_labels_filter = NULL;

438 + char *group_options = NULL;

439

440 int group = RRDR_GROUPING_AVERAGE;

441 int show_dimensions = 0;

@@ -468,6 +470,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c

470 else if(!strcmp(name, "points")) points_str = value;

471 else if(!strcmp(name, "timeout")) timeout_str = value;

472 else if(!strcmp(name, "gtime")) group_time_str = value;

473 + else if(!strcmp(name, "group_options")) group_options = value;

474 else if(!strcmp(name, "group")) {

475 group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE);

476 }

@@ -675,7 +678,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c

678 .wb = w->response.data};

679

680 ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,

678 - points, after, before, group, group_time, options, &last_timestamp_in_data);

681 + points, after, before, group, group_options, group_time, options, &last_timestamp_in_data);

682

683 free_context_param_list(owa, &context_param_list);

684

@@ -1337,6 +1340,7 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie

1340 METRIC_CORRELATIONS_METHOD method = default_metric_correlations_method;

1341 RRDR_GROUPING group = RRDR_GROUPING_AVERAGE;

1342 int timeout = 0;

1343 + const char *group_options = NULL;

1344

1345 while (url) {

1346 char *value = mystrsep(&url, "&");

@@ -1383,7 +1387,7 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie

1387 wb->contenttype = CT_APPLICATION_JSON;

1388 buffer_no_cacheable(wb);

1389

1386 - return metric_correlations(host, wb, method, group, baseline_after, baseline_before, after, before, points, options, timeout);

1390 + return metric_correlations(host, wb, method, group, group_options, baseline_after, baseline_before, after, before, points, options, timeout);

1391 }

1392

1393 static struct api_command {