@cryptotaxi247 / netdata-1 / commits / 87c0cc2d6

One way allocator to double the speed of parallel context queries (#12787)

* one way allocator to speed up context queries * fixed a bug while expanding memory pages * reworked for clarity and finally fixed the bug of allocating memory beyond the page size * further optimize allocation step to minimize the number of allocations made * implement strdup with memcpy instead of strcpy * added documentation * prevent an uninitialized use of owa * added callocz() interface * integrate onewayalloc everywhere - apart sql queries * one way allocator is now used in context queries using archived charts in sql * align on the size of pointers * forgotten freez() * removed not needed memcpys * give unique names to global variables to avoid conflicts with system definitions

Costa Tsaousis committed May 3, 2022 at 00:31 UTC 87c0cc2d6049c46f38b9c866668a0a24a3e962c0
18 files changed +389 -95
CMakeLists.txt
+2
@@ -383,6 +383,8 @@ set(LIBNETDATA_FILES
383 libnetdata/log/log.h
384 libnetdata/os.c
385 libnetdata/os.h
386 + libnetdata/onewayalloc/onewayalloc.c
387 + libnetdata/onewayalloc/onewayalloc.h
388 libnetdata/popen/popen.c
389 libnetdata/popen/popen.h
390 libnetdata/procfile/procfile.c
Makefile.am
+2
@@ -158,6 +158,8 @@ LIBNETDATA_FILES = \
158 libnetdata/locks/locks.h \
159 libnetdata/log/log.c \
160 libnetdata/log/log.h \
161 + libnetdata/onewayalloc/onewayalloc.c \
162 + libnetdata/onewayalloc/onewayalloc.h \
163 libnetdata/popen/popen.c \
164 libnetdata/popen/popen.h \
165 libnetdata/procfile/procfile.c \
configure.ac
+1
@@ -1758,6 +1758,7 @@ AC_CONFIG_FILES([
1758 libnetdata/eval/Makefile
1759 libnetdata/locks/Makefile
1760 libnetdata/log/Makefile
1761 + libnetdata/onewayalloc/Makefile
1762 libnetdata/popen/Makefile
1763 libnetdata/procfile/Makefile
1764 libnetdata/simple_pattern/Makefile
daemon/unit_test.c
+8 -4
@@ -1732,7 +1732,8 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
1732 update_every = REGION_UPDATE_EVERY[current_region];
1733 long points = (time_end - time_start) / update_every;
1734 for (i = 0 ; i < CHARTS ; ++i) {
1735 - RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);
1735 + ONEWAYALLOC *owa = onewayalloc_create(0);
1736 + RRDR *r = rrd2rrdr(owa, st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);
1737 if (!r) {
1738 fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
1739 return ++errors;
@@ -1766,8 +1767,9 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
1767 }
1768 }
1769 }
1769 - rrdr_free(r);
1770 + rrdr_free(owa, r);
1771 }
1772 + onewayalloc_destroy(owa);
1773 }
1774 return errors;
1775 }
@@ -1851,7 +1853,8 @@ int test_dbengine(void)
1853 long points = (time_end[REGIONS - 1] - time_start[0]) / update_every; // cover all time regions with RRDR
1854 long point_offset = (time_start[current_region] - time_start[0]) / update_every;
1855 for (i = 0 ; i < CHARTS ; ++i) {
1854 - RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);
1856 + ONEWAYALLOC *owa = onewayalloc_create(0);
1857 + RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0);
1858 if (!r) {
1859 fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
1860 ++errors;
@@ -1888,8 +1891,9 @@ int test_dbengine(void)
1891 }
1892 }
1893 }
1891 - rrdr_free(r);
1894 + rrdr_free(owa, r);
1895 }
1896 + onewayalloc_destroy(owa);
1897 }
1898 error_out:
1899 rrd_wrlock();
database/sqlite/sqlite_functions.c
+19 -19
@@ -1446,13 +1446,13 @@ int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id
1446 }
1447
1448 #ifdef ENABLE_DBENGINE
1449 -static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
1449 +static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
1450 {
1451 - RRDDIM *rd = callocz(1, sizeof(*rd));
1451 + RRDDIM *rd = onewayalloc_callocz(owa, 1, sizeof(*rd));
1452 rd->rrdset = st;
1453 rd->last_stored_value = NAN;
1454 rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
1455 - rd->state = mallocz(sizeof(*rd->state));
1455 + rd->state = onewayalloc_mallocz(owa, sizeof(*rd->state));
1456 rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
1457 rd->state->query_ops.init = rrdeng_load_metric_init;
1458 rd->state->query_ops.next_metric = rrdeng_load_metric_next;
@@ -1460,11 +1460,11 @@ static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metr
1460 rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
1461 rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
1462 rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
1463 - rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t));
1463 + rd->state->rrdeng_uuid = onewayalloc_mallocz(owa, sizeof(uuid_t));
1464 uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid);
1465 uuid_copy(rd->state->metric_uuid, *metric_uuid);
1466 - rd->id = strdupz(id);
1467 - rd->name = strdupz(name);
1466 + rd->id = onewayalloc_strdupz(owa, id);
1467 + rd->name = onewayalloc_strdupz(owa, name);
1468 return rd;
1469 }
1470 #endif
@@ -1481,7 +1481,7 @@ static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metr
1481 "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \
1482 "order by c.chart_id asc, c.type||'.'||c.id desc;"
1483
1484 -void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart)
1484 +void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart)
1485 {
1486 #ifdef ENABLE_DBENGINE
1487 int rc;
@@ -1490,7 +1490,7 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
1490 return;
1491
1492 if (unlikely(!(*param_list))) {
1493 - *param_list = mallocz(sizeof(struct context_param));
1493 + *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
1494 (*param_list)->first_entry_t = LONG_MAX;
1495 (*param_list)->last_entry_t = 0;
1496 (*param_list)->rd = NULL;
@@ -1539,21 +1539,21 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
1539
1540 if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) {
1541 if (unlikely(st && !st->counter)) {
1542 - freez(st->context);
1543 - freez((char *) st->name);
1544 - freez(st);
1542 + onewayalloc_freez(owa, st->context);
1543 + onewayalloc_freez(owa, (char *) st->name);
1544 + onewayalloc_freez(owa, st);
1545 }
1546 - st = callocz(1, sizeof(*st));
1546 + st = onewayalloc_callocz(owa, 1, sizeof(*st));
1547 char n[RRD_ID_LENGTH_MAX + 1];
1548
1549 snprintfz(
1550 n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4),
1551 (char *)sqlite3_column_text(res, 3));
1552 - st->name = strdupz(n);
1552 + st->name = onewayalloc_strdupz(owa, n);
1553 st->update_every = sqlite3_column_int(res, 6);
1554 st->counter = 0;
1555 if (chart) {
1556 - st->context = strdupz((char *)sqlite3_column_text(res, 8));
1556 + st->context = onewayalloc_strdupz(owa, (char *)sqlite3_column_text(res, 8));
1557 strncpyz(st->id, chart, RRD_ID_LENGTH_MAX);
1558 }
1559 uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7));
@@ -1569,7 +1569,7 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
1569 st->counter++;
1570 st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t);
1571
1572 - RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
1572 + RRDDIM *rd = create_rrdim_entry(owa, st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
1573 if (sqlite3_column_int(res, 9) == 1)
1574 rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
1575 rd->next = (*param_list)->rd;
@@ -1577,13 +1577,13 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
1577 }
1578 if (st) {
1579 if (!st->counter) {
1580 - freez(st->context);
1581 - freez((char *)st->name);
1582 - freez(st);
1580 + onewayalloc_freez(owa,st->context);
1581 + onewayalloc_freez(owa,(char *)st->name);
1582 + onewayalloc_freez(owa,st);
1583 }
1584 else
1585 if (!st->context && context)
1586 - st->context = strdupz(context);
1586 + st->context = onewayalloc_strdupz(owa,context);
1587 }
1588
1589 failed:
database/sqlite/sqlite_functions.h
+1 -1
@@ -89,7 +89,7 @@ extern void db_unlock(void);
89 extern void db_lock(void);
90 extern void delete_dimension_uuid(uuid_t *dimension_uuid);
91 extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value);
92 -extern void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart);
92 +extern void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart);
93 extern void store_claim_id(uuid_t *host_id, uuid_t *claim_id);
94 extern int update_node_id(uuid_t *host_id, uuid_t *node_id);
95 extern int get_node_id(uuid_t *host_id, uuid_t *node_id);
libnetdata/Makefile.am
+1
@@ -17,6 +17,7 @@ SUBDIRS = \
17 health \
18 locks \
19 log \
20 + onewayalloc \
21 popen \
22 procfile \
23 simple_pattern \
libnetdata/libnetdata.h
+1
@@ -345,6 +345,7 @@ extern char *netdata_configured_host_prefix;
345 #include "json/json.h"
346 #include "health/health.h"
347 #include "string/utf8.h"
348 +#include "onewayalloc/onewayalloc.h"
349
350 // BEWARE: Outside of the C code this also exists in alarm-notify.sh
351 #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
libnetdata/onewayalloc/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
libnetdata/onewayalloc/README.md new
+71
@@ -0,0 +1,71 @@
1 +<!--
2 +title: "One Way Allocator"
3 +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/onewayallocator/README.md
4 +-->
5 +
6 +# One Way Allocator
7 +
8 +This is a very fast single-threaded-only memory allocator, that minimized system calls
9 +when a lot of memory allocations needs to be made to perform a task, which all of them
10 +can be freed together when the task finishes.
11 +
12 +It has been designed to be used for netdata context queries.
13 +
14 +For netdata to perform a context query, it builds a virtual chart, a chart that contains
15 +all the dimensions of the charts having the same context. This process requires allocating
16 +several structures for each of the dimensions to attach them to the virtual chart. All
17 +these data can be freed immediately after the query finishes.
18 +
19 +## How it works
20 +
21 +1. The caller calls `ONEWAYALLOC *owa = onewayalloc_create(sizehint)` to create an OWA.
22 + Internally this allocates the first memory buffer with size >= `sizehint`.
23 + If `sizehint` is zero, it will allocate 1 hardware page (usually 4kb).
24 + No need to check for success or failure. As with `mallocz()` in netdata, a `fatal()`
25 + will be called if the allocation fails - although this will never fail, since Linux
26 + does not really check if there is memory available for `mmap()` calls.
27 +
28 +2. The caller can then perform any number of the following calls to acquire memory:
29 + - `onewayalloc_mallocz(owa, size)`, similar to `mallocz()`
30 + - `onewayalloc_callocz(owa, nmemb, size)`, similar to `callocz()`
31 + - `onewayalloc_strdupz(owa, string)`, similar to `strdupz()`
32 + - `onewayalloc_memdupz(owa, ptr, size)`, similar to `mallocz()` and then `memcpy()`
33 +
34 +3. Once the caller has done all the work with the allocated buffers, all memory allocated
35 + can be freed with `onewayalloc_destroy(owa)`.
36 +
37 +## How faster it is?
38 +
39 +On modern hardware, for any single query the performance improvement is marginal and not
40 +noticeable at all.
41 +
42 +We performed the following tests using the same huge context query (1000 charts,
43 +100 dimensions each = 100k dimensions)
44 +
45 +1. using `mallocz()`, 1 caller, 256 queries (sequential)
46 +2. using `mallocz()`, 256 callers, 1 query each (parallel)
47 +3. using `OWA`, 1 caller, 256 queries (sequential)
48 +4. using `OWA`, 256 callers, 1 query each (parallel)
49 +
50 +Netdata was configured to use 24 web threads on the 24 core server we used.
51 +
52 +The results are as follows:
53 +
54 +### sequential test
55 +
56 +branch|transactions|time to complete|transaction rate|average response time|min response time|max response time
57 +:---:|:---:|:---:|:---:|:---:|:---:|:---:|
58 +`malloc()`|256|322.35s|0.79/sec|1.26s|1.01s|1.87s
59 +`OWA`|256|310.19s|0.83/sec|1.21s|1.04s|1.63s
60 +
61 +For a single query, the improvement is just marginal and not noticeable at all.
62 +
63 +### parallel test
64 +
65 +branch|transactions|time to complete|transaction rate|average response time|min response time|max response time
66 +:---:|:---:|:---:|:---:|:---:|:---:|:---:|
67 +`malloc()`|256|84.72s|3.02/sec|68.43s|50.20s|84.71s
68 +`OWA`|256|39.35s|6.51/sec|34.48s|20.55s|39.34s
69 +
70 +For parallel workload, like the one executed by netdata.cloud, `OWA` provides a 54% overall speed improvement (more than double the overall
71 +user-experienced speed, including the data query itself).
libnetdata/onewayalloc/onewayalloc.c new
+173
@@ -0,0 +1,173 @@
1 +#include "onewayalloc.h"
2 +
3 +static size_t OWA_NATURAL_PAGE_SIZE = 0;
4 +static size_t OWA_NATURAL_ALIGNMENT = sizeof(int*);
5 +
6 +typedef struct owa_page {
7 + size_t stats_pages;
8 + size_t stats_pages_size;
9 + size_t stats_mallocs_made;
10 + size_t stats_mallocs_size;
11 + size_t size; // the total size of the page
12 + size_t offset; // the first free byte of the page
13 + struct owa_page *next; // the next page on the list
14 + struct owa_page *last; // the last page on the list - we currently allocate on this
15 +} OWA_PAGE;
16 +
17 +// allocations need to be aligned to CPU register width
18 +// https://en.wikipedia.org/wiki/Data_structure_alignment
19 +static inline size_t natural_alignment(size_t size) {
20 + if(unlikely(size % OWA_NATURAL_ALIGNMENT))
21 + size = size + OWA_NATURAL_ALIGNMENT - (size % OWA_NATURAL_ALIGNMENT);
22 +
23 + return size;
24 +}
25 +
26 +// Create an OWA
27 +// Once it is created, the called may call the onewayalloc_mallocz()
28 +// any number of times, for any amount of memory.
29 +
30 +static OWA_PAGE *onewayalloc_create_internal(OWA_PAGE *head, size_t size_hint) {
31 + if(unlikely(!OWA_NATURAL_PAGE_SIZE))
32 + OWA_NATURAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
33 +
34 + // our default page size
35 + size_t size = OWA_NATURAL_PAGE_SIZE;
36 +
37 + // make sure the new page will fit both the requested size
38 + // and the OWA_PAGE structure at its beginning
39 + size_hint += sizeof(OWA_PAGE);
40 +
41 + // prefer the user size if it is bigger than our size
42 + if(size_hint > size) size = size_hint;
43 +
44 + // try to allocate half of the total we have allocated already
45 + if(likely(head)) {
46 + size_t optimal_size = head->stats_pages_size / 2;
47 + if(optimal_size > size) size = optimal_size;
48 + }
49 +
50 + // Make sure our allocations are always a multiple of the hardware page size
51 + if(size % OWA_NATURAL_PAGE_SIZE) size = size + OWA_NATURAL_PAGE_SIZE - (size % OWA_NATURAL_PAGE_SIZE);
52 +
53 + OWA_PAGE *page = (OWA_PAGE *)netdata_mmap(NULL, size, MAP_ANONYMOUS|MAP_PRIVATE, 0);
54 + if(unlikely(!page)) fatal("Cannot allocate onewayalloc buffer of size %zu", size);
55 +
56 + page->size = size;
57 + page->offset = natural_alignment(sizeof(OWA_PAGE));
58 + page->next = page->last = NULL;
59 +
60 + if(unlikely(!head)) {
61 + // this is the first time we are called
62 + head = page;
63 + head->stats_pages = 0;
64 + head->stats_pages_size = 0;
65 + head->stats_mallocs_made = 0;
66 + head->stats_mallocs_size = 0;
67 + }
68 + else {
69 + // link this page into our existing linked list
70 + head->last->next = page;
71 + }
72 +
73 + head->last = page;
74 + head->stats_pages++;
75 + head->stats_pages_size += size;
76 +
77 + return (ONEWAYALLOC *)page;
78 +}
79 +
80 +ONEWAYALLOC *onewayalloc_create(size_t size_hint) {
81 + return onewayalloc_create_internal(NULL, size_hint);
82 +}
83 +
84 +void *onewayalloc_mallocz(ONEWAYALLOC *owa, size_t size) {
85 + OWA_PAGE *head = (OWA_PAGE *)owa;
86 + OWA_PAGE *page = head->last;
87 +
88 + // update stats
89 + head->stats_mallocs_made++;
90 + head->stats_mallocs_size += size;
91 +
92 + // make sure the size is aligned
93 + size = natural_alignment(size);
94 +
95 + if(unlikely(page->size - page->offset < size)) {
96 + // we don't have enough space to fit the data
97 + // let's get another page
98 + page = onewayalloc_create_internal(head, (size > page->size)?size:page->size);
99 + }
100 +
101 + char *mem = (char *)page;
102 + mem = &mem[page->offset];
103 + page->offset += size;
104 +
105 + return (void *)mem;
106 +}
107 +
108 +void *onewayalloc_callocz(ONEWAYALLOC *owa, size_t nmemb, size_t size) {
109 + size_t total = nmemb * size;
110 + void *mem = onewayalloc_mallocz(owa, total);
111 + memset(mem, 0, total);
112 + return mem;
113 +}
114 +
115 +char *onewayalloc_strdupz(ONEWAYALLOC *owa, const char *s) {
116 + size_t size = strlen(s) + 1;
117 + char *d = onewayalloc_mallocz((OWA_PAGE *)owa, size);
118 + memcpy(d, s, size);
119 + return d;
120 +}
121 +
122 +void *onewayalloc_memdupz(ONEWAYALLOC *owa, const void *src, size_t size) {
123 + void *mem = onewayalloc_mallocz((OWA_PAGE *)owa, size);
124 + // memcpy() is way faster than strcpy() since it does not check for '\0'
125 + memcpy(mem, src, size);
126 + return mem;
127 +}
128 +
129 +void onewayalloc_freez(ONEWAYALLOC *owa __maybe_unused, const void *ptr __maybe_unused) {
130 +#ifdef NETDATA_INTERNAL_CHECKS
131 + // allow the caller to call us for a mallocz() allocation
132 + // so try to find it in our memory and if it is not there
133 + // log an error
134 +
135 + OWA_PAGE *head = (OWA_PAGE *)owa;
136 + OWA_PAGE *page;
137 + size_t seeking = (size_t)ptr;
138 +
139 + for(page = head; page ;page = page->next) {
140 + size_t start = (size_t)page;
141 + size_t end = start + page->size;
142 +
143 + if(seeking >= start && seeking <= end) {
144 + // found it - it is ours
145 + // just return to let the caller think we actually did something
146 + return;
147 + }
148 + }
149 +
150 + // not found - it is not ours
151 + // let's free it with the system allocator
152 + error("ONEWAYALLOC: request to free address 0x%p that is not allocated by this OWA", ptr);
153 +#endif
154 +
155 + return;
156 +}
157 +
158 +void onewayalloc_destroy(ONEWAYALLOC *owa) {
159 + if(!owa) return;
160 +
161 + OWA_PAGE *head = (OWA_PAGE *)owa;
162 +
163 + //info("OWA: %zu allocations of %zu total bytes, in %zu pages of %zu total bytes",
164 + // head->stats_mallocs_made, head->stats_mallocs_size,
165 + // head->stats_pages, head->stats_pages_size);
166 +
167 + OWA_PAGE *page = head;
168 + while(page) {
169 + OWA_PAGE *p = page;
170 + page = page->next;
171 + munmap(p, p->size);
172 + }
173 +}
libnetdata/onewayalloc/onewayalloc.h new
+17
@@ -0,0 +1,17 @@
1 +#ifndef ONEWAYALLOC_H
2 +#define ONEWAYALLOC_H 1
3 +
4 +#include "../libnetdata.h"
5 +
6 +typedef void ONEWAYALLOC;
7 +
8 +extern ONEWAYALLOC *onewayalloc_create(size_t size_hint);
9 +extern void onewayalloc_destroy(ONEWAYALLOC *owa);
10 +
11 +extern void *onewayalloc_mallocz(ONEWAYALLOC *owa, size_t size);
12 +extern void *onewayalloc_callocz(ONEWAYALLOC *owa, size_t nmemb, size_t size);
13 +extern char *onewayalloc_strdupz(ONEWAYALLOC *owa, const char *s);
14 +extern void *onewayalloc_memdupz(ONEWAYALLOC *owa, const void *src, size_t size);
15 +extern void onewayalloc_freez(ONEWAYALLOC *owa, const void *ptr);
16 +
17 +#endif // ONEWAYALLOC_H
web/api/formatters/rrd2json.c
+40 -35
@@ -2,27 +2,27 @@
2
3 #include "web/api/web_api_v1.h"
4
5 -static inline void free_single_rrdrim(RRDDIM *temp_rd, int archive_mode)
5 +static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
6 {
7 if (unlikely(!temp_rd))
8 return;
9
10 - freez((char *)temp_rd->id);
11 - freez((char *)temp_rd->name);
10 + onewayalloc_freez(owa, (char *)temp_rd->id);
11
12 if (unlikely(archive_mode)) {
13 temp_rd->rrdset->counter--;
14 if (!temp_rd->rrdset->counter) {
16 - freez((char *)temp_rd->rrdset->name);
17 - freez(temp_rd->rrdset->context);
18 - freez(temp_rd->rrdset);
15 + onewayalloc_freez(owa, (char *)temp_rd->rrdset->name);
16 + onewayalloc_freez(owa, temp_rd->rrdset->context);
17 + onewayalloc_freez(owa, temp_rd->rrdset);
18 }
19 }
21 - freez(temp_rd->state);
22 - freez(temp_rd);
20 +
21 + onewayalloc_freez(owa, temp_rd->state);
22 + onewayalloc_freez(owa, temp_rd);
23 }
24
25 -static inline void free_rrddim_list(RRDDIM *temp_rd, int archive_mode)
25 +static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
26 {
27 if (unlikely(!temp_rd))
28 return;
@@ -30,22 +30,22 @@ static inline void free_rrddim_list(RRDDIM *temp_rd, int archive_mode)
30 RRDDIM *t;
31 while (temp_rd) {
32 t = temp_rd->next;
33 - free_single_rrdrim(temp_rd, archive_mode);
33 + free_single_rrdrim(owa, temp_rd, archive_mode);
34 temp_rd = t;
35 }
36 }
37
38 -void free_context_param_list(struct context_param **param_list)
38 +void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list)
39 {
40 if (unlikely(!param_list || !*param_list))
41 return;
42
43 - free_rrddim_list(((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE);
44 - freez((*param_list));
43 + free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE);
44 + onewayalloc_freez(owa, (*param_list));
45 *param_list = NULL;
46 }
47
48 -void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested)
48 +void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested)
49 {
50 RRDDIM *temp_rd = context_param_list->rd;
51 RRDDIM *new_rd_list = NULL, *t;
@@ -59,19 +59,19 @@ void rebuild_context_param_list(struct context_param *context_param_list, time_t
59 temp_rd->next = new_rd_list;
60 new_rd_list = temp_rd;
61 } else
62 - free_single_rrdrim(temp_rd, is_archived);
62 + free_single_rrdrim(owa, temp_rd, is_archived);
63 temp_rd = t;
64 }
65 context_param_list->rd = new_rd_list;
66 };
67
68 -void build_context_param_list(struct context_param **param_list, RRDSET *st)
68 +void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st)
69 {
70 if (unlikely(!param_list || !st))
71 return;
72
73 if (unlikely(!(*param_list))) {
74 - *param_list = mallocz(sizeof(struct context_param));
74 + *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
75 (*param_list)->first_entry_t = LONG_MAX;
76 (*param_list)->last_entry_t = 0;
77 (*param_list)->flags = CONTEXT_FLAGS_CONTEXT;
@@ -86,14 +86,10 @@ void build_context_param_list(struct context_param **param_list, RRDSET *st)
86 (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st));
87
88 rrddim_foreach_read(rd1, st) {
89 - RRDDIM *rd = mallocz(rd1->memsize);
90 - memcpy(rd, rd1, rd1->memsize);
91 - rd->id = strdupz(rd1->id);
92 - rd->name = strdupz(rd1->name);
93 - rd->state = mallocz(sizeof(*rd->state));
94 - memcpy(rd->state, rd1->state, sizeof(*rd->state));
95 - memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops));
96 - memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops));
89 + RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize);
90 + rd->id = onewayalloc_strdupz(owa, rd1->id);
91 + rd->name = onewayalloc_strdupz(owa, rd1->name);
92 + rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state));
93 rd->next = (*param_list)->rd;
94 (*param_list)->rd = rd;
95 }
@@ -169,22 +165,27 @@ int rrdset2value_api_v1(
165 , int *value_is_null
166 , int timeout
167 ) {
168 + int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
169 +
170 + ONEWAYALLOC *owa = onewayalloc_create(0);
171
173 - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout);
172 + RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout);
173
174 if(!r) {
175 if(value_is_null) *value_is_null = 1;
177 - return HTTP_RESP_INTERNAL_SERVER_ERROR;
176 + ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
177 + goto cleanup;
178 }
179
180 if(rrdr_rows(r) == 0) {
181 - rrdr_free(r);
181 + rrdr_free(owa, r);
182
183 if(db_after) *db_after = 0;
184 if(db_before) *db_before = 0;
185 if(value_is_null) *value_is_null = 1;
186
187 - return HTTP_RESP_BAD_REQUEST;
187 + ret = HTTP_RESP_BAD_REQUEST;
188 + goto cleanup;
189 }
190
191 if(wb) {
@@ -199,13 +200,17 @@ int rrdset2value_api_v1(
200
201 long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
202 *n = rrdr2value(r, i, options, value_is_null, NULL);
203 + ret = HTTP_RESP_OK;
204
203 - rrdr_free(r);
204 - return HTTP_RESP_OK;
205 +cleanup:
206 + if(r) rrdr_free(owa, r);
207 + onewayalloc_destroy(owa);
208 + return ret;
209 }
210
211 int rrdset2anything_api_v1(
208 - RRDSET *st
212 + ONEWAYALLOC *owa
213 + , RRDSET *st
214 , BUFFER *wb
215 , BUFFER *dimensions
216 , uint32_t format
@@ -225,14 +230,14 @@ int rrdset2anything_api_v1(
230 if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
231 st->last_accessed_time = now_realtime_sec();
232
228 - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
233 + RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
234 if(!r) {
235 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
236 return HTTP_RESP_INTERNAL_SERVER_ERROR;
237 }
238
239 if (r->result_options & RRDR_RESULT_OPTION_CANCEL) {
235 - rrdr_free(r);
240 + rrdr_free(owa, r);
241 return HTTP_RESP_BACKEND_FETCH_FAILED;
242 }
243
@@ -411,6 +416,6 @@ int rrdset2anything_api_v1(
416 break;
417 }
418
414 - rrdr_free(r);
419 + rrdr_free(owa, r);
420 return HTTP_RESP_OK;
421 }
web/api/formatters/rrd2json.h
+5 -4
@@ -54,7 +54,8 @@ extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
54 extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
55
56 extern int rrdset2anything_api_v1(
57 - RRDSET *st
57 + ONEWAYALLOC *owa
58 + , RRDSET *st
59 , BUFFER *wb
60 , BUFFER *dimensions
61 , uint32_t format
@@ -88,8 +89,8 @@ extern int rrdset2value_api_v1(
89 , int timeout
90 );
91
91 -extern void build_context_param_list(struct context_param **param_list, RRDSET *st);
92 -extern void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested);
93 -extern void free_context_param_list(struct context_param **param_list);
92 +extern void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st);
93 +extern void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested);
94 +extern void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list);
95
96 #endif /* NETDATA_RRD2JSON_H */
web/api/queries/query.c
+14 -11
@@ -831,7 +831,8 @@ static int rrdr_convert_before_after_to_absolute(
831 }
832
833 static RRDR *rrd2rrdr_fixedstep(
834 - RRDSET *st
834 + ONEWAYALLOC *owa
835 + , RRDSET *st
836 , long points_requested
837 , long long after_requested
838 , long long before_requested
@@ -855,7 +856,7 @@ static RRDR *rrd2rrdr_fixedstep(
856 RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
857
858 if(duration <= 0 || available_points <= 0)
858 - return rrdr_create(st, 1, context_param_list);
859 + return rrdr_create(owa, st, 1, context_param_list);
860
861 // check the number of wanted points in the result
862 if(unlikely(points_requested < 0)) points_requested = -points_requested;
@@ -1013,7 +1014,7 @@ static RRDR *rrd2rrdr_fixedstep(
1014 // initialize our result set
1015 // this also locks the chart for us
1016
1016 - RRDR *r = rrdr_create(st, points_wanted, context_param_list);
1017 + RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);
1018 if(unlikely(!r)) {
1019 #ifdef NETDATA_INTERNAL_CHECKS
1020 error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
@@ -1216,7 +1217,8 @@ static RRDR *rrd2rrdr_fixedstep(
1217
1218 #ifdef ENABLE_DBENGINE
1219 static RRDR *rrd2rrdr_variablestep(
1219 - RRDSET *st
1220 + ONEWAYALLOC *owa
1221 + , RRDSET *st
1222 , long points_requested
1223 , long long after_requested
1224 , long long before_requested
@@ -1242,7 +1244,7 @@ static RRDR *rrd2rrdr_variablestep(
1244
1245 if(duration <= 0 || available_points <= 0) {
1246 freez(region_info_array);
1245 - return rrdr_create(st, 1, context_param_list);
1247 + return rrdr_create(owa, st, 1, context_param_list);
1248 }
1249
1250 // check the number of wanted points in the result
@@ -1401,7 +1403,7 @@ static RRDR *rrd2rrdr_variablestep(
1403 // initialize our result set
1404 // this also locks the chart for us
1405
1404 - RRDR *r = rrdr_create(st, points_wanted, context_param_list);
1406 + RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);
1407 if(unlikely(!r)) {
1408 #ifdef NETDATA_INTERNAL_CHECKS
1409 error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
@@ -1608,7 +1610,8 @@ static RRDR *rrd2rrdr_variablestep(
1610 #endif //#ifdef ENABLE_DBENGINE
1611
1612 RRDR *rrd2rrdr(
1611 - RRDSET *st
1613 + ONEWAYALLOC *owa
1614 + , RRDSET *st
1615 , long points_requested
1616 , long long after_requested
1617 , long long before_requested
@@ -1644,7 +1647,7 @@ RRDR *rrd2rrdr(
1647 first_entry_t = after_requested;
1648
1649 if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
1647 - rebuild_context_param_list(context_param_list, after_requested);
1650 + rebuild_context_param_list(owa, context_param_list, after_requested);
1651 st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
1652 if (unlikely(!st))
1653 return NULL;
@@ -1669,7 +1672,7 @@ RRDR *rrd2rrdr(
1672 }
1673 freez(region_info_array);
1674 }
1672 - return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
1675 + return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method,
1676 resampling_time_requested, options, dimensions, rrd_update_every,
1677 first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);
1678 } else {
@@ -1680,13 +1683,13 @@ RRDR *rrd2rrdr(
1683 rrd_update_every, first_entry_t,
1684 last_entry_t, options);
1685 }
1683 - return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method,
1686 + return rrd2rrdr_variablestep(owa, st, points_requested, after_requested, before_requested, group_method,
1687 resampling_time_requested, options, dimensions, rrd_update_every,
1688 first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout);
1689 }
1690 }
1691 #endif
1689 - return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
1692 + return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method,
1693 resampling_time_requested, options, dimensions,
1694 rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);
1695 }
web/api/queries/rrdr.c
+12 -12
@@ -83,7 +83,7 @@ inline static void rrdr_unlock_rrdset(RRDR *r) {
83 }
84 }
85
86 -inline void rrdr_free(RRDR *r)
86 +inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r)
87 {
88 if(unlikely(!r)) {
89 error("NULL value given!");
@@ -91,21 +91,21 @@ inline void rrdr_free(RRDR *r)
91 }
92
93 rrdr_unlock_rrdset(r);
94 - freez(r->t);
95 - freez(r->v);
96 - freez(r->o);
97 - freez(r->od);
98 - freez(r);
94 + onewayalloc_freez(owa, r->t);
95 + onewayalloc_freez(owa, r->v);
96 + onewayalloc_freez(owa, r->o);
97 + onewayalloc_freez(owa, r->od);
98 + onewayalloc_freez(owa, r);
99 }
100
101 -RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list)
101 +RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list)
102 {
103 if (unlikely(!st)) {
104 error("NULL value given!");
105 return NULL;
106 }
107
108 - RRDR *r = callocz(1, sizeof(RRDR));
108 + RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
109 r->st = st;
110
111 if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
@@ -126,10 +126,10 @@ RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param
126
127 r->n = n;
128
129 - r->t = callocz((size_t)n, sizeof(time_t));
130 - r->v = mallocz(n * r->d * sizeof(calculated_number));
131 - r->o = mallocz(n * r->d * sizeof(RRDR_VALUE_FLAGS));
132 - r->od = mallocz(r->d * sizeof(RRDR_DIMENSION_FLAGS));
129 + r->t = onewayalloc_callocz(owa, (size_t)n, sizeof(time_t));
130 + r->v = onewayalloc_mallocz(owa, n * r->d * sizeof(calculated_number));
131 + r->o = onewayalloc_mallocz(owa, n * r->d * sizeof(RRDR_VALUE_FLAGS));
132 + r->od = onewayalloc_mallocz(owa, r->d * sizeof(RRDR_DIMENSION_FLAGS));
133
134 // set the hidden flag on hidden dimensions
135 int c;
web/api/queries/rrdr.h
+3 -2
@@ -102,13 +102,14 @@ typedef struct rrdresult {
102 #define rrdr_rows(r) ((r)->rows)
103
104 #include "database/rrd.h"
105 -extern void rrdr_free(RRDR *r);
106 -extern RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list);
105 +extern void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
106 +extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list);
107
108 #include "../web_api_v1.h"
109 #include "web/api/queries/query.h"
110
111 extern RRDR *rrd2rrdr(
112 + ONEWAYALLOC *owa,
113 RRDSET *st, long points_requested, long long after_requested, long long before_requested,
114 RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
115 struct context_param *context_param_list, int timeout);
web/api/web_api_v1.c
+11 -7
@@ -512,6 +512,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
512 fix_google_param(outFileName);
513
514 RRDSET *st = NULL;
515 + ONEWAYALLOC *owa = onewayalloc_create(0);
516
517 if((!chart || !*chart) && (!context)) {
518 buffer_sprintf(w->response.data, "No chart id is given at the request.");
@@ -519,8 +520,10 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
520 }
521
522 struct context_param *context_param_list = NULL;
523 +
524 if (context && !chart) {
525 RRDSET *st1;
526 +
527 uint32_t context_hash = simple_hash(context);
528
529 rrdhost_rdlock(host);
@@ -532,14 +535,14 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
535 (!chart_label_key || rrdset_contains_label_keylist(st1, chart_label_key)) &&
536 (!chart_labels_filter ||
537 rrdset_matches_label_keys(st1, chart_labels_filter, words, hash_key_list, &word_count, MAX_CHART_LABELS_FILTER)))
535 - build_context_param_list(&context_param_list, st1);
538 + build_context_param_list(owa, &context_param_list, st1);
539 }
540 rrdhost_unlock(host);
541 if (likely(context_param_list && context_param_list->rd)) // Just set the first one
542 st = context_param_list->rd->rrdset;
543 else {
544 if (!chart_label_key && !chart_labels_filter)
542 - sql_build_context_param_list(&context_param_list, host, context, NULL);
545 + sql_build_context_param_list(owa, &context_param_list, host, context, NULL);
546 }
547 }
548 else {
@@ -549,14 +552,14 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
552 if (likely(st))
553 st->last_accessed_time = now_realtime_sec();
554 else
552 - sql_build_context_param_list(&context_param_list, host, NULL, chart);
555 + sql_build_context_param_list(owa, &context_param_list, host, NULL, chart);
556 }
557
558 if (!st) {
559 if (likely(context_param_list && context_param_list->rd && context_param_list->rd->rrdset))
560 st = context_param_list->rd->rrdset;
561 else {
559 - free_context_param_list(&context_param_list);
562 + free_context_param_list(owa, &context_param_list);
563 context_param_list = NULL;
564 }
565 }
@@ -630,12 +633,12 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
633 buffer_strcat(w->response.data, "(");
634 }
635
633 - ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format,
636 + ret = rrdset2anything_api_v1(owa, st, w->response.data, dimensions, format,
637 points, after, before, group, group_time,
638 options, &last_timestamp_in_data, context_param_list,
639 chart_label_key, max_anomaly_rates, timeout);
640
638 - free_context_param_list(&context_param_list);
641 + free_context_param_list(owa, &context_param_list);
642
643 if(format == DATASOURCE_DATATABLE_JSONP) {
644 if(google_timestamp < last_timestamp_in_data)
@@ -652,7 +655,8 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
655 else if(format == DATASOURCE_JSONP)
656 buffer_strcat(w->response.data, ");");
657
655 - cleanup:
658 +cleanup:
659 + onewayalloc_destroy(owa);
660 buffer_free(dimensions);
661 return ret;
662 }