@cryptotaxi247 / netdata-1 / commits / 5eed0545d

/api/v2/X part 5 (#14718)

* query timestamps are now pre-determined and alignment on timestamps is guarranteed * turn internal_fatal() to internal_error() to investigate the issue * handle query when no data exist in the db * check for non NULL dict when running dictionary garbage collect * support API v2 requests via ACLK * add nodes detailed information to /api/v2/nodes * fixed keys and added dummy nodes for completeness * added nodes_hard_hash, alerts_hard_hash, alerts_soft_hash; started building a nodes status object to reflect the current status of a node * make sure replication does not double count charts that are already being replicated * expose min and max in sts structures * added view_minimum_value and view_maximum_value; percentage calculation is now an additional pass on the data, removed from formatters; absolute value calculation is now done at the query level, removed from formatters * respect trimming in percentage calculation; updated swagger * api/v2/weights preparative work to support multi-node queries - still single node though * multi-node /api/v2/weights endpoint, supporting all the filtering parameters of /api/v2/data * when passing the raw option, the query exposes the hidden dimensions * fix compilation issues on older systems * the query engine now calculates per dimension min, max, sum, count, anomaly count * use the macro to calculate storage point anomaly rate * weights endpoint exposing version hashes * weights method=value shows min, max, average, sum, count, anomaly count, anomaly rate * query: expose RESET flag; do not add the same point multiple times to the aggregated point * weights: more compact output * weights requests can be interrupted * all /api/v2 requests can be interrupted and timeout * allow relative timestamps in weights * fix macos compilation warnings * Revert "fix macos compilation warnings" This reverts commit 8a1d24e41e9b58de566ac59f0c4b1c465bcc0592. * /api/v2/data group-by now works on dimension names, not ids * /api/v2/weights does not query metrics without retention and new output format * /api/v2/weights value and anomaly queries do context queries when contexts are filtered; query timeout is now always in ms

Costa Tsaousis committed Mar 21, 2023 at 21:53 UTC 5eed0545d46c3d227b87a23b0d0362a4f63e1bf0

36 files changed +1979 -907

aclk/aclk_query.c

+18 -3

@@ -11,12 +11,17 @@ pthread_mutex_t query_lock_wait = PTHREAD_MUTEX_INITIALIZER;

11 #define QUERY_THREAD_LOCK pthread_mutex_lock(&query_lock_wait)

12 #define QUERY_THREAD_UNLOCK pthread_mutex_unlock(&query_lock_wait)

14 -static usec_t aclk_web_api_v1_request(RRDHOST *host, struct web_client *w, char *url)

14 +static usec_t aclk_web_api_request(RRDHOST *host, struct web_client *w, char *url, const size_t api_version)

15 {

16 usec_t t;

18 t = now_monotonic_high_precision_usec();

19 - w->response.code = web_client_api_request_v1(host, w, url);

19 +

20 + if(api_version == 2)

21 + w->response.code = web_client_api_request_v2(host, w, url);

22 + else

23 + w->response.code = web_client_api_request_v1(host, w, url);

24 +

25 t = now_monotonic_high_precision_usec() - t;

27 if (aclk_stats_enabled) {

@@ -119,6 +124,16 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)

124 }

125 }

126

127 + size_t api_version = 1;

128 + {

129 + char *s = strstr(query->data.http_api_v2.query, "/api/v");

130 + if(s && s[6]) {

131 + api_version = str2u(&s[6]);

132 + if(api_version != 1 && api_version != 2)

133 + api_version = 1;

134 + }

135 + }

136 +

137 char *mysep = strchr(query->data.http_api_v2.query, '?');

138 if (mysep) {

139 url_decode_r(w->decoded_query_string, mysep, NETDATA_WEB_REQUEST_URL_SIZE + 1);

@@ -136,7 +151,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)

151 }

152

153 // execute the query

139 - t = aclk_web_api_v1_request(query_host, w, mysep ? mysep + 1 : "noop");

154 + t = aclk_web_api_request(query_host, w, mysep ? mysep + 1 : "noop", api_version);

155 size = (w->mode == WEB_CLIENT_MODE_FILECOPY) ? w->response.rlen : w->response.data->len;

156 sent = size;

157

database/contexts/api_v2.c

+117 -23

@@ -92,8 +92,7 @@ struct rrdcontext_to_json_v2_data {

92 DICTIONARY *ctx;

94 CONTEXTS_V2_OPTIONS options;

95 - uint64_t hard_hash;

96 - uint64_t soft_hash;

95 + struct query_versions versions;

97 struct {

98 SIMPLE_PATTERN *scope_pattern;

@@ -177,7 +176,7 @@ static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_jso

176 return matched;

177 }

178

180 -static bool rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context __maybe_unused) {

179 +static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context __maybe_unused) {

180 struct rrdcontext_to_json_v2_data *ctl = data;

181

182 RRDCONTEXT *rc = rrdcontext_acquired_value(rca);

@@ -187,7 +186,7 @@ static bool rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *r

186 match = rrdcontext_to_json_v2_full_text_search(ctl, rc, ctl->q.pattern);

187

188 if(match == FTS_MATCHED_NONE)

190 - return false;

189 + return 0;

190 }

191

192 struct rrdcontext_to_json_v2_entry t = {

@@ -220,18 +219,26 @@ static bool rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *r

219 z->last_time_s = rc->last_time_s;

220 }

221

223 - return true;

222 + return 1;

223 }

224

226 -static bool rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) {

225 +static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) {

226 if(!queryable_host || !host->rrdctx.contexts)

227 // the host matches the 'scope_host' but does not match the 'host' patterns

228 // or the host does not have any contexts

230 - return false;

229 + return 0;

230

231 struct rrdcontext_to_json_v2_data *ctl = data;

232 BUFFER *wb = ctl->wb;

233

234 + if(ctl->request->timeout_ms && now_monotonic_usec() > ctl->request->timings.received_ut + ctl->request->timeout_ms * USEC_PER_MS)

235 + // timed out

236 + return -2;

237 +

238 + if(ctl->request->interrupt_callback && ctl->request->interrupt_callback(ctl->request->interrupt_callback_data))

239 + // interrupted

240 + return -1;

241 +

242 bool host_matched = (ctl->options & CONTEXTS_V2_NODES);

243 bool do_contexts = (ctl->options & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH));

244

@@ -255,7 +262,7 @@ static bool rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool query

262 // do not do pattern matching on contexts - we matched the host itself

263 ctl->q.pattern = NULL;

264

258 - size_t added = query_scope_foreach_context(

265 + ssize_t added = query_scope_foreach_context(

266 host, ctl->request->scope_contexts,

267 ctl->contexts.scope_pattern, ctl->contexts.pattern,

268 rrdcontext_to_json_v2_add_context, queryable_host, ctl);

@@ -263,26 +270,99 @@ static bool rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool query

270 // restore it

271 ctl->q.pattern = old_q;

272

273 + if(added == -1)

274 + return -1;

275 +

276 if(added)

277 host_matched = true;

278 }

279

270 - if(host_matched && (ctl->options & (CONTEXTS_V2_NODES | CONTEXTS_V2_DEBUG))) {

280 + if(host_matched && (ctl->options & (CONTEXTS_V2_NODES | CONTEXTS_V2_NODES_DETAILED | CONTEXTS_V2_DEBUG))) {

281 buffer_json_add_array_item_object(wb);

282 buffer_json_member_add_string(wb, "mg", host->machine_guid);

283 buffer_json_member_add_uuid(wb, "nd", host->node_id);

284 buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));

285 +

286 + if(ctl->options & CONTEXTS_V2_NODES_DETAILED) {

287 + buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));

288 + buffer_json_member_add_uint64(wb, "hops", host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);

289 + buffer_json_member_add_string(wb, "state", (host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)) ? "reachable" : "stale");

290 + buffer_json_member_add_boolean(wb, "isDeleted", false);

291 +

292 + buffer_json_member_add_array(wb, "services");

293 + buffer_json_array_close(wb);

294 +

295 + buffer_json_member_add_array(wb, "capabilities");

296 + buffer_json_add_array_item_object(wb);

297 + buffer_json_member_add_string(wb, "name", "funcs");

298 + buffer_json_member_add_uint64(wb, "version", 1);

299 + buffer_json_member_add_boolean(wb, "enabled", true);

300 + buffer_json_object_close(wb);

301 + buffer_json_add_array_item_object(wb);

302 + buffer_json_member_add_string(wb, "name", "mc");

303 + buffer_json_member_add_uint64(wb, "version", 1);

304 + buffer_json_member_add_boolean(wb, "enabled", true);

305 + buffer_json_object_close(wb);

306 + buffer_json_add_array_item_object(wb);

307 + buffer_json_member_add_string(wb, "name", "ml");

308 + buffer_json_member_add_uint64(wb, "version", 1);

309 + buffer_json_member_add_boolean(wb, "enabled", true);

310 + buffer_json_object_close(wb);

311 + buffer_json_array_close(wb);

312 +

313 + web_client_api_request_v1_info_summary_alarm_statuses(host, wb, "alarmCounters");

314 +

315 + host_labels2json(host, wb, "hostLabels");

316 +

317 + buffer_json_member_add_object(wb, "mlInfo");

318 + buffer_json_member_add_boolean(wb, "mlCapable", ml_capable(host));

319 + buffer_json_member_add_boolean(wb, "mlEnabled", ml_enabled(host));

320 + buffer_json_object_close(wb);

321 +

322 + if(host->system_info) {

323 + buffer_json_member_add_string_or_empty(wb, "architecture", host->system_info->architecture);

324 + buffer_json_member_add_string_or_empty(wb, "kernelName", host->system_info->kernel_name);

325 + buffer_json_member_add_string_or_empty(wb, "kernelVersion", host->system_info->kernel_version);

326 + buffer_json_member_add_string_or_empty(wb, "cpuFrequency", host->system_info->host_cpu_freq);

327 + buffer_json_member_add_string_or_empty(wb, "cpus", host->system_info->host_cores);

328 + buffer_json_member_add_string_or_empty(wb, "memory", host->system_info->host_ram_total);

329 + buffer_json_member_add_string_or_empty(wb, "diskSpace", host->system_info->host_disk_space);

330 + buffer_json_member_add_string_or_empty(wb, "container", host->system_info->container);

331 + buffer_json_member_add_string_or_empty(wb, "virtualization", host->system_info->virtualization);

332 + buffer_json_member_add_string_or_empty(wb, "os", host->system_info->host_os_id);

333 + buffer_json_member_add_string_or_empty(wb, "osName", host->system_info->host_os_name);

334 + buffer_json_member_add_string_or_empty(wb, "osVersion", host->system_info->host_os_version);

335 + }

336 +

337 + buffer_json_member_add_object(wb, "status");

338 +

339 + size_t receiver_hops = host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1;

340 + buffer_json_member_add_object(wb, "collection");

341 + buffer_json_member_add_uint64(wb, "hops", receiver_hops);

342 + buffer_json_member_add_boolean(wb, "online", host == localhost || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED));

343 + buffer_json_member_add_boolean(wb, "replicating", rrdhost_receiver_replicating_charts(host));

344 + buffer_json_object_close(wb); // collection

345 +

346 + buffer_json_member_add_object(wb, "streaming");

347 + buffer_json_member_add_uint64(wb, "hops", host->sender ? host->sender->hops : receiver_hops + 1);

348 + buffer_json_member_add_boolean(wb, "online", rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED));

349 + buffer_json_member_add_boolean(wb, "replicating", rrdhost_sender_replicating_charts(host));

350 + buffer_json_object_close(wb); // streaming

351 +

352 + buffer_json_object_close(wb); // status

353 + }

354 +

355 buffer_json_object_close(wb);

356 }

357

278 - return host_matched;

358 + return host_matched ? 1 : 0;

359 }

360

361 static void buffer_json_contexts_v2_options_to_array(BUFFER *wb, CONTEXTS_V2_OPTIONS options) {

362 if(options & CONTEXTS_V2_DEBUG)

363 buffer_json_add_array_item_string(wb, "debug");

364

285 - if(options & CONTEXTS_V2_NODES)

365 + if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_NODES_DETAILED))

366 buffer_json_add_array_item_string(wb, "nodes");

367

368 if(options & CONTEXTS_V2_CONTEXTS)

@@ -293,6 +373,8 @@ static void buffer_json_contexts_v2_options_to_array(BUFFER *wb, CONTEXTS_V2_OPT

373 }

374

375 int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options) {

376 + int resp = HTTP_RESP_OK;

377 +

378 req->timings.processing_ut = now_monotonic_usec();

379

380 if(options & CONTEXTS_V2_SEARCH)

@@ -303,8 +385,7 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE

385 .request = req,

386 .ctx = NULL,

387 .options = options,

306 - .hard_hash = 0,

307 - .soft_hash = 0,

388 + .versions = { 0 },

389 .nodes.scope_pattern = string_to_simple_pattern(req->scope_nodes),

390 .nodes.pattern = string_to_simple_pattern(req->nodes),

391 .contexts.pattern = string_to_simple_pattern(req->contexts),

@@ -348,20 +429,32 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE

429 buffer_json_object_close(wb);

430 }

431

351 - if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_DEBUG))

432 + if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_NODES_DETAILED | CONTEXTS_V2_DEBUG))

433 buffer_json_member_add_array(wb, "nodes");

434

354 - query_scope_foreach_host(ctl.nodes.scope_pattern, ctl.nodes.pattern,

355 - rrdcontext_to_json_v2_add_host, &ctl, &ctl.hard_hash, &ctl.soft_hash,

356 - ctl.q.host_uuid_buffer);

357 - if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_DEBUG))

435 + ssize_t ret = query_scope_foreach_host(ctl.nodes.scope_pattern, ctl.nodes.pattern,

436 + rrdcontext_to_json_v2_add_host, &ctl,

437 + &ctl.versions, ctl.q.host_uuid_buffer);

438 +

439 + if(unlikely(ret < 0)) {

440 + buffer_flush(wb);

441 +

442 + if(ret == -2) {

443 + buffer_strcat(wb, "query timeout");

444 + resp = HTTP_RESP_GATEWAY_TIMEOUT;

445 + }

446 + else {

447 + buffer_strcat(wb, "query interrupted");

448 + resp = HTTP_RESP_BACKEND_FETCH_FAILED;

449 + }

450 + goto cleanup;

451 + }

452 +

453 + if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_NODES_DETAILED | CONTEXTS_V2_DEBUG))

454 buffer_json_array_close(wb);

455

456 req->timings.output_ut = now_monotonic_usec();

361 - buffer_json_member_add_object(wb, "versions");

362 - buffer_json_member_add_uint64(wb, "contexts_hard_hash", ctl.hard_hash);

363 - buffer_json_member_add_uint64(wb, "contexts_soft_hash", ctl.soft_hash);

364 - buffer_json_object_close(wb);

457 + version_hashes_api_v2(wb, &ctl.versions);

458

459 if(options & CONTEXTS_V2_CONTEXTS) {

460 buffer_json_member_add_object(wb, "contexts");

@@ -402,6 +495,7 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE

495 buffer_json_object_close(wb);

496 buffer_json_finalize(wb);

497

498 +cleanup:

499 dictionary_destroy(ctl.ctx);

500 simple_pattern_free(ctl.nodes.scope_pattern);

501 simple_pattern_free(ctl.nodes.pattern);

@@ -409,6 +503,6 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE

503 simple_pattern_free(ctl.contexts.scope_pattern);

504 simple_pattern_free(ctl.q.pattern);

505

412 - return HTTP_RESP_OK;

506 + return resp;

507 }

508

database/contexts/instance.c

@@ -20,6 +20,11 @@ inline const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria) {

20 return string2str(ri->name);

21 }

23 +inline bool rrdinstance_acquired_has_name(RRDINSTANCE_ACQUIRED *ria) {

24 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);

25 + return (ri->name && ri->name != ri->id);

26 +}

27 +

28 inline const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria) {

29 RRDINSTANCE *ri = rrdinstance_acquired_value(ria);

30 return string2str(ri->units);

database/contexts/internal.h

-9

@@ -377,13 +377,4 @@ uint64_t rrdcontext_version_hash_with_callback(

377

378 void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused);

379

380 -// ----------------------------------------------------------------------------

381 -// scope

382 -

383 -typedef bool (*foreach_host_cb_t)(void *data, RRDHOST *host, bool queryable);

384 -uint64_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN *hosts_sp, foreach_host_cb_t cb, void *data, uint64_t *hard_hash, uint64_t *soft_hash, char *host_uuid_buffer);

385 -

386 -typedef bool (*foreach_context_cb_t)(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context);

387 -size_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp, SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data);

388 -

380 #endif //NETDATA_RRDCONTEXT_INTERNAL_H

database/contexts/metric.c

@@ -14,6 +14,11 @@ inline const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma) {

14 return string2str(rm->name);

15 }

17 +inline bool rrdmetric_acquired_has_name(RRDMETRIC_ACQUIRED *rma) {

18 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);

19 + return (rm->name && rm->name != rm->id);

20 +}

21 +

22 inline STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma) {

23 RRDMETRIC *rm = rrdmetric_acquired_value(rma);

24 return string_dup(rm->id);

database/contexts/query_scope.c

+57 -34

@@ -2,61 +2,77 @@

3 #include "internal.h"

5 -uint64_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN *hosts_sp, foreach_host_cb_t cb, void *data, uint64_t *hard_hash, uint64_t *soft_hash, char *host_uuid_buffer) {

5 +ssize_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN *hosts_sp,

6 + foreach_host_cb_t cb, void *data,

7 + struct query_versions *versions,

8 + char *host_uuid_buffer) {

9 char uuid[UUID_STR_LEN];

10 if(!host_uuid_buffer) host_uuid_buffer = uuid;

11 host_uuid_buffer[0] = '\0';

13 RRDHOST *host;

11 - size_t count = 0;

14 + ssize_t added = 0;

15 uint64_t v_hash = 0;

16 uint64_t h_hash = 0;

17 + uint64_t a_hash = 0;

18 + uint64_t t_hash = 0;

15 - dfe_start_reentrant(rrdhost_root_index, host) {

20 + dfe_start_read(rrdhost_root_index, host) {

21 if(host->node_id)

22 uuid_unparse_lower(*host->node_id, host_uuid_buffer);

19 - SIMPLE_PATTERN_RESULT ret = SP_MATCHED_POSITIVE;

24 + SIMPLE_PATTERN_RESULT match = SP_MATCHED_POSITIVE;

25 if(scope_hosts_sp) {

21 - ret = simple_pattern_matches_string_extract(scope_hosts_sp, host->hostname, NULL, 0);

22 - if(ret == SP_NOT_MATCHED) {

23 - ret = simple_pattern_matches_extract(scope_hosts_sp, host->machine_guid, NULL, 0);

24 - if(ret == SP_NOT_MATCHED && *host_uuid_buffer)

25 - ret = simple_pattern_matches_extract(scope_hosts_sp, host_uuid_buffer, NULL, 0);

26 + match = simple_pattern_matches_string_extract(scope_hosts_sp, host->hostname, NULL, 0);

27 + if(match == SP_NOT_MATCHED) {

28 + match = simple_pattern_matches_extract(scope_hosts_sp, host->machine_guid, NULL, 0);

29 + if(match == SP_NOT_MATCHED && *host_uuid_buffer)

30 + match = simple_pattern_matches_extract(scope_hosts_sp, host_uuid_buffer, NULL, 0);

31 }

32 }

29 - if(ret == SP_MATCHED_POSITIVE) {

30 - if(hosts_sp) {

31 - ret = simple_pattern_matches_string_extract(hosts_sp, host->hostname, NULL, 0);

32 - if(ret == SP_NOT_MATCHED) {

33 - ret = simple_pattern_matches_extract(hosts_sp, host->machine_guid, NULL, 0);

34 - if(ret == SP_NOT_MATCHED && *host_uuid_buffer)

35 - ret = simple_pattern_matches_extract(hosts_sp, host_uuid_buffer, NULL, 0);

36 - }

34 + if(match != SP_MATCHED_POSITIVE)

35 + continue;

36 +

37 + dfe_unlock(host);

38 +

39 + if(hosts_sp) {

40 + match = simple_pattern_matches_string_extract(hosts_sp, host->hostname, NULL, 0);

41 + if(match == SP_NOT_MATCHED) {

42 + match = simple_pattern_matches_extract(hosts_sp, host->machine_guid, NULL, 0);

43 + if(match == SP_NOT_MATCHED && *host_uuid_buffer)

44 + match = simple_pattern_matches_extract(hosts_sp, host_uuid_buffer, NULL, 0);

45 }

46 + }

39 - bool queryable_host = (ret == SP_MATCHED_POSITIVE);

48 + bool queryable_host = (match == SP_MATCHED_POSITIVE);

41 - count++;

42 - v_hash += dictionary_version(host->rrdctx.contexts);

43 - h_hash += dictionary_version(host->rrdctx.hub_queue);

44 - cb(data, host, queryable_host);

50 + v_hash += dictionary_version(host->rrdctx.contexts);

51 + h_hash += dictionary_version(host->rrdctx.hub_queue);

52 + a_hash += dictionary_version(host->rrdcalc_root_index);

53 + t_hash += __atomic_load_n(&host->health_transitions, __ATOMIC_RELAXED);

54 + ssize_t ret = cb(data, host, queryable_host);

55 + if(ret < 0) {

56 + added = ret;

57 + break;

58 }

59 + added += ret;

60 }

61 dfe_done(host);

49 - if(hard_hash)

50 - *hard_hash = v_hash;

51 -

52 - if(soft_hash)

53 - *soft_hash = h_hash;

63 + if(versions) {

64 + versions->contexts_hard_hash = v_hash;

65 + versions->contexts_soft_hash = h_hash;

66 + versions->alerts_hard_hash = a_hash;

67 + versions->alerts_soft_hash = t_hash;

68 + }

55 - return count;

70 + return added;

71 }

58 -size_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp, SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data) {

59 - size_t added = 0;

73 +ssize_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp,

74 + SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data) {

75 + ssize_t added = 0;

77 RRDCONTEXT_ACQUIRED *rca = NULL;

@@ -71,8 +87,7 @@ size_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SI

87 if(queryable_context && contexts_sp && !simple_pattern_matches_string(contexts_sp, rc->id))

88 queryable_context = false;

74 - if(cb(data, rca, queryable_context))

75 - added++;

90 + added = cb(data, rca, queryable_context);

92 rrdcontext_release(rca);

93 }

@@ -83,12 +98,20 @@ size_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SI

98 if(scope_contexts_sp && !simple_pattern_matches_string(scope_contexts_sp, rc->id))

99 continue;

100

101 + dfe_unlock(rc);

102 +

103 bool queryable_context = queryable_host;

104 if(queryable_context && contexts_sp && !simple_pattern_matches_string(contexts_sp, rc->id))

105 queryable_context = false;

106

90 - if(cb(data, (RRDCONTEXT_ACQUIRED *)rc_dfe.item, queryable_context))

91 - added++;

107 + ssize_t ret = cb(data, (RRDCONTEXT_ACQUIRED *)rc_dfe.item, queryable_context);

108 +

109 + if(ret < 0) {

110 + added = ret;

111 + break;

112 + }

113 +

114 + added += ret;

115 }

116 dfe_done(rc);

117 }

database/contexts/query_target.c

+155 -54

@@ -29,7 +29,7 @@ typedef struct query_target_locals {

30 const char *nodes;

31 const char *contexts;

32 - const char *charts;

32 + const char *instances;

33 const char *dimensions;

34 const char *chart_label_key;

35 const char *labels;

@@ -151,10 +151,6 @@ void query_target_free(void) {

151 qt->nodes.size = 0;

152 }

153

154 -#define query_target_retention_matches_query(qt, first_entry_s, last_entry_s, update_every_s) \

155 - (((first_entry_s) - ((update_every_s) * 2) <= (qt)->window.before) && \

156 - ((last_entry_s) + ((update_every_s) * 2) >= (qt)->window.after))

157 -

154 struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {

155 QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);

156 return qn->rrdhost->db[tier].eng;

@@ -242,7 +238,7 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON

238

239 bool timeframe_matches =

240 (tiers_added &&

245 - query_target_retention_matches_query(qt, common_first_time_s, common_last_time_s, common_update_every_s))

241 + query_matches_retention(qt->window.after, qt->window.before, common_first_time_s, common_last_time_s, common_update_every_s))

242 ? true : false;

243

244 if(timeframe_matches) {

@@ -297,7 +293,7 @@ static inline bool rrdmetric_retention_matches_query(QUERY_TARGET *qt, RRDMETRIC

293 time_t first_time_s = rm->first_time_s;

294 time_t last_time_s = rrd_flag_is_collected(rm) ? now_s : rm->last_time_s;

295 time_t update_every_s = rm->ri->update_every_s;

300 - return query_target_retention_matches_query(qt, first_time_s, last_time_s, update_every_s);

296 + return query_matches_retention(qt->window.after, qt->window.before, first_time_s, last_time_s, update_every_s);

297 }

298

299 static inline void query_dimension_release(QUERY_DIMENSION *qd) {

@@ -476,9 +472,9 @@ static inline STRING *rrdinstance_create_name_fqdn_v2(RRDINSTANCE_ACQUIRED *ria)

472 return string_strdupz(buffer);

473 }

474

479 -inline STRING *query_instance_id_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi) {

475 +inline STRING *query_instance_id_fqdn(QUERY_INSTANCE *qi, size_t version) {

476 if(!qi->id_fqdn) {

481 - if (qt->request.version <= 1)

477 + if (version <= 1)

478 qi->id_fqdn = rrdinstance_create_id_fqdn_v1(qi->ria);

479 else

480 qi->id_fqdn = rrdinstance_create_id_fqdn_v2(qi->ria);

@@ -487,9 +483,9 @@ inline STRING *query_instance_id_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi) {

483 return qi->id_fqdn;

484 }

485

490 -inline STRING *query_instance_name_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi) {

486 +inline STRING *query_instance_name_fqdn(QUERY_INSTANCE *qi, size_t version) {

487 if(!qi->name_fqdn) {

492 - if (qt->request.version <= 1)

488 + if (version <= 1)

489 qi->name_fqdn = rrdinstance_create_name_fqdn_v1(qi->ria);

490 else

491 qi->name_fqdn = rrdinstance_create_name_fqdn_v2(qi->ria);

@@ -557,11 +553,11 @@ static void query_target_eval_instance_rrdcalc(QUERY_TARGET_LOCALS *qtl __maybe_

553 }

554 }

555

560 -static bool query_target_match_alert_pattern(QUERY_INSTANCE *qi, SIMPLE_PATTERN *pattern) {

556 +static bool query_target_match_alert_pattern(RRDINSTANCE_ACQUIRED *ria, SIMPLE_PATTERN *pattern) {

557 if(!pattern)

558 return true;

559

564 - RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);

560 + RRDSET *st = rrdinstance_acquired_rrdset(ria);

561 if (!st)

562 return false;

563

@@ -605,8 +601,10 @@ static bool query_target_match_alert_pattern(QUERY_INSTANCE *qi, SIMPLE_PATTERN

601 }

602

603 static inline void query_instance_release(QUERY_INSTANCE *qi) {

608 - rrdinstance_release(qi->ria);

609 - qi->ria = NULL;

604 + if(qi->ria) {

605 + rrdinstance_release(qi->ria);

606 + qi->ria = NULL;

607 + }

608

609 string_freez(qi->id_fqdn);

610 qi->id_fqdn = NULL;

@@ -635,6 +633,40 @@ static inline QUERY_INSTANCE *query_instance_allocate(QUERY_TARGET *qt, RRDINSTA

633 return qi;

634 }

635

636 +static inline SIMPLE_PATTERN_RESULT query_instance_matches(QUERY_INSTANCE *qi,

637 + RRDINSTANCE *ri,

638 + SIMPLE_PATTERN *instances_sp,

639 + bool match_ids,

640 + bool match_names,

641 + size_t version) {

642 + SIMPLE_PATTERN_RESULT ret = SP_MATCHED_POSITIVE;

643 +

644 + if(instances_sp) {

645 + ret = SP_NOT_MATCHED;

646 +

647 + if(match_ids)

648 + ret = simple_pattern_matches_string_extract(instances_sp, ri->id, NULL, 0);

649 + if (ret == SP_NOT_MATCHED && match_names && (ri->name != ri->id || !match_ids))

650 + ret = simple_pattern_matches_string_extract(instances_sp, ri->name, NULL, 0);

651 + if (ret == SP_NOT_MATCHED && match_ids)

652 + ret = simple_pattern_matches_string_extract(instances_sp, query_instance_id_fqdn(qi, version), NULL, 0);

653 + if (ret == SP_NOT_MATCHED && match_names)

654 + ret = simple_pattern_matches_string_extract(instances_sp, query_instance_name_fqdn(qi, version), NULL, 0);

655 + }

656 +

657 + return ret;

658 +}

659 +

660 +static inline bool query_instance_matches_labels(RRDINSTANCE *ri, SIMPLE_PATTERN *chart_label_key_sp, SIMPLE_PATTERN *labels_sp) {

661 + if ((chart_label_key_sp && !rrdlabels_match_simple_pattern_parsed(

662 + ri->rrdlabels, chart_label_key_sp, '\0', NULL)) ||

663 + (labels_sp && !rrdlabels_match_simple_pattern_parsed(

664 + ri->rrdlabels, labels_sp, ':', NULL)))

665 + return false;

666 +

667 + return true;

668 +}

669 +

670 static bool query_instance_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CONTEXT *qc,

671 RRDINSTANCE_ACQUIRED *ria, bool queryable_instance, bool filter_instances) {

672 RRDINSTANCE *ri = rrdinstance_acquired_value(ria);

@@ -647,35 +679,15 @@ static bool query_instance_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_C

679 if(qt->db.minimum_latest_update_every_s == 0 || ri->update_every_s < qt->db.minimum_latest_update_every_s)

680 qt->db.minimum_latest_update_every_s = ri->update_every_s;

681

650 - if(queryable_instance && filter_instances) {

651 - SIMPLE_PATTERN_RESULT ret = SP_MATCHED_POSITIVE;

682 + if(queryable_instance && filter_instances)

683 + queryable_instance = (SP_MATCHED_POSITIVE == query_instance_matches(

684 + qi, ri, qt->instances.pattern, qtl->match_ids, qtl->match_names, qt->request.version));

685

653 - if(qt->instances.pattern) {

654 - ret = SP_NOT_MATCHED;

655 -

656 - if(qtl->match_ids)

657 - ret = simple_pattern_matches_string_extract(qt->instances.pattern, ri->id, NULL, 0);

658 - if (ret == SP_NOT_MATCHED && qtl->match_names && (ri->name != ri->id || !qtl->match_ids))

659 - ret = simple_pattern_matches_string_extract(qt->instances.pattern, ri->name, NULL, 0);

660 - if (ret == SP_NOT_MATCHED && qtl->match_ids)

661 - ret = simple_pattern_matches_string_extract(qt->instances.pattern, query_instance_id_fqdn(qt, qi), NULL, 0);

662 - if (ret == SP_NOT_MATCHED && qtl->match_names)

663 - ret = simple_pattern_matches_string_extract(qt->instances.pattern, query_instance_name_fqdn(qt, qi), NULL, 0);

664 - }

665 -

666 - queryable_instance = (ret == SP_MATCHED_POSITIVE);

667 - }

668 -

669 - if(queryable_instance) {

670 - if ((qt->instances.chart_label_key_pattern && !rrdlabels_match_simple_pattern_parsed(

671 - ri->rrdlabels, qt->instances.chart_label_key_pattern,'\0', NULL)) ||

672 - (qt->instances.labels_pattern && !rrdlabels_match_simple_pattern_parsed(

673 - ri->rrdlabels, qt->instances.labels_pattern, ':', NULL)))

674 - queryable_instance = false;

675 - }

686 + if(queryable_instance)

687 + queryable_instance = query_instance_matches_labels(ri, qt->instances.chart_label_key_pattern, qt->instances.labels_pattern);

688

689 if(queryable_instance) {

678 - if(qt->instances.alerts_pattern && !query_target_match_alert_pattern(qi, qt->instances.alerts_pattern))

690 + if(qt->instances.alerts_pattern && !query_target_match_alert_pattern(ria, qt->instances.alerts_pattern))

691 queryable_instance = false;

692 }

693

@@ -739,18 +751,18 @@ static inline QUERY_CONTEXT *query_context_allocate(QUERY_TARGET *qt, RRDCONTEXT

751 return qc;

752 }

753

742 -static bool query_context_add(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context) {

754 +static ssize_t query_context_add(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context) {

755 QUERY_TARGET_LOCALS *qtl = data;

756

757 RRDCONTEXT *rc = rrdcontext_acquired_value(rca);

758 if(rrd_flag_is_deleted(rc))

747 - return false;

759 + return 0;

760

761 QUERY_NODE *qn = qtl->qn;

762 QUERY_TARGET *qt = qtl->qt;

763 QUERY_CONTEXT *qc = query_context_allocate(qt, rca);

764

753 - size_t added = 0;

765 + ssize_t added = 0;

766 if(unlikely(qt->request.ria)) {

767 if(query_instance_add(qtl, qn, qc, qt->request.ria, queryable_context, false))

768 added++;

@@ -771,10 +783,10 @@ static bool query_context_add(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryab

783 if(!added) {

784 query_context_release(qc);

785 qt->contexts.used--;

774 - return false;

786 + return 0;

787 }

788

777 - return true;

789 + return added;

790 }

791

792 static inline void query_node_release(QUERY_NODE *qn) {

@@ -799,7 +811,7 @@ static inline QUERY_NODE *query_node_allocate(QUERY_TARGET *qt, RRDHOST *host) {

811 return qn;

812 }

813

802 -static bool query_node_add(void *data, RRDHOST *host, bool queryable_host) {

814 +static ssize_t query_node_add(void *data, RRDHOST *host, bool queryable_host) {

815 QUERY_TARGET_LOCALS *qtl = data;

816 QUERY_TARGET *qt = qtl->qt;

817 QUERY_NODE *qn = query_node_allocate(qt, host);

@@ -822,8 +834,8 @@ static bool query_node_add(void *data, RRDHOST *host, bool queryable_host) {

834 error("QUERY TARGET: RRDSET '%s' given, but failed to be linked to rrdcontext structures. Switching to context query.",

835 rrdset_name(qtl->st));

836

825 - if (!is_valid_sp(qtl->charts))

826 - qtl->charts = rrdset_name(qtl->st);

837 + if (!is_valid_sp(qtl->instances))

838 + qtl->instances = rrdset_name(qtl->st);

839

840 qtl->st = NULL;

841 }

@@ -831,7 +843,7 @@ static bool query_node_add(void *data, RRDHOST *host, bool queryable_host) {

843

844 qtl->qn = qn;

845

834 - size_t added = 0;

846 + ssize_t added = 0;

847 if(unlikely(qt->request.rca)) {

848 if(query_context_add(qtl, qt->request.rca, true))

849 added++;

@@ -847,6 +859,9 @@ static bool query_node_add(void *data, RRDHOST *host, bool queryable_host) {

859 host, qtl->scope_contexts,

860 qt->contexts.scope_pattern, qt->contexts.pattern,

861 query_context_add, queryable_host, qtl);

862 +

863 + if(added < 0)

864 + added = 0;

865 }

866

867 qtl->qn = NULL;

@@ -981,7 +996,7 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {

996 .scope_contexts = qt->request.scope_contexts,

997 .nodes = qt->request.nodes,

998 .contexts = qt->request.contexts,

984 - .charts = qt->request.instances,

999 + .instances = qt->request.instances,

1000 .dimensions = qt->request.dimensions,

1001 .chart_label_key = qt->request.chart_label_key,

1002 .labels = qt->request.labels,

@@ -997,7 +1012,7 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {

1012 qt->contexts.pattern = string_to_simple_pattern(qtl.contexts);

1013 qt->contexts.scope_pattern = string_to_simple_pattern(qtl.scope_contexts);

1014

1000 - qt->instances.pattern = string_to_simple_pattern(qtl.charts);

1015 + qt->instances.pattern = string_to_simple_pattern(qtl.instances);

1016 qt->query.pattern = string_to_simple_pattern(qtl.dimensions);

1017 qt->instances.chart_label_key_pattern = string_to_simple_pattern(qtl.chart_label_key);

1018 qt->instances.labels_pattern = string_to_simple_pattern(qtl.labels);

@@ -1026,12 +1041,15 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {

1041 // single host query

1042 qt->versions.contexts_hard_hash = dictionary_version(host->rrdctx.contexts);

1043 qt->versions.contexts_soft_hash = dictionary_version(host->rrdctx.hub_queue);

1044 + qt->versions.alerts_hard_hash = dictionary_version(host->rrdcalc_root_index);

1045 + qt->versions.alerts_soft_hash = __atomic_load_n(&host->health_transitions, __ATOMIC_RELAXED);

1046 query_node_add(&qtl, host, true);

1047 qtl.nodes = rrdhost_hostname(host);

1048 }

1049 else

1033 - query_scope_foreach_host(qt->nodes.scope_pattern, qt->nodes.pattern, query_node_add, &qtl,

1034 - &qt->versions.contexts_hard_hash, &qt->versions.contexts_soft_hash,

1050 + query_scope_foreach_host(qt->nodes.scope_pattern, qt->nodes.pattern,

1051 + query_node_add, &qtl,

1052 + &qt->versions,

1053 qtl.host_uuid_buffer);

1054

1055 // we need the available db retention for this call

@@ -1042,3 +1060,86 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {

1060

1061 return qt;

1062 }

1063 +

1064 +ssize_t weights_foreach_rrdmetric_in_context(RRDCONTEXT_ACQUIRED *rca,

1065 + SIMPLE_PATTERN *instances_sp,

1066 + SIMPLE_PATTERN *chart_label_key_sp,

1067 + SIMPLE_PATTERN *labels_sp,

1068 + SIMPLE_PATTERN *alerts_sp,

1069 + SIMPLE_PATTERN *dimensions_sp,

1070 + bool match_ids, bool match_names,

1071 + size_t version,

1072 + weights_add_metric_t cb,

1073 + void *data

1074 + ) {

1075 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);

1076 + if(!rc || rrd_flag_is_deleted(rc))

1077 + return 0;

1078 +

1079 + bool proceed = true;

1080 +

1081 + ssize_t count = 0;

1082 + RRDINSTANCE *ri;

1083 + dfe_start_read(rc->rrdinstances, ri) {

1084 + if(rrd_flag_is_deleted(ri))

1085 + continue;

1086 +

1087 + RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *) ri_dfe.item;

1088 +

1089 + if(instances_sp) {

1090 + QUERY_INSTANCE qi = { .ria = ria, };

1091 + SIMPLE_PATTERN_RESULT ret = query_instance_matches(&qi, ri, instances_sp, match_ids, match_names, version);

1092 + qi.ria = NULL;

1093 + query_instance_release(&qi);

1094 +

1095 + if (ret != SP_MATCHED_POSITIVE)

1096 + continue;

1097 + }

1098 +

1099 + if(!query_instance_matches_labels(ri, chart_label_key_sp, labels_sp))

1100 + continue;

1101 +

1102 + if(alerts_sp && !query_target_match_alert_pattern(ria, alerts_sp))

1103 + continue;

1104 +

1105 + dfe_unlock(ri);

1106 +

1107 + RRDMETRIC *rm;

1108 + dfe_start_read(ri->rrdmetrics, rm) {

1109 + if(rrd_flag_is_deleted(rm))

1110 + continue;

1111 +

1112 + if(dimensions_sp) {

1113 + SIMPLE_PATTERN_RESULT ret = SP_NOT_MATCHED;

1114 +

1115 + if (match_ids)

1116 + ret = simple_pattern_matches_string_extract(dimensions_sp, rm->id, NULL, 0);

1117 +

1118 + if (ret == SP_NOT_MATCHED && match_names && (rm->name != rm->id || !match_ids))

1119 + ret = simple_pattern_matches_string_extract(dimensions_sp, rm->name, NULL, 0);

1120 +

1121 + if(ret != SP_MATCHED_POSITIVE)

1122 + continue;

1123 + }

1124 +

1125 + dfe_unlock(rm);

1126 +

1127 + RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)rm_dfe.item;

1128 + ssize_t ret = cb(data, rc->rrdhost, rca, ria, rma);

1129 +

1130 + if(ret < 0) {

1131 + proceed = false;

1132 + break;

1133 + }

1134 +

1135 + count += ret;

1136 + }

1137 + dfe_done(rm);

1138 +

1139 + if(unlikely(!proceed))

1140 + break;

1141 + }

1142 + dfe_done(ri);

1143 +

1144 + return count;

1145 +}

database/contexts/rrdcontext.c

+8 -63

@@ -312,68 +312,13 @@ void rrdcontext_hub_stop_streaming_command(void *ptr) {

312 rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);

313 }

314

315 -// ----------------------------------------------------------------------------

316 -// weights API

317 -

318 -static void metric_entry_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {

319 - struct metric_entry *t = value;

320 - t->rca = rrdcontext_acquired_dup(t->rca);

321 - t->ria = rrdinstance_acquired_dup(t->ria);

322 - t->rma = rrdmetric_acquired_dup(t->rma);

323 -}

324 -static void metric_entry_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {

325 - struct metric_entry *t = value;

326 - rrdcontext_release(t->rca);

327 - rrdinstance_release(t->ria);

328 - rrdmetric_release(t->rma);

329 -}

330 -static bool metric_entry_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value __maybe_unused, void *new_value __maybe_unused, void *data __maybe_unused) {

331 - internal_fatal("RRDCONTEXT: %s() detected a conflict on a metric pointer!", __FUNCTION__);

332 - return false;

333 -}

334 -

335 -DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contexts) {

336 - if(!host || !host->rrdctx.contexts)

337 - return NULL;

315 +bool rrdcontext_retention_match(RRDCONTEXT_ACQUIRED *rca, time_t after, time_t before) {

316 + if(unlikely(!rca)) return false;

317

339 - DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext, 0);

340 - dictionary_register_insert_callback(dict, metric_entry_insert_callback, NULL);

341 - dictionary_register_delete_callback(dict, metric_entry_delete_callback, NULL);

342 - dictionary_register_conflict_callback(dict, metric_entry_conflict_callback, NULL);

343 -

344 - RRDCONTEXT *rc;

345 - dfe_start_reentrant(host->rrdctx.contexts, rc) {

346 - if(rrd_flag_is_deleted(rc))

347 - continue;

348 -

349 - if(contexts && !simple_pattern_matches_string(contexts, rc->id))

350 - continue;

351 -

352 - RRDINSTANCE *ri;

353 - dfe_start_read(rc->rrdinstances, ri) {

354 - if(rrd_flag_is_deleted(ri))

355 - continue;

356 -

357 - RRDMETRIC *rm;

358 - dfe_start_read(ri->rrdmetrics, rm) {

359 - if(rrd_flag_is_deleted(rm))

360 - continue;

361 -

362 - struct metric_entry tmp = {

363 - .rca = (RRDCONTEXT_ACQUIRED *)rc_dfe.item,

364 - .ria = (RRDINSTANCE_ACQUIRED *)ri_dfe.item,

365 - .rma = (RRDMETRIC_ACQUIRED *)rm_dfe.item,

366 - };

367 -

368 - char buffer[20 + 1];

369 - ssize_t len = snprintfz(buffer, 20, "%p", rm);

370 - dictionary_set_advanced(dict, buffer, len + 1, &tmp, sizeof(struct metric_entry), NULL);

371 - }

372 - dfe_done(rm);

373 - }

374 - dfe_done(ri);

375 - }

376 - dfe_done(rc);

318 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);

319

378 - return dict;

379 -}

320 + if(rrd_flag_is_collected(rc))

321 + return query_matches_retention(after, before, rc->first_time_s, before > rc->last_time_s ? before : rc->last_time_s, 1);

322 + else

323 + return query_matches_retention(after, before, rc->first_time_s, rc->last_time_s, 1);

324 +}

\ No newline at end of file

database/contexts/rrdcontext.h

+58 -24

@@ -25,6 +25,7 @@ typedef struct rrdcontext_acquired RRDCONTEXT_ACQUIRED;

25 bool rrdinstance_acquired_id_and_name_are_same(RRDINSTANCE_ACQUIRED *ria);

26 const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma);

27 const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma);

28 +bool rrdmetric_acquired_has_name(RRDMETRIC_ACQUIRED *rma);

30 STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma);

31 STRING *rrdmetric_acquired_name_dup(RRDMETRIC_ACQUIRED *rma);

@@ -36,6 +37,7 @@ bool rrdmetric_acquired_belongs_to_instance(RRDMETRIC_ACQUIRED *rma, RRDINSTANCE

38 const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria);

39 const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria);

40 +bool rrdinstance_acquired_has_name(RRDINSTANCE_ACQUIRED *ria);

41 const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria);

42 STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria);

43 DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria);

@@ -123,17 +125,6 @@ void rrdcontext_hub_stop_streaming_command(void *cmd);

125 void rrdcontext_db_rotation(void);

126 void *rrdcontext_main(void *);

127

126 -// ----------------------------------------------------------------------------

127 -// public API for weights

128 -

129 -struct metric_entry {

130 - RRDCONTEXT_ACQUIRED *rca;

131 - RRDINSTANCE_ACQUIRED *ria;

132 - RRDMETRIC_ACQUIRED *rma;

133 -};

134 -

135 -DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contexts);

136 -

128 // ----------------------------------------------------------------------------

129 // public API for queries

130

@@ -258,7 +249,7 @@ typedef struct query_metric {

249

250 #define MAX_QUERY_TARGET_ID_LENGTH 255

251

261 -typedef bool (*interrupt_callback_t)(void *data);

252 +typedef bool (*qt_interrupt_callback_t)(void *data);

253

254 typedef struct query_target_request {

255 size_t version;

@@ -286,7 +277,7 @@ typedef struct query_target_request {

277

278 uint32_t format; // DATASOURCE_FORMAT

279 RRDR_OPTIONS options;

289 - time_t timeout; // the timeout of the query in seconds

280 + time_t timeout_ms; // the timeout of the query in milliseconds

281

282 size_t tier;

283 QUERY_SOURCE query_source;

@@ -306,7 +297,7 @@ typedef struct query_target_request {

297

298 usec_t received_ut;

299

309 - interrupt_callback_t interrupt_callback;

300 + qt_interrupt_callback_t interrupt_callback;

301 void *interrupt_callback_data;

302 } QUERY_TARGET_REQUEST;

303

@@ -322,6 +313,13 @@ struct query_tier_statistics {

313 } retention;

314 };

315

316 +struct query_versions {

317 + uint64_t contexts_hard_hash;

318 + uint64_t contexts_soft_hash;

319 + uint64_t alerts_hard_hash;

320 + uint64_t alerts_soft_hash;

321 +};

322 +

323 typedef struct query_target {

324 char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging)

325 QUERY_TARGET_REQUEST request;

@@ -400,10 +398,7 @@ typedef struct query_target {

398

399 struct query_data_statistics query_stats;

400

403 - struct {

404 - uint64_t contexts_hard_hash;

405 - uint64_t contexts_soft_hash;

406 - } versions;

401 + struct query_versions versions;

402

403 struct {

404 usec_t received_ut;

@@ -451,8 +446,8 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)

446

447 struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);

448

454 -STRING *query_instance_id_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);

455 -STRING *query_instance_name_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);

449 +STRING *query_instance_id_fqdn(QUERY_INSTANCE *qi, size_t version);

450 +STRING *query_instance_name_fqdn(QUERY_INSTANCE *qi, size_t version);

451

452 void query_target_free(void);

453 void query_target_release(QUERY_TARGET *qt);

@@ -472,18 +467,57 @@ struct api_v2_contexts_request {

467 usec_t output_ut;

468 usec_t finished_ut;

469 } timings;

470 +

471 + time_t timeout_ms;

472 +

473 + qt_interrupt_callback_t interrupt_callback;

474 + void *interrupt_callback_data;

475 };

476

477 typedef enum __attribute__ ((__packed__)) {

478 - CONTEXTS_V2_DEBUG = (1 << 0),

479 - CONTEXTS_V2_SEARCH = (1 << 1),

480 - CONTEXTS_V2_NODES = (1 << 2),

481 - CONTEXTS_V2_CONTEXTS = (1 << 3),

478 + CONTEXTS_V2_DEBUG = (1 << 0),

479 + CONTEXTS_V2_SEARCH = (1 << 1),

480 + CONTEXTS_V2_NODES = (1 << 2),

481 + CONTEXTS_V2_NODES_DETAILED = (1 << 3),

482 + CONTEXTS_V2_CONTEXTS = (1 << 4),

483 } CONTEXTS_V2_OPTIONS;

484

485 int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options);

486

487 RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o);

488

489 +// ----------------------------------------------------------------------------

490 +// scope

491 +

492 +typedef ssize_t (*foreach_host_cb_t)(void *data, RRDHOST *host, bool queryable);

493 +ssize_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN *hosts_sp,

494 + foreach_host_cb_t cb, void *data,

495 + struct query_versions *versions,

496 + char *host_uuid_buffer);

497 +

498 +typedef ssize_t (*foreach_context_cb_t)(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context);

499 +ssize_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp, SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data);

500 +

501 +// ----------------------------------------------------------------------------

502 +// public API for weights

503 +

504 +typedef ssize_t (*weights_add_metric_t)(void *data, RRDHOST *host, RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma);

505 +ssize_t weights_foreach_rrdmetric_in_context(RRDCONTEXT_ACQUIRED *rca,

506 + SIMPLE_PATTERN *instances_sp,

507 + SIMPLE_PATTERN *chart_label_key_sp,

508 + SIMPLE_PATTERN *labels_sp,

509 + SIMPLE_PATTERN *alerts_sp,

510 + SIMPLE_PATTERN *dimensions_sp,

511 + bool match_ids, bool match_names,

512 + size_t version,

513 + weights_add_metric_t cb,

514 + void *data);

515 +

516 +bool rrdcontext_retention_match(RRDCONTEXT_ACQUIRED *rca, time_t after, time_t before);

517 +

518 +#define query_matches_retention(after, before, first_entry_s, last_entry_s, update_every_s) \

519 + (((first_entry_s) - ((update_every_s) * 2) <= (before)) && \

520 + ((last_entry_s) + ((update_every_s) * 2) >= (after)))

521 +

522 #endif // NETDATA_RRDCONTEXT_H

523

database/rrd.h

+1 -40

@@ -117,21 +117,6 @@ struct storage_engine_query_handle {

117 STORAGE_QUERY_HANDLE* handle;

118 };

119

120 -typedef struct storage_point {

121 - NETDATA_DOUBLE min; // when count > 1, this is the minimum among them

122 - NETDATA_DOUBLE max; // when count > 1, this is the maximum among them

123 - NETDATA_DOUBLE sum; // the point sum - divided by count gives the average

124 -

125 - // end_time - start_time = point duration

126 - time_t start_time_s; // the time the point starts

127 - time_t end_time_s; // the time the point ends

128 -

129 - size_t count; // the number of original points aggregated

130 - size_t anomaly_count; // the number of original points found anomalous

131 -

132 - SN_FLAGS flags; // flags stored with the point

133 -} STORAGE_POINT;

134 -

120 // ----------------------------------------------------------------------------

121 // chart types

122

@@ -426,31 +411,6 @@ size_t rrddim_memory_file_header_size(void);

411

412 void rrddim_memory_file_save(RRDDIM *rd);

413

429 -// ----------------------------------------------------------------------------

430 -

431 -#define storage_point_unset(x) do { \

432 - (x).min = (x).max = (x).sum = NAN; \

433 - (x).count = 0; \

434 - (x).anomaly_count = 0; \

435 - (x).flags = SN_FLAG_NONE; \

436 - (x).start_time_s = 0; \

437 - (x).end_time_s = 0; \

438 - } while(0)

439 -

440 -#define storage_point_empty(x, start_s, end_s) do { \

441 - (x).min = (x).max = (x).sum = NAN; \

442 - (x).count = 1; \

443 - (x).anomaly_count = 0; \

444 - (x).flags = SN_FLAG_NONE; \

445 - (x).start_time_s = start_s; \

446 - (x).end_time_s = end_s; \

447 - } while(0)

448 -

449 -#define STORAGE_POINT_UNSET { .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }

450 -

451 -#define storage_point_is_unset(x) (!(x).count)

452 -#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))

453 -

414 // ------------------------------------------------------------------------

415 // function pointers that handle data collection

416 struct storage_engine_collect_ops {

@@ -1065,6 +1025,7 @@ struct rrdhost {

1025 uint32_t health_last_processed_id; // the last processed health id from the log

1026 uint32_t health_max_unique_id; // the max alarm log unique id given for the host

1027 uint32_t health_max_alarm_id; // the max alarm id given for the host

1028 + size_t health_transitions; // the number of times an alert changed state

1029

1030 // ------------------------------------------------------------------------

1031 // locks

health/health_log.c

@@ -95,6 +95,8 @@ inline void health_alarm_log_add_entry(

95 ) {

96 debug(D_HEALTH, "Health adding alarm log entry with id: %u", ae->unique_id);

98 + __atomic_add_fetch(&host->health_transitions, 1, __ATOMIC_RELAXED);

99 +

100 // link it

101 netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock);

102 ae->next = host->health_log.alarms;

libnetdata/buffer/buffer.h

@@ -755,6 +755,14 @@ static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE

755 wb->json.stack[wb->json.depth].count++;

756 }

757

758 +static inline void buffer_json_add_array_item_int64(BUFFER *wb, int64_t value) {

759 + if(wb->json.stack[wb->json.depth].count)

760 + buffer_fast_strcat(wb, ",", 1);

761 +

762 + buffer_print_int64(wb, value);

763 + wb->json.stack[wb->json.depth].count++;

764 +}

765 +

766 static inline void buffer_json_add_array_item_uint64(BUFFER *wb, uint64_t value) {

767 if(wb->json.stack[wb->json.depth].count)

768 buffer_fast_strcat(wb, ",", 1);

libnetdata/dictionary/dictionary.c

+21 -5

@@ -812,6 +812,7 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {

812 }

813

814 void dictionary_garbage_collect(DICTIONARY *dict) {

815 + if(!dict) return;

816 garbage_collect_pending_deletes(dict);

817 }

818

@@ -2277,7 +2278,7 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {

2278 dfe->counter = 0;

2279 dfe->dict = dict;

2280 dfe->rw = rw;

2280 -

2281 + dfe->locked = true;

2282 ll_recursive_lock(dict, dfe->rw);

2283

2284 DICTIONARY_STATS_TRAVERSALS_PLUS1(dict);

@@ -2300,8 +2301,10 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {

2301 dfe->value = NULL;

2302 }

2303

2303 - if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))

2304 + if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {

2305 ll_recursive_unlock(dfe->dict, dfe->rw);

2306 + dfe->locked = false;

2307 + }

2308

2309 return dfe->value;

2310 }

@@ -2317,8 +2320,10 @@ void *dictionary_foreach_next(DICTFE *dfe) {

2320 return NULL;

2321 }

2322

2320 - if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))

2323 + if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT) || !dfe->locked) {

2324 ll_recursive_lock(dfe->dict, dfe->rw);

2325 + dfe->locked = true;

2326 + }

2327

2328 // the item we just did

2329 DICTIONARY_ITEM *item = dfe->item;

@@ -2348,12 +2353,21 @@ void *dictionary_foreach_next(DICTFE *dfe) {

2353 dfe->value = NULL;

2354 }

2355

2351 - if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))

2356 + if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {

2357 ll_recursive_unlock(dfe->dict, dfe->rw);

2358 + dfe->locked = false;

2359 + }

2360

2361 return dfe->value;

2362 }

2363

2364 +void dictionary_foreach_unlock(DICTFE *dfe) {

2365 + if(dfe->locked) {

2366 + ll_recursive_unlock(dfe->dict, dfe->rw);

2367 + dfe->locked = false;

2368 + }

2369 +}

2370 +

2371 void dictionary_foreach_done(DICTFE *dfe) {

2372 if(unlikely(!dfe || !dfe->dict)) return;

2373

@@ -2371,8 +2385,10 @@ void dictionary_foreach_done(DICTFE *dfe) {

2385 // item_release(dfe->dict, item);

2386 }

2387

2374 - if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT))

2388 + if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT) && dfe->locked) {

2389 ll_recursive_unlock(dfe->dict, dfe->rw);

2390 + dfe->locked = false;

2391 + }

2392

2393 dfe->dict = NULL;

2394 dfe->item = NULL;

libnetdata/dictionary/dictionary.h

+6 -2

@@ -263,19 +263,20 @@ void dictionary_write_lock(DICTIONARY *dict);

263 void dictionary_write_unlock(DICTIONARY *dict);

264

265 typedef DICTFE_CONST struct dictionary_foreach {

266 - DICTIONARY *dict; // the dictionary upon we work

266 + DICTIONARY *dict; // the dictionary upon we work

267

268 DICTIONARY_ITEM *item; // the item we work on, to remember the position we are at

269 // this can be used with dictionary_acquired_item_dup() to

270 // acquire the currently working item.

271

272 - DICTFE_CONST char *name; // the dictionary name of the last item used

272 + const char *name; // the dictionary name of the last item used

273 void *value; // the dictionary value of the last item used

274 // same as the return value of dictfe_start() and dictfe_next()

275

276 size_t counter; // counts the number of iterations made, starting from zero

277

278 char rw; // the lock mode 'r' or 'w'

279 + bool locked; // true when the dictionary is locked

280 } DICTFE;

281

282 #define dfe_start_read(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_READ)

@@ -296,9 +297,12 @@ typedef DICTFE_CONST struct dictionary_foreach {

297 dictionary_foreach_done(&value ## _dfe); \

298 } while(0)

299

300 +#define dfe_unlock(value) dictionary_foreach_unlock(&value ## _dfe);

301 +

302 void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw);

303 void *dictionary_foreach_next(DICTFE *dfe);

304 void dictionary_foreach_done(DICTFE *dfe);

305 +void dictionary_foreach_unlock(DICTFE *dfe);

306

307 // ----------------------------------------------------------------------------

308 // Get statistics about the dictionary

libnetdata/libnetdata.h

+117 -1

@@ -366,6 +366,123 @@ size_t judy_aral_structures(void);

366

367 // ---------------------------------------------------------------------------------------------

368

369 +#include "storage_number/storage_number.h"

370 +

371 +typedef struct storage_point {

372 + NETDATA_DOUBLE min; // when count > 1, this is the minimum among them

373 + NETDATA_DOUBLE max; // when count > 1, this is the maximum among them

374 + NETDATA_DOUBLE sum; // the point sum - divided by count gives the average

375 +

376 + // end_time - start_time = point duration

377 + time_t start_time_s; // the time the point starts

378 + time_t end_time_s; // the time the point ends

379 +

380 + size_t count; // the number of original points aggregated

381 + size_t anomaly_count; // the number of original points found anomalous

382 +

383 + SN_FLAGS flags; // flags stored with the point

384 +} STORAGE_POINT;

385 +

386 +#define storage_point_unset(x) do { \

387 + (x).min = (x).max = (x).sum = NAN; \

388 + (x).count = 0; \

389 + (x).anomaly_count = 0; \

390 + (x).flags = SN_FLAG_NONE; \

391 + (x).start_time_s = 0; \

392 + (x).end_time_s = 0; \

393 + } while(0)

394 +

395 +#define storage_point_empty(x, start_s, end_s) do { \

396 + (x).min = (x).max = (x).sum = NAN; \

397 + (x).count = 1; \

398 + (x).anomaly_count = 0; \

399 + (x).flags = SN_FLAG_NONE; \

400 + (x).start_time_s = start_s; \

401 + (x).end_time_s = end_s; \

402 + } while(0)

403 +

404 +#define STORAGE_POINT_UNSET (STORAGE_POINT){ .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }

405 +

406 +#define storage_point_is_unset(x) (!(x).count)

407 +#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))

408 +

409 +#define storage_point_merge_to(dst, src) do { \

410 + if(storage_point_is_unset(dst)) \

411 + (dst) = (src); \

412 + \

413 + else if(!storage_point_is_unset(src) && \

414 + !storage_point_is_gap(src)) { \

415 + \

416 + if((src).start_time_s < (dst).start_time_s) \

417 + (dst).start_time_s = (src).start_time_s;\

418 + \

419 + if((src).end_time_s > (dst).end_time_s) \

420 + (dst).end_time_s = (src).end_time_s; \

421 + \

422 + if((src).min < (dst).min) \

423 + (dst).min = (src).min; \

424 + \

425 + if((src).max > (dst).max) \

426 + (dst).max = (src).max; \

427 + \

428 + (dst).sum += (src).sum; \

429 + \

430 + (dst).count += (src).count; \

431 + (dst).anomaly_count += (src).anomaly_count; \

432 + \

433 + (dst).flags |= (src).flags & SN_FLAG_RESET; \

434 + } \

435 +} while(0)

436 +

437 +#define storage_point_add_to(dst, src) do { \

438 + if(storage_point_is_unset(dst)) \

439 + (dst) = (src); \

440 + \

441 + else if(!storage_point_is_unset(src) && \

442 + !storage_point_is_gap(src)) { \

443 + \

444 + if((src).start_time_s < (dst).start_time_s) \

445 + (dst).start_time_s = (src).start_time_s;\

446 + \

447 + if((src).end_time_s > (dst).end_time_s) \

448 + (dst).end_time_s = (src).end_time_s; \

449 + \

450 + (dst).min += (src).min; \

451 + (dst).max += (src).max; \

452 + (dst).sum += (src).sum; \

453 + \

454 + (dst).count += (src).count; \

455 + (dst).anomaly_count += (src).anomaly_count; \

456 + \

457 + (dst).flags |= (src).flags & SN_FLAG_RESET; \

458 + } \

459 +} while(0)

460 +

461 +#define storage_point_make_positive(sp) do { \

462 + if(!storage_point_is_unset(sp) && \

463 + !storage_point_is_gap(sp)) { \

464 + \

465 + if(unlikely(signbit((sp).sum))) \

466 + (sp).sum = -(sp).sum; \

467 + \

468 + if(unlikely(signbit((sp).min))) \

469 + (sp).min = -(sp).min; \

470 + \

471 + if(unlikely(signbit((sp).max))) \

472 + (sp).max = -(sp).max; \

473 + \

474 + if(unlikely((sp).min > (sp).max)) { \

475 + NETDATA_DOUBLE t = (sp).min; \

476 + (sp).min = (sp).max; \

477 + (sp).max = t; \

478 + } \

479 + } \

480 +} while(0)

481 +

482 +#define storage_point_anomaly_rate(sp) \

483 + (NETDATA_DOUBLE)(storage_point_is_unset(sp) ? 0.0 : (NETDATA_DOUBLE)((sp).anomaly_count) * 100.0 / (NETDATA_DOUBLE)((sp).count))

484 +

485 +// ---------------------------------------------------------------------------------------------

486

487 void netdata_fix_chart_id(char *s);

488 void netdata_fix_chart_name(char *s);

@@ -517,7 +634,6 @@ extern char *netdata_configured_host_prefix;

634 #include "libjudy/src/Judy.h"

635 #include "july/july.h"

636 #include "os.h"

520 -#include "storage_number/storage_number.h"

637 #include "threads/threads.h"

638 #include "buffer/buffer.h"

639 #include "locks/locks.h"

streaming/rrdpush.c

+5 -3

@@ -321,9 +321,11 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {

321 (unsigned long long)db_last_time_t,

322 (unsigned long long)now);

323

324 - rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS);

325 - rrdset_flag_clear(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);

326 - rrdhost_sender_replicating_charts_plus_one(st->rrdhost);

324 + if(!rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS)) {

325 + rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS);

326 + rrdset_flag_clear(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);

327 + rrdhost_sender_replicating_charts_plus_one(st->rrdhost);

328 + }

329 replication_progress = true;

330

331 #ifdef NETDATA_LOG_REPLICATION_REQUESTS

web/api/formatters/csv/csv.c

+1 -34

@@ -61,7 +61,6 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const

61 }

63 // for each line in the array

64 - NETDATA_DOUBLE total = 1;

64 for(i = start; i != end ;i += step) {

65 NETDATA_DOUBLE *cn = &r->v[ i * r->d ];

66 RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];

@@ -84,22 +83,6 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const

83 buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);

84 }

87 - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {

88 - total = 0;

89 - for(c = 0; c < used ;c++) {

90 - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

91 -

92 - NETDATA_DOUBLE n = cn[c];

93 -

94 - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

95 - n = -n;

96 -

97 - total += n;

98 - }

99 - // prevent a division by zero

100 - if(total == 0) total = 1;

101 - }

102 -

86 // for each dimension

87 for(c = 0; c < used ;c++) {

88 if(!rrdr_dimension_should_be_exposed(r->od[c], options))

@@ -115,24 +98,8 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const

98 else

99 buffer_strcat(wb, "null");

100 }

118 - else {

119 - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

120 - n = -n;

121 -

122 - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {

123 - n = n * 100 / total;

124 -

125 - if(unlikely(i == start && c == 0)) {

126 - r->view.min = r->view.max = n;

127 - }

128 - else {

129 - if (n < r->view.min) r->view.min = n;

130 - if (n > r->view.max) r->view.max = n;

131 - }

132 - }

133 -

101 + else

102 buffer_print_netdata_double(wb, n);

135 - }

103 }

104

105 buffer_strcat(wb, endline);

web/api/formatters/json/json.c

+2 -72

@@ -149,7 +149,6 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {

149 );

150

151 // for each line in the array

152 - NETDATA_DOUBLE total = 1;

152 for(i = start; i != end ;i += step) {

153 NETDATA_DOUBLE *cn = &r->v[ i * r->d ];

154 RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];

@@ -210,26 +209,6 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {

209 buffer_fast_strcat(wb, post_date, post_date_len);

210 }

211

213 - if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_AR)))) {

214 - total = 0;

215 - for(c = 0; c < used ;c++) {

216 - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

217 -

218 - NETDATA_DOUBLE n;

219 - if(unlikely(options & RRDR_OPTION_INTERNAL_AR))

220 - n = ar[c];

221 - else

222 - n = cn[c];

223 -

224 - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

225 - n = -n;

226 -

227 - total += n;

228 - }

229 - // prevent a division by zero

230 - if(total == 0) total = 1;

231 - }

232 -

212 // for each dimension

213 for(c = 0; c < used ;c++) {

214 if(!rrdr_dimension_should_be_exposed(r->od[c], options))

@@ -252,24 +231,8 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {

231 else

232 buffer_fast_strcat(wb, "null", 4);

233 }

255 - else {

256 - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

257 - n = -n;

258 -

259 - if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_AR)))) {

260 - n = n * 100 / total;

261 -

262 - if(unlikely(i == start && c == 0)) {

263 - r->view.min = r->view.max = n;

264 - }

265 - else {

266 - if (n < r->view.min) r->view.min = n;

267 - if (n > r->view.max) r->view.max = n;

268 - }

269 - }

270 -

234 + else

235 buffer_print_netdata_double(wb, n);

272 - }

236

237 buffer_fast_strcat(wb, post_value, post_value_len);

238 }

@@ -335,23 +298,6 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {

298 else

299 buffer_json_add_array_item_time_t(wb, now); // the time

300

338 - NETDATA_DOUBLE total = 1;

339 - if(unlikely((options & RRDR_OPTION_PERCENTAGE))) {

340 - total = 0;

341 - for(d = 0; d < used ; d++) {

342 - if(unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) continue;

343 -

344 - NETDATA_DOUBLE n = cn[d];

345 - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

346 - n = -n;

347 -

348 - total += n;

349 - }

350 -

351 - // prevent a division by zero

352 - if(total == 0) total = 1;

353 - }

354 -

301 for (d = 0; d < used; d++) {

302 if (!rrdr_dimension_should_be_exposed(r->od[d], options))

303 continue;

@@ -369,24 +315,8 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {

315 else

316 buffer_json_add_array_item_double(wb, NAN);

317 }

372 - else {

373 - if (unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

374 - n = -n;

375 -

376 - if (unlikely((options & RRDR_OPTION_PERCENTAGE))) {

377 - n = n * 100 / total;

378 - }

379 -

380 - if(unlikely(i == start && d == 0)) {

381 - r->view.min = r->view.max = n;

382 - }

383 - else {

384 - if (n < r->view.min) r->view.min = n;

385 - if (n > r->view.max) r->view.max = n;

386 - }

387 -

318 + else

319 buffer_json_add_array_item_double(wb, n);

389 - }

320

321 // add the anomaly

322 buffer_json_add_array_item_double(wb, ar[d]);

web/api/formatters/json_wrapper.c

+76 -61

@@ -213,6 +213,10 @@ static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, st

213 return;

214

215 buffer_json_member_add_object(wb, "sts");

216 +

217 + buffer_json_member_add_double(wb, "min", d->min);

218 + buffer_json_member_add_double(wb, "max", d->max);

219 +

220 if(query_target_aggregatable(qt)) {

221 buffer_json_member_add_uint64(wb, "cnt", d->group_points);

222

@@ -397,15 +401,24 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co

401 qm = tqm;

402 }

403

400 - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",

401 - rrdmetric_acquired_id(rma),

402 - rrdmetric_acquired_name(rma));

403 -

404 - z = dictionary_set(dict, name, NULL, sizeof(*z));

405 - if(!z->id)

406 - z->id = rrdmetric_acquired_id(rma);

407 - if(!z->name)

408 - z->name = rrdmetric_acquired_name(rma);

404 + if(v2) {

405 + z = dictionary_set(dict, rrdmetric_acquired_name(rma), NULL, sizeof(*z));

406 + if(!z->id)

407 + z->id = rrdmetric_acquired_name(rma);

408 + if(!z->name)

409 + z->name = rrdmetric_acquired_name(rma);

410 + }

411 + else {

412 + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",

413 + rrdmetric_acquired_id(rma),

414 + rrdmetric_acquired_name(rma));

415 +

416 + z = dictionary_set(dict, name, NULL, sizeof(*z));

417 + if (!z->id)

418 + z->id = rrdmetric_acquired_id(rma);

419 + if (!z->name)

420 + z->name = rrdmetric_acquired_name(rma);

421 + }

422

423 if(qm) {

424 z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0;

@@ -423,7 +436,9 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co

436 if(v2) {

437 buffer_json_add_array_item_object(wb);

438 buffer_json_member_add_string(wb, "id", z->id);

426 - buffer_json_member_add_string(wb, "nm", z->name);

439 + if(z->id != z->name)

440 + buffer_json_member_add_string(wb, "nm", z->name);

441 +

442 query_target_metric_counts(wb, &z->metrics);

443 query_target_data_statistics(wb, qt, &z->query_stats);

444 buffer_json_object_close(wb);

@@ -676,30 +691,10 @@ static inline long query_target_metrics_latest_values(BUFFER *wb, const char *ke

691 return i;

692 }

693

679 -static inline size_t rrdr_dimension_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {

680 - size_t c, i;

681 -

694 +static inline size_t rrdr_dimension_view_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {

695 buffer_json_member_add_array(wb, key);

696

684 - NETDATA_DOUBLE total = 1;

685 -

686 - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {

687 - total = 0;

688 - for(c = 0; c < r->d ; c++) {

689 - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

690 -

691 - NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];

692 - NETDATA_DOUBLE n = cn[c];

693 -

694 - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

695 - n = -n;

696 -

697 - total += n;

698 - }

699 - // prevent a division by zero

700 - if(total == 0) total = 1;

701 - }

702 -

697 + size_t c, i;

698 for(c = 0, i = 0; c < r->d ; c++) {

699 if(!rrdr_dimension_should_be_exposed(r->od[c], options))

700 continue;

@@ -716,15 +711,8 @@ static inline size_t rrdr_dimension_latest_values(BUFFER *wb, const char *key, R

711 else

712 buffer_json_add_array_item_double(wb, NAN);

713 }

719 - else {

720 - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

721 - n = -n;

722 -

723 - if(unlikely(options & RRDR_OPTION_PERCENTAGE))

724 - n = n * 100 / total;

725 -

714 + else

715 buffer_json_add_array_item_double(wb, n);

727 - }

716 }

717

718 buffer_json_array_close(wb);

@@ -732,31 +720,49 @@ static inline size_t rrdr_dimension_latest_values(BUFFER *wb, const char *key, R

720 return i;

721 }

722

735 -static inline void rrdr_dimension_average_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {

723 +static inline void rrdr_dimension_view_average_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {

724 if(!r->dv)

725 return;

726

727 buffer_json_member_add_array(wb, key);

728

741 - bool percentage = r->internal.qt->request.options & RRDR_OPTION_PERCENTAGE;

742 - NETDATA_DOUBLE total = 0;

743 - if(percentage) {

744 - for(size_t c = 0; c < r->d ; c++) {

745 - if(!(r->od[c] & RRDR_DIMENSION_QUERIED))

746 - continue;

729 + for(size_t c = 0; c < r->d ; c++) {

730 + if(!rrdr_dimension_should_be_exposed(r->od[c], options))

731 + continue;

732

748 - total += r->dv[c];

749 - }

733 + buffer_json_add_array_item_double(wb, r->dv[c]);

734 }

735

736 + buffer_json_array_close(wb);

737 +}

738 +

739 +static inline void rrdr_dimension_view_minimum_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {

740 + if(!r->dmin)

741 + return;

742 +

743 + buffer_json_member_add_array(wb, key);

744 +

745 for(size_t c = 0; c < r->d ; c++) {

746 if(!rrdr_dimension_should_be_exposed(r->od[c], options))

747 continue;

748

756 - if(percentage)

757 - buffer_json_add_array_item_double(wb, r->dv[c] * 100.0 / total);

758 - else

759 - buffer_json_add_array_item_double(wb, r->dv[c]);

749 + buffer_json_add_array_item_double(wb, r->dmin[c]);

750 + }

751 +

752 + buffer_json_array_close(wb);

753 +}

754 +

755 +static inline void rrdr_dimension_view_maximum_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {

756 + if(!r->dmin)

757 + return;

758 +

759 + buffer_json_member_add_array(wb, key);

760 +

761 + for(size_t c = 0; c < r->d ; c++) {

762 + if(!rrdr_dimension_should_be_exposed(r->od[c], options))

763 + continue;

764 +

765 + buffer_json_add_array_item_double(wb, r->dmax[c]);

766 }

767

768 buffer_json_array_close(wb);

@@ -831,7 +837,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {

837 if(!query_target_metrics_latest_values(wb, "latest_values", r, options))

838 rows = 0;

839

834 - size_t dimensions = rrdr_dimension_latest_values(wb, "view_latest_values", r, options);

840 + size_t dimensions = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options);

841 if(!dimensions)

842 rows = 0;

843

@@ -1172,6 +1178,16 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS

1178 buffer_json_object_close(wb); // hosts

1179 }

1180

1181 +void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) {

1182 + buffer_json_member_add_object(wb, "versions");

1183 + buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdhost_root_index));

1184 + buffer_json_member_add_uint64(wb, "contexts_hard_hash", versions->contexts_hard_hash);

1185 + buffer_json_member_add_uint64(wb, "contexts_soft_hash", versions->contexts_soft_hash);

1186 + buffer_json_member_add_uint64(wb, "alerts_hard_hash", versions->alerts_hard_hash);

1187 + buffer_json_member_add_uint64(wb, "alerts_soft_hash", versions->alerts_soft_hash);

1188 + buffer_json_object_close(wb);

1189 +}

1190 +

1191 void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {

1192 QUERY_TARGET *qt = r->internal.qt;

1193 RRDR_OPTIONS options = qt->request.options;

@@ -1251,15 +1267,12 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {

1267 }

1268 buffer_json_object_close(wb); // aggregations

1269

1254 - buffer_json_member_add_uint64(wb, "timeout", qt->request.timeout);

1270 + buffer_json_member_add_uint64(wb, "timeout", qt->request.timeout_ms);

1271 }

1272 buffer_json_object_close(wb); // request

1273 }

1274

1259 - buffer_json_member_add_object(wb, "versions");

1260 - buffer_json_member_add_uint64(wb, "contexts_hard_hash", qt->versions.contexts_hard_hash);

1261 - buffer_json_member_add_uint64(wb, "contexts_soft_hash", qt->versions.contexts_soft_hash);

1262 - buffer_json_object_close(wb);

1275 + version_hashes_api_v2(wb, &qt->versions);

1276

1277 buffer_json_member_add_object(wb, "summary");

1278 struct summary_total_counts

@@ -1432,8 +1445,10 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {

1445 rrdr_dimension_units_array_v2(wb, "units", r, options);

1446 rrdr_dimension_priority_array_v2(wb, "priorities", r, options);

1447 rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options);

1435 - rrdr_dimension_average_values(wb, "view_average_values", r, options);

1436 - size_t dims = rrdr_dimension_latest_values(wb, "view_latest_values", r, options);

1448 + rrdr_dimension_view_minimum_values(wb, "view_minimum_values", r, options);

1449 + rrdr_dimension_view_maximum_values(wb, "view_maximum_values", r, options);

1450 + rrdr_dimension_view_average_values(wb, "view_average_values", r, options);

1451 + size_t dims = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options);

1452 buffer_json_member_add_uint64(wb, "count", dims);

1453 rrdr_json_group_by_labels(wb, "labels", r, options);

1454 }

web/api/formatters/json_wrapper.h

@@ -15,4 +15,7 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb);

15 void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb);

16 void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb);

18 +struct query_versions;

19 +void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions);

20 +

21 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H

web/api/formatters/rrd2json.c

-4

@@ -11,10 +11,6 @@ inline bool query_target_has_percentage_units(struct query_target *qt) {

11 return false;

12 }

14 -bool query_target_aggregatable(struct query_target *qt) {

15 - return (qt->request.options & RRDR_OPTION_RETURN_RAW);

16 -}

17 -

14 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {

15 rrdset2json(st, wb, NULL, NULL, 0);

16 }

web/api/formatters/rrd2json.h

+5 -1

@@ -63,7 +63,7 @@ void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTION

64 struct query_target;

65 bool query_target_has_percentage_units(struct query_target *qt);

66 -bool query_target_aggregatable(struct query_target *qt);

66 +#define query_target_aggregatable(qt) (qt->request.options & RRDR_OPTION_RETURN_RAW)

68 int rrdset2value_api_v1(

69 RRDSET *st

@@ -91,9 +91,13 @@ int rrdset2value_api_v1(

91 );

93 static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) {

94 + if(unlikely(options & RRDR_OPTION_RETURN_RAW))

95 + return true;

96 +

97 if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false;

98 if(unlikely(!(rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) return false;

99 if(unlikely((options & RRDR_OPTION_NONZERO) && !(rrdr_dim_flags & RRDR_DIMENSION_NONZERO))) return false;

100 +

101 return true;

102 }

103

web/api/formatters/value/value.c

+17 -35

@@ -13,24 +13,8 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all

13 NETDATA_DOUBLE sum = 0, min = 0, max = 0, v;

14 int all_null = 1, init = 1;

16 - NETDATA_DOUBLE total = 1;

16 NETDATA_DOUBLE total_anomaly_rate = 0;

19 - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {

20 - total = 0;

21 - for (c = 0; c < r->d ; c++) {

22 - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

23 - NETDATA_DOUBLE n = cn[c];

24 -

25 - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

26 - n = -n;

27 -

28 - total += n;

29 - }

30 - // prevent a division by zero

31 - if(total == 0) total = 1;

32 - }

33 -

18 // for each dimension

19 for (c = 0; c < r->d ; c++) {

20 if(!rrdr_dimension_should_be_exposed(r->od[c], options))

@@ -38,21 +22,6 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all

23 NETDATA_DOUBLE n = cn[c];

41 - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))

42 - n = -n;

43 -

44 - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {

45 - n = n * 100 / total;

46 -

47 - if(unlikely(c == 0)) {

48 - r->view.min = r->view.max = n;

49 - }

50 - else {

51 - if (n < r->view.min) r->view.min = n;

52 - if (n > r->view.max) r->view.max = n;

53 - }

54 - }

55 -

25 if(unlikely(init)) {

26 if(n > 0) {

27 min = 0;

@@ -102,7 +71,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all

71 QUERY_VALUE rrdmetric2value(RRDHOST *host,

72 struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,

73 time_t after, time_t before,

105 - RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method, const char *group_options,

74 + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options,

75 size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority

76 ) {

77 QUERY_TARGET_REQUEST qtr = {

@@ -115,10 +84,10 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,

84 .before = before,

85 .points = 1,

86 .options = options,

118 - .time_group_method = group_method,

119 - .time_group_options = group_options,

87 + .time_group_method = time_group_method,

88 + .time_group_options = time_group_options,

89 .tier = tier,

121 - .timeout = timeout,

90 + .timeout_ms = timeout,

91 .query_source = query_source,

92 .priority = priority,

93 };

@@ -132,6 +101,13 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,

101 qv = (QUERY_VALUE) {

102 .value = NAN,

103 .anomaly_rate = NAN,

104 + .sp = {

105 + .count = 0,

106 + .min = NAN,

107 + .max = NAN,

108 + .sum = NAN,

109 + .anomaly_count = 0,

110 + }

111 };

112 }

113 else {

@@ -140,8 +116,14 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,

116 .before = r->view.before,

117 .points_read = r->stats.db_points_read,

118 .result_points = r->stats.result_points_generated,

119 + .sp = {

120 + .count = 0,

121 + }

122 };

123

124 + for(size_t d = 0; d < r->d ;d++)

125 + storage_point_merge_to(qv.sp, r->drs[d]);

126 +

127 for(size_t t = 0; t < storage_tiers ;t++)

128 qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points;

129

web/api/formatters/value/value.h

+2 -1

@@ -13,6 +13,7 @@ typedef struct storage_value {

13 size_t points_read;

14 size_t storage_points_per_tier[RRD_STORAGE_TIERS];

15 size_t result_points;

16 + STORAGE_POINT sp;

17 } QUERY_VALUE;

19 struct rrdmetric_acquired;

@@ -22,7 +23,7 @@ struct rrdcontext_acquired;

23 QUERY_VALUE rrdmetric2value(RRDHOST *host,

24 struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,

25 time_t after, time_t before,

25 - RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method, const char *group_options,

26 + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options,

27 size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority

28 );

web/api/netdata-swagger.yaml

+37

@@ -388,6 +388,7 @@ paths:

388 in: query

389 description: |

390 The aggregation function to apply when grouping metrics together.

391 + When option `raw` is given, `average` and `avg` behave like `sum` and the caller is expected to calculate the average.

392 required: false

393 schema:

394 type: string

@@ -2414,6 +2415,10 @@ components:

2415 Hashes that allow the caller to detect important database changes of Netdata agents.

2416 type: object

2417 properties:

2418 + nodes_hard_hash:

2419 + description: |

2420 + An auto-increment value that reflects the number of changes to the number of nodes maintained by the server. Everytime a node is added or removed, this number gets incremented.

2421 + type: integer

2422 contexts_hard_hash:

2423 description: |

2424 An auto-increment value that reflects the number of changes to the number of contexts maintained by the server. Everytime a context is added or removed, this number gets incremented.

@@ -2422,6 +2427,14 @@ components:

2427 description: |

2428 An auto-increment value that reflects the number of changes to the queue that sends contexts updates to Netdata Cloud. Everytime the contents of a context are updated, this number gets incremented.

2429 type: integer

2430 + alerts_hard_hash:

2431 + description: |

2432 + An auto-increment value that reflects the number of changes to the number of alerts. Everytime an alert is added or removed, this number gets incremented.

2433 + type: integer

2434 + alerts_soft_hash:

2435 + description: |

2436 + An auto-increment value that reflects the number of alerts transitions. Everytime an alert transitions to a new state, this number gets incremented.

2437 + type: integer

2438 summary:

2439 description: |

2440 Summarized information about nodes, contexts, instances, labels, alerts, and dimensions. The items returned are determined by the scope of the query only, however the statistical data in them are influenced by the filters of the query. Using this information the dashboard allows users to slice and dice the data by filtering and grouping.

@@ -2747,6 +2760,18 @@ components:

2760 type: array

2761 items:

2762 type: integer

2763 + view_minimum_values:

2764 + description: |

2765 + An array of the minimum value of each dimension across the entire query.

2766 + type: array

2767 + items:

2768 + type: number

2769 + view_maximum_values:

2770 + description: |

2771 + An array of the maximum value of each dimension across the entire query.

2772 + type: array

2773 + items:

2774 + type: number

2775 view_average_values:

2776 description: |

2777 An array of the average value of each dimension across the entire query.

@@ -2837,6 +2862,12 @@ components:

2862 Statistical values

2863 type: object

2864 properties:

2865 + min:

2866 + description: The minimum value of all metrics aggregated

2867 + type: number

2868 + max:

2869 + description: The maximum value of all metrics aggregated

2870 + type: number

2871 avg:

2872 description: The average value of all metrics aggregated

2873 type: number

@@ -2851,6 +2882,12 @@ components:

2882 Statistical values when `raw` option is given.

2883 type: object

2884 properties:

2885 + min:

2886 + description: The minimum value of all metrics aggregated

2887 + type: number

2888 + max:

2889 + description: The maximum value of all metrics aggregated

2890 + type: number

2891 sum:

2892 description: The sum value of all metrics aggregated

2893 type: number

web/api/queries/query.c

+342 -194

@@ -775,16 +775,13 @@ static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rr

775 static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) {

776 rrdr_line++;

777

778 - internal_error(rrdr_line >= (long)r->n,

778 + internal_fatal(rrdr_line >= (long)r->n,

779 "QUERY: requested to step above RRDR size for query '%s'",

780 r->internal.qt->id);

781

782 - internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t,

783 - "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of query '%s'",

784 - (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->internal.qt->id);

785 -

786 - // save the time

787 - r->t[rrdr_line] = t;

782 + internal_fatal(r->t[rrdr_line] != t,

783 + "QUERY: wrong timestamp at RRDR line %ld, expected %ld, got %ld, of query '%s'",

784 + rrdr_line, r->t[rrdr_line], t, r->internal.qt->id);

785

786 return rrdr_line;

787 }

@@ -986,26 +983,20 @@ static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, t

983 // query ops

984

985 typedef struct query_point {

989 - time_t end_time;

990 - time_t start_time;

986 + STORAGE_POINT sp;

987 NETDATA_DOUBLE value;

992 - size_t anomaly_outlier_points;

993 - size_t anomaly_all_points;

994 - SN_FLAGS flags;

988 + bool added;

989 #ifdef NETDATA_INTERNAL_CHECKS

990 size_t id;

991 #endif

992 } QUERY_POINT;

993

994 QUERY_POINT QUERY_POINT_EMPTY = {

1001 - .end_time = 0,

1002 - .start_time = 0,

1003 - .value = NAN,

1004 - .anomaly_outlier_points = 0,

1005 - .anomaly_all_points = 0,

1006 - .flags = SN_FLAG_NONE,

995 + .sp = STORAGE_POINT_UNSET,

996 + .value = NAN,

997 + .added = false,

998 #ifdef NETDATA_INTERNAL_CHECKS

1008 - .id = 0,

999 + .id = 0,

1000 #endif

1001 };

1002

@@ -1042,8 +1033,7 @@ typedef struct query_engine_ops {

1033 NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);

1034 size_t group_points_non_zero;

1035 size_t group_points_added;

1045 - size_t group_anomaly_outlier_points;

1046 - size_t group_anomaly_all_points;

1036 + STORAGE_POINT group_point; // aggregates min, max, sum, count, anomaly count for each group point

1037 RRDR_VALUE_FLAGS group_value_flags;

1038

1039 // statistics

@@ -1360,18 +1350,18 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before

1350 #define query_interpolate_point(this_point, last_point, now) do { \

1351 if(likely( \

1352 /* the point to interpolate is more than 1s wide */ \

1363 - (this_point).end_time - (this_point).start_time > 1 \

1353 + (this_point).sp.end_time_s - (this_point).sp.start_time_s > 1 \

1354 \

1355 /* the two points are exactly next to each other */ \

1366 - && (last_point).end_time == (this_point).start_time \

1356 + && (last_point).sp.end_time_s == (this_point).sp.start_time_s \

1357 \

1358 /* both points are valid numbers */ \

1359 && netdata_double_isnumber((this_point).value) \

1360 && netdata_double_isnumber((last_point).value) \

1361 \

1362 )) { \

1373 - (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).end_time - (now)) / (NETDATA_DOUBLE)((this_point).end_time - (this_point).start_time)); \

1374 - (this_point).end_time = now; \

1363 + (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).sp.end_time_s - (now)) / (NETDATA_DOUBLE)((this_point).sp.end_time_s - (this_point).sp.start_time_s)); \

1364 + (this_point).sp.end_time_s = now; \

1365 } \

1366 } while(0)

1367

@@ -1380,15 +1370,16 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before

1370 if(likely(fpclassify((point).value) != FP_ZERO)) \

1371 (ops)->group_points_non_zero++; \

1372 \

1383 - if(unlikely((point).flags & SN_FLAG_RESET)) \

1373 + if(unlikely((point).sp.flags & SN_FLAG_RESET)) \

1374 (ops)->group_value_flags |= RRDR_VALUE_RESET; \

1375 \

1376 (ops)->grouping_add(r, (point).value); \

1377 + \

1378 + if(!(point).added) \

1379 + storage_point_merge_to((ops)->group_point, (point).sp); \

1380 } \

1381 \

1382 (ops)->group_points_added++; \

1390 - (ops)->group_anomaly_outlier_points = (point).anomaly_outlier_points; \

1391 - (ops)->group_anomaly_all_points = (point).anomaly_all_points; \

1383 } while(0)

1384

1385 static __thread QUERY_ENGINE_OPS *released_ops = NULL;

@@ -1450,6 +1441,10 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1441 QUERY_TARGET *qt = r->internal.qt;

1442 QUERY_METRIC *qm = ops->qm;

1443

1444 + r->drs[dim_id_in_rrdr] = STORAGE_POINT_UNSET;

1445 + ops->group_point = STORAGE_POINT_UNSET;

1446 +

1447 + RRDR_OPTIONS options = qt->request.options;

1448 size_t points_wanted = qt->window.points;

1449 time_t after_wanted = qt->window.after;

1450 time_t before_wanted = qt->window.before; (void)before_wanted;

@@ -1458,9 +1453,6 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1453 // if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)

1454 // debug_this = true;

1455

1461 - time_t max_date = 0,

1462 - min_date = 0;

1463 -

1456 size_t points_added = 0;

1457

1458 long rrdr_line = -1;

@@ -1484,10 +1476,11 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1476 size_t query_is_finished_counter = 0;

1477

1478 // The main loop, based on the query granularity we need

1487 - for( ; points_added < points_wanted && query_is_finished_counter <= 10 ; now_start_time = now_end_time, now_end_time += ops->view_update_every) {

1479 + for( ; points_added < points_wanted && query_is_finished_counter <= 10 ;

1480 + now_start_time = now_end_time, now_end_time += ops->view_update_every) {

1481

1482 if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) {

1490 - query_planer_next_plan(ops, now_end_time, new_point.end_time);

1483 + query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s);

1484 db_points_read_since_plan_switch = 0;

1485 }

1486

@@ -1508,8 +1501,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1501 last1_point = new_point;

1502 }

1503 new_point = QUERY_POINT_EMPTY;

1511 - new_point.start_time = last1_point.end_time;

1512 - new_point.end_time = now_end_time;

1504 + new_point.sp.start_time_s = last1_point.sp.end_time_s;

1505 + new_point.sp.end_time_s = now_end_time;

1506 //

1507 // if(debug_this) info("QUERY: is finished() returned true");

1508 //

@@ -1524,6 +1517,11 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1517 if(likely(storage_point_is_unset(next1_point))) {

1518 db_points_read_since_plan_switch++;

1519 sp = ops->next_metric(ops->handle);

1520 + ops->db_points_read_per_tier[ops->tier]++;

1521 + ops->db_total_points_read++;

1522 +

1523 + if(unlikely(options & RRDR_OPTION_ABSOLUTE))

1524 + storage_point_make_positive(sp);

1525 }

1526 else {

1527 // ONE POINT READ-AHEAD

@@ -1534,7 +1532,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1532

1533 // ONE POINT READ-AHEAD

1534 if(unlikely(query_plan_should_switch_plan(ops, sp.end_time_s) &&

1537 - query_planer_next_plan(ops, now_end_time, new_point.end_time))) {

1535 + query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s))) {

1536

1537 // The end time of the current point, crosses our plans (tiers)

1538 // so, we switched plan (tier)

@@ -1545,6 +1543,11 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1543 // B. part of the point of the previous plan overlaps with the point from the next plan

1544

1545 STORAGE_POINT sp2 = ops->next_metric(ops->handle);

1546 + ops->db_points_read_per_tier[ops->tier]++;

1547 + ops->db_total_points_read++;

1548 +

1549 + if(unlikely(options & RRDR_OPTION_ABSOLUTE))

1550 + storage_point_make_positive(sp);

1551

1552 if(sp.start_time_s > sp2.start_time_s)

1553 // the point from the previous plan is useless

@@ -1556,13 +1559,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1559 next1_point = sp2;

1560 }

1561

1559 - ops->db_points_read_per_tier[ops->tier]++;

1560 - ops->db_total_points_read++;

1561 -

1562 - new_point.start_time = sp.start_time_s;

1563 - new_point.end_time = sp.end_time_s;

1564 - new_point.anomaly_outlier_points = sp.anomaly_count;

1565 - new_point.anomaly_all_points = sp.count;

1562 + new_point.sp = sp;

1563 + new_point.added = false;

1564 query_point_set_id(new_point, ops->db_total_points_read);

1565

1566 // if(debug_this)

@@ -1573,7 +1571,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1571 if(likely(!storage_point_is_unset(sp) && !storage_point_is_gap(sp))) {

1572

1573 if(unlikely(use_anomaly_bit_as_value))

1576 - new_point.value = new_point.anomaly_all_points ? (NETDATA_DOUBLE)new_point.anomaly_outlier_points * 100.0 / (NETDATA_DOUBLE)new_point.anomaly_all_points : 0.0;

1574 + new_point.value = storage_point_anomaly_rate(new_point.sp);

1575

1576 else {

1577 switch (ops->tier_query_fetch) {

@@ -1596,27 +1594,25 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1594 };

1595 }

1596 }

1599 - else {

1597 + else

1598 new_point.value = NAN;

1601 - new_point.flags = SN_FLAG_NONE;

1602 - }

1599 }

1600

1601 // check if the db is giving us zero duration points

1602 if(unlikely(db_points_read_since_plan_switch > 1 &&

1607 - new_point.start_time == new_point.end_time)) {

1603 + new_point.sp.start_time_s == new_point.sp.end_time_s)) {

1604

1605 internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned "

1606 "point %zu from %ld to %ld, that are both equal",

1611 - qt->id, query_metric_id(qt, qm),

1612 - new_point.id, new_point.start_time, new_point.end_time);

1607 + qt->id, query_metric_id(qt, qm),

1608 + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s);

1609

1614 - new_point.start_time = new_point.end_time - ops->tier_ptr->db_update_every_s;

1610 + new_point.sp.start_time_s = new_point.sp.end_time_s - ops->tier_ptr->db_update_every_s;

1611 }

1612

1613 // check if the db is advancing the query

1614 if(unlikely(db_points_read_since_plan_switch > 1 &&

1619 - new_point.end_time <= last1_point.end_time)) {

1615 + new_point.sp.end_time_s <= last1_point.sp.end_time_s)) {

1616

1617 internal_error(true,

1618 "QUERY: '%s', dimension '%s' next_metric() returned "

@@ -1624,8 +1620,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1620 "last point %zu from %ld to %ld, "

1621 "now is %ld to %ld",

1622 qt->id, query_metric_id(qt, qm),

1627 - new_point.id, new_point.start_time, new_point.end_time,

1628 - last1_point.id, last1_point.start_time, last1_point.end_time,

1623 + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s,

1624 + last1_point.id, last1_point.sp.start_time_s, last1_point.sp.end_time_s,

1625 now_start_time, now_end_time);

1626

1627 count_same_end_time++;

@@ -1634,13 +1630,14 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1630 count_same_end_time = 0;

1631

1632 // decide how to use this point

1637 - if(likely(new_point.end_time < now_end_time)) { // likely to favor tier0

1633 + if(likely(new_point.sp.end_time_s < now_end_time)) { // likely to favor tier0

1634 // this db point ends before our now_end_time

1635

1640 - if(likely(new_point.end_time >= now_start_time)) { // likely to favor tier0

1636 + if(likely(new_point.sp.end_time_s >= now_start_time)) { // likely to favor tier0

1637 // this db point ends after our now_start time

1638

1639 query_add_point_to_group(r, new_point, ops);

1640 + new_point.added = true;

1641 }

1642 else {

1643 // we don't need this db point

@@ -1651,14 +1648,14 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1648 // at exactly the time we will want

1649

1650 // we only log if this is not point 1

1654 - internal_error(new_point.end_time < ops->plan_expanded_after &&

1651 + internal_error(new_point.sp.end_time_s < ops->plan_expanded_after &&

1652 db_points_read_since_plan_switch > 1,

1653 "QUERY: '%s', dimension '%s' next_metric() "

1654 "returned point %zu from %ld time %ld, "

1655 "which is entirely before our current timeframe %ld to %ld "

1656 "(and before the entire query, after %ld, before %ld)",

1657 qt->id, query_metric_id(qt, qm),

1661 - new_point.id, new_point.start_time, new_point.end_time,

1658 + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s,

1659 now_start_time, now_end_time,

1660 ops->plan_expanded_after, ops->plan_expanded_before);

1661 }

@@ -1677,14 +1674,14 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1674 " it returned an end time less or equal to the end time of the last "

1675 "point we got %ld, %zu times",

1676 qt->id, query_metric_id(qt, qm),

1680 - last1_point.end_time, count_same_end_time);

1677 + last1_point.sp.end_time_s, count_same_end_time);

1678

1682 - if(unlikely(new_point.end_time <= last1_point.end_time))

1683 - new_point.end_time = now_end_time;

1679 + if(unlikely(new_point.sp.end_time_s <= last1_point.sp.end_time_s))

1680 + new_point.sp.end_time_s = now_end_time;

1681 }

1682

1686 - time_t stop_time = new_point.end_time;

1687 - if(unlikely(!storage_point_is_unset(next1_point))) {

1683 + time_t stop_time = new_point.sp.end_time_s;

1684 + if(unlikely(!storage_point_is_unset(next1_point) && next1_point.start_time_s >= now_end_time)) {

1685 // ONE POINT READ-AHEAD

1686 // the point crosses the start time of the

1687 // read ahead storage point we have read

@@ -1695,18 +1692,20 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1692 // we have 3 points in memory: last2, last1, new

1693 // we select the one to use based on their timestamps

1694

1698 - size_t iterations = 0;

1699 - for ( ; now_end_time <= stop_time && points_added < points_wanted ;

1700 - now_end_time += ops->view_update_every, iterations++) {

1695 + internal_fatal(now_end_time > stop_time || points_added >= points_wanted,

1696 + "QUERY: first part of query provides invalid point to interpolate (now_end_time %ld, stop_time %ld",

1697 + now_end_time, stop_time);

1698

1699 + do {

1700 // now_start_time is wrong in this loop

1701 // but, we don't need it

1702

1703 QUERY_POINT current_point;

1704

1707 - if(likely(now_end_time > new_point.start_time)) {

1705 + if(likely(now_end_time > new_point.sp.start_time_s)) {

1706 // it is time for our NEW point to be used

1707 current_point = new_point;

1708 + new_point.added = true; // first copy, then set it, so that new_point will not be added again

1709 query_interpolate_point(current_point, last1_point, now_end_time);

1710

1711 // internal_error(current_point.id > 0

@@ -1722,9 +1721,10 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1721 // current_point.id, current_point.start_time, current_point.end_time,

1722 // now_end_time);

1723 }

1725 - else if(likely(now_end_time <= last1_point.end_time)) {

1724 + else if(likely(now_end_time <= last1_point.sp.end_time_s)) {

1725 // our LAST point is still valid

1726 current_point = last1_point;

1727 + last1_point.added = true; // first copy, then set it, so that last1_point will not be added again

1728 query_interpolate_point(current_point, last2_point, now_end_time);

1729

1730 // internal_error(current_point.id > 0

@@ -1749,9 +1749,6 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1749 rrdr_line = rrdr_line_init(r, now_end_time, rrdr_line);

1750 size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;

1751

1752 - if(unlikely(!min_date)) min_date = now_end_time;

1753 - max_date = now_end_time;

1754 -

1752 // find the place to store our values

1753 RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];

1754

@@ -1766,9 +1763,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1763 NETDATA_DOUBLE group_value = ops->grouping_flush(r, rrdr_value_options_ptr);

1764 r->v[rrdr_o_v_index] = group_value;

1765

1769 - NETDATA_DOUBLE group_ar = r->ar[rrdr_o_v_index] = ops->group_anomaly_all_points ?

1770 - (NETDATA_DOUBLE)ops->group_anomaly_outlier_points * 100.0 / (NETDATA_DOUBLE)ops->group_anomaly_all_points

1771 - : 0.0;

1766 + NETDATA_DOUBLE group_ar = r->ar[rrdr_o_v_index] = storage_point_anomaly_rate(ops->group_point);

1767

1768 if(likely(points_added || r->internal.queries_count)) {

1769 // find the min/max across all dimensions

@@ -1783,6 +1778,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1778 min = max = group_value;

1779 }

1780

1781 + // for volume contribution calculation we need the absolute value

1782 NETDATA_DOUBLE stats_value = group_value < 0 ? -group_value : group_value;

1783

1784 if(unlikely(!points_added)) {

@@ -1806,49 +1802,38 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_

1802 ops->group_points_added = 0;

1803 ops->group_value_flags = RRDR_VALUE_NOTHING;

1804 ops->group_points_non_zero = 0;

1809 - ops->group_anomaly_outlier_points = 0;

1810 - ops->group_anomaly_all_points = 0;

1811 - }

1812 - // the loop above increased "now" by query_granularity,

1805 +

1806 + storage_point_merge_to(r->drs[dim_id_in_rrdr], ops->group_point);

1807 + ops->group_point = STORAGE_POINT_UNSET;

1808 +

1809 + now_end_time += ops->view_update_every;

1810 + } while(now_end_time <= stop_time && points_added < points_wanted);

1811 +

1812 + // the loop above increased "now" by ops->view_update_every,

1813 // but the main loop will increase it too,

1814 // so, let's undo the last iteration of this loop

1815 - if(iterations)

1816 - now_end_time -= ops->view_update_every;

1815 + now_end_time -= ops->view_update_every;

1816 }

1817 query_planer_finalize_remaining_plans(ops);

1818

1819 // fill the rest of the points with empty values

1820 while (points_added < points_wanted) {

1822 - if(!max_date)

1823 - min_date = max_date = after_wanted + ops->view_update_every - ops->query_granularity;

1824 - else

1825 - max_date += ops->view_update_every;

1826 -

1827 - rrdr_line = rrdr_line_init(r, max_date, rrdr_line);

1821 + rrdr_line++;

1822 size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;

1823 r->o[rrdr_o_v_index] = RRDR_VALUE_EMPTY;

1824 r->v[rrdr_o_v_index] = 0.0;

1825 r->ar[rrdr_o_v_index] = 0.0;

1832 -

1826 points_added++;

1827 }

1828

1829 r->internal.queries_count++;

1830 r->view.min = min;

1831 r->view.max = max;

1839 - r->view.before = max_date;

1840 - r->view.after = min_date - ops->view_update_every + ops->query_granularity;

1841 - r->rows = rrdr_line + 1;

1832

1833 r->stats.result_points_generated += points_added;

1834 r->stats.db_points_read += ops->db_total_points_read;

1835 for(size_t tr = 0; tr < storage_tiers ; tr++)

1836 qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];

1847 -

1848 - internal_error(points_added != points_wanted,

1849 - "QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).",

1850 - qt->id, query_metric_id(qt, qm),

1851 - (size_t)points_wanted, (size_t)points_added, ops->db_total_points_read);

1837 }

1838

1839 // ----------------------------------------------------------------------------

@@ -2092,7 +2077,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {

2077 time_t resampling_time_requested = qt->request.resampling_time;

2078 RRDR_OPTIONS options = qt->request.options;

2079 size_t tier = qt->request.tier;

2095 - time_t update_every = qt->db.minimum_latest_update_every_s;

2080 + time_t update_every = qt->db.minimum_latest_update_every_s ? qt->db.minimum_latest_update_every_s : 1;

2081

2082 // RULES

2083 // points_requested = 0

@@ -2151,27 +2136,36 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {

2136 time_t last_entry_s = qt->db.last_time_s;

2137

2138 if (first_entry_s == 0 || last_entry_s == 0) {

2154 - internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld", qt->id, first_entry_s, last_entry_s);

2155 - query_debug_log_free();

2156 - return false;

2157 - }

2139 + internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld)", qt->id, first_entry_s, last_entry_s);

2140 + after_wanted = qt->window.after;

2141 + before_wanted = qt->window.before;

2142

2159 - query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_s, last_entry_s);

2143 + if(after_wanted == before_wanted)

2144 + after_wanted = before_wanted - update_every;

2145

2161 - if (after_wanted == 0) {

2162 - after_wanted = first_entry_s;

2163 - query_debug_log(":zero after_wanted %ld", after_wanted);

2146 + if (points_wanted == 0) {

2147 + points_wanted = (before_wanted - after_wanted) / update_every;

2148 + query_debug_log(":zero points_wanted %zu", points_wanted);

2149 + }

2150 }

2151 + else {

2152 + query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_s, last_entry_s);

2153

2166 - if (before_wanted == 0) {

2167 - before_wanted = last_entry_s;

2168 - before_is_aligned_to_db_end = true;

2169 - query_debug_log(":zero before_wanted %ld", before_wanted);

2170 - }

2154 + if (after_wanted == 0) {

2155 + after_wanted = first_entry_s;

2156 + query_debug_log(":zero after_wanted %ld", after_wanted);

2157 + }

2158

2172 - if (points_wanted == 0) {

2173 - points_wanted = (last_entry_s - first_entry_s) / update_every;

2174 - query_debug_log(":zero points_wanted %zu", points_wanted);

2159 + if (before_wanted == 0) {

2160 + before_wanted = last_entry_s;

2161 + before_is_aligned_to_db_end = true;

2162 + query_debug_log(":zero before_wanted %ld", before_wanted);

2163 + }

2164 +

2165 + if (points_wanted == 0) {

2166 + points_wanted = (last_entry_s - first_entry_s) / update_every;

2167 + query_debug_log(":zero points_wanted %zu", points_wanted);

2168 + }

2169 }

2170 }

2171

@@ -2443,6 +2437,45 @@ static int group_by_label_is_space(char c) {

2437 return 0;

2438 }

2439

2440 +static void rrd2rrdr_set_timestamps(RRDR *r) {

2441 + QUERY_TARGET *qt = r->internal.qt;

2442 +

2443 + internal_fatal(qt->window.points != r->n, "QUERY: mismatch to the number of points in qt and r");

2444 +

2445 + r->view.group = qt->window.group;

2446 + r->view.update_every = (int) (qt->window.group * qt->window.query_granularity);

2447 + r->view.before = qt->window.before;

2448 + r->view.after = qt->window.after;

2449 + r->view.options = qt->window.options;

2450 +

2451 + r->time_grouping.points_wanted = qt->window.points;

2452 + r->time_grouping.resampling_group = qt->window.resampling_group;

2453 + r->time_grouping.resampling_divisor = qt->window.resampling_divisor;

2454 +

2455 + r->rows = qt->window.points;

2456 +

2457 + size_t points_wanted = qt->window.points;

2458 + time_t after_wanted = qt->window.after;

2459 + time_t before_wanted = qt->window.before;

2460 +

2461 + time_t view_update_every = r->view.update_every;

2462 + time_t query_granularity = (time_t)(r->view.update_every / r->view.group);

2463 +

2464 + size_t rrdr_line = 0;

2465 + time_t first_point_end_time = after_wanted + view_update_every - query_granularity;

2466 + time_t now_end_time = first_point_end_time;

2467 +

2468 + while (rrdr_line < points_wanted) {

2469 + r->t[rrdr_line++] = now_end_time;

2470 + now_end_time += view_update_every;

2471 + }

2472 +

2473 + internal_fatal(r->t[0] != first_point_end_time, "QUERY: wrong first timestamp in the query");

2474 + internal_error(r->t[points_wanted - 1] != before_wanted,

2475 + "QUERY: wrong last timestamp in the query, expected %ld, found %ld",

2476 + before_wanted, r->t[points_wanted - 1]);

2477 +}

2478 +

2479 static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2480 RRDR_OPTIONS options = qt->request.options;

2481

@@ -2463,6 +2496,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2496 r->dn[d] = rrdmetric_acquired_name_dup(qd->rma);

2497 }

2498

2499 + rrd2rrdr_set_timestamps(r);

2500 return r;

2501 }

2502

@@ -2517,12 +2551,12 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2551 else {

2552 if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {

2553 buffer_fast_strcat(key, "|", 1);

2520 - buffer_strcat(key, query_metric_id(qt, qm));

2554 + buffer_strcat(key, query_metric_name(qt, qm));

2555 }

2556

2557 if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {

2558 buffer_fast_strcat(key, "|", 1);

2525 - buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));

2559 + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));

2560 }

2561

2562 if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {

@@ -2570,7 +2604,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2604 }

2605 else {

2606 if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {

2573 - buffer_strcat(key, query_metric_id(qt, qm));

2607 + buffer_strcat(key, query_metric_name(qt, qm));

2608 }

2609

2610 if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {

@@ -2580,7 +2614,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2614 if (qt->request.group_by & RRDR_GROUP_BY_NODE)

2615 buffer_strcat(key, rrdinstance_acquired_id(qi->ria));

2616 else

2583 - buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));

2617 + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));

2618 }

2619

2620 if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {

@@ -2638,7 +2672,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2672 if (qt->request.group_by & RRDR_GROUP_BY_NODE)

2673 buffer_strcat(key, rrdinstance_acquired_name(qi->ria));

2674 else

2641 - buffer_strcat(key, string2str(query_instance_name_fqdn(qt, qi)));

2675 + buffer_strcat(key, string2str(query_instance_name_fqdn(qi, qt->request.version)));

2676 }

2677

2678 if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {

@@ -2748,8 +2782,13 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2782 goto cleanup;

2783 }

2784

2785 + rrd2rrdr_set_timestamps(r);

2786 + rrd2rrdr_set_timestamps(r->group_by.r);

2787 +

2788 r->dp = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dp));

2789 r->dv = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dv));

2790 + r->dmin = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dmin));

2791 + r->dmax = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dmax));

2792 r->dgbc = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dgbc));

2793 r->gbc = onewayalloc_callocz(r->internal.owa, r->n * r->d, sizeof(*r->gbc));

2794

@@ -2826,6 +2865,9 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r, size_t query_metric_id) {

2865 RRDR_OPTIONS options = qt->request.options;

2866 RRDR *r_tmp = r->group_by.r;

2867

2868 + QUERY_METRIC *qm = query_metric(qt, query_metric_id);

2869 + size_t d = qm->grouped_as.slot;

2870 +

2871 // do the group_by

2872 for(size_t i = 0; i != rrdr_rows(r_tmp) ; i++) {

2873

@@ -2855,12 +2897,6 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r, size_t query_metric_id) {

2897 continue;

2898 }

2899

2858 - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n_tmp < 0))

2859 - n_tmp = -n_tmp;

2860 -

2861 - QUERY_METRIC *qm = query_metric(qt, query_metric_id);

2862 - size_t d = qm->grouped_as.slot;

2863 -

2900 r->od[d] |= RRDR_DIMENSION_QUERIED;

2901

2902 NETDATA_DOUBLE *cn = &cn_base[d];

@@ -2891,86 +2927,198 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r, size_t query_metric_id) {

2927 (*gbc)++;

2928 }

2929 }

2930 +

2931 + for(size_t d_tmp = 0; d_tmp < r_tmp->d ; d_tmp++) {

2932 + if (unlikely(!(r_tmp->od[d_tmp] & RRDR_DIMENSION_QUERIED)))

2933 + continue;

2934 +

2935 + switch(qt->request.group_by_aggregate_function) {

2936 + case RRDR_GROUP_BY_FUNCTION_SUM:

2937 + storage_point_add_to(r->drs[d], r_tmp->drs[d_tmp]);

2938 + break;

2939 +

2940 + default:

2941 + case RRDR_GROUP_BY_FUNCTION_AVERAGE:

2942 + case RRDR_GROUP_BY_FUNCTION_MIN:

2943 + case RRDR_GROUP_BY_FUNCTION_MAX:

2944 + storage_point_merge_to(r->drs[d], r_tmp->drs[d_tmp]);

2945 + break;

2946 + }

2947 + }

2948 }

2949

2896 -static void rrd2rrdr_group_by_finalize(RRDR *r) {

2897 - if(!r->group_by.r)

2898 - return;

2950 +static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {

2951 + // FIXME - this is not optimal, we should not traverse the entire array to go to the end of it

2952

2900 - QUERY_TARGET *qt = r->internal.qt;

2901 - RRDR_OPTIONS options = qt->request.options;

2953 + size_t last_row_gbc = 0;

2954 + for (size_t i = 0; i != r->n; i++) {

2955 + size_t row_gbc = 0;

2956 + for (size_t d = 0; d < r->d; d++) {

2957 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))

2958 + continue;

2959

2903 - // copy the timestamps

2904 - for(size_t i = 0; i != r->n ;i++) {

2905 - r->t[i] = r->group_by.r->t[i];

2960 + row_gbc += r->gbc[ i * r->d + d ];

2961 + }

2962 +

2963 + if (unlikely(r->t[i] > r->partial_data_trimming.expected_after && row_gbc < last_row_gbc)) {

2964 + // discard the rest of the points

2965 + r->partial_data_trimming.trimmed_after = r->t[i];

2966 + r->rows = i;

2967 + break;

2968 + }

2969 + else

2970 + last_row_gbc = row_gbc;

2971 }

2972 +}

2973

2908 - if(!(options & RRDR_OPTION_RETURN_RAW)) {

2909 - // partial trimming

2910 - size_t last_row_gbc = 0;

2911 - for (size_t i = 0; i != r->n; i++) {

2912 - size_t row_gbc = 0;

2913 - for (size_t d = 0; d < r->d; d++) {

2914 - if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))

2915 - continue;

2974 +static void rrd2rrdr_convert_to_percentage(RRDR *r) {

2975 + size_t global_min_max_values = 0;

2976 + NETDATA_DOUBLE global_min = NAN, global_max = NAN;

2977 +

2978 + for(size_t i = 0; i != r->n ;i++) {

2979 + NETDATA_DOUBLE *cn = &r->v[ i * r->d ];

2980 + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];

2981

2917 - row_gbc += r->gbc[ i * r->d + d ];

2982 + NETDATA_DOUBLE total = 0;

2983 + for (size_t d = 0; d < r->d; d++) {

2984 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))

2985 + continue;

2986 +

2987 + if(co[d] & RRDR_VALUE_EMPTY)

2988 + continue;

2989 +

2990 + total += cn[d];

2991 + }

2992 +

2993 + if(total == 0.0)

2994 + total = 1.0;

2995 +

2996 + for (size_t d = 0; d < r->d; d++) {

2997 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))

2998 + continue;

2999 +

3000 + if(co[d] & RRDR_VALUE_EMPTY)

3001 + continue;

3002 +

3003 + NETDATA_DOUBLE n = cn[d];

3004 + n = cn[d] = n * 100.0 / total;

3005 +

3006 + if(unlikely(!global_min_max_values++))

3007 + global_min = global_max = n;

3008 + else {

3009 + if(n < global_min)

3010 + global_min = n;

3011 + if(n > global_max)

3012 + global_max = n;

3013 }

3014 + }

3015 + }

3016

2920 - if (unlikely(r->t[i] > r->partial_data_trimming.expected_after && row_gbc < last_row_gbc)) {

2921 - // discard the rest of the points

2922 - r->partial_data_trimming.trimmed_after = r->t[i];

2923 - r->rows = i;

2924 - break;

3017 + r->view.min = global_min;

3018 + r->view.max = global_max;

3019 +

3020 + if(!r->dv || !r->dmin || !r->dmax)

3021 + // v1 query

3022 + return;

3023 +

3024 + // v2 query

3025 +

3026 + for (size_t d = 0; d < r->d; d++) {

3027 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))

3028 + continue;

3029 +

3030 + size_t count = 0;

3031 + NETDATA_DOUBLE min = 0, max = 0, sum = 0;

3032 + for(size_t i = 0; i != r->rows ;i++) { // we use r->rows to respect trimming

3033 + size_t idx = i * r->d + d;

3034 +

3035 + RRDR_VALUE_FLAGS o = r->o[ idx ];

3036 +

3037 + if (o & RRDR_VALUE_EMPTY)

3038 + continue;

3039 +

3040 + NETDATA_DOUBLE n = r->v[ idx ];

3041 + sum += n;

3042 +

3043 + if(!count++)

3044 + min = max = n;

3045 + else {

3046 + if(n < min)

3047 + min = n;

3048 + if(n > max)

3049 + max = n;

3050 }

2926 - else

2927 - last_row_gbc = row_gbc;

3051 }

3052 +

3053 + r->dv[d] = (count) ? sum / (NETDATA_DOUBLE )count : 0.0;

3054 + r->dmin[d] = min;

3055 + r->dmax[d] = max;

3056 }

3057 +}

3058 +

3059 +static void rrd2rrdr_group_by_finalize(RRDR *r) {

3060 + QUERY_TARGET *qt = r->internal.qt;

3061 + RRDR_OPTIONS options = qt->request.options;

3062 +

3063 + if(!r->group_by.r) {

3064 + // v1 query

3065 + if(options & RRDR_OPTION_PERCENTAGE)

3066 + rrd2rrdr_convert_to_percentage(r);

3067 + return;

3068 + }

3069 + // v2 query

3070 +

3071 + // copy the timestamps

3072 + for(size_t i = 0; i != r->n ;i++) {

3073 + r->t[i] = r->group_by.r->t[i];

3074 + }

3075 +

3076 + if(!(options & RRDR_OPTION_RETURN_RAW) && r->partial_data_trimming.expected_after < qt->window.before)

3077 + rrdr2rrdr_group_by_partial_trimming(r);

3078

3079 // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max

2932 - size_t min_max_values = 0;

2933 - NETDATA_DOUBLE min = NAN, max = NAN;

3080 + size_t global_min_max_values = 0;

3081 + NETDATA_DOUBLE global_min = NAN, global_max = NAN;

3082 for (size_t d = 0; d < r->d; d++) {

3083 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))

3084 + continue;

3085 +

3086 size_t non_zero = 0;

3087

2937 - NETDATA_DOUBLE sum = 0;

3088 + NETDATA_DOUBLE min = 0, max = 0, sum = 0;

3089 size_t count = 0;

3090

3091 for(size_t i = 0; i != r->n ;i++) {

2941 - size_t idx2 = i * r->d + d;

3092 + size_t idx = i * r->d + d;

3093

2943 - NETDATA_DOUBLE *cn2 = &r->v[ idx2 ];

2944 - RRDR_VALUE_FLAGS *co2 = &r->o[ idx2 ];

2945 - NETDATA_DOUBLE *ar2 = &r->ar[ idx2 ];

2946 - uint32_t gbc2 = r->gbc[ idx2 ];

3094 + NETDATA_DOUBLE *cn = &r->v[ idx ];

3095 + RRDR_VALUE_FLAGS *co = &r->o[ idx ];

3096 + NETDATA_DOUBLE *ar = &r->ar[ idx ];

3097 + uint32_t gbc = r->gbc[ idx ];

3098

2948 - if(likely(gbc2)) {

2949 - *co2 &= ~RRDR_VALUE_EMPTY;

3099 + if(likely(gbc)) {

3100 + *co &= ~RRDR_VALUE_EMPTY;

3101

2951 - if(gbc2 != r->dgbc[d])

2952 - *co2 |= RRDR_VALUE_PARTIAL;

3102 + if(gbc != r->dgbc[d])

3103 + *co |= RRDR_VALUE_PARTIAL;

3104

3105 NETDATA_DOUBLE n;

3106

2956 - sum += *cn2;

2957 - count += gbc2;

3107 + sum += *cn;

3108

2959 - if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_AVERAGE)

2960 - n = (*cn2 /= gbc2);

3109 + if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_AVERAGE && !query_target_aggregatable(qt))

3110 + n = (*cn /= gbc);

3111 else

2962 - n = *cn2;

3112 + n = *cn;

3113

3114 if(!query_target_aggregatable(qt))

2965 - *ar2 /= gbc2;

3115 + *ar /= gbc;

3116

3117 if(islessgreater(n, 0.0))

3118 non_zero++;

3119

2970 - if(unlikely(!min_max_values++)) {

2971 - min = n;

2972 - max = n;

2973 - }

3120 + if(unlikely(!count))

3121 + min = max = n;

3122 else {

3123 if(n < min)

3124 min = n;

@@ -2978,6 +3126,18 @@ static void rrd2rrdr_group_by_finalize(RRDR *r) {

3126 if(n > max)

3127 max = n;

3128 }

3129 +

3130 + if(unlikely(!global_min_max_values++))

3131 + global_min = global_max = n;

3132 + else {

3133 + if(n < global_min)

3134 + global_min = n;

3135 +

3136 + if(n > global_max)

3137 + global_max = n;

3138 + }

3139 +

3140 + count += gbc;

3141 }

3142 }

3143

@@ -2985,10 +3145,15 @@ static void rrd2rrdr_group_by_finalize(RRDR *r) {

3145 r->od[d] |= RRDR_DIMENSION_NONZERO;

3146

3147 r->dv[d] = (count) ? sum / (NETDATA_DOUBLE)count : 0.0;

3148 + r->dmin[d] = min;

3149 + r->dmax[d] = max;

3150 }

3151

2990 - r->view.min = min;

2991 - r->view.max = max;

3152 + r->view.min = global_min;

3153 + r->view.max = global_max;

3154 +

3155 + if(options & RRDR_OPTION_PERCENTAGE)

3156 + rrd2rrdr_convert_to_percentage(r);

3157

3158 // update query instance counts in query host and query context

3159 {

@@ -3029,7 +3194,7 @@ RRDR *rrd2rrdr_legacy(

3194 ONEWAYALLOC *owa,

3195 RRDSET *st, size_t points, time_t after, time_t before,

3196 RRDR_TIME_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,

3032 - const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source,

3197 + const char *group_options, time_t timeout_ms, size_t tier, QUERY_SOURCE query_source,

3198 STORAGE_PRIORITY priority) {

3199

3200 QUERY_TARGET_REQUEST qtr = {

@@ -3043,7 +3208,7 @@ RRDR *rrd2rrdr_legacy(

3208 .options = options,

3209 .dimensions = dimensions,

3210 .time_group_options = group_options,

3046 - .timeout = timeout,

3211 + .timeout_ms = timeout_ms,

3212 .tier = tier,

3213 .query_source = query_source,

3214 .priority = priority,

@@ -3073,23 +3238,6 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

3238 else

3239 r->view.flags |= RRDR_RESULT_FLAG_ABSOLUTE;

3240

3076 - // -------------------------------------------------------------------------

3077 - // initialize RRDR

3078 -

3079 - r->view.group = qt->window.group;

3080 - r->view.update_every = (int) (qt->window.group * qt->window.query_granularity);

3081 - r->view.before = qt->window.before;

3082 - r->view.after = qt->window.after;

3083 - r->view.options = qt->window.options;

3084 - r->time_grouping.points_wanted = qt->window.points;

3085 - r->time_grouping.resampling_group = qt->window.resampling_group;

3086 - r->time_grouping.resampling_divisor = qt->window.resampling_divisor;

3087 -

3088 - if(r->group_by.r) {

3089 - r->group_by.r->view = r->view;

3090 - r->group_by.r->time_grouping = r->time_grouping;

3091 - }

3092 -

3241 RRDR *r_tmp = r->group_by.r ? r->group_by.r : r;

3242

3243 // -------------------------------------------------------------------------

@@ -3108,7 +3256,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

3256 long dimensions_used = 0, dimensions_nonzero = 0;

3257 struct timeval query_start_time;

3258 struct timeval query_current_time;

3111 - if (qt->request.timeout)

3259 + if (qt->request.timeout_ms)

3260 now_realtime_timeval(&query_start_time);

3261

3262 size_t last_db_points_read = 0;

@@ -3207,7 +3355,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

3355 last_db_points_read = r_tmp->stats.db_points_read;

3356 last_result_points_generated = r_tmp->stats.result_points_generated;

3357

3210 - if (qt->request.timeout)

3358 + if (qt->request.timeout_ms)

3359 now_realtime_timeval(&query_current_time);

3360

3361 if(qm->status & RRDR_DIMENSION_NONZERO)

@@ -3250,10 +3398,10 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

3398 log_access("QUERY INTERRUPTED");

3399 }

3400

3253 - if (qt->request.timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout) {

3401 + if (qt->request.timeout_ms && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout_ms) {

3402 cancel = true;

3403 log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)",

3256 - (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout);

3404 + (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout_ms);

3405 }

3406

3407 if(cancel) {

web/api/queries/rrdr.c

@@ -78,6 +78,9 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {

78 onewayalloc_freez(owa, r->du);

79 onewayalloc_freez(owa, r->dp);

80 onewayalloc_freez(owa, r->dv);

81 + onewayalloc_freez(owa, r->dmin);

82 + onewayalloc_freez(owa, r->dmax);

83 + onewayalloc_freez(owa, r->drs);

84 onewayalloc_freez(owa, r->ar);

85 onewayalloc_freez(owa, r->gbc);

86 onewayalloc_freez(owa, r->dgbc);

@@ -134,6 +137,7 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt, size_t dimensions, size_t

137 r->di = onewayalloc_callocz(owa, dimensions, sizeof(STRING *));

138 r->dn = onewayalloc_callocz(owa, dimensions, sizeof(STRING *));

139 r->du = onewayalloc_callocz(owa, dimensions, sizeof(STRING *));

140 + r->drs = onewayalloc_callocz(owa, dimensions, sizeof(STORAGE_POINT));

141 }

142

143 r->view.group = 1;

web/api/queries/rrdr.h

+4 -1

@@ -105,7 +105,10 @@ typedef struct rrdresult {

105 uint32_t *dgbc; // array of d dimension units - NOT ALLOCATED when RRDR is created

106 uint32_t *dp; // array of d dimension priority - NOT ALLOCATED when RRDR is created

107 NETDATA_DOUBLE *dv; // array of d dimension averages - NOT ALLOCATED when RRDR is created

108 + NETDATA_DOUBLE *dmin; // array of d dimension minimums - NOT ALLOCATED when RRDR is created

109 + NETDATA_DOUBLE *dmax; // array of d dimension maximums - NOT ALLOCATED when RRDR is created

110 DICTIONARY **dl; // array of d dimension labels - NOT ALLOCATED when RRDR is created

111 + STORAGE_POINT *drs; // array of d dimensions raw statistics (storage points)

112

113 DICTIONARY *label_keys;

114

@@ -183,7 +186,7 @@ RRDR *rrd2rrdr_legacy(

186 ONEWAYALLOC *owa,

187 RRDSET *st, size_t points, time_t after, time_t before,

188 RRDR_TIME_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,

186 - const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source,

189 + const char *group_options, time_t timeout_ms, size_t tier, QUERY_SOURCE query_source,

190 STORAGE_PRIORITY priority);

191

192 RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);

web/api/queries/weights.c

+714 -141

@@ -24,10 +24,11 @@ static struct {

24 const char *name;

25 WEIGHTS_METHOD value;

26 } weights_methods[] = {

27 - { "ks2" , WEIGHTS_METHOD_MC_KS2}

28 - , { "volume" , WEIGHTS_METHOD_MC_VOLUME}

29 - , { "anomaly-rate" , WEIGHTS_METHOD_ANOMALY_RATE}

30 - , { NULL , 0 }

27 + { "ks2" , WEIGHTS_METHOD_MC_KS2}

28 + , { "volume" , WEIGHTS_METHOD_MC_VOLUME}

29 + , { "anomaly-rate" , WEIGHTS_METHOD_ANOMALY_RATE}

30 + , { "value" , WEIGHTS_METHOD_VALUE}

31 + , { NULL , 0 }

32 };

34 WEIGHTS_METHOD weights_string_to_method(const char *method) {

@@ -56,14 +57,17 @@ typedef enum {

58 struct register_result {

59 RESULT_FLAGS flags;

60 + RRDHOST *host;

61 RRDCONTEXT_ACQUIRED *rca;

62 RRDINSTANCE_ACQUIRED *ria;

63 RRDMETRIC_ACQUIRED *rma;

64 NETDATA_DOUBLE value;

65 + STORAGE_POINT highlighted;

66 + STORAGE_POINT baseline;

67 };

69 static DICTIONARY *register_result_init() {

66 - DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED);

70 + DICTIONARY *results = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct register_result));

71 return results;

72 }

@@ -72,11 +76,14 @@ static void register_result_destroy(DICTIONARY *results) {

76 }

78 static void register_result(DICTIONARY *results,

79 + RRDHOST *host,

80 RRDCONTEXT_ACQUIRED *rca,

81 RRDINSTANCE_ACQUIRED *ria,

82 RRDMETRIC_ACQUIRED *rma,

83 NETDATA_DOUBLE value,

84 RESULT_FLAGS flags,

85 + STORAGE_POINT *highlighted,

86 + STORAGE_POINT *baseline,

87 WEIGHTS_STATS *stats,

88 bool register_zero) {

@@ -95,12 +102,19 @@ static void register_result(DICTIONARY *results,

102

103 struct register_result t = {

104 .flags = flags,

105 + .host = host,

106 .rca = rca,

107 .ria = ria,

108 .rma = rma,

101 - .value = v

109 + .value = v,

110 };

111

112 + if(highlighted)

113 + t.highlighted = *highlighted;

114 +

115 + if(baseline)

116 + t.baseline = *baseline;

117 +

118 // we can use the pointer address or RMA as a unique key for each metric

119 char buf[20 + 1];

120 ssize_t len = snprintfz(buf, 20, "%p", rma);

@@ -118,7 +132,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w

132 size_t examined_dimensions __maybe_unused, usec_t duration,

133 WEIGHTS_STATS *stats) {

134

121 - buffer_json_initialize(wb, "\"", "\"", 0, true, false);

135 + buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);

136 buffer_json_member_add_time_t(wb, "after", after);

137 buffer_json_member_add_time_t(wb, "before", before);

138 buffer_json_member_add_time_t(wb, "duration", before - after);

@@ -291,6 +305,344 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w

305 return total_dimensions;

306 }

307

308 +typedef enum {

309 + WPT_DIMENSION = 0,

310 + WPT_INSTANCE = 1,

311 + WPT_CONTEXT = 2,

312 + WPT_NODE = 3,

313 +} WEIGHTS_POINT_TYPE;

314 +

315 +static inline void storage_point_to_json(BUFFER *wb, WEIGHTS_POINT_TYPE type, ssize_t di, ssize_t ii, ssize_t ci, ssize_t ni, NETDATA_DOUBLE weight, STORAGE_POINT *highlighted_sp, STORAGE_POINT *baseline_sp, RRDR_OPTIONS options __maybe_unused, bool baseline) {

316 + buffer_json_add_array_item_array(wb);

317 +

318 + buffer_json_add_array_item_uint64(wb, type); // "type"

319 + buffer_json_add_array_item_array(wb);

320 + if(type == WPT_DIMENSION)

321 + buffer_json_add_array_item_int64(wb, di);

322 + if(type == WPT_DIMENSION || type == WPT_INSTANCE)

323 + buffer_json_add_array_item_int64(wb, ii);

324 + if(type == WPT_CONTEXT)

325 + buffer_json_add_array_item_int64(wb, ci);

326 + buffer_json_add_array_item_int64(wb, ni);

327 + buffer_json_array_close(wb);

328 + buffer_json_add_array_item_double(wb, weight); // "weight"

329 +

330 + buffer_json_add_array_item_array(wb);

331 + buffer_json_add_array_item_double(wb, highlighted_sp->min); // "min"

332 + buffer_json_add_array_item_double(wb, (highlighted_sp->count) ? highlighted_sp->sum / (NETDATA_DOUBLE) highlighted_sp->count : 0.0); // "avg"

333 + buffer_json_add_array_item_double(wb, highlighted_sp->max); // "max"

334 + buffer_json_add_array_item_double(wb, highlighted_sp->sum); // "sum"

335 + buffer_json_add_array_item_uint64(wb, highlighted_sp->count); // "count"

336 + buffer_json_add_array_item_uint64(wb, highlighted_sp->anomaly_count); // "anomaly_count"

337 + buffer_json_array_close(wb);

338 +

339 + if(baseline) {

340 + buffer_json_add_array_item_array(wb);

341 + buffer_json_add_array_item_double(wb, baseline_sp->min); // "min"

342 + buffer_json_add_array_item_double(wb, (baseline_sp->count) ? baseline_sp->sum / (NETDATA_DOUBLE) baseline_sp->count : 0.0); // "avg"

343 + buffer_json_add_array_item_double(wb, baseline_sp->max); // "max"

344 + buffer_json_add_array_item_double(wb, baseline_sp->sum); // "sum"

345 + buffer_json_add_array_item_uint64(wb, baseline_sp->count); // "count"

346 + buffer_json_add_array_item_uint64(wb, baseline_sp->anomaly_count); // "anomaly_count"

347 + buffer_json_array_close(wb);

348 + }

349 +

350 + buffer_json_array_close(wb); // key

351 +}

352 +

353 +static void multinode_data_schema(BUFFER *wb, RRDR_OPTIONS options __maybe_unused, const char *key, bool baseline) {

354 + size_t idx = 0;

355 + buffer_json_member_add_object(wb, key); // schema

356 +

357 + buffer_json_member_add_object(wb, "type");

358 + buffer_json_member_add_uint64(wb, "idx", idx++);

359 + buffer_json_object_close(wb); // type

360 +

361 + buffer_json_member_add_object(wb, "link");

362 + buffer_json_member_add_uint64(wb, "idx", idx++);

363 + buffer_json_member_add_object(wb, "dimension");

364 + {

365 + buffer_json_member_add_uint64(wb, "type", WPT_DIMENSION);

366 + size_t pidx = 0;

367 + buffer_json_member_add_uint64(wb, "di", pidx++);

368 + buffer_json_member_add_uint64(wb, "ii", pidx++);

369 + buffer_json_member_add_uint64(wb, "ni", pidx++);

370 + }

371 + buffer_json_object_close(wb); // dimension

372 + buffer_json_member_add_object(wb, "instance");

373 + {

374 + buffer_json_member_add_uint64(wb, "type", WPT_INSTANCE);

375 + size_t pidx = 0;

376 + buffer_json_member_add_uint64(wb, "ii", pidx++);

377 + buffer_json_member_add_uint64(wb, "ni", pidx++);

378 + }

379 + buffer_json_object_close(wb); // context

380 + buffer_json_member_add_object(wb, "context");

381 + {

382 + buffer_json_member_add_uint64(wb, "type", WPT_CONTEXT);

383 + size_t pidx = 0;

384 + buffer_json_member_add_uint64(wb, "ci", pidx++);

385 + buffer_json_member_add_uint64(wb, "ni", pidx++);

386 + }

387 + buffer_json_object_close(wb); // context

388 + buffer_json_member_add_object(wb, "node");

389 + {

390 + buffer_json_member_add_uint64(wb, "type", WPT_NODE);

391 + size_t pidx = 0;

392 + buffer_json_member_add_uint64(wb, "ni", pidx++);

393 + }

394 + buffer_json_object_close(wb); // node

395 + buffer_json_object_close(wb); // link

396 +

397 + buffer_json_member_add_object(wb, "weight");

398 + buffer_json_member_add_uint64(wb, "idx", idx++);

399 + buffer_json_object_close(wb); // weight

400 +

401 + for(size_t i = 0; i < ((baseline) ? 2 : 1) ; i++) {

402 + if(i == 0)

403 + buffer_json_member_add_object(wb, "highlighted");

404 + else

405 + buffer_json_member_add_object(wb, "baseline");

406 +

407 + buffer_json_member_add_uint64(wb, "idx", idx++);

408 + size_t pidx = 0;

409 + buffer_json_member_add_uint64(wb, "min", pidx++);

410 + buffer_json_member_add_uint64(wb, "avg", pidx++);

411 + buffer_json_member_add_uint64(wb, "max", pidx++);

412 + buffer_json_member_add_uint64(wb, "sum", pidx++);

413 + buffer_json_member_add_uint64(wb, "count", pidx++);

414 + buffer_json_member_add_uint64(wb, "anomaly_count", pidx++);

415 + buffer_json_object_close(wb); // point

416 + }

417 +

418 + buffer_json_object_close(wb); // schema

419 +}

420 +

421 +struct dict_unique_name {

422 + bool existing;

423 + uint32_t i;

424 +};

425 +

426 +struct dict_unique_id_name {

427 + bool existing;

428 + uint32_t i;

429 + const char *id;

430 + const char *name;

431 +};

432 +

433 +static inline ssize_t dict_unique_name_add(DICTIONARY *dict, const char *name, ssize_t *max_id) {

434 + struct dict_unique_name *dun = dictionary_set(dict, name, NULL, sizeof(struct dict_unique_name));

435 + if(!dun->existing) {

436 + dun->existing = true;

437 + dun->i = *max_id;

438 + (*max_id)++;

439 + }

440 +

441 + return (ssize_t)dun->i;

442 +}

443 +

444 +static inline ssize_t dict_unique_id_name_add(DICTIONARY *dict, const char *id, const char *name, ssize_t *max_id) {

445 + char key[1024 + 1];

446 + snprintfz(key, 1024, "%s:%s", id, name);

447 + struct dict_unique_id_name *dun = dictionary_set(dict, key, NULL, sizeof(struct dict_unique_id_name));

448 + if(!dun->existing) {

449 + dun->existing = true;

450 + dun->i = *max_id;

451 + (*max_id)++;

452 + dun->id = id;

453 + dun->name = name;

454 + }

455 +

456 + return (ssize_t)dun->i;

457 +}

458 +

459 +static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *wb,

460 + time_t after, time_t before,

461 + time_t baseline_after, time_t baseline_before,

462 + size_t points, WEIGHTS_METHOD method,

463 + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,

464 + size_t examined_dimensions, usec_t duration,

465 + WEIGHTS_STATS *stats,

466 + struct query_versions *versions) {

467 + results_header_to_json(results, wb, after, before, baseline_after, baseline_before,

468 + points, method, group, options, shifts, examined_dimensions, duration, stats);

469 +

470 + version_hashes_api_v2(wb, versions);

471 +

472 + bool baseline = method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME;

473 + multinode_data_schema(wb, options, "schema", baseline);

474 +

475 + DICTIONARY *dict_nodes = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_name));

476 + DICTIONARY *dict_contexts = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_name));

477 + DICTIONARY *dict_instances = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_id_name));

478 + DICTIONARY *dict_dimensions = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_id_name));

479 +

480 + buffer_json_member_add_array(wb, "points");

481 +

482 + size_t total_dimensions = 0, node_dims = 0, context_dims = 0, instance_dims = 0;

483 + NETDATA_DOUBLE context_total_weight = 0.0, instance_total_weight = 0.0, node_total_weight = 0.0;

484 + STORAGE_POINT context_hsp = STORAGE_POINT_UNSET, instance_hsp = STORAGE_POINT_UNSET, node_hsp = STORAGE_POINT_UNSET;

485 + STORAGE_POINT context_bsp = STORAGE_POINT_UNSET, instance_bsp = STORAGE_POINT_UNSET, node_bsp = STORAGE_POINT_UNSET;

486 + struct register_result *t;

487 + RRDHOST *last_host = NULL;

488 + RRDCONTEXT_ACQUIRED *last_rca = NULL;

489 + RRDINSTANCE_ACQUIRED *last_ria = NULL;

490 + ssize_t di = -1, ii = -1, ci = -1, ni = -1;

491 + ssize_t di_max = 0, ii_max = 0, ci_max = 0, ni_max = 0;

492 + dfe_start_read(results, t) {

493 +

494 + // close instance

495 + if(t->ria != last_ria && last_ria) {

496 + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, instance_total_weight / (double) instance_dims, &instance_hsp, &instance_bsp, options, baseline);

497 +

498 + last_ria = NULL;

499 + instance_dims = 0;

500 + instance_total_weight = 0.0;

501 + instance_hsp = instance_bsp = STORAGE_POINT_UNSET;

502 + }

503 +

504 + // close context

505 + if(t->rca != last_rca && last_rca) {

506 + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, context_total_weight / (double) context_dims, &context_hsp, &instance_bsp, options, baseline);

507 + last_rca = NULL;

508 + context_dims = 0;

509 + context_total_weight = 0.0;

510 + context_hsp = context_bsp = STORAGE_POINT_UNSET;

511 + }

512 +

513 + // close node

514 + if(t->host != last_host && last_host) {

515 + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, node_total_weight / (double) node_dims, &node_hsp, &node_bsp, options, baseline);

516 + last_host = NULL;

517 + node_dims = 0;

518 + node_total_weight = 0.0;

519 + node_hsp = node_bsp = STORAGE_POINT_UNSET;

520 + }

521 +

522 + // open node

523 + if(t->host != last_host) {

524 + last_host = t->host;

525 + ni = dict_unique_name_add(dict_nodes, t->host->machine_guid, &ni_max);

526 + }

527 +

528 + // open context

529 + if(t->rca != last_rca) {

530 + last_rca = t->rca;

531 + ci = dict_unique_name_add(dict_contexts, rrdcontext_acquired_id(t->rca), &ci_max);

532 + }

533 +

534 + // open instance

535 + if(t->ria != last_ria) {

536 + last_ria = t->ria;

537 + ii = dict_unique_id_name_add(dict_instances, rrdinstance_acquired_id(t->ria), rrdinstance_acquired_name(t->ria), &ii_max);

538 + }

539 +

540 + di = dict_unique_id_name_add(dict_dimensions, rrdmetric_acquired_id(t->rma), rrdmetric_acquired_name(t->rma), &di_max);

541 + storage_point_to_json(wb, WPT_DIMENSION, di, ii, ci, ni, t->value, &t->highlighted, &t->baseline, options, baseline);

542 +

543 + instance_total_weight += t->value;

544 + context_total_weight += t->value;

545 + node_total_weight += t->value;

546 +

547 + storage_point_merge_to(instance_hsp, t->highlighted);

548 + storage_point_merge_to(context_hsp, t->highlighted);

549 + storage_point_merge_to(node_hsp, t->highlighted);

550 +

551 + if(baseline) {

552 + storage_point_merge_to(instance_bsp, t->baseline);

553 + storage_point_merge_to(context_bsp, t->baseline);

554 + storage_point_merge_to(node_bsp, t->baseline);

555 + }

556 +

557 + instance_dims++;

558 + context_dims++;

559 + node_dims++;

560 + total_dimensions++;

561 + }

562 + dfe_done(t);

563 +

564 + // close instance

565 + if(last_ria)

566 + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, instance_total_weight / (double) instance_dims, &instance_hsp, &instance_bsp, options, baseline);

567 +

568 + // close context

569 + if(last_rca)

570 + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, context_total_weight / (double) context_dims, &context_hsp, &instance_bsp, options, baseline);

571 +

572 + // close node

573 + if(last_host)

574 + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, node_total_weight / (double) node_dims, &node_hsp, &node_bsp, options, baseline);

575 +

576 + buffer_json_array_close(wb); // points

577 +

578 + buffer_json_member_add_array(wb, "nodes");

579 + {

580 + struct dict_unique_name *dun;

581 + dfe_start_read(dict_nodes, dun) {

582 + buffer_json_add_array_item_object(wb);

583 + buffer_json_member_add_string(wb, "mg", dun_dfe.name);

584 + buffer_json_member_add_int64(wb, "ni", dun->i);

585 + buffer_json_object_close(wb);

586 + }

587 + dfe_done(dun);

588 + }

589 + buffer_json_array_close(wb);

590 +

591 + buffer_json_member_add_array(wb, "contexts");

592 + {

593 + struct dict_unique_name *dun;

594 + dfe_start_read(dict_contexts, dun) {

595 + buffer_json_add_array_item_object(wb);

596 + buffer_json_member_add_string(wb, "id", dun_dfe.name);

597 + buffer_json_member_add_int64(wb, "ci", dun->i);

598 + buffer_json_object_close(wb);

599 + }

600 + dfe_done(dun);

601 + }

602 + buffer_json_array_close(wb);

603 +

604 + buffer_json_member_add_array(wb, "instances");

605 + {

606 + struct dict_unique_id_name *dun;

607 + dfe_start_read(dict_instances, dun) {

608 + buffer_json_add_array_item_object(wb);

609 + buffer_json_member_add_string(wb, "id", dun->id);

610 + if(dun->id != dun->name)

611 + buffer_json_member_add_string(wb, "nm", dun->name);

612 + buffer_json_member_add_int64(wb, "ii", dun->i);

613 + buffer_json_object_close(wb);

614 + }

615 + dfe_done(dun);

616 + }

617 + buffer_json_array_close(wb);

618 +

619 + buffer_json_member_add_array(wb, "dimensions");

620 + {

621 + struct dict_unique_id_name *dun;

622 + dfe_start_read(dict_dimensions, dun) {

623 + buffer_json_add_array_item_object(wb);

624 + buffer_json_member_add_string(wb, "id", dun->id);

625 + if(dun->id != dun->name)

626 + buffer_json_member_add_string(wb, "nm", dun->name);

627 + buffer_json_member_add_int64(wb, "di", dun->i);

628 + buffer_json_object_close(wb);

629 + }

630 + dfe_done(dun);

631 + }

632 + buffer_json_array_close(wb);

633 +

634 + buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions);

635 + buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions);

636 + buffer_json_finalize(wb);

637 +

638 + dictionary_destroy(dict_nodes);

639 + dictionary_destroy(dict_contexts);

640 + dictionary_destroy(dict_instances);

641 + dictionary_destroy(dict_dimensions);

642 +

643 + return total_dimensions;

644 +}

645 +

646 // ----------------------------------------------------------------------------

647 // KS2 algorithm functions

648

@@ -469,9 +821,10 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(

821 ONEWAYALLOC *owa, RRDHOST *host,

822 RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,

823 time_t after, time_t before, size_t points, RRDR_OPTIONS options,

472 - RRDR_TIME_GROUPING group_method, const char *group_options, size_t tier,

824 + RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier,

825 WEIGHTS_STATS *stats,

474 - size_t *entries

826 + size_t *entries,

827 + STORAGE_POINT *sp

828 ) {

829

830 NETDATA_DOUBLE *ret = NULL;

@@ -486,8 +839,8 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(

839 .before = before,

840 .points = points,

841 .options = options,

489 - .time_group_method = group_method,

490 - .time_group_options = group_options,

842 + .time_group_method = time_group_method,

843 + .time_group_options = time_group_options,

844 .tier = tier,

845 .query_source = QUERY_SOURCE_API_WEIGHTS,

846 .priority = STORAGE_PRIORITY_SYNCHRONOUS,

@@ -523,6 +876,9 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(

876 *entries = rrdr_rows(r);

877 ret = onewayalloc_mallocz(owa, sizeof(NETDATA_DOUBLE) * rrdr_rows(r));

878

879 + if(sp)

880 + *sp = r->drs[0];

881 +

882 // copy the points of the dimension to a contiguous array

883 // there is no need to check for empty values, since empty values are already zero

884 // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43

@@ -540,7 +896,7 @@ static void rrdset_metric_correlations_ks2(

896 time_t baseline_after, time_t baseline_before,

897 time_t after, time_t before,

898 size_t points, RRDR_OPTIONS options,

543 - RRDR_TIME_GROUPING group_method, const char *group_options, size_t tier,

899 + RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier,

900 uint32_t shifts,

901 WEIGHTS_STATS *stats, bool register_zero

902 ) {

@@ -550,17 +906,19 @@ static void rrdset_metric_correlations_ks2(

906 ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);

907

908 size_t high_points = 0;

909 + STORAGE_POINT highlighted_sp;

910 NETDATA_DOUBLE *highlight = rrd2rrdr_ks2(

911 owa, host, rca, ria, rma, after, before, points,

555 - options, group_method, group_options, tier, stats, &high_points);

912 + options, time_group_method, time_group_options, tier, stats, &high_points, &highlighted_sp);

913

914 if(!highlight)

915 goto cleanup;

916

917 size_t base_points = 0;

918 + STORAGE_POINT baseline_sp;

919 NETDATA_DOUBLE *baseline = rrd2rrdr_ks2(

920 owa, host, rca, ria, rma, baseline_after, baseline_before, high_points << shifts,

563 - options, group_method, group_options, tier, stats, &base_points);

921 + options, time_group_method, time_group_options, tier, stats, &base_points, &baseline_sp);

922

923 if(!baseline)

924 goto cleanup;

@@ -582,7 +940,7 @@ static void rrdset_metric_correlations_ks2(

940

941 // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated

942 // so, we flip the result of kstwo()

585 - register_result(results, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero);

943 + register_result(results, host, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, &highlighted_sp, &baseline_sp, stats, register_zero);

944 }

945

946 cleanup:

@@ -592,8 +950,8 @@ cleanup:

950 // ----------------------------------------------------------------------------

951 // VOLUME algorithm functions

952

595 -static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats) {

596 - stats->db_queries++;

953 +static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats, size_t queries) {

954 + stats->db_queries += queries;

955 stats->result_points += qv->result_points;

956 stats->db_points += qv->points_read;

957 for(size_t tier = 0; tier < storage_tiers ; tier++)

@@ -606,16 +964,16 @@ static void rrdset_metric_correlations_volume(

964 DICTIONARY *results,

965 time_t baseline_after, time_t baseline_before,

966 time_t after, time_t before,

609 - RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method, const char *group_options,

967 + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options,

968 size_t tier,

969 WEIGHTS_STATS *stats, bool register_zero) {

970

971 options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS;

972

973 QUERY_VALUE baseline_average = rrdmetric2value(host, rca, ria, rma, baseline_after, baseline_before,

616 - options, group_method, group_options, tier, 0,

974 + options, time_group_method, time_group_options, tier, 0,

975 QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS);

618 - merge_query_value_to_stats(&baseline_average, stats);

976 + merge_query_value_to_stats(&baseline_average, stats, 1);

977

978 if(!netdata_double_isnumber(baseline_average.value)) {

979 // this means no data for the baseline window, but we may have data for the highlighted one - assume zero

@@ -623,9 +981,9 @@ static void rrdset_metric_correlations_volume(

981 }

982

983 QUERY_VALUE highlight_average = rrdmetric2value(host, rca, ria, rma, after, before,

626 - options, group_method, group_options, tier, 0,

984 + options, time_group_method, time_group_options, tier, 0,

985 QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS);

628 - merge_query_value_to_stats(&highlight_average, stats);

986 + merge_query_value_to_stats(&highlight_average, stats, 1);

987

988 if(!netdata_double_isnumber(highlight_average.value))

989 return;

@@ -640,7 +998,7 @@ static void rrdset_metric_correlations_volume(

998 QUERY_VALUE highlight_countif = rrdmetric2value(host, rca, ria, rma, after, before,

999 options, RRDR_GROUPING_COUNTIF, highlight_countif_options, tier, 0,

1000 QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS);

643 - merge_query_value_to_stats(&highlight_countif, stats);

1001 + merge_query_value_to_stats(&highlight_countif, stats, 1);

1002

1003 if(!netdata_double_isnumber(highlight_countif.value)) {

1004 info("WEIGHTS: highlighted countif query failed, but highlighted average worked - strange...");

@@ -663,31 +1021,130 @@ static void rrdset_metric_correlations_volume(

1021 pcent = highlight_countif.value;

1022 }

1023

666 - register_result(results, rca, ria, rma, pcent, flags, stats, register_zero);

1024 + register_result(results, host, rca, ria, rma, pcent, flags, &highlight_average.sp, &baseline_average.sp, stats, register_zero);

1025 }

1026

1027 // ----------------------------------------------------------------------------

670 -// ANOMALY RATE algorithm functions

1028 +// VALUE / ANOMALY RATE algorithm functions

1029

672 -static void rrdset_weights_anomaly_rate(

1030 +static void rrdset_weights_value(

1031 RRDHOST *host,

1032 RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,

1033 DICTIONARY *results,

1034 time_t after, time_t before,

677 - RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method, const char *group_options,

1035 + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options,

1036 size_t tier,

1037 WEIGHTS_STATS *stats, bool register_zero) {

1038

681 - options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ANOMALY_BIT | RRDR_OPTION_NATURAL_POINTS;

1039 + options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_NATURAL_POINTS;

1040

1041 QUERY_VALUE qv = rrdmetric2value(host, rca, ria, rma, after, before,

684 - options, group_method, group_options, tier, 0,

1042 + options, time_group_method, time_group_options, tier, 0,

1043 QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS);

1044

687 - merge_query_value_to_stats(&qv, stats);

1045 + merge_query_value_to_stats(&qv, stats, 1);

1046

1047 if(netdata_double_isnumber(qv.value))

690 - register_result(results, rca, ria, rma, qv.value, 0, stats, register_zero);

1048 + register_result(results, host, rca, ria, rma, qv.value, 0, &qv.sp, NULL, stats, register_zero);

1049 +}

1050 +

1051 +struct query_weights_data {

1052 + QUERY_WEIGHTS_REQUEST *qwr;

1053 +

1054 + SIMPLE_PATTERN *scope_nodes_sp;

1055 + SIMPLE_PATTERN *scope_contexts_sp;

1056 + SIMPLE_PATTERN *nodes_sp;

1057 + SIMPLE_PATTERN *contexts_sp;

1058 + SIMPLE_PATTERN *instances_sp;

1059 + SIMPLE_PATTERN *dimensions_sp;

1060 + SIMPLE_PATTERN *labels_sp;

1061 + SIMPLE_PATTERN *alerts_sp;

1062 +

1063 + usec_t now_us;

1064 + usec_t started_us;

1065 + usec_t timeout_us;

1066 + bool timed_out;

1067 + bool interrupted;

1068 +

1069 + size_t examined_dimensions;

1070 + bool register_zero;

1071 +

1072 + DICTIONARY *results;

1073 + WEIGHTS_STATS stats;

1074 +

1075 + uint32_t shifts;

1076 +

1077 + struct query_versions versions;

1078 +};

1079 +

1080 +static void rrdset_weights_multi_dimensional_value(struct query_weights_data *qwd) {

1081 + QUERY_TARGET_REQUEST qtr = {

1082 + .version = 1,

1083 + .scope_nodes = qwd->qwr->scope_nodes,

1084 + .scope_contexts = qwd->qwr->scope_contexts,

1085 + .nodes = qwd->qwr->nodes,

1086 + .contexts = qwd->qwr->contexts,

1087 + .instances = qwd->qwr->instances,

1088 + .dimensions = qwd->qwr->dimensions,

1089 + .labels = qwd->qwr->labels,

1090 + .alerts = qwd->qwr->alerts,

1091 + .after = qwd->qwr->after,

1092 + .before = qwd->qwr->before,

1093 + .points = 1,

1094 + .options = qwd->qwr->options | RRDR_OPTION_NATURAL_POINTS,

1095 + .time_group_method = qwd->qwr->time_group_method,

1096 + .time_group_options = qwd->qwr->time_group_options,

1097 + .tier = qwd->qwr->tier,

1098 + .timeout_ms = qwd->qwr->timeout_ms,

1099 + .query_source = QUERY_SOURCE_API_WEIGHTS,

1100 + .priority = STORAGE_PRIORITY_NORMAL,

1101 + };

1102 +

1103 + ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);

1104 + RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));

1105 +

1106 + if(rrdr_rows(r) != 1 || !r->d)

1107 + goto cleanup;

1108 +

1109 + QUERY_VALUE qv = {

1110 + .after = r->view.after,

1111 + .before = r->view.before,

1112 + .points_read = r->stats.db_points_read,

1113 + .result_points = r->stats.result_points_generated,

1114 + };

1115 +

1116 + size_t queries = 0;

1117 + for(size_t d = 0; d < r->d ;d++) {

1118 + if(!rrdr_dimension_should_be_exposed(r->od[d], qwd->qwr->options))

1119 + continue;

1120 +

1121 + long i = 0; // only one row

1122 + NETDATA_DOUBLE *cn = &r->v[ i * r->d ];

1123 + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];

1124 +

1125 + qv.value = cn[d];

1126 + qv.anomaly_rate = ar[d];

1127 + qv.sp = *r->drs;

1128 +

1129 + if(netdata_double_isnumber(qv.value)) {

1130 + QUERY_METRIC *qm = query_metric(r->internal.qt, d);

1131 + QUERY_DIMENSION *qd = query_dimension(r->internal.qt, qm->link.query_dimension_id);

1132 + QUERY_INSTANCE *qi = query_instance(r->internal.qt, qm->link.query_instance_id);

1133 + QUERY_CONTEXT *qc = query_context(r->internal.qt, qm->link.query_context_id);

1134 + QUERY_NODE *qn = query_node(r->internal.qt, qm->link.query_node_id);

1135 +

1136 + register_result(qwd->results, qn->rrdhost, qc->rca, qi->ria, qd->rma, qv.value, 0, &qv.sp,

1137 + NULL, &qwd->stats, qwd->register_zero);

1138 + }

1139 +

1140 + queries++;

1141 + }

1142 +

1143 + merge_query_value_to_stats(&qv, &qwd->stats, queries);

1144 +

1145 +cleanup:

1146 + rrdr_free(owa, r);

1147 + onewayalloc_destroy(owa);

1148 }

1149

1150 // ----------------------------------------------------------------------------

@@ -777,60 +1234,182 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) {

1234 // ----------------------------------------------------------------------------

1235 // The main function

1236

780 -int web_api_v1_weights(

781 - RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,

782 - RRDR_TIME_GROUPING group, const char *group_options,

783 - time_t baseline_after, time_t baseline_before,

784 - time_t after, time_t before,

785 - size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout) {

1237 +static ssize_t weights_for_rrdmetric(void *data, RRDHOST *host, RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma) {

1238 + struct query_weights_data *qwd = data;

1239 + QUERY_WEIGHTS_REQUEST *qwr = qwd->qwr;

1240

787 - WEIGHTS_STATS stats = {};

1241 + qwd->now_us = now_realtime_usec();

1242 + if(qwd->now_us - qwd->started_us > qwd->timeout_us) {

1243 + qwd->timed_out = true;

1244 + return -1;

1245 + }

1246 +

1247 + if(qwd->qwr->interrupt_callback && qwd->qwr->interrupt_callback(qwd->qwr->interrupt_callback_data)) {

1248 + qwd->interrupted = true;

1249 + return -1;

1250 + }

1251 +

1252 + qwd->examined_dimensions++;

1253 +

1254 + switch(qwr->method) {

1255 + case WEIGHTS_METHOD_VALUE:

1256 + rrdset_weights_value(

1257 + host, rca, ria, rma,

1258 + qwd->results,

1259 + qwr->after, qwr->before,

1260 + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier,

1261 + &qwd->stats, qwd->register_zero

1262 + );

1263 + break;

1264 +

1265 + case WEIGHTS_METHOD_ANOMALY_RATE:

1266 + qwr->options |= RRDR_OPTION_ANOMALY_BIT;

1267 + rrdset_weights_value(

1268 + host, rca, ria, rma,

1269 + qwd->results,

1270 + qwr->after, qwr->before,

1271 + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier,

1272 + &qwd->stats, qwd->register_zero

1273 + );

1274 + break;

1275 +

1276 + case WEIGHTS_METHOD_MC_VOLUME:

1277 + rrdset_metric_correlations_volume(

1278 + host, rca, ria, rma,

1279 + qwd->results,

1280 + qwr->baseline_after, qwr->baseline_before,

1281 + qwr->after, qwr->before,

1282 + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier,

1283 + &qwd->stats, qwd->register_zero

1284 + );

1285 + break;

1286 +

1287 + default:

1288 + case WEIGHTS_METHOD_MC_KS2:

1289 + rrdset_metric_correlations_ks2(

1290 + host, rca, ria, rma,

1291 + qwd->results,

1292 + qwr->baseline_after, qwr->baseline_before,

1293 + qwr->after, qwr->before, qwr->points,

1294 + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier, qwd->shifts,

1295 + &qwd->stats, qwd->register_zero

1296 + );

1297 + break;

1298 + }

1299 +

1300 + return 1;

1301 +}

1302 +

1303 +static ssize_t weights_do_context_callback(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context) {

1304 + if(!queryable_context)

1305 + return false;

1306 +

1307 + struct query_weights_data *qwd = data;

1308 +

1309 + bool has_retention = false;

1310 + switch(qwd->qwr->method) {

1311 + case WEIGHTS_METHOD_VALUE:

1312 + case WEIGHTS_METHOD_ANOMALY_RATE:

1313 + has_retention = rrdcontext_retention_match(rca, qwd->qwr->after, qwd->qwr->before);

1314 + break;

1315 +

1316 + case WEIGHTS_METHOD_MC_KS2:

1317 + case WEIGHTS_METHOD_MC_VOLUME:

1318 + has_retention = rrdcontext_retention_match(rca, qwd->qwr->after, qwd->qwr->before);

1319 + if(has_retention)

1320 + has_retention = rrdcontext_retention_match(rca, qwd->qwr->baseline_after, qwd->qwr->baseline_before);

1321 + break;

1322 + }

1323 +

1324 + if(!has_retention)

1325 + return 0;

1326 +

1327 + ssize_t ret = weights_foreach_rrdmetric_in_context(rca,

1328 + qwd->instances_sp,

1329 + NULL,

1330 + qwd->labels_sp,

1331 + qwd->alerts_sp,

1332 + qwd->dimensions_sp,

1333 + true, true, qwd->qwr->version,

1334 + weights_for_rrdmetric, qwd);

1335 + return ret;

1336 +}

1337 +

1338 +ssize_t weights_do_node_callback(void *data, RRDHOST *host, bool queryable) {

1339 + if(!queryable)

1340 + return 0;

1341 +

1342 + struct query_weights_data *qwd = data;

1343 +

1344 + ssize_t ret = query_scope_foreach_context(host, qwd->qwr->scope_contexts,

1345 + qwd->scope_contexts_sp, qwd->contexts_sp,

1346 + weights_do_context_callback, queryable, qwd);

1347 +

1348 + return ret;

1349 +}

1350 +

1351 +int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {

1352

789 - DICTIONARY *results = register_result_init();

790 - DICTIONARY *metrics = NULL;

1353 char *error = NULL;

1354 int resp = HTTP_RESP_OK;

1355

1356 // if the user didn't give a timeout

1357 // assume 60 seconds

796 - if(!timeout)

797 - timeout = 60 * MSEC_PER_SEC;

1358 + if(!qwr->timeout_ms)

1359 + qwr->timeout_ms = 5 * 60 * MSEC_PER_SEC;

1360

1361 // if the timeout is less than 1 second

1362 // make it at least 1 second

801 - if(timeout < (long)(1 * MSEC_PER_SEC))

802 - timeout = 1 * MSEC_PER_SEC;

803 -

804 - usec_t timeout_usec = timeout * USEC_PER_MS;

805 - usec_t started_usec = now_realtime_usec();

1363 + if(qwr->timeout_ms < (long)(1 * MSEC_PER_SEC))

1364 + qwr->timeout_ms = 1 * MSEC_PER_SEC;

1365 +

1366 + struct query_weights_data qwd = {

1367 + .qwr = qwr,

1368 +

1369 + .scope_nodes_sp = string_to_simple_pattern(qwr->scope_nodes),

1370 + .scope_contexts_sp = string_to_simple_pattern(qwr->scope_contexts),

1371 + .nodes_sp = string_to_simple_pattern(qwr->nodes),

1372 + .contexts_sp = string_to_simple_pattern(qwr->contexts),

1373 + .instances_sp = string_to_simple_pattern(qwr->instances),

1374 + .dimensions_sp = string_to_simple_pattern(qwr->dimensions),

1375 + .labels_sp = string_to_simple_pattern(qwr->labels),

1376 + .alerts_sp = string_to_simple_pattern(qwr->alerts),

1377 + .timeout_us = qwr->timeout_ms * USEC_PER_MS,

1378 + .started_us = now_realtime_usec(),

1379 + .timed_out = false,

1380 + .examined_dimensions = 0,

1381 + .register_zero = true,

1382 + .results = register_result_init(),

1383 + .stats = {},

1384 + .shifts = 0,

1385 + };

1386

807 - if(!rrdr_relative_window_to_absolute(&after, &before, NULL))

1387 + if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL))

1388 buffer_no_cacheable(wb);

1389

810 - if (before <= after) {

1390 + if (qwr->before <= qwr->after) {

1391 resp = HTTP_RESP_BAD_REQUEST;

1392 error = "Invalid selected time-range.";

1393 goto cleanup;

1394 }

1395

816 - uint32_t shifts = 0;

817 - if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) {

818 - if(!points) points = 500;

1396 + if(qwr->method == WEIGHTS_METHOD_MC_KS2 || qwr->method == WEIGHTS_METHOD_MC_VOLUME) {

1397 + if(!qwr->points) qwr->points = 500;

1398

820 - if(baseline_before <= API_RELATIVE_TIME_MAX)

821 - baseline_before += after;

1399 + if(qwr->baseline_before <= API_RELATIVE_TIME_MAX)

1400 + qwr->baseline_before += qwr->after;

1401

823 - rrdr_relative_window_to_absolute(&baseline_after, &baseline_before, NULL);

1402 + rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL);

1403

825 - if (baseline_before <= baseline_after) {

1404 + if (qwr->baseline_before <= qwr->baseline_after) {

1405 resp = HTTP_RESP_BAD_REQUEST;

1406 error = "Invalid baseline time-range.";

1407 goto cleanup;

1408 }

1409

1410 // baseline should be a power of two multiple of highlight

832 - long long base_delta = baseline_before - baseline_after;

833 - long long high_delta = before - after;

1411 + long long base_delta = qwr->baseline_before - qwr->baseline_after;

1412 + long long high_delta = qwr->before - qwr->after;

1413 uint32_t multiplier = (uint32_t)round((double)base_delta / (double)high_delta);

1414

1415 // check if the multiplier is a power of two

@@ -852,98 +1431,73 @@ int web_api_v1_weights(

1431 // we need to do, to divide baseline numbers to match

1432 // the highlight ones

1433 while(multiplier > 1) {

855 - shifts++;

1434 + qwd.shifts++;

1435 multiplier = multiplier >> 1;

1436 }

1437

1438 // if the baseline size will not comply to MAX_POINTS

1439 // lower the window of the baseline

861 - while(shifts && (points << shifts) > MAX_POINTS)

862 - shifts--;

1440 + while(qwd.shifts && (qwr->points << qwd.shifts) > MAX_POINTS)

1441 + qwd.shifts--;

1442

1443 // if the baseline size still does not comply to MAX_POINTS

1444 // lower the resolution of the highlight and the baseline

866 - while((points << shifts) > MAX_POINTS)

867 - points = points >> 1;

1445 + while((qwr->points << qwd.shifts) > MAX_POINTS)

1446 + qwr->points = qwr->points >> 1;

1447

869 - if(points < 15) {

1448 + if(qwr->points < 15) {

1449 resp = HTTP_RESP_BAD_REQUEST;

1450 error = "Too few points available, at least 15 are needed.";

1451 goto cleanup;

1452 }

1453

1454 // adjust the baseline to be multiplier times bigger than the highlight

876 - baseline_after = baseline_before - (high_delta << shifts);

1455 + qwr->baseline_after = qwr->baseline_before - (high_delta << qwd.shifts);

1456 }

1457

879 - size_t examined_dimensions = 0;

1458 + if(qwr->options & RRDR_OPTION_NONZERO) {

1459 + qwd.register_zero = false;

1460

881 - bool register_zero = true;

882 - if(options & RRDR_OPTION_NONZERO) {

883 - register_zero = false;

884 - options &= ~RRDR_OPTION_NONZERO;

1461 + // remove it to run the queries without it

1462 + qwr->options &= ~RRDR_OPTION_NONZERO;

1463 }

1464

887 - metrics = rrdcontext_all_metrics_to_dict(host, contexts);

888 - struct metric_entry *me;

889 -

890 - // for every metric_entry in the dictionary

891 - dfe_start_read(metrics, me) {

892 - usec_t now_usec = now_realtime_usec();

893 - if(now_usec - started_usec > timeout_usec) {

894 - error = "timed out";

895 - resp = HTTP_RESP_GATEWAY_TIMEOUT;

896 - goto cleanup;

1465 + if(qwr->host && qwr->version == 1)

1466 + weights_do_node_callback(&qwd, qwr->host, true);

1467 + else {

1468 + if((qwd.qwr->method == WEIGHTS_METHOD_VALUE || qwd.qwr->method == WEIGHTS_METHOD_ANOMALY_RATE) && (qwd.contexts_sp || qwd.scope_contexts_sp)) {

1469 + rrdset_weights_multi_dimensional_value(&qwd);

1470 }

898 -

899 - examined_dimensions++;

900 -

901 - switch(method) {

902 - case WEIGHTS_METHOD_ANOMALY_RATE:

903 - options |= RRDR_OPTION_ANOMALY_BIT;

904 - rrdset_weights_anomaly_rate(

905 - host,

906 - me->rca, me->ria, me->rma,

907 - results,

908 - after, before,

909 - options, group, group_options, tier,

910 - &stats, register_zero

911 - );

912 - break;

913 -

914 - case WEIGHTS_METHOD_MC_VOLUME:

915 - rrdset_metric_correlations_volume(

916 - host,

917 - me->rca, me->ria, me->rma,

918 - results,

919 - baseline_after, baseline_before,

920 - after, before,

921 - options, group, group_options, tier,

922 - &stats, register_zero

923 - );

924 - break;

925 -

926 - default:

927 - case WEIGHTS_METHOD_MC_KS2:

928 - rrdset_metric_correlations_ks2(

929 - host,

930 - me->rca, me->ria, me->rma,

931 - results,

932 - baseline_after, baseline_before,

933 - after, before, points,

934 - options, group, group_options, tier, shifts,

935 - &stats, register_zero

936 - );

937 - break;

1471 + else {

1472 + query_scope_foreach_host(qwd.scope_nodes_sp, qwd.nodes_sp,

1473 + weights_do_node_callback, &qwd,

1474 + &qwd.versions,

1475 + NULL);

1476 }

1477 }

940 - dfe_done(me);

1478

942 - if(!register_zero)

943 - options |= RRDR_OPTION_NONZERO;

1479 + if(!qwd.register_zero) {

1480 + // put it back, to show it in the response

1481 + qwr->options |= RRDR_OPTION_NONZERO;

1482 + }

1483 +

1484 + if(qwd.timed_out) {

1485 + error = "timed out";

1486 + resp = HTTP_RESP_GATEWAY_TIMEOUT;

1487 + goto cleanup;

1488 + }

1489 +

1490 + if(qwd.interrupted) {

1491 + error = "interrupted";

1492 + resp = HTTP_RESP_BACKEND_FETCH_FAILED;

1493 + goto cleanup;

1494 + }

1495 +

1496 + if(!qwd.register_zero)

1497 + qwr->options |= RRDR_OPTION_NONZERO;

1498

945 - if(!(options & RRDR_OPTION_RETURN_RAW))

946 - spread_results_evenly(results, &stats);

1499 + if(!(qwr->options & RRDR_OPTION_RETURN_RAW) && qwr->method != WEIGHTS_METHOD_VALUE)

1500 + spread_results_evenly(qwd.results, &qwd.stats);

1501

1502 usec_t ended_usec = now_realtime_usec();

1503

@@ -951,28 +1505,39 @@ int web_api_v1_weights(

1505 buffer_flush(wb);

1506

1507 size_t added_dimensions = 0;

954 - switch(format) {

1508 + switch(qwr->format) {

1509 case WEIGHTS_FORMAT_CHARTS:

1510 added_dimensions =

1511 registered_results_to_json_charts(

958 - results, wb,

959 - after, before,

960 - baseline_after, baseline_before,

961 - points, method, group, options, shifts,

962 - examined_dimensions,

963 - ended_usec - started_usec, &stats);

1512 + qwd.results, wb,

1513 + qwr->after, qwr->before,

1514 + qwr->baseline_after, qwr->baseline_before,

1515 + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,

1516 + qwd.examined_dimensions,

1517 + ended_usec - qwd.started_us, &qwd.stats);

1518 break;

1519

966 - default:

1520 case WEIGHTS_FORMAT_CONTEXTS:

1521 added_dimensions =

1522 registered_results_to_json_contexts(

970 - results, wb,

971 - after, before,

972 - baseline_after, baseline_before,

973 - points, method, group, options, shifts,

974 - examined_dimensions,

975 - ended_usec - started_usec, &stats);

1523 + qwd.results, wb,

1524 + qwr->after, qwr->before,

1525 + qwr->baseline_after, qwr->baseline_before,

1526 + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,

1527 + qwd.examined_dimensions,

1528 + ended_usec - qwd.started_us, &qwd.stats);

1529 + break;

1530 +

1531 + default:

1532 + case WEIGHTS_FORMAT_MULTINODE:

1533 + added_dimensions =

1534 + registered_results_to_json_multinode(

1535 + qwd.results, wb,

1536 + qwr->after, qwr->before,

1537 + qwr->baseline_after, qwr->baseline_before,

1538 + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,

1539 + qwd.examined_dimensions,

1540 + ended_usec - qwd.started_us, &qwd.stats, &qwd.versions);

1541 break;

1542 }

1543

@@ -982,8 +1547,16 @@ int web_api_v1_weights(

1547 }

1548

1549 cleanup:

985 - if(metrics) dictionary_destroy(metrics);

986 - if(results) register_result_destroy(results);

1550 + simple_pattern_free(qwd.scope_nodes_sp);

1551 + simple_pattern_free(qwd.scope_contexts_sp);

1552 + simple_pattern_free(qwd.nodes_sp);

1553 + simple_pattern_free(qwd.contexts_sp);

1554 + simple_pattern_free(qwd.instances_sp);

1555 + simple_pattern_free(qwd.dimensions_sp);

1556 + simple_pattern_free(qwd.labels_sp);

1557 + simple_pattern_free(qwd.alerts_sp);

1558 +

1559 + register_result_destroy(qwd.results);

1560

1561 if(error) {

1562 buffer_flush(wb);

web/api/queries/weights.h

+33 -5

@@ -9,22 +9,50 @@ typedef enum {

9 WEIGHTS_METHOD_MC_KS2 = 1,

10 WEIGHTS_METHOD_MC_VOLUME = 2,

11 WEIGHTS_METHOD_ANOMALY_RATE = 3,

12 + WEIGHTS_METHOD_VALUE = 4,

13 } WEIGHTS_METHOD;

15 typedef enum {

16 WEIGHTS_FORMAT_CHARTS = 1,

17 WEIGHTS_FORMAT_CONTEXTS = 2,

18 + WEIGHTS_FORMAT_MULTINODE = 3,

19 } WEIGHTS_FORMAT;

21 extern int enable_metric_correlations;

22 extern int metric_correlations_version;

23 extern WEIGHTS_METHOD default_metric_correlations_method;

23 -int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,

24 - RRDR_TIME_GROUPING group, const char *group_options,

25 - time_t baseline_after, time_t baseline_before,

26 - time_t after, time_t before,

27 - size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout);

25 +typedef bool (*weights_interrupt_callback_t)(void *data);

26 +

27 +typedef struct query_weights_request {

28 + size_t version;

29 + RRDHOST *host;

30 + const char *scope_nodes;

31 + const char *scope_contexts;

32 + const char *nodes;

33 + const char *contexts;

34 + const char *instances;

35 + const char *dimensions;

36 + const char *labels;

37 + const char *alerts;

38 + WEIGHTS_METHOD method;

39 + WEIGHTS_FORMAT format;

40 + RRDR_TIME_GROUPING time_group_method;

41 + const char *time_group_options;

42 + time_t baseline_after;

43 + time_t baseline_before;

44 + time_t after;

45 + time_t before;

46 + size_t points;

47 + RRDR_OPTIONS options;

48 + size_t tier;

49 + time_t timeout_ms;

50 +

51 + weights_interrupt_callback_t interrupt_callback;

52 + void *interrupt_callback_data;

53 +} QUERY_WEIGHTS_REQUEST;

54 +

55 +int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr);

57 WEIGHTS_METHOD weights_string_to_method(const char *method);

58 const char *weights_method_to_string(WEIGHTS_METHOD method);

web/api/web_api.c

+123

@@ -58,6 +58,129 @@ RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {

58 return options;

59 }

61 +int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *url, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, size_t api_version) {

62 + if (!netdata_ready)

63 + return HTTP_RESP_BACKEND_FETCH_FAILED;

64 +

65 + time_t baseline_after = 0, baseline_before = 0, after = 0, before = 0;

66 + size_t points = 0;

67 + RRDR_OPTIONS options = 0;

68 + RRDR_TIME_GROUPING time_group_method = RRDR_GROUPING_AVERAGE;

69 + time_t timeout_ms = 0;

70 + size_t tier = 0;

71 + const char *time_group_options = NULL, *scope_contexts = NULL, *scope_nodes = NULL, *contexts = NULL, *nodes = NULL,

72 + *instances = NULL, *dimensions = NULL, *labels = NULL, *alerts = NULL;

73 +

74 + while (url) {

75 + char *value = mystrsep(&url, "&");

76 + if (!value || !*value)

77 + continue;

78 +

79 + char *name = mystrsep(&value, "=");

80 + if (!name || !*name)

81 + continue;

82 + if (!value || !*value)

83 + continue;

84 +

85 + if (!strcmp(name, "baseline_after"))

86 + baseline_after = str2l(value);

87 +

88 + else if (!strcmp(name, "baseline_before"))

89 + baseline_before = str2l(value);

90 +

91 + else if (!strcmp(name, "after") || !strcmp(name, "highlight_after"))

92 + after = str2l(value);

93 +

94 + else if (!strcmp(name, "before") || !strcmp(name, "highlight_before"))

95 + before = str2l(value);

96 +

97 + else if (!strcmp(name, "points") || !strcmp(name, "max_points"))

98 + points = str2ul(value);

99 +

100 + else if (!strcmp(name, "timeout"))

101 + timeout_ms = str2l(value);

102 +

103 + else if((api_version == 1 && !strcmp(name, "group")) || (api_version >= 2 && !strcmp(name, "time_group")))

104 + time_group_method = time_grouping_parse(value, RRDR_GROUPING_AVERAGE);

105 +

106 + else if((api_version == 1 && !strcmp(name, "group_options")) || (api_version >= 2 && !strcmp(name, "time_group_options")))

107 + time_group_options = value;

108 +

109 + else if(!strcmp(name, "options"))

110 + options |= web_client_api_request_v1_data_options(value);

111 +

112 + else if(!strcmp(name, "method"))

113 + method = weights_string_to_method(value);

114 +

115 + else if(api_version == 1 && (!strcmp(name, "context") || !strcmp(name, "contexts")))

116 + scope_contexts = value;

117 +

118 + else if(api_version >= 2 && !strcmp(name, "scope_nodes")) scope_nodes = value;

119 + else if(api_version >= 2 && !strcmp(name, "scope_contexts")) scope_contexts = value;

120 + else if(api_version >= 2 && !strcmp(name, "nodes")) nodes = value;

121 + else if(api_version >= 2 && !strcmp(name, "contexts")) contexts = value;

122 + else if(api_version >= 2 && !strcmp(name, "instances")) instances = value;

123 + else if(api_version >= 2 && !strcmp(name, "dimensions")) dimensions = value;

124 + else if(api_version >= 2 && !strcmp(name, "labels")) labels = value;

125 + else if(api_version >= 2 && !strcmp(name, "alerts")) alerts = value;

126 +

127 + else if(!strcmp(name, "tier")) {

128 + tier = str2ul(value);

129 + if(tier < storage_tiers)

130 + options |= RRDR_OPTION_SELECTED_TIER;

131 + else

132 + tier = 0;

133 + }

134 + }

135 +

136 + if(options == 0)

137 + // the user did not set any options

138 + options = RRDR_OPTION_NOT_ALIGNED | RRDR_OPTION_NULL2ZERO | RRDR_OPTION_NONZERO;

139 + else

140 + // the user set some options, add also these

141 + options |= RRDR_OPTION_NOT_ALIGNED | RRDR_OPTION_NULL2ZERO;

142 +

143 + if(options & RRDR_OPTION_PERCENTAGE)

144 + options |= RRDR_OPTION_ABSOLUTE;

145 +

146 + if(options & RRDR_OPTION_DEBUG)

147 + options &= ~RRDR_OPTION_MINIFY;

148 +

149 + BUFFER *wb = w->response.data;

150 + buffer_flush(wb);

151 + wb->content_type = CT_APPLICATION_JSON;

152 +

153 + QUERY_WEIGHTS_REQUEST qwr = {

154 + .version = api_version,

155 + .host = (api_version == 1) ? NULL : host,

156 + .scope_nodes = scope_nodes,

157 + .scope_contexts = scope_contexts,

158 + .nodes = nodes,

159 + .contexts = contexts,

160 + .instances = instances,

161 + .dimensions = dimensions,

162 + .labels = labels,

163 + .alerts = alerts,

164 + .method = method,

165 + .format = format,

166 + .time_group_method = time_group_method,

167 + .time_group_options = time_group_options,

168 + .baseline_after = baseline_after,

169 + .baseline_before = baseline_before,

170 + .after = after,

171 + .before = before,

172 + .points = points,

173 + .options = options,

174 + .tier = tier,

175 + .timeout_ms = timeout_ms,

176 +

177 + .interrupt_callback = web_client_interrupt_callback,

178 + .interrupt_callback_data = w,

179 + };

180 +

181 + return web_api_v12_weights(wb, &qwr);

182 +}

183 +

184 bool web_client_interrupt_callback(void *data) {

185 struct web_client *w = data;

186 return sock_has_output_error(w->ofd);

web/api/web_api.h

@@ -29,6 +29,8 @@ static inline void fix_google_param(char *s) {

29 }

30 }

32 +int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *url, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, size_t api_version);

33 +

34 bool web_client_interrupt_callback(void *data);

36 #include "web_api_v1.h"

web/api/web_api_v1.c

+11 -87

@@ -700,7 +700,7 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien

700 .contexts = context,

701 .instances = chart,

702 .dimensions = (dimensions)?buffer_tostring(dimensions):NULL,

703 - .timeout = timeout,

703 + .timeout_ms = timeout,

704 .points = points,

705 .format = format,

706 .options = options,

@@ -977,8 +977,8 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *

977 }

978 }

979

980 -static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST *host, BUFFER *wb) {

981 - buffer_json_member_add_object(wb, "alarms");

980 +void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST *host, BUFFER *wb, const char *key) {

981 + buffer_json_member_add_object(wb, key);

982

983 size_t normal = 0, warning = 0, critical = 0;

984 RRDCALC *rc;

@@ -1048,8 +1048,8 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {

1048 rrd_unlock();

1049 }

1050

1051 -static inline void host_labels2json(RRDHOST *host, BUFFER *wb) {

1052 - buffer_json_member_add_object(wb, "host_labels");

1051 +void host_labels2json(RRDHOST *host, BUFFER *wb, const char *key) {

1052 + buffer_json_member_add_object(wb, key);

1053 rrdlabels_to_buffer_json_members(host->rrdlabels, wb);

1054 buffer_json_object_close(wb);

1055 }

@@ -1096,7 +1096,7 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)

1096 buffer_json_member_add_uint64(wb, "hosts-available", rrdhost_hosts_available());

1097 web_client_api_request_v1_info_mirrored_hosts(wb);

1098

1099 - web_client_api_request_v1_info_summary_alarm_statuses(host, wb);

1099 + web_client_api_request_v1_info_summary_alarm_statuses(host, wb, "alarms");

1100

1101 buffer_json_member_add_string_or_empty(wb, "os_name", host->system_info->host_os_name);

1102 buffer_json_member_add_string_or_empty(wb, "os_id", host->system_info->host_os_id);

@@ -1129,7 +1129,7 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)

1129 buffer_json_member_add_string_or_omit(wb, "cloud_instance_type", host->system_info->cloud_instance_type);

1130 buffer_json_member_add_string_or_omit(wb, "cloud_instance_region", host->system_info->cloud_instance_region);

1131

1132 - host_labels2json(host, wb);

1132 + host_labels2json(host, wb, "host_labels");

1133 host_functions2json(host, wb);

1134 host_collectors(host, wb);

1135

@@ -1268,90 +1268,14 @@ static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client

1268 return HTTP_RESP_OK;

1269 }

1270

1271 -static int web_client_api_request_v1_weights_internal(RRDHOST *host, struct web_client *w, char *url, WEIGHTS_METHOD method, WEIGHTS_FORMAT format) {

1272 - if (!netdata_ready)

1273 - return HTTP_RESP_BACKEND_FETCH_FAILED;

1274 -

1275 - long long baseline_after = 0, baseline_before = 0, after = 0, before = 0, points = 0;

1276 - RRDR_OPTIONS options = RRDR_OPTION_NOT_ALIGNED | RRDR_OPTION_NONZERO | RRDR_OPTION_NULL2ZERO;

1277 - int options_count = 0;

1278 - RRDR_TIME_GROUPING group = RRDR_GROUPING_AVERAGE;

1279 - int timeout = 0;

1280 - size_t tier = 0;

1281 - const char *group_options = NULL, *contexts_str = NULL;

1282 -

1283 - while (url) {

1284 - char *value = mystrsep(&url, "&");

1285 - if (!value || !*value)

1286 - continue;

1287 -

1288 - char *name = mystrsep(&value, "=");

1289 - if (!name || !*name)

1290 - continue;

1291 - if (!value || !*value)

1292 - continue;

1293 -

1294 - if (!strcmp(name, "baseline_after"))

1295 - baseline_after = (long long) strtoul(value, NULL, 0);

1296 -

1297 - else if (!strcmp(name, "baseline_before"))

1298 - baseline_before = (long long) strtoul(value, NULL, 0);

1299 -

1300 - else if (!strcmp(name, "after") || !strcmp(name, "highlight_after"))

1301 - after = (long long) strtoul(value, NULL, 0);

1302 -

1303 - else if (!strcmp(name, "before") || !strcmp(name, "highlight_before"))

1304 - before = (long long) strtoul(value, NULL, 0);

1305 -

1306 - else if (!strcmp(name, "points") || !strcmp(name, "max_points"))

1307 - points = (long long) strtoul(value, NULL, 0);

1308 -

1309 - else if (!strcmp(name, "timeout"))

1310 - timeout = (int) strtoul(value, NULL, 0);

1311 -

1312 - else if(!strcmp(name, "group"))

1313 - group = time_grouping_parse(value, RRDR_GROUPING_AVERAGE);

1314 -

1315 - else if(!strcmp(name, "options")) {

1316 - if(!options_count) options = RRDR_OPTION_NOT_ALIGNED | RRDR_OPTION_NULL2ZERO;

1317 - options |= web_client_api_request_v1_data_options(value);

1318 - options_count++;

1319 - }

1320 -

1321 - else if(!strcmp(name, "method"))

1322 - method = weights_string_to_method(value);

1323 -

1324 - else if(!strcmp(name, "context") || !strcmp(name, "contexts"))

1325 - contexts_str = value;

1326 -

1327 - else if(!strcmp(name, "tier")) {

1328 - tier = str2ul(value);

1329 - if(tier < storage_tiers)

1330 - options |= RRDR_OPTION_SELECTED_TIER;

1331 - else

1332 - tier = 0;

1333 - }

1334 - }

1335 -

1336 - BUFFER *wb = w->response.data;

1337 - buffer_flush(wb);

1338 - wb->content_type = CT_APPLICATION_JSON;

1339 -

1340 - SIMPLE_PATTERN *contexts = (contexts_str) ? simple_pattern_create(contexts_str, ",|\t\r\n\f\v",

1341 - SIMPLE_PATTERN_EXACT, true) : NULL;

1342 -

1343 - int ret = web_api_v1_weights(host, wb, method, format, group, group_options, baseline_after, baseline_before, after, before, points, options, contexts, tier, timeout);

1344 -

1345 - simple_pattern_free(contexts);

1346 - return ret;

1347 -}

1348 -

1271 int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_client *w, char *url) {

1350 - return web_client_api_request_v1_weights_internal(host, w, url, default_metric_correlations_method, WEIGHTS_FORMAT_CHARTS);

1272 + return web_client_api_request_weights(host, w, url, default_metric_correlations_method,

1273 + WEIGHTS_FORMAT_CHARTS, 1);

1274 }

1275

1276 int web_client_api_request_v1_weights(RRDHOST *host, struct web_client *w, char *url) {

1354 - return web_client_api_request_v1_weights_internal(host, w, url, WEIGHTS_METHOD_ANOMALY_RATE, WEIGHTS_FORMAT_CONTEXTS);

1277 + return web_client_api_request_weights(host, w, url, WEIGHTS_METHOD_ANOMALY_RATE,

1278 + WEIGHTS_FORMAT_CONTEXTS, 1);

1279 }

1280

1281 int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char *url) {

web/api/web_api_v1.h

@@ -30,6 +30,9 @@ int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb);

30 void web_client_api_v1_init(void);

31 void web_client_api_v1_management_init(void);

33 +void host_labels2json(RRDHOST *host, BUFFER *wb, const char *key);

34 +void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST *host, BUFFER *wb, const char *key);

35 +

36 extern char *api_secret;

38 #endif //NETDATA_WEB_API_V1_H

web/api/web_api_v2.c

+15 -8

@@ -22,6 +22,7 @@ static int web_client_api_request_v2_contexts_internal(RRDHOST *host __maybe_unu

22 else if((options & CONTEXTS_V2_CONTEXTS) && !strcmp(name, "scope_contexts")) req.scope_contexts = value;

23 else if((options & CONTEXTS_V2_CONTEXTS) && !strcmp(name, "contexts")) req.contexts = value;

24 else if((options & CONTEXTS_V2_SEARCH) && !strcmp(name, "q")) req.q = value;

25 + else if(!strcmp(name, "timeout")) req.timeout_ms = str2l(value);

26 }

28 options |= CONTEXTS_V2_DEBUG;

@@ -40,7 +41,12 @@ static int web_client_api_request_v2_contexts(RRDHOST *host __maybe_unused, stru

41 }

43 static int web_client_api_request_v2_nodes(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {

43 - return web_client_api_request_v2_contexts_internal(host, w, url, CONTEXTS_V2_NODES);

44 + return web_client_api_request_v2_contexts_internal(host, w, url, CONTEXTS_V2_NODES | CONTEXTS_V2_NODES_DETAILED);

45 +}

46 +

47 +static int web_client_api_request_v2_weights(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {

48 + return web_client_api_request_weights(host, w, url, WEIGHTS_METHOD_VALUE,

49 + WEIGHTS_FORMAT_MULTINODE, 2);

50 }

52 static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {

@@ -180,11 +186,11 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w

186 tier = 0;

187 }

188

183 - long long before = (before_str && *before_str)?str2l(before_str):0;

184 - long long after = (after_str && *after_str) ?str2l(after_str):-600;

185 - int points = (points_str && *points_str)?str2i(points_str):0;

186 - int timeout = (timeout_str && *timeout_str)?str2i(timeout_str): 0;

187 - long group_time = (resampling_time_str && *resampling_time_str) ? str2l(resampling_time_str) : 0;

189 + time_t before = (before_str && *before_str)?str2l(before_str):0;

190 + time_t after = (after_str && *after_str) ?str2l(after_str):-600;

191 + size_t points = (points_str && *points_str)?str2u(points_str):0;

192 + time_t timeout = (timeout_str && *timeout_str)?str2l(timeout_str): 0;

193 + time_t resampling_time = (resampling_time_str && *resampling_time_str) ? str2l(resampling_time_str) : 0;

194

195 QUERY_TARGET_REQUEST qtr = {

196 .version = 2,

@@ -199,7 +205,7 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w

205 .instances = instances,

206 .dimensions = dimensions,

207 .alerts = alerts,

202 - .timeout = timeout,

208 + .timeout_ms = timeout,

209 .points = points,

210 .format = format,

211 .options = options,

@@ -208,7 +214,7 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w

214 .group_by_aggregate_function = group_by_aggregate,

215 .time_group_method = time_group,

216 .time_group_options = time_group_options,

211 - .resampling_time = group_time,

217 + .resampling_time = resampling_time,

218 .tier = tier,

219 .chart_label_key = NULL,

220 .labels = labels,

@@ -300,6 +306,7 @@ static struct web_api_command api_commands_v2[] = {

306 {"data", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_data},

307 {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_nodes},

308 {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_contexts},

309 + {"weights", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_weights},

310 {"q", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_q},

311

312 // terminator

web/server/web_client.h

+2 -2

@@ -27,8 +27,8 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy;

28 // HTTP_CODES 5XX Server Errors

29 #define HTTP_RESP_INTERNAL_SERVER_ERROR 500

30 -#define HTTP_RESP_BACKEND_FETCH_FAILED 503 // 503 is right

31 -#define HTTP_RESP_SERVICE_UNAVAILABLE 503 // 503 is right

30 +#define HTTP_RESP_BACKEND_FETCH_FAILED 503

31 +#define HTTP_RESP_SERVICE_UNAVAILABLE 503

32 #define HTTP_RESP_GATEWAY_TIMEOUT 504

33 #define HTTP_RESP_BACKEND_RESPONSE_INVALID 591