@cryptotaxi247 / netdata-1 / commits / 66fd79777

/api/v2 part 9 (#14888)

* /api/v2/weights now supports group-by * /api/v2/weights now follows the same principles for describing responses as /api/v2/data

Costa Tsaousis committed Apr 11, 2023 at 17:32 UTC 66fd79777c90e737887d76409cf7bdaa6cb9dbd4

3 files changed +493 -159

web/api/queries/weights.c

+466 -159

@@ -303,116 +303,325 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w

303 return total_dimensions;

304 }

305

306 +struct query_weights_data {

307 + QUERY_WEIGHTS_REQUEST *qwr;

308 +

309 + SIMPLE_PATTERN *scope_nodes_sp;

310 + SIMPLE_PATTERN *scope_contexts_sp;

311 + SIMPLE_PATTERN *nodes_sp;

312 + SIMPLE_PATTERN *contexts_sp;

313 + SIMPLE_PATTERN *instances_sp;

314 + SIMPLE_PATTERN *dimensions_sp;

315 + SIMPLE_PATTERN *labels_sp;

316 + SIMPLE_PATTERN *alerts_sp;

317 +

318 + usec_t timeout_us;

319 + bool timed_out;

320 + bool interrupted;

321 +

322 + struct query_timings timings;

323 +

324 + size_t examined_dimensions;

325 + bool register_zero;

326 +

327 + DICTIONARY *results;

328 + WEIGHTS_STATS stats;

329 +

330 + uint32_t shifts;

331 +

332 + struct query_versions versions;

333 +};

334 +

335 +#define AGGREGATED_WEIGHT_EMPTY (struct aggregated_weight) { \

336 + .min = NAN, \

337 + .max = NAN, \

338 + .sum = NAN, \

339 + .count = 0, \

340 + .hsp = STORAGE_POINT_UNSET, \

341 + .bsp = STORAGE_POINT_UNSET, \

342 +}

343 +

344 +#define merge_into_aw(aw, t) do { \

345 + if(!(aw).count) { \

346 + (aw).count = 1; \

347 + (aw).min = (aw).max = (aw).sum = (t)->value; \

348 + (aw).hsp = (t)->highlighted; \

349 + if(baseline) \

350 + (aw).bsp = (t)->baseline; \

351 + } \

352 + else { \

353 + (aw).count++; \

354 + (aw).sum += (t)->value; \

355 + if((t)->value < (aw).min) \

356 + (aw).min = (t)->value; \

357 + if((t)->value > (aw).max) \

358 + (aw).max = (t)->value; \

359 + storage_point_merge_to((aw).hsp, (t)->highlighted); \

360 + if(baseline) \

361 + storage_point_merge_to((aw).bsp, (t)->baseline); \

362 + } \

363 +} while(0)

364 +

365 +static void results_header_to_json_v2(DICTIONARY *results __maybe_unused, BUFFER *wb, struct query_weights_data *qwd,

366 + time_t after, time_t before,

367 + time_t baseline_after, time_t baseline_before,

368 + size_t points, WEIGHTS_METHOD method,

369 + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,

370 + size_t examined_dimensions __maybe_unused, usec_t duration,

371 + WEIGHTS_STATS *stats, bool group_by) {

372 +

373 + buffer_json_member_add_object(wb, "request");

374 + buffer_json_member_add_string(wb, "method", weights_method_to_string(method));

375 + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options);

376 +

377 + buffer_json_member_add_object(wb, "scope");

378 + buffer_json_member_add_string(wb, "scope_nodes", qwd->qwr->scope_nodes ? qwd->qwr->scope_nodes : "*");

379 + buffer_json_member_add_string(wb, "scope_contexts", qwd->qwr->scope_contexts ? qwd->qwr->scope_contexts : "*");

380 + buffer_json_object_close(wb);

381 +

382 + buffer_json_member_add_object(wb, "selectors");

383 + buffer_json_member_add_string(wb, "nodes", qwd->qwr->nodes ? qwd->qwr->nodes : "*");

384 + buffer_json_member_add_string(wb, "contexts", qwd->qwr->contexts ? qwd->qwr->contexts : "*");

385 + buffer_json_member_add_string(wb, "instances", qwd->qwr->instances ? qwd->qwr->instances : "*");

386 + buffer_json_member_add_string(wb, "dimensions", qwd->qwr->dimensions ? qwd->qwr->dimensions : "*");

387 + buffer_json_member_add_string(wb, "labels", qwd->qwr->labels ? qwd->qwr->labels : "*");

388 + buffer_json_member_add_string(wb, "alerts", qwd->qwr->alerts ? qwd->qwr->alerts : "*");

389 + buffer_json_object_close(wb);

390 +

391 + buffer_json_member_add_object(wb, "window");

392 + buffer_json_member_add_time_t(wb, "after", qwd->qwr->after);

393 + buffer_json_member_add_time_t(wb, "before", qwd->qwr->before);

394 + buffer_json_member_add_uint64(wb, "points", qwd->qwr->points);

395 + if(qwd->qwr->options & RRDR_OPTION_SELECTED_TIER)

396 + buffer_json_member_add_uint64(wb, "tier", qwd->qwr->tier);

397 + else

398 + buffer_json_member_add_string(wb, "tier", NULL);

399 + buffer_json_object_close(wb);

400 +

401 + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) {

402 + buffer_json_member_add_object(wb, "baseline");

403 + buffer_json_member_add_time_t(wb, "baseline_after", qwd->qwr->baseline_after);

404 + buffer_json_member_add_time_t(wb, "baseline_before", qwd->qwr->baseline_before);

405 + buffer_json_object_close(wb);

406 + }

407 +

408 + buffer_json_member_add_object(wb, "aggregations");

409 + buffer_json_member_add_object(wb, "time");

410 + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qwd->qwr->time_group_method));

411 + buffer_json_member_add_string(wb, "time_group_options", qwd->qwr->time_group_options);

412 + buffer_json_object_close(wb); // time

413 +

414 + buffer_json_member_add_array(wb, "metrics");

415 + buffer_json_add_array_item_object(wb);

416 + {

417 + buffer_json_member_add_array(wb, "group_by");

418 + buffer_json_group_by_to_array(wb, qwd->qwr->group_by.group_by);

419 + buffer_json_array_close(wb);

420 +

421 +// buffer_json_member_add_array(wb, "group_by_label");

422 +// buffer_json_array_close(wb);

423 +

424 + buffer_json_member_add_string(wb, "aggregation", group_by_aggregate_function_to_string(qwd->qwr->group_by.aggregation));

425 + }

426 + buffer_json_object_close(wb); // 1st group by

427 + buffer_json_array_close(wb); // array

428 + buffer_json_object_close(wb); // aggregations

429 +

430 + buffer_json_member_add_uint64(wb, "timeout", qwd->qwr->timeout_ms);

431 + buffer_json_object_close(wb); // request

432 +

433 + buffer_json_member_add_object(wb, "view");

434 + buffer_json_member_add_string(wb, "format", (group_by)?"grouped":"full");

435 + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(group));

436 +

437 + buffer_json_member_add_object(wb, "window");

438 + buffer_json_member_add_time_t(wb, "after", after);

439 + buffer_json_member_add_time_t(wb, "before", before);

440 + buffer_json_member_add_time_t(wb, "duration", before - after);

441 + buffer_json_member_add_uint64(wb, "points", points);

442 + buffer_json_object_close(wb);

443 +

444 + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) {

445 + buffer_json_member_add_object(wb, "baseline");

446 + buffer_json_member_add_time_t(wb, "baseline_after", baseline_after);

447 + buffer_json_member_add_time_t(wb, "baseline_before", baseline_before);

448 + buffer_json_member_add_time_t(wb, "baseline_duration", baseline_before - baseline_after);

449 + buffer_json_member_add_uint64(wb, "baseline_points", points << shifts);

450 + buffer_json_object_close(wb);

451 + }

452 +

453 + buffer_json_object_close(wb); // view

454 +

455 + buffer_json_member_add_object(wb, "db");

456 + {

457 + buffer_json_member_add_double(wb, "query_time_ms", (double) duration / (double) USEC_PER_MS);

458 + buffer_json_member_add_uint64(wb, "db_queries", stats->db_queries);

459 + buffer_json_member_add_uint64(wb, "query_result_points", stats->result_points);

460 + buffer_json_member_add_uint64(wb, "binary_searches", stats->binary_searches);

461 + buffer_json_member_add_uint64(wb, "db_points_read", stats->db_points);

462 +

463 + buffer_json_member_add_array(wb, "db_points_per_tier");

464 + {

465 + for (size_t tier = 0; tier < storage_tiers; tier++)

466 + buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]);

467 + }

468 + buffer_json_array_close(wb);

469 + }

470 + buffer_json_object_close(wb); // db

471 +}

472 +

473 typedef enum {

474 WPT_DIMENSION = 0,

475 WPT_INSTANCE = 1,

476 WPT_CONTEXT = 2,

477 WPT_NODE = 3,

478 + WPT_GROUP = 4,

479 } WEIGHTS_POINT_TYPE;

480

313 -static inline void storage_point_to_json(BUFFER *wb, WEIGHTS_POINT_TYPE type, ssize_t di, ssize_t ii, ssize_t ci, ssize_t ni, NETDATA_DOUBLE weight, STORAGE_POINT *highlighted_sp, STORAGE_POINT *baseline_sp, RRDR_OPTIONS options __maybe_unused, bool baseline) {

314 - buffer_json_add_array_item_array(wb);

481 +struct aggregated_weight {

482 + const char *name;

483 + NETDATA_DOUBLE min;

484 + NETDATA_DOUBLE max;

485 + NETDATA_DOUBLE sum;

486 + size_t count;

487 + STORAGE_POINT hsp;

488 + STORAGE_POINT bsp;

489 +};

490

316 - buffer_json_add_array_item_uint64(wb, type); // "type"

317 - buffer_json_add_array_item_int64(wb, ni);

318 - if(type != WPT_NODE) {

319 - buffer_json_add_array_item_int64(wb, ci);

320 - if(type != WPT_CONTEXT) {

321 - buffer_json_add_array_item_int64(wb, ii);

322 - if(type != WPT_INSTANCE)

323 - buffer_json_add_array_item_int64(wb, di);

324 - else

491 +static inline void storage_point_to_json(BUFFER *wb, WEIGHTS_POINT_TYPE type, ssize_t di, ssize_t ii, ssize_t ci, ssize_t ni, struct aggregated_weight *aw, RRDR_OPTIONS options __maybe_unused, bool baseline) {

492 + if(type != WPT_GROUP) {

493 + buffer_json_add_array_item_array(wb);

494 + buffer_json_add_array_item_uint64(wb, type); // "type"

495 + buffer_json_add_array_item_int64(wb, ni);

496 + if (type != WPT_NODE) {

497 + buffer_json_add_array_item_int64(wb, ci);

498 + if (type != WPT_CONTEXT) {

499 + buffer_json_add_array_item_int64(wb, ii);

500 + if (type != WPT_INSTANCE)

501 + buffer_json_add_array_item_int64(wb, di);

502 + else

503 + buffer_json_add_array_item_string(wb, NULL);

504 + }

505 + else {

506 + buffer_json_add_array_item_string(wb, NULL);

507 buffer_json_add_array_item_string(wb, NULL);

326 - } else {

508 + }

509 + }

510 + else {

511 + buffer_json_add_array_item_string(wb, NULL);

512 buffer_json_add_array_item_string(wb, NULL);

513 buffer_json_add_array_item_string(wb, NULL);

514 }

515 + buffer_json_add_array_item_double(wb, (aw->count) ? aw->sum / (NETDATA_DOUBLE)aw->count : 0.0); // "weight"

516 }

517 else {

332 - buffer_json_add_array_item_string(wb, NULL);

333 - buffer_json_add_array_item_string(wb, NULL);

334 - buffer_json_add_array_item_string(wb, NULL);

518 + buffer_json_member_add_array(wb, "v");

519 + buffer_json_add_array_item_array(wb);

520 + buffer_json_add_array_item_double(wb, aw->min); // "min"

521 + buffer_json_add_array_item_double(wb, (aw->count) ? aw->sum / (NETDATA_DOUBLE)aw->count : 0.0); // "avg"

522 + buffer_json_add_array_item_double(wb, aw->max); // "max"

523 + buffer_json_add_array_item_double(wb, aw->sum); // "sum"

524 + buffer_json_add_array_item_uint64(wb, aw->count); // "count"

525 + buffer_json_array_close(wb);

526 }

336 - buffer_json_add_array_item_double(wb, weight); // "weight"

527

528 buffer_json_add_array_item_array(wb);

339 - buffer_json_add_array_item_double(wb, highlighted_sp->min); // "min"

340 - buffer_json_add_array_item_double(wb, (highlighted_sp->count) ? highlighted_sp->sum / (NETDATA_DOUBLE) highlighted_sp->count : 0.0); // "avg"

341 - buffer_json_add_array_item_double(wb, highlighted_sp->max); // "max"

342 - buffer_json_add_array_item_double(wb, highlighted_sp->sum); // "sum"

343 - buffer_json_add_array_item_uint64(wb, highlighted_sp->count); // "count"

344 - buffer_json_add_array_item_uint64(wb, highlighted_sp->anomaly_count); // "anomaly_count"

529 + buffer_json_add_array_item_double(wb, aw->hsp.min); // "min"

530 + buffer_json_add_array_item_double(wb, (aw->hsp.count) ? aw->hsp.sum / (NETDATA_DOUBLE) aw->hsp.count : 0.0); // "avg"

531 + buffer_json_add_array_item_double(wb, aw->hsp.max); // "max"

532 + buffer_json_add_array_item_double(wb, aw->hsp.sum); // "sum"

533 + buffer_json_add_array_item_uint64(wb, aw->hsp.count); // "count"

534 + buffer_json_add_array_item_uint64(wb, aw->hsp.anomaly_count); // "anomaly_count"

535 buffer_json_array_close(wb);

536

537 if(baseline) {

538 buffer_json_add_array_item_array(wb);

349 - buffer_json_add_array_item_double(wb, baseline_sp->min); // "min"

350 - buffer_json_add_array_item_double(wb, (baseline_sp->count) ? baseline_sp->sum / (NETDATA_DOUBLE) baseline_sp->count : 0.0); // "avg"

351 - buffer_json_add_array_item_double(wb, baseline_sp->max); // "max"

352 - buffer_json_add_array_item_double(wb, baseline_sp->sum); // "sum"

353 - buffer_json_add_array_item_uint64(wb, baseline_sp->count); // "count"

354 - buffer_json_add_array_item_uint64(wb, baseline_sp->anomaly_count); // "anomaly_count"

539 + buffer_json_add_array_item_double(wb, aw->bsp.min); // "min"

540 + buffer_json_add_array_item_double(wb, (aw->bsp.count) ? aw->bsp.sum / (NETDATA_DOUBLE) aw->bsp.count : 0.0); // "avg"

541 + buffer_json_add_array_item_double(wb, aw->bsp.max); // "max"

542 + buffer_json_add_array_item_double(wb, aw->bsp.sum); // "sum"

543 + buffer_json_add_array_item_uint64(wb, aw->bsp.count); // "count"

544 + buffer_json_add_array_item_uint64(wb, aw->bsp.anomaly_count); // "anomaly_count"

545 buffer_json_array_close(wb);

546 }

547

358 - buffer_json_array_close(wb); // key

548 + buffer_json_array_close(wb);

549 }

550

361 -static void multinode_data_schema(BUFFER *wb, RRDR_OPTIONS options __maybe_unused, const char *key, bool baseline) {

551 +static void multinode_data_schema(BUFFER *wb, RRDR_OPTIONS options __maybe_unused, const char *key, bool baseline, bool group_by) {

552 buffer_json_member_add_object(wb, key); // schema

553

554 buffer_json_member_add_string(wb, "type", "array");

555 buffer_json_member_add_array(wb, "items");

556

367 - buffer_json_add_array_item_object(wb);

368 - buffer_json_member_add_string(wb, "name", "row_type");

369 - buffer_json_member_add_string(wb, "type", "integer");

370 - buffer_json_member_add_array(wb, "value");

371 - buffer_json_add_array_item_string(wb, "dimension");

372 - buffer_json_add_array_item_string(wb, "instance");

373 - buffer_json_add_array_item_string(wb, "context");

374 - buffer_json_add_array_item_string(wb, "node");

375 - buffer_json_array_close(wb);

376 - buffer_json_object_close(wb);

377 -

378 - buffer_json_add_array_item_object(wb);

379 - {

380 - buffer_json_member_add_string(wb, "name", "ni");

381 - buffer_json_member_add_string(wb, "type", "integer");

382 - buffer_json_member_add_string(wb, "dictionary", "nodes");

557 + if(group_by) {

558 + buffer_json_add_array_item_object(wb);

559 + {

560 + buffer_json_member_add_string(wb, "name", "weight");

561 + buffer_json_member_add_string(wb, "type", "array");

562 + buffer_json_member_add_array(wb, "labels");

563 + {

564 + buffer_json_add_array_item_string(wb, "min");

565 + buffer_json_add_array_item_string(wb, "avg");

566 + buffer_json_add_array_item_string(wb, "max");

567 + buffer_json_add_array_item_string(wb, "sum");

568 + buffer_json_add_array_item_string(wb, "count");

569 + }

570 + buffer_json_array_close(wb);

571 + }

572 + buffer_json_object_close(wb);

573 }

384 - buffer_json_object_close(wb);

385 -

386 - buffer_json_add_array_item_object(wb);

387 - {

388 - buffer_json_member_add_string(wb, "name", "ci");

574 + else {

575 + buffer_json_add_array_item_object(wb);

576 + buffer_json_member_add_string(wb, "name", "row_type");

577 buffer_json_member_add_string(wb, "type", "integer");

390 - buffer_json_member_add_string(wb, "dictionary", "contexts");

391 - }

392 - buffer_json_object_close(wb);

578 + buffer_json_member_add_array(wb, "value");

579 + buffer_json_add_array_item_string(wb, "dimension");

580 + buffer_json_add_array_item_string(wb, "instance");

581 + buffer_json_add_array_item_string(wb, "context");

582 + buffer_json_add_array_item_string(wb, "node");

583 + buffer_json_array_close(wb);

584 + buffer_json_object_close(wb);

585

394 - buffer_json_add_array_item_object(wb);

395 - {

396 - buffer_json_member_add_string(wb, "name", "ii");

397 - buffer_json_member_add_string(wb, "type", "integer");

398 - buffer_json_member_add_string(wb, "dictionary", "instances");

399 - }

400 - buffer_json_object_close(wb);

586 + buffer_json_add_array_item_object(wb);

587 + {

588 + buffer_json_member_add_string(wb, "name", "ni");

589 + buffer_json_member_add_string(wb, "type", "integer");

590 + buffer_json_member_add_string(wb, "dictionary", "nodes");

591 + }

592 + buffer_json_object_close(wb);

593

402 - buffer_json_add_array_item_object(wb);

403 - {

404 - buffer_json_member_add_string(wb, "name", "di");

405 - buffer_json_member_add_string(wb, "type", "integer");

406 - buffer_json_member_add_string(wb, "dictionary", "dimensions");

407 - }

408 - buffer_json_object_close(wb);

594 + buffer_json_add_array_item_object(wb);

595 + {

596 + buffer_json_member_add_string(wb, "name", "ci");

597 + buffer_json_member_add_string(wb, "type", "integer");

598 + buffer_json_member_add_string(wb, "dictionary", "contexts");

599 + }

600 + buffer_json_object_close(wb);

601

410 - buffer_json_add_array_item_object(wb);

411 - {

412 - buffer_json_member_add_string(wb, "name", "weight");

413 - buffer_json_member_add_string(wb, "type", "number");

602 + buffer_json_add_array_item_object(wb);

603 + {

604 + buffer_json_member_add_string(wb, "name", "ii");

605 + buffer_json_member_add_string(wb, "type", "integer");

606 + buffer_json_member_add_string(wb, "dictionary", "instances");

607 + }

608 + buffer_json_object_close(wb);

609 +

610 + buffer_json_add_array_item_object(wb);

611 + {

612 + buffer_json_member_add_string(wb, "name", "di");

613 + buffer_json_member_add_string(wb, "type", "integer");

614 + buffer_json_member_add_string(wb, "dictionary", "dimensions");

615 + }

616 + buffer_json_object_close(wb);

617 +

618 + buffer_json_add_array_item_object(wb);

619 + {

620 + buffer_json_member_add_string(wb, "name", "weight");

621 + buffer_json_member_add_string(wb, "type", "number");

622 + }

623 + buffer_json_object_close(wb);

624 }

415 - buffer_json_object_close(wb);

625

626 buffer_json_add_array_item_object(wb);

627 {

@@ -518,53 +727,26 @@ static inline ssize_t dict_unique_id_name_add(DICTIONARY *dict, const char *id,

727

728 return (ssize_t)dun->i;

729 }

521 -struct query_weights_data {

522 - QUERY_WEIGHTS_REQUEST *qwr;

523 -

524 - SIMPLE_PATTERN *scope_nodes_sp;

525 - SIMPLE_PATTERN *scope_contexts_sp;

526 - SIMPLE_PATTERN *nodes_sp;

527 - SIMPLE_PATTERN *contexts_sp;

528 - SIMPLE_PATTERN *instances_sp;

529 - SIMPLE_PATTERN *dimensions_sp;

530 - SIMPLE_PATTERN *labels_sp;

531 - SIMPLE_PATTERN *alerts_sp;

532 -

533 - usec_t timeout_us;

534 - bool timed_out;

535 - bool interrupted;

536 -

537 - struct query_timings timings;

538 -

539 - size_t examined_dimensions;

540 - bool register_zero;

541 -

542 - DICTIONARY *results;

543 - WEIGHTS_STATS stats;

544 -

545 - uint32_t shifts;

546 -

547 - struct query_versions versions;

548 -};

549 -

550 -static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *wb,

551 - time_t after, time_t before,

552 - time_t baseline_after, time_t baseline_before,

553 - size_t points, WEIGHTS_METHOD method,

554 - RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,

555 - size_t examined_dimensions, struct query_weights_data *qwd,

556 - WEIGHTS_STATS *stats,

557 - struct query_versions *versions) {

730 +static size_t registered_results_to_json_multinode_no_group_by(

731 + DICTIONARY *results, BUFFER *wb,

732 + time_t after, time_t before,

733 + time_t baseline_after, time_t baseline_before,

734 + size_t points, WEIGHTS_METHOD method,

735 + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,

736 + size_t examined_dimensions, struct query_weights_data *qwd,

737 + WEIGHTS_STATS *stats,

738 + struct query_versions *versions) {

739 buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);

740 buffer_json_member_add_uint64(wb, "api", 2);

741

561 - results_header_to_json(results, wb, after, before, baseline_after, baseline_before,

562 - points, method, group, options, shifts, examined_dimensions, qwd->timings.executed_ut - qwd->timings.received_ut, stats);

742 + results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,

743 + points, method, group, options, shifts, examined_dimensions,

744 + qwd->timings.executed_ut - qwd->timings.received_ut, stats, false);

745

746 version_hashes_api_v2(wb, versions);

747

748 bool baseline = method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME;

567 - multinode_data_schema(wb, options, "schema", baseline);

749 + multinode_data_schema(wb, options, "schema", baseline, false);

750

751 DICTIONARY *dict_nodes = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_node));

752 DICTIONARY *dict_contexts = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_name_units));

@@ -573,10 +755,7 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *

755

756 buffer_json_member_add_array(wb, "result");

757

576 - size_t total_dimensions = 0, node_dims = 0, context_dims = 0, instance_dims = 0;

577 - NETDATA_DOUBLE context_total_weight = 0.0, instance_total_weight = 0.0, node_total_weight = 0.0;

578 - STORAGE_POINT context_hsp = STORAGE_POINT_UNSET, instance_hsp = STORAGE_POINT_UNSET, node_hsp = STORAGE_POINT_UNSET;

579 - STORAGE_POINT context_bsp = STORAGE_POINT_UNSET, instance_bsp = STORAGE_POINT_UNSET, node_bsp = STORAGE_POINT_UNSET;

758 + struct aggregated_weight node_aw = AGGREGATED_WEIGHT_EMPTY, context_aw = AGGREGATED_WEIGHT_EMPTY, instance_aw = AGGREGATED_WEIGHT_EMPTY;

759 struct register_result *t;

760 RRDHOST *last_host = NULL;

761 RRDCONTEXT_ACQUIRED *last_rca = NULL;

@@ -584,34 +763,28 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *

763 struct dict_unique_node *node_dun = NULL;

764 ssize_t di = -1, ii = -1, ci = -1, ni = -1;

765 ssize_t di_max = 0, ii_max = 0, ci_max = 0, ni_max = 0;

766 + size_t total_dimensions = 0;

767 dfe_start_read(results, t) {

768

769 // close instance

770 if(t->ria != last_ria && last_ria) {

591 - storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, instance_total_weight / (double) instance_dims, &instance_hsp, &instance_bsp, options, baseline);

592 -

771 + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, &instance_aw, options, baseline);

772 last_ria = NULL;

594 - instance_dims = 0;

595 - instance_total_weight = 0.0;

596 - instance_hsp = instance_bsp = STORAGE_POINT_UNSET;

773 + instance_aw = AGGREGATED_WEIGHT_EMPTY;

774 }

775

776 // close context

777 if(t->rca != last_rca && last_rca) {

601 - storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, context_total_weight / (double) context_dims, &context_hsp, &instance_bsp, options, baseline);

778 + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, &context_aw, options, baseline);

779 last_rca = NULL;

603 - context_dims = 0;

604 - context_total_weight = 0.0;

605 - context_hsp = context_bsp = STORAGE_POINT_UNSET;

780 + context_aw = AGGREGATED_WEIGHT_EMPTY;

781 }

782

783 // close node

784 if(t->host != last_host && last_host) {

610 - storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, node_total_weight / (double) node_dims, &node_hsp, &node_bsp, options, baseline);

785 + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, &node_aw, options, baseline);

786 last_host = NULL;

612 - node_dims = 0;

613 - node_total_weight = 0.0;

614 - node_hsp = node_bsp = STORAGE_POINT_UNSET;

787 + node_aw = AGGREGATED_WEIGHT_EMPTY;

788 }

789

790 // open node

@@ -635,42 +808,37 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *

808 }

809

810 di = dict_unique_id_name_add(dict_dimensions, rrdmetric_acquired_id(t->rma), rrdmetric_acquired_name(t->rma), &di_max);

638 - storage_point_to_json(wb, WPT_DIMENSION, di, ii, ci, ni, t->value, &t->highlighted, &t->baseline, options, baseline);

811

640 - instance_total_weight += t->value;

641 - context_total_weight += t->value;

642 - node_total_weight += t->value;

812 + struct aggregated_weight aw = {

813 + .min = t->value,

814 + .max = t->value,

815 + .sum = t->value,

816 + .count = 1,

817 + .hsp = t->highlighted,

818 + .bsp = t->baseline,

819 + };

820 + storage_point_to_json(wb, WPT_DIMENSION, di, ii, ci, ni, &aw, options, baseline);

821

644 - node_dun->duration_ut += t->duration_ut;

645 -

646 - storage_point_merge_to(instance_hsp, t->highlighted);

647 - storage_point_merge_to(context_hsp, t->highlighted);

648 - storage_point_merge_to(node_hsp, t->highlighted);

822 + merge_into_aw(instance_aw, t);

823 + merge_into_aw(context_aw, t);

824 + merge_into_aw(node_aw, t);

825

650 - if(baseline) {

651 - storage_point_merge_to(instance_bsp, t->baseline);

652 - storage_point_merge_to(context_bsp, t->baseline);

653 - storage_point_merge_to(node_bsp, t->baseline);

654 - }

655 -

656 - instance_dims++;

657 - context_dims++;

658 - node_dims++;

826 + node_dun->duration_ut += t->duration_ut;

827 total_dimensions++;

828 }

829 dfe_done(t);

830

831 // close instance

832 if(last_ria)

665 - storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, instance_total_weight / (double) instance_dims, &instance_hsp, &instance_bsp, options, baseline);

833 + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, &instance_aw, options, baseline);

834

835 // close context

836 if(last_rca)

669 - storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, context_total_weight / (double) context_dims, &context_hsp, &instance_bsp, options, baseline);

837 + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, &context_aw, options, baseline);

838

839 // close node

840 if(last_host)

673 - storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, node_total_weight / (double) node_dims, &node_hsp, &node_bsp, options, baseline);

841 + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, &node_aw, options, baseline);

842

843 buffer_json_array_close(wb); // points

844

@@ -746,6 +914,131 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *

914 return total_dimensions;

915 }

916

917 +static size_t registered_results_to_json_multinode_group_by(

918 + DICTIONARY *results, BUFFER *wb,

919 + time_t after, time_t before,

920 + time_t baseline_after, time_t baseline_before,

921 + size_t points, WEIGHTS_METHOD method,

922 + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,

923 + size_t examined_dimensions, struct query_weights_data *qwd,

924 + WEIGHTS_STATS *stats,

925 + struct query_versions *versions) {

926 + buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);

927 + buffer_json_member_add_uint64(wb, "api", 2);

928 +

929 + results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,

930 + points, method, group, options, shifts, examined_dimensions,

931 + qwd->timings.executed_ut - qwd->timings.received_ut, stats, true);

932 +

933 + version_hashes_api_v2(wb, versions);

934 +

935 + bool baseline = method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME;

936 + multinode_data_schema(wb, options, "v_schema", baseline, true);

937 +

938 + DICTIONARY *group_by = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,

939 + NULL, sizeof(struct aggregated_weight));

940 +

941 + struct register_result *t;

942 + size_t total_dimensions = 0;

943 + BUFFER *key = buffer_create(0, NULL);

944 + BUFFER *name = buffer_create(0, NULL);

945 + dfe_start_read(results, t) {

946 +

947 + buffer_flush(key);

948 + buffer_flush(name);

949 +

950 + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_DIMENSION) {

951 + buffer_strcat(key, rrdmetric_acquired_name(t->rma));

952 + buffer_strcat(name, rrdmetric_acquired_name(t->rma));

953 + }

954 + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_INSTANCE) {

955 + if(buffer_strlen(key)) {

956 + buffer_fast_strcat(key, ",", 1);

957 + buffer_fast_strcat(name, ",", 1);

958 + }

959 +

960 + buffer_strcat(key, rrdinstance_acquired_id(t->ria));

961 + buffer_strcat(name, rrdinstance_acquired_name(t->ria));

962 +

963 + if(!(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_NODE)) {

964 + buffer_fast_strcat(key, "@", 1);

965 + buffer_fast_strcat(name, "@", 1);

966 + buffer_strcat(key, t->host->machine_guid);

967 + buffer_strcat(name, rrdhost_hostname(t->host));

968 + }

969 + }

970 + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_NODE) {

971 + if(buffer_strlen(key)) {

972 + buffer_fast_strcat(key, ",", 1);

973 + buffer_fast_strcat(name, ",", 1);

974 + }

975 +

976 + buffer_strcat(key, t->host->machine_guid);

977 + buffer_strcat(name, rrdhost_hostname(t->host));

978 + }

979 + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_CONTEXT) {

980 + if(buffer_strlen(key)) {

981 + buffer_fast_strcat(key, ",", 1);

982 + buffer_fast_strcat(name, ",", 1);

983 + }

984 +

985 + buffer_strcat(key, rrdcontext_acquired_id(t->rca));

986 + buffer_strcat(name, rrdcontext_acquired_id(t->rca));

987 + }

988 + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_UNITS) {

989 + if(buffer_strlen(key)) {

990 + buffer_fast_strcat(key, ",", 1);

991 + buffer_fast_strcat(name, ",", 1);

992 + }

993 +

994 + buffer_strcat(key, rrdcontext_acquired_units(t->rca));

995 + buffer_strcat(name, rrdcontext_acquired_units(t->rca));

996 + }

997 +

998 + struct aggregated_weight *aw = dictionary_set(group_by, buffer_tostring(key), NULL, sizeof(struct aggregated_weight));

999 + if(!aw->name) {

1000 + aw->name = strdupz(buffer_tostring(name));

1001 + aw->min = aw->max = aw->sum = t->value;

1002 + aw->count = 1;

1003 + aw->hsp = t->highlighted;

1004 + aw->bsp = t->baseline;

1005 + }

1006 + else

1007 + merge_into_aw(*aw, t);

1008 +

1009 + total_dimensions++;

1010 + }

1011 + dfe_done(t);

1012 + buffer_free(key); key = NULL;

1013 + buffer_free(name); name = NULL;

1014 +

1015 + struct aggregated_weight *aw;

1016 + buffer_json_member_add_array(wb, "result");

1017 + dfe_start_read(group_by, aw) {

1018 + const char *k = aw_dfe.name;

1019 + const char *n = aw->name;

1020 +

1021 + buffer_json_add_array_item_object(wb);

1022 + buffer_json_member_add_string(wb, "id", k);

1023 + buffer_json_member_add_string(wb, "nm", n);

1024 + storage_point_to_json(wb, WPT_GROUP, 0, 0, 0, 0, aw, options, baseline);

1025 + buffer_json_object_close(wb);

1026 +

1027 + freez((void *)aw->name);

1028 + }

1029 + dfe_done(aw);

1030 + buffer_json_array_close(wb); // result

1031 +

1032 + buffer_json_agents_array_v2(wb, &qwd->timings, 0);

1033 + buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions);

1034 + buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions);

1035 + buffer_json_finalize(wb);

1036 +

1037 + dictionary_destroy(group_by);

1038 +

1039 + return total_dimensions;

1040 +}

1041 +

1042 // ----------------------------------------------------------------------------

1043 // KS2 algorithm functions

1044

@@ -1616,14 +1909,28 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {

1909

1910 default:

1911 case WEIGHTS_FORMAT_MULTINODE:

1619 - added_dimensions =

1620 - registered_results_to_json_multinode(

1621 - qwd.results, wb,

1622 - qwr->after, qwr->before,

1623 - qwr->baseline_after, qwr->baseline_before,

1624 - qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,

1625 - qwd.examined_dimensions,

1626 - &qwd, &qwd.stats, &qwd.versions);

1912 + // we don't support these groupings in weights

1913 + qwr->group_by.group_by &= ~(RRDR_GROUP_BY_LABEL|RRDR_GROUP_BY_SELECTED|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE);

1914 + if(qwr->group_by.group_by == RRDR_GROUP_BY_NONE) {

1915 + added_dimensions =

1916 + registered_results_to_json_multinode_no_group_by(

1917 + qwd.results, wb,

1918 + qwr->after, qwr->before,

1919 + qwr->baseline_after, qwr->baseline_before,

1920 + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,

1921 + qwd.examined_dimensions,

1922 + &qwd, &qwd.stats, &qwd.versions);

1923 + }

1924 + else {

1925 + added_dimensions =

1926 + registered_results_to_json_multinode_group_by(

1927 + qwd.results, wb,

1928 + qwr->after, qwr->before,

1929 + qwr->baseline_after, qwr->baseline_before,

1930 + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,

1931 + qwd.examined_dimensions,

1932 + &qwd, &qwd.stats, &qwd.versions);

1933 + }

1934 break;

1935 }

1936

web/api/queries/weights.h

@@ -35,6 +35,13 @@ typedef struct query_weights_request {

35 const char *dimensions;

36 const char *labels;

37 const char *alerts;

38 +

39 + struct {

40 + RRDR_GROUP_BY group_by;

41 + char *group_by_label;

42 + RRDR_GROUP_BY_FUNCTION aggregation;

43 + } group_by;

44 +

45 WEIGHTS_METHOD method;

46 WEIGHTS_FORMAT format;

47 RRDR_TIME_GROUPING time_group_method;

web/api/web_api.c

+20

@@ -71,6 +71,12 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur

71 const char *time_group_options = NULL, *scope_contexts = NULL, *scope_nodes = NULL, *contexts = NULL, *nodes = NULL,

72 *instances = NULL, *dimensions = NULL, *labels = NULL, *alerts = NULL;

74 + struct group_by_pass group_by = {

75 + .group_by = RRDR_GROUP_BY_NONE,

76 + .group_by_label = NULL,

77 + .aggregation = RRDR_GROUP_BY_FUNCTION_AVERAGE,

78 + };

79 +

80 while (url) {

81 char *value = mystrsep(&url, "&");

82 if (!value || !*value)

@@ -123,6 +129,15 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur

129 else if(api_version >= 2 && !strcmp(name, "dimensions")) dimensions = value;

130 else if(api_version >= 2 && !strcmp(name, "labels")) labels = value;

131 else if(api_version >= 2 && !strcmp(name, "alerts")) alerts = value;

132 + else if(api_version >= 2 && (!strcmp(name, "group_by") || !strcmp(name, "group_by[0]"))) {

133 + group_by.group_by = group_by_parse(value);

134 + }

135 + else if(api_version >= 2 && (!strcmp(name, "group_by_label") || !strcmp(name, "group_by_label[0]"))) {

136 + group_by.group_by_label = value;

137 + }

138 + else if(api_version >= 2 && (!strcmp(name, "aggregation") || !strcmp(name, "aggregation[0]"))) {

139 + group_by.aggregation = group_by_aggregate_function_parse(value);

140 + }

141

142 else if(!strcmp(name, "tier")) {

143 tier = str2ul(value);

@@ -161,6 +176,11 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur

176 .dimensions = dimensions,

177 .labels = labels,

178 .alerts = alerts,

179 + .group_by = {

180 + .group_by = group_by.group_by,

181 + .group_by_label = group_by.group_by_label,

182 + .aggregation = group_by.aggregation,

183 + },

184 .method = method,

185 .format = format,

186 .time_group_method = time_group_method,