@cryptotaxi247 / netdata-1 / commits / 021e252fc

/api/v2/contexts (#14592)

* preparation for /api/v2/contexts * working /api/v2/contexts * add anomaly rate information in all statistics; when sum-count is requested, return sums and counts instead of averages * minor fix * query targegt now accurately counts hosts, contexts, instances, dimensions, metrics * cleanup /api/v2/contexts * full text search with /api/v2/contexts * simple patterns now support the option to search ignoring case * full text search API with /api/v2/q * simple pattern execution optimization * do not show q when not given * full text search accounting * separated /api/v2/nodes from /api/v2/contexts * fix ssv queries for group_by * count query instances queried and failed per context and host * split rrdcontext.c to multiple files * add query totals * fix anomaly rate calculation; provide "ni" for indexing hosts * do not generate zero valued members * faster calculation of anomaly rate; by just summing integers for each db points and doing math once for every generated point * fix typo when printing dimensions totals * added option minify to remove spaces and newlines fron JSON output * send instance ids and names when they differ * do not add in query target dimensions, instances, contexts and hosts for which there is no retention in the current timeframe * fix for the previous + renames and code cleanup * when a dimension is filtered, include in the response all the other dimensions that are selectable * do not add nodes that do not have retention in the current window * move selection of dimensions to query_dimension_add(), instead of query_metric_add() * increase the pre-processing capacity of queries * generate instance fqdn ids and names only when they are needed * provide detailed statistics about tiers retention, queries, points, update_every * late allocation of query dimensions * cleanup * more cleanup * support for annotations per displayed point, RESET and PARTIAL * new type annotations * if a chart is not linked to contexts and it is collected, link it when it is collected * make ML run reentrant * make ML rrdr query synchronous * optimize replication memory allocation of replication_sort_entry * change units to percentage, when requesting a coefficinet of variation, or a percentage query * initialize replication before starting main threads * properly decrement no room requests counter * propagate the non-zero flag to group-by * the same by avoiding the extra loop * respect non-zero in all dimension arrays * remove dictionary garbage collection from dictionary_entries() and dictionary_version() * be more verbose when jv2 indexing is postponed * prevent infinite loop * use hidden dimensions even when dimensions pattern is unset * traverse hosts using dictionaries * fix dictionary unittests

Costa Tsaousis committed Mar 2, 2023 at 22:50 UTC 021e252fc5d18a7225c0f4c975b3281016861d3c
63 files changed +6190 -4948
Makefile.am
+11 -2
@@ -410,12 +410,21 @@ PLUGINSD_PLUGIN_FILES = \
410 $(NULL)
411
412 RRD_PLUGIN_FILES = \
413 + database/contexts/rrdcontext.c \
414 + database/contexts/rrdcontext.h \
415 + database/contexts/metric.c \
416 + database/contexts/instance.c \
417 + database/contexts/context.c \
418 + database/contexts/worker.c \
419 + database/contexts/query_target.c \
420 + database/contexts/query_scope.c \
421 + database/contexts/api_v1.c \
422 + database/contexts/api_v2.c \
423 + database/contexts/internal.h \
424 database/rrdcalc.c \
425 database/rrdcalc.h \
426 database/rrdcalctemplate.c \
427 database/rrdcalctemplate.h \
417 - database/rrdcontext.c \
418 - database/rrdcontext.h \
428 database/rrddim.c \
429 database/rrddimvar.c \
430 database/rrddimvar.h \
collectors/apps.plugin/apps_plugin.c
+1 -1
@@ -4400,7 +4400,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
4400 unsigned int io_divisor = 1024 * RATES_DETAIL;
4401
4402 BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
4403 - buffer_json_initialize(wb, "\"", "\"", 0, true);
4403 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
4404 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
4405 buffer_json_member_add_string(wb, "type", "table");
4406 buffer_json_member_add_time_t(wb, "update_every", update_every);
collectors/cgroups.plugin/sys_fs_cgroup.c
+89 -89
@@ -449,70 +449,70 @@ void read_cgroup_plugin_configuration() {
449 config_get("plugin:cgroups", "enable by default cgroups matching",
450 // ----------------------------------------------------------------
451
452 - " !*/init.scope " // ignore init.scope
453 - " !/system.slice/run-*.scope " // ignore system.slice/run-XXXX.scope
454 - " *.scope " // we need all other *.scope for sure
455 -
456 - // ----------------------------------------------------------------
457 -
458 - " /machine.slice/*.service " // #3367 systemd-nspawn
459 -
460 - // ----------------------------------------------------------------
461 -
462 - " */kubepods/pod*/* " // k8s containers
463 - " */kubepods/*/pod*/* " // k8s containers
464 - " */*-kubepods-pod*/* " // k8s containers
465 - " */*-kubepods-*-pod*/* " // k8s containers
466 - " !*kubepods* !*kubelet* " // all other k8s cgroups
467 -
468 - // ----------------------------------------------------------------
469 -
470 - " !*/vcpu* " // libvirtd adds these sub-cgroups
471 - " !*/emulator " // libvirtd adds these sub-cgroups
472 - " !*.mount "
473 - " !*.partition "
474 - " !*.service "
475 - " !*.socket "
476 - " !*.slice "
477 - " !*.swap "
478 - " !*.user "
479 - " !/ "
480 - " !/docker "
481 - " !*/libvirt "
482 - " !/lxc "
483 - " !/lxc/*/* " // #1397 #2649
484 - " !/lxc.monitor* "
485 - " !/lxc.pivot "
486 - " !/lxc.payload "
487 - " !/machine "
488 - " !/qemu "
489 - " !/system "
490 - " !/systemd "
491 - " !/user "
492 - " * " // enable anything else
493 - ), NULL, SIMPLE_PATTERN_EXACT);
452 + " !*/init.scope " // ignore init.scope
453 + " !/system.slice/run-*.scope " // ignore system.slice/run-XXXX.scope
454 + " *.scope " // we need all other *.scope for sure
455 +
456 + // ----------------------------------------------------------------
457 +
458 + " /machine.slice/*.service " // #3367 systemd-nspawn
459 +
460 + // ----------------------------------------------------------------
461 +
462 + " */kubepods/pod*/* " // k8s containers
463 + " */kubepods/*/pod*/* " // k8s containers
464 + " */*-kubepods-pod*/* " // k8s containers
465 + " */*-kubepods-*-pod*/* " // k8s containers
466 + " !*kubepods* !*kubelet* " // all other k8s cgroups
467 +
468 + // ----------------------------------------------------------------
469 +
470 + " !*/vcpu* " // libvirtd adds these sub-cgroups
471 + " !*/emulator " // libvirtd adds these sub-cgroups
472 + " !*.mount "
473 + " !*.partition "
474 + " !*.service "
475 + " !*.socket "
476 + " !*.slice "
477 + " !*.swap "
478 + " !*.user "
479 + " !/ "
480 + " !/docker "
481 + " !*/libvirt "
482 + " !/lxc "
483 + " !/lxc/*/* " // #1397 #2649
484 + " !/lxc.monitor* "
485 + " !/lxc.pivot "
486 + " !/lxc.payload "
487 + " !/machine "
488 + " !/qemu "
489 + " !/system "
490 + " !/systemd "
491 + " !/user "
492 + " * " // enable anything else
493 + ), NULL, SIMPLE_PATTERN_EXACT, true);
494
495 enabled_cgroup_names = simple_pattern_create(
496 config_get("plugin:cgroups", "enable by default cgroups names matching",
497 - " * "
498 - ), NULL, SIMPLE_PATTERN_EXACT);
497 + " * "
498 + ), NULL, SIMPLE_PATTERN_EXACT, true);
499
500 search_cgroup_paths = simple_pattern_create(
501 config_get("plugin:cgroups", "search for cgroups in subpaths matching",
502 - " !*/init.scope " // ignore init.scope
503 - " !*-qemu " // #345
504 - " !*.libvirt-qemu " // #3010
505 - " !/init.scope "
506 - " !/system "
507 - " !/systemd "
508 - " !/user "
509 - " !/user.slice "
510 - " !/lxc/*/* " // #2161 #2649
511 - " !/lxc.monitor "
512 - " !/lxc.payload/*/* "
513 - " !/lxc.payload.* "
514 - " * "
515 - ), NULL, SIMPLE_PATTERN_EXACT);
502 + " !*/init.scope " // ignore init.scope
503 + " !*-qemu " // #345
504 + " !*.libvirt-qemu " // #3010
505 + " !/init.scope "
506 + " !/system "
507 + " !/systemd "
508 + " !/user "
509 + " !/user.slice "
510 + " !/lxc/*/* " // #2161 #2649
511 + " !/lxc.monitor "
512 + " !/lxc.payload/*/* "
513 + " !/lxc.payload.* "
514 + " * "
515 + ), NULL, SIMPLE_PATTERN_EXACT, true);
516
517 snprintfz(filename, FILENAME_MAX, "%s/cgroup-name.sh", netdata_configured_primary_plugins_dir);
518 cgroups_rename_script = config_get("plugin:cgroups", "script to get cgroup names", filename);
@@ -522,37 +522,37 @@ void read_cgroup_plugin_configuration() {
522
523 enabled_cgroup_renames = simple_pattern_create(
524 config_get("plugin:cgroups", "run script to rename cgroups matching",
525 - " !/ "
526 - " !*.mount "
527 - " !*.socket "
528 - " !*.partition "
529 - " /machine.slice/*.service " // #3367 systemd-nspawn
530 - " !*.service "
531 - " !*.slice "
532 - " !*.swap "
533 - " !*.user "
534 - " !init.scope "
535 - " !*.scope/vcpu* " // libvirtd adds these sub-cgroups
536 - " !*.scope/emulator " // libvirtd adds these sub-cgroups
537 - " *.scope "
538 - " *docker* "
539 - " *lxc* "
540 - " *qemu* "
541 - " */kubepods/pod*/* " // k8s containers
542 - " */kubepods/*/pod*/* " // k8s containers
543 - " */*-kubepods-pod*/* " // k8s containers
544 - " */*-kubepods-*-pod*/* " // k8s containers
545 - " !*kubepods* !*kubelet* " // all other k8s cgroups
546 - " *.libvirt-qemu " // #3010
547 - " * "
548 - ), NULL, SIMPLE_PATTERN_EXACT);
525 + " !/ "
526 + " !*.mount "
527 + " !*.socket "
528 + " !*.partition "
529 + " /machine.slice/*.service " // #3367 systemd-nspawn
530 + " !*.service "
531 + " !*.slice "
532 + " !*.swap "
533 + " !*.user "
534 + " !init.scope "
535 + " !*.scope/vcpu* " // libvirtd adds these sub-cgroups
536 + " !*.scope/emulator " // libvirtd adds these sub-cgroups
537 + " *.scope "
538 + " *docker* "
539 + " *lxc* "
540 + " *qemu* "
541 + " */kubepods/pod*/* " // k8s containers
542 + " */kubepods/*/pod*/* " // k8s containers
543 + " */*-kubepods-pod*/* " // k8s containers
544 + " */*-kubepods-*-pod*/* " // k8s containers
545 + " !*kubepods* !*kubelet* " // all other k8s cgroups
546 + " *.libvirt-qemu " // #3010
547 + " * "
548 + ), NULL, SIMPLE_PATTERN_EXACT, true);
549
550 if(cgroup_enable_systemd_services) {
551 systemd_services_cgroups = simple_pattern_create(
552 config_get("plugin:cgroups", "cgroups to match as systemd services",
553 - " !/system.slice/*/*.service "
554 - " /system.slice/*.service "
555 - ), NULL, SIMPLE_PATTERN_EXACT);
553 + " !/system.slice/*/*.service "
554 + " /system.slice/*.service "
555 + ), NULL, SIMPLE_PATTERN_EXACT, true);
556 }
557
558 mountinfo_free_all(root);
@@ -2784,10 +2784,10 @@ void cgroup_discovery_worker(void *ptr)
2784 worker_register_job_name(WORKER_DISCOVERY_LOCK, "lock");
2785
2786 entrypoint_parent_process_comm = simple_pattern_create(
2787 - " runc:[* " // http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2021/12/28/runc-internals-3)
2788 - " exe ", // https://github.com/falcosecurity/falco/blob/9d41b0a151b83693929d3a9c84f7c5c85d070d3a/rules/falco_rules.yaml#L1961
2789 - NULL,
2790 - SIMPLE_PATTERN_EXACT);
2787 + " runc:[* " // http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2021/12/28/runc-internals-3)
2788 + " exe ", // https://github.com/falcosecurity/falco/blob/9d41b0a151b83693929d3a9c84f7c5c85d070d3a/rules/falco_rules.yaml#L1961
2789 + NULL,
2790 + SIMPLE_PATTERN_EXACT, true);
2791
2792 while (service_running(SERVICE_COLLECTORS)) {
2793 worker_is_idle();
collectors/diskspace.plugin/plugin_diskspace.c
+4 -8
@@ -308,16 +308,12 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
308 }
309
310 excluded_mountpoints = simple_pattern_create(
311 - config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on paths", DEFAULT_EXCLUDED_PATHS)
312 - , NULL
313 - , mode
314 - );
311 + config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on paths", DEFAULT_EXCLUDED_PATHS), NULL,
312 + mode, true);
313
314 excluded_filesystems = simple_pattern_create(
317 - config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on filesystems", DEFAULT_EXCLUDED_FILESYSTEMS)
318 - , NULL
319 - , SIMPLE_PATTERN_EXACT
320 - );
315 + config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on filesystems",
316 + DEFAULT_EXCLUDED_FILESYSTEMS), NULL, SIMPLE_PATTERN_EXACT, true);
317
318 dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors, 0);
319 }
collectors/ebpf.plugin/ebpf_socket.c
+1 -1
@@ -3767,7 +3767,7 @@ static void link_hostnames(char *parse)
3767 ebpf_network_viewer_hostname_list_t *hostname = callocz(1 , sizeof(ebpf_network_viewer_hostname_list_t));
3768 hostname->value = strdupz(parse);
3769 hostname->hash = simple_hash(parse);
3770 - hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT);
3770 + hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT, true);
3771
3772 link_hostname((!neg)?&network_viewer_opt.included_hostnames:&network_viewer_opt.excluded_hostnames,
3773 hostname);
collectors/proc.plugin/proc_diskstats.c
+4 -8
@@ -934,16 +934,12 @@ int do_proc_diskstats(int update_every, usec_t dt) {
934 name_disks_by_id = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "name disks by id", name_disks_by_id);
935
936 preferred_ids = simple_pattern_create(
937 - config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "preferred disk ids", DEFAULT_PREFERRED_IDS)
938 - , NULL
939 - , SIMPLE_PATTERN_EXACT
940 - );
937 + config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "preferred disk ids", DEFAULT_PREFERRED_IDS), NULL,
938 + SIMPLE_PATTERN_EXACT, true);
939
940 excluded_disks = simple_pattern_create(
943 - config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "exclude disks", DEFAULT_EXCLUDED_DISKS)
944 - , NULL
945 - , SIMPLE_PATTERN_EXACT
946 - );
941 + config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "exclude disks", DEFAULT_EXCLUDED_DISKS), NULL,
942 + SIMPLE_PATTERN_EXACT, true);
943 }
944
945 // --------------------------------------------------------------------------
collectors/proc.plugin/proc_net_dev.c
+3 -1
@@ -725,7 +725,9 @@ int do_proc_net_dev(int update_every, usec_t dt) {
725 do_carrier = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "carrier for all interfaces", CONFIG_BOOLEAN_AUTO);
726 do_mtu = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "mtu for all interfaces", CONFIG_BOOLEAN_AUTO);
727
728 - disabled_list = simple_pattern_create(config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "disable by default interfaces matching", "lo fireqos* *-ifb fwpr* fwbr* fwln*"), NULL, SIMPLE_PATTERN_EXACT);
728 + disabled_list = simple_pattern_create(
729 + config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "disable by default interfaces matching",
730 + "lo fireqos* *-ifb fwpr* fwbr* fwln*"), NULL, SIMPLE_PATTERN_EXACT, true);
731 }
732
733 if(unlikely(!ff)) {
collectors/proc.plugin/proc_pagetypeinfo.c
+2 -4
@@ -120,10 +120,8 @@ int do_proc_pagetypeinfo(int update_every, usec_t dt) {
120 do_global = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable system summary", CONFIG_BOOLEAN_YES);
121 do_detail = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable detail per-type", CONFIG_BOOLEAN_AUTO);
122 filter_types = simple_pattern_create(
123 - config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "hide charts id matching", "")
124 - , NULL
125 - , SIMPLE_PATTERN_SUFFIX
126 - );
123 + config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "hide charts id matching", ""), NULL,
124 + SIMPLE_PATTERN_SUFFIX, true);
125
126 pagelines_cnt = 0;
127
collectors/proc.plugin/sys_class_infiniband.c
+3 -2
@@ -327,8 +327,9 @@ int do_sys_class_infiniband(int update_every, usec_t dt)
327 enable_only_active = config_get_boolean_ondemand(
328 CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "monitor only active ports", CONFIG_BOOLEAN_AUTO);
329 disabled_list = simple_pattern_create(
330 - config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""), NULL,
331 - SIMPLE_PATTERN_EXACT);
330 + config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""),
331 + NULL,
332 + SIMPLE_PATTERN_EXACT, true);
333
334 dt_to_refresh_ports =
335 config_get_number(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "refresh ports state every seconds", 30) *
collectors/statsd.plugin/statsd.c
+5 -3
@@ -1418,7 +1418,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1418 }
1419 else if (!strcmp(name, "metrics")) {
1420 simple_pattern_free(app->metrics);
1421 - app->metrics = simple_pattern_create(value, NULL, SIMPLE_PATTERN_EXACT);
1421 + app->metrics = simple_pattern_create(value, NULL, SIMPLE_PATTERN_EXACT, true);
1422 }
1423 else if (!strcmp(name, "private charts")) {
1424 if (!strcmp(value, "yes") || !strcmp(value, "on"))
@@ -1533,7 +1533,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
1533 );
1534
1535 if(pattern)
1536 - dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT);
1536 + dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT, true);
1537 }
1538 else {
1539 error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [%s] section.", line, name, filename, chart->id);
@@ -2462,7 +2462,9 @@ void *statsd_main(void *ptr) {
2462 statsd.recvmmsg_size = (size_t)config_get_number(CONFIG_SECTION_STATSD, "udp messages to process at once", (long long)statsd.recvmmsg_size);
2463 #endif
2464
2465 - statsd.charts_for = simple_pattern_create(config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
2465 + statsd.charts_for = simple_pattern_create(
2466 + config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL,
2467 + SIMPLE_PATTERN_EXACT, true);
2468 statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts_hard);
2469 statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
2470 statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
configure.ac
+1
@@ -1720,6 +1720,7 @@ AC_CONFIG_FILES([
1720 collectors/perf.plugin/Makefile
1721 daemon/Makefile
1722 database/Makefile
1723 + database/contexts/Makefile
1724 database/engine/Makefile
1725 database/ram/Makefile
1726 database/sqlite/Makefile
daemon/main.c
+20 -14
@@ -530,38 +530,41 @@ void web_server_config_options(void)
530 web_x_frame_options = NULL;
531
532 web_allow_connections_from =
533 - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"),
534 - NULL, SIMPLE_PATTERN_EXACT);
533 + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"),
534 + NULL, SIMPLE_PATTERN_EXACT, true);
535 web_allow_connections_dns =
536 make_dns_decision(CONFIG_SECTION_WEB, "allow connections by dns", "heuristic", web_allow_connections_from);
537 web_allow_dashboard_from =
538 - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"),
539 - NULL, SIMPLE_PATTERN_EXACT);
538 + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"),
539 + NULL, SIMPLE_PATTERN_EXACT, true);
540 web_allow_dashboard_dns =
541 make_dns_decision(CONFIG_SECTION_WEB, "allow dashboard by dns", "heuristic", web_allow_dashboard_from);
542 web_allow_badges_from =
543 - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT);
543 + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT,
544 + true);
545 web_allow_badges_dns =
546 make_dns_decision(CONFIG_SECTION_WEB, "allow badges by dns", "heuristic", web_allow_badges_from);
547 web_allow_registry_from =
547 - simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT);
548 + simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT,
549 + true);
550 web_allow_registry_dns = make_dns_decision(CONFIG_SECTION_REGISTRY, "allow by dns", "heuristic",
551 web_allow_registry_from);
552 web_allow_streaming_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow streaming from", "*"),
551 - NULL, SIMPLE_PATTERN_EXACT);
553 + NULL, SIMPLE_PATTERN_EXACT, true);
554 web_allow_streaming_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow streaming by dns", "heuristic",
555 web_allow_streaming_from);
556 // Note the default is not heuristic, the wildcards could match DNS but the intent is ip-addresses.
557 web_allow_netdataconf_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow netdata.conf from",
556 - "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*"
557 - " 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*"
558 - " 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*"
559 - " 172.31.* UNKNOWN"), NULL, SIMPLE_PATTERN_EXACT);
558 + "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*"
559 + " 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*"
560 + " 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*"
561 + " 172.31.* UNKNOWN"), NULL, SIMPLE_PATTERN_EXACT,
562 + true);
563 web_allow_netdataconf_dns =
564 make_dns_decision(CONFIG_SECTION_WEB, "allow netdata.conf by dns", "no", web_allow_netdataconf_from);
565 web_allow_mgmt_from =
563 - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"),
564 - NULL, SIMPLE_PATTERN_EXACT);
566 + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"),
567 + NULL, SIMPLE_PATTERN_EXACT, true);
568 web_allow_mgmt_dns =
569 make_dns_decision(CONFIG_SECTION_WEB, "allow management by dns","heuristic",web_allow_mgmt_from);
570
@@ -1325,6 +1328,7 @@ int pgc_unittest(void);
1328 int mrg_unittest(void);
1329 int julytest(void);
1330 int pluginsd_parser_unittest(void);
1331 +void replication_initialize(void);
1332
1333 int main(int argc, char **argv) {
1334 // initialize the system clocks
@@ -1612,7 +1616,7 @@ int main(int argc, char **argv) {
1616 size_t len = strlen(needle) + 1;
1617 char wildcarded[len];
1618
1615 - SIMPLE_PATTERN *p = simple_pattern_create(haystack, NULL, SIMPLE_PATTERN_EXACT);
1619 + SIMPLE_PATTERN *p = simple_pattern_create(haystack, NULL, SIMPLE_PATTERN_EXACT, true);
1620 int ret = simple_pattern_matches_extract(p, needle, wildcarded, len);
1621 simple_pattern_free(p);
1622
@@ -1897,6 +1901,8 @@ int main(int argc, char **argv) {
1901
1902 get_system_timezone();
1903
1904 + replication_initialize();
1905 +
1906 // --------------------------------------------------------------------
1907 // get the certificate and start security
1908
database/Makefile.am
+1
@@ -7,6 +7,7 @@ SUBDIRS = \
7 engine \
8 ram \
9 sqlite \
10 + contexts \
11 $(NULL)
12
13 dist_noinst_DATA = \
database/contexts/Makefile.am new
+11
@@ -0,0 +1,11 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +SUBDIRS = \
7 + $(NULL)
8 +
9 +dist_noinst_DATA = \
10 + README.md \
11 + $(NULL)
database/contexts/README.md
database/contexts/api_v1.c new
+439
@@ -0,0 +1,439 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +static void rrd_flags_to_buffer_json_array_items(RRD_FLAGS flags, BUFFER *wb) {
6 + if(flags & RRD_FLAG_QUEUED_FOR_HUB)
7 + buffer_json_add_array_item_string(wb, "QUEUED");
8 +
9 + if(flags & RRD_FLAG_DELETED)
10 + buffer_json_add_array_item_string(wb, "DELETED");
11 +
12 + if(flags & RRD_FLAG_COLLECTED)
13 + buffer_json_add_array_item_string(wb, "COLLECTED");
14 +
15 + if(flags & RRD_FLAG_UPDATED)
16 + buffer_json_add_array_item_string(wb, "UPDATED");
17 +
18 + if(flags & RRD_FLAG_ARCHIVED)
19 + buffer_json_add_array_item_string(wb, "ARCHIVED");
20 +
21 + if(flags & RRD_FLAG_OWN_LABELS)
22 + buffer_json_add_array_item_string(wb, "OWN_LABELS");
23 +
24 + if(flags & RRD_FLAG_LIVE_RETENTION)
25 + buffer_json_add_array_item_string(wb, "LIVE_RETENTION");
26 +
27 + if(flags & RRD_FLAG_HIDDEN)
28 + buffer_json_add_array_item_string(wb, "HIDDEN");
29 +
30 + if(flags & RRD_FLAG_QUEUED_FOR_PP)
31 + buffer_json_add_array_item_string(wb, "PENDING_UPDATES");
32 +}
33 +
34 +// ----------------------------------------------------------------------------
35 +// /api/v1/context(s) API
36 +
37 +struct rrdcontext_to_json {
38 + BUFFER *wb;
39 + RRDCONTEXT_TO_JSON_OPTIONS options;
40 + time_t after;
41 + time_t before;
42 + SIMPLE_PATTERN *chart_label_key;
43 + SIMPLE_PATTERN *chart_labels_filter;
44 + SIMPLE_PATTERN *chart_dimensions;
45 + size_t written;
46 + time_t now;
47 + time_t combined_first_time_s;
48 + time_t combined_last_time_s;
49 + RRD_FLAGS combined_flags;
50 +};
51 +
52 +static inline int rrdmetric_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
53 + const char *id = dictionary_acquired_item_name(item);
54 + struct rrdcontext_to_json * t = data;
55 + RRDMETRIC *rm = value;
56 + BUFFER *wb = t->wb;
57 + RRDCONTEXT_TO_JSON_OPTIONS options = t->options;
58 + time_t after = t->after;
59 + time_t before = t->before;
60 +
61 + if(unlikely(rrd_flag_is_deleted(rm) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
62 + return 0;
63 +
64 + if(after && (!rm->last_time_s || after > rm->last_time_s))
65 + return 0;
66 +
67 + if(before && (!rm->first_time_s || before < rm->first_time_s))
68 + return 0;
69 +
70 + if(t->chart_dimensions
71 + && !simple_pattern_matches_string(t->chart_dimensions, rm->id)
72 + && rm->name != rm->id
73 + && !simple_pattern_matches_string(t->chart_dimensions, rm->name))
74 + return 0;
75 +
76 + if(t->written) {
77 + t->combined_first_time_s = MIN(t->combined_first_time_s, rm->first_time_s);
78 + t->combined_last_time_s = MAX(t->combined_last_time_s, rm->last_time_s);
79 + t->combined_flags |= rrd_flags_get(rm);
80 + }
81 + else {
82 + t->combined_first_time_s = rm->first_time_s;
83 + t->combined_last_time_s = rm->last_time_s;
84 + t->combined_flags = rrd_flags_get(rm);
85 + }
86 +
87 + buffer_json_member_add_object(wb, id);
88 +
89 + if(options & RRDCONTEXT_OPTION_SHOW_UUIDS) {
90 + char uuid[UUID_STR_LEN];
91 + uuid_unparse(rm->uuid, uuid);
92 + buffer_json_member_add_string(wb, "uuid", uuid);
93 + }
94 +
95 + buffer_json_member_add_string(wb, "name", string2str(rm->name));
96 + buffer_json_member_add_time_t(wb, "first_time_t", rm->first_time_s);
97 + buffer_json_member_add_time_t(wb, "last_time_t", rrd_flag_is_collected(rm) ? (long long)t->now : (long long)rm->last_time_s);
98 + buffer_json_member_add_boolean(wb, "collected", rrd_flag_is_collected(rm));
99 +
100 + if(options & RRDCONTEXT_OPTION_SHOW_DELETED)
101 + buffer_json_member_add_boolean(wb, "deleted", rrd_flag_is_deleted(rm));
102 +
103 + if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
104 + buffer_json_member_add_array(wb, "flags");
105 + rrd_flags_to_buffer_json_array_items(rrd_flags_get(rm), wb);
106 + buffer_json_array_close(wb);
107 + }
108 +
109 + buffer_json_object_close(wb);
110 + t->written++;
111 + return 1;
112 +}
113 +
114 +static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
115 + const char *id = dictionary_acquired_item_name(item);
116 +
117 + struct rrdcontext_to_json *t_parent = data;
118 + RRDINSTANCE *ri = value;
119 + BUFFER *wb = t_parent->wb;
120 + RRDCONTEXT_TO_JSON_OPTIONS options = t_parent->options;
121 + time_t after = t_parent->after;
122 + time_t before = t_parent->before;
123 + bool has_filter = t_parent->chart_label_key || t_parent->chart_labels_filter || t_parent->chart_dimensions;
124 +
125 + if(unlikely(rrd_flag_is_deleted(ri) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
126 + return 0;
127 +
128 + if(after && (!ri->last_time_s || after > ri->last_time_s))
129 + return 0;
130 +
131 + if(before && (!ri->first_time_s || before < ri->first_time_s))
132 + return 0;
133 +
134 + if(t_parent->chart_label_key && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, t_parent->chart_label_key,
135 + '\0', NULL))
136 + return 0;
137 +
138 + if(t_parent->chart_labels_filter && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels,
139 + t_parent->chart_labels_filter, ':',
140 + NULL))
141 + return 0;
142 +
143 + time_t first_time_s = ri->first_time_s;
144 + time_t last_time_s = ri->last_time_s;
145 + RRD_FLAGS flags = rrd_flags_get(ri);
146 +
147 + BUFFER *wb_metrics = NULL;
148 + if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) {
149 +
150 + wb_metrics = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
151 + buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, false);
152 +
153 + struct rrdcontext_to_json t_metrics = {
154 + .wb = wb_metrics,
155 + .options = options,
156 + .chart_label_key = t_parent->chart_label_key,
157 + .chart_labels_filter = t_parent->chart_labels_filter,
158 + .chart_dimensions = t_parent->chart_dimensions,
159 + .after = after,
160 + .before = before,
161 + .written = 0,
162 + .now = t_parent->now,
163 + };
164 + dictionary_walkthrough_read(ri->rrdmetrics, rrdmetric_to_json_callback, &t_metrics);
165 +
166 + if(has_filter && !t_metrics.written) {
167 + buffer_free(wb_metrics);
168 + return 0;
169 + }
170 +
171 + first_time_s = t_metrics.combined_first_time_s;
172 + last_time_s = t_metrics.combined_last_time_s;
173 + flags = t_metrics.combined_flags;
174 + }
175 +
176 + if(t_parent->written) {
177 + t_parent->combined_first_time_s = MIN(t_parent->combined_first_time_s, first_time_s);
178 + t_parent->combined_last_time_s = MAX(t_parent->combined_last_time_s, last_time_s);
179 + t_parent->combined_flags |= flags;
180 + }
181 + else {
182 + t_parent->combined_first_time_s = first_time_s;
183 + t_parent->combined_last_time_s = last_time_s;
184 + t_parent->combined_flags = flags;
185 + }
186 +
187 + buffer_json_member_add_object(wb, id);
188 +
189 + if(options & RRDCONTEXT_OPTION_SHOW_UUIDS) {
190 + char uuid[UUID_STR_LEN];
191 + uuid_unparse(ri->uuid, uuid);
192 + buffer_json_member_add_string(wb, "uuid", uuid);
193 + }
194 +
195 + buffer_json_member_add_string(wb, "name", string2str(ri->name));
196 + buffer_json_member_add_string(wb, "context", string2str(ri->rc->id));
197 + buffer_json_member_add_string(wb, "title", string2str(ri->title));
198 + buffer_json_member_add_string(wb, "units", string2str(ri->units));
199 + buffer_json_member_add_string(wb, "family", string2str(ri->family));
200 + buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(ri->chart_type));
201 + buffer_json_member_add_uint64(wb, "priority", ri->priority);
202 + buffer_json_member_add_time_t(wb, "update_every", ri->update_every_s);
203 + buffer_json_member_add_time_t(wb, "first_time_t", first_time_s);
204 + buffer_json_member_add_time_t(wb, "last_time_t", (flags & RRD_FLAG_COLLECTED) ? (long long)t_parent->now : (long long)last_time_s);
205 + buffer_json_member_add_boolean(wb, "collected", flags & RRD_FLAG_COLLECTED);
206 +
207 + if(options & RRDCONTEXT_OPTION_SHOW_DELETED)
208 + buffer_json_member_add_boolean(wb, "deleted", rrd_flag_is_deleted(ri));
209 +
210 + if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
211 + buffer_json_member_add_array(wb, "flags");
212 + rrd_flags_to_buffer_json_array_items(rrd_flags_get(ri), wb);
213 + buffer_json_array_close(wb);
214 + }
215 +
216 + if(options & RRDCONTEXT_OPTION_SHOW_LABELS && ri->rrdlabels && dictionary_entries(ri->rrdlabels)) {
217 + buffer_json_member_add_object(wb, "labels");
218 + rrdlabels_to_buffer_json_members(ri->rrdlabels, wb);
219 + buffer_json_object_close(wb);
220 + }
221 +
222 + if(wb_metrics) {
223 + buffer_json_member_add_object(wb, "dimensions");
224 + buffer_fast_strcat(wb, buffer_tostring(wb_metrics), buffer_strlen(wb_metrics));
225 + buffer_json_object_close(wb);
226 +
227 + buffer_free(wb_metrics);
228 + }
229 +
230 + buffer_json_object_close(wb);
231 + t_parent->written++;
232 + return 1;
233 +}
234 +
235 +static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
236 + const char *id = dictionary_acquired_item_name(item);
237 + struct rrdcontext_to_json *t_parent = data;
238 + RRDCONTEXT *rc = value;
239 + BUFFER *wb = t_parent->wb;
240 + RRDCONTEXT_TO_JSON_OPTIONS options = t_parent->options;
241 + time_t after = t_parent->after;
242 + time_t before = t_parent->before;
243 + bool has_filter = t_parent->chart_label_key || t_parent->chart_labels_filter || t_parent->chart_dimensions;
244 +
245 + if(unlikely(rrd_flag_check(rc, RRD_FLAG_HIDDEN) && !(options & RRDCONTEXT_OPTION_SHOW_HIDDEN)))
246 + return 0;
247 +
248 + if(unlikely(rrd_flag_is_deleted(rc) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
249 + return 0;
250 +
251 + if(options & RRDCONTEXT_OPTION_DEEPSCAN)
252 + rrdcontext_recalculate_context_retention(rc, RRD_FLAG_NONE, false);
253 +
254 + if(after && (!rc->last_time_s || after > rc->last_time_s))
255 + return 0;
256 +
257 + if(before && (!rc->first_time_s || before < rc->first_time_s))
258 + return 0;
259 +
260 + time_t first_time_s = rc->first_time_s;
261 + time_t last_time_s = rc->last_time_s;
262 + RRD_FLAGS flags = rrd_flags_get(rc);
263 +
264 + BUFFER *wb_instances = NULL;
265 + if((options & (RRDCONTEXT_OPTION_SHOW_LABELS|RRDCONTEXT_OPTION_SHOW_INSTANCES|RRDCONTEXT_OPTION_SHOW_METRICS))
266 + || t_parent->chart_label_key
267 + || t_parent->chart_labels_filter
268 + || t_parent->chart_dimensions) {
269 +
270 + wb_instances = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
271 + buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, false);
272 +
273 + struct rrdcontext_to_json t_instances = {
274 + .wb = wb_instances,
275 + .options = options,
276 + .chart_label_key = t_parent->chart_label_key,
277 + .chart_labels_filter = t_parent->chart_labels_filter,
278 + .chart_dimensions = t_parent->chart_dimensions,
279 + .after = after,
280 + .before = before,
281 + .written = 0,
282 + .now = t_parent->now,
283 + };
284 + dictionary_walkthrough_read(rc->rrdinstances, rrdinstance_to_json_callback, &t_instances);
285 +
286 + if(has_filter && !t_instances.written) {
287 + buffer_free(wb_instances);
288 + return 0;
289 + }
290 +
291 + first_time_s = t_instances.combined_first_time_s;
292 + last_time_s = t_instances.combined_last_time_s;
293 + flags = t_instances.combined_flags;
294 + }
295 +
296 + if(!(options & RRDCONTEXT_OPTION_SKIP_ID))
297 + buffer_json_member_add_object(wb, id);
298 +
299 + rrdcontext_lock(rc);
300 +
301 + buffer_json_member_add_string(wb, "title", string2str(rc->title));
302 + buffer_json_member_add_string(wb, "units", string2str(rc->units));
303 + buffer_json_member_add_string(wb, "family", string2str(rc->family));
304 + buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rc->chart_type));
305 + buffer_json_member_add_uint64(wb, "priority", rc->priority);
306 + buffer_json_member_add_time_t(wb, "first_time_t", first_time_s);
307 + buffer_json_member_add_time_t(wb, "last_time_t", (flags & RRD_FLAG_COLLECTED) ? (long long)t_parent->now : (long long)last_time_s);
308 + buffer_json_member_add_boolean(wb, "collected", (flags & RRD_FLAG_COLLECTED));
309 +
310 + if(options & RRDCONTEXT_OPTION_SHOW_DELETED)
311 + buffer_json_member_add_boolean(wb, "deleted", rrd_flag_is_deleted(rc));
312 +
313 + if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
314 + buffer_json_member_add_array(wb, "flags");
315 + rrd_flags_to_buffer_json_array_items(rrd_flags_get(rc), wb);
316 + buffer_json_array_close(wb);
317 + }
318 +
319 + if(options & RRDCONTEXT_OPTION_SHOW_QUEUED) {
320 + buffer_json_member_add_array(wb, "queued_reasons");
321 + rrd_reasons_to_buffer_json_array_items(rc->queue.queued_flags, wb);
322 + buffer_json_array_close(wb);
323 +
324 + buffer_json_member_add_time_t(wb, "last_queued", (time_t)(rc->queue.queued_ut / USEC_PER_SEC));
325 + buffer_json_member_add_time_t(wb, "scheduled_dispatch", (time_t)(rc->queue.scheduled_dispatch_ut / USEC_PER_SEC));
326 + buffer_json_member_add_time_t(wb, "last_dequeued", (time_t)(rc->queue.dequeued_ut / USEC_PER_SEC));
327 + buffer_json_member_add_uint64(wb, "dispatches", rc->queue.dispatches);
328 + buffer_json_member_add_uint64(wb, "hub_version", rc->hub.version);
329 + buffer_json_member_add_uint64(wb, "version", rc->version);
330 +
331 + buffer_json_member_add_array(wb, "pp_reasons");
332 + rrd_reasons_to_buffer_json_array_items(rc->pp.queued_flags, wb);
333 + buffer_json_array_close(wb);
334 +
335 + buffer_json_member_add_time_t(wb, "pp_last_queued", (time_t)(rc->pp.queued_ut / USEC_PER_SEC));
336 + buffer_json_member_add_time_t(wb, "pp_last_dequeued", (time_t)(rc->pp.dequeued_ut / USEC_PER_SEC));
337 + buffer_json_member_add_uint64(wb, "pp_executed", rc->pp.executions);
338 + }
339 +
340 + rrdcontext_unlock(rc);
341 +
342 + if(wb_instances) {
343 + buffer_json_member_add_object(wb, "charts");
344 + buffer_fast_strcat(wb, buffer_tostring(wb_instances), buffer_strlen(wb_instances));
345 + buffer_json_object_close(wb);
346 +
347 + buffer_free(wb_instances);
348 + }
349 +
350 + if(!(options & RRDCONTEXT_OPTION_SKIP_ID))
351 + buffer_json_object_close(wb);
352 +
353 + t_parent->written++;
354 + return 1;
355 +}
356 +
357 +int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) {
358 + if(!host->rrdctx.contexts) {
359 + error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host));
360 + return HTTP_RESP_NOT_FOUND;
361 + }
362 +
363 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, context);
364 + if(!rca) return HTTP_RESP_NOT_FOUND;
365 +
366 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
367 +
368 + if(after != 0 && before != 0)
369 + rrdr_relative_window_to_absolute(&after, &before);
370 +
371 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
372 + struct rrdcontext_to_json t_contexts = {
373 + .wb = wb,
374 + .options = options|RRDCONTEXT_OPTION_SKIP_ID,
375 + .chart_label_key = chart_label_key,
376 + .chart_labels_filter = chart_labels_filter,
377 + .chart_dimensions = chart_dimensions,
378 + .after = after,
379 + .before = before,
380 + .written = 0,
381 + .now = now_realtime_sec(),
382 + };
383 + rrdcontext_to_json_callback((DICTIONARY_ITEM *)rca, rc, &t_contexts);
384 + buffer_json_finalize(wb);
385 +
386 + rrdcontext_release(rca);
387 +
388 + if(!t_contexts.written)
389 + return HTTP_RESP_NOT_FOUND;
390 +
391 + return HTTP_RESP_OK;
392 +}
393 +
394 +int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) {
395 + if(!host->rrdctx.contexts) {
396 + error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host));
397 + return HTTP_RESP_NOT_FOUND;
398 + }
399 +
400 + char node_uuid[UUID_STR_LEN] = "";
401 +
402 + if(host->node_id)
403 + uuid_unparse(*host->node_id, node_uuid);
404 +
405 + if(after != 0 && before != 0)
406 + rrdr_relative_window_to_absolute(&after, &before);
407 +
408 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
409 + buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
410 + buffer_json_member_add_string(wb, "machine_guid", host->machine_guid);
411 + buffer_json_member_add_string(wb, "node_id", node_uuid);
412 + buffer_json_member_add_string(wb, "claim_id", host->aclk_state.claimed_id ? host->aclk_state.claimed_id : "");
413 +
414 + if(options & RRDCONTEXT_OPTION_SHOW_LABELS) {
415 + buffer_json_member_add_object(wb, "host_labels");
416 + rrdlabels_to_buffer_json_members(host->rrdlabels, wb);
417 + buffer_json_object_close(wb);
418 + }
419 +
420 + buffer_json_member_add_object(wb, "contexts");
421 + struct rrdcontext_to_json t_contexts = {
422 + .wb = wb,
423 + .options = options,
424 + .chart_label_key = chart_label_key,
425 + .chart_labels_filter = chart_labels_filter,
426 + .chart_dimensions = chart_dimensions,
427 + .after = after,
428 + .before = before,
429 + .written = 0,
430 + .now = now_realtime_sec(),
431 + };
432 + dictionary_walkthrough_read(host->rrdctx.contexts, rrdcontext_to_json_callback, &t_contexts);
433 + buffer_json_object_close(wb);
434 +
435 + buffer_json_finalize(wb);
436 +
437 + return HTTP_RESP_OK;
438 +}
439 +
database/contexts/api_v2.c new
+414
@@ -0,0 +1,414 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +
6 +// ----------------------------------------------------------------------------
7 +// /api/v2/contexts API
8 +
9 +typedef enum __attribute__ ((__packed__)) {
10 + FTS_MATCHED_NONE = 0,
11 + FTS_MATCHED_HOST,
12 + FTS_MATCHED_CONTEXT,
13 + FTS_MATCHED_INSTANCE,
14 + FTS_MATCHED_DIMENSION,
15 + FTS_MATCHED_LABEL,
16 + FTS_MATCHED_ALERT,
17 + FTS_MATCHED_ALERT_INFO,
18 + FTS_MATCHED_FAMILY,
19 + FTS_MATCHED_TITLE,
20 + FTS_MATCHED_UNITS,
21 +} FTS_MATCH;
22 +
23 +static const char *fts_match_to_string(FTS_MATCH match) {
24 + switch(match) {
25 + case FTS_MATCHED_HOST:
26 + return "HOST";
27 +
28 + case FTS_MATCHED_CONTEXT:
29 + return "CONTEXT";
30 +
31 + case FTS_MATCHED_INSTANCE:
32 + return "INSTANCE";
33 +
34 + case FTS_MATCHED_DIMENSION:
35 + return "DIMENSION";
36 +
37 + case FTS_MATCHED_ALERT:
38 + return "ALERT";
39 +
40 + case FTS_MATCHED_ALERT_INFO:
41 + return "ALERT_INFO";
42 +
43 + case FTS_MATCHED_LABEL:
44 + return "LABEL";
45 +
46 + case FTS_MATCHED_FAMILY:
47 + return "FAMILY";
48 +
49 + case FTS_MATCHED_TITLE:
50 + return "TITLE";
51 +
52 + case FTS_MATCHED_UNITS:
53 + return "UNITS";
54 +
55 + default:
56 + return "NONE";
57 + }
58 +}
59 +
60 +struct rrdcontext_to_json_v2_entry {
61 + size_t count;
62 + STRING *id;
63 + STRING *family;
64 + uint32_t priority;
65 + time_t first_time_s;
66 + time_t last_time_s;
67 + RRD_FLAGS flags;
68 + FTS_MATCH match;
69 +};
70 +
71 +typedef struct full_text_search_index {
72 + size_t searches;
73 + size_t string_searches;
74 + size_t char_searches;
75 +} FTS_INDEX;
76 +
77 +static inline bool full_text_search_string(FTS_INDEX *fts, SIMPLE_PATTERN *q, STRING *ptr) {
78 + fts->searches++;
79 + fts->string_searches++;
80 + return simple_pattern_matches_string(q, ptr);
81 +}
82 +
83 +static inline bool full_text_search_char(FTS_INDEX *fts, SIMPLE_PATTERN *q, char *ptr) {
84 + fts->searches++;
85 + fts->char_searches++;
86 + return simple_pattern_matches(q, ptr);
87 +}
88 +
89 +struct rrdcontext_to_json_v2_data {
90 + BUFFER *wb;
91 + struct api_v2_contexts_request *request;
92 + DICTIONARY *ctx;
93 +
94 + CONTEXTS_V2_OPTIONS options;
95 + uint64_t hard_hash;
96 + uint64_t soft_hash;
97 +
98 + struct {
99 + SIMPLE_PATTERN *scope_pattern;
100 + SIMPLE_PATTERN *pattern;
101 + } nodes;
102 +
103 + struct {
104 + SIMPLE_PATTERN *scope_pattern;
105 + SIMPLE_PATTERN *pattern;
106 + } contexts;
107 +
108 + struct {
109 + FTS_MATCH host_match;
110 + char host_uuid_buffer[UUID_STR_LEN];
111 + SIMPLE_PATTERN *pattern;
112 + FTS_INDEX fts;
113 + } q;
114 +};
115 +
116 +static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc, SIMPLE_PATTERN *q) {
117 + if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->id) ||
118 + full_text_search_string(&ctl->q.fts, q, rc->family)))
119 + return FTS_MATCHED_CONTEXT;
120 +
121 + if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->title)))
122 + return FTS_MATCHED_TITLE;
123 +
124 + if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->units)))
125 + return FTS_MATCHED_UNITS;
126 +
127 + FTS_MATCH matched = FTS_MATCHED_NONE;
128 + RRDINSTANCE *ri;
129 + dfe_start_read(rc->rrdinstances, ri) {
130 + if(matched) break;
131 +
132 + if(unlikely(full_text_search_string(&ctl->q.fts, q, ri->id)) ||
133 + (ri->name != ri->id && full_text_search_string(&ctl->q.fts, q, ri->name))) {
134 + matched = FTS_MATCHED_INSTANCE;
135 + break;
136 + }
137 +
138 + RRDMETRIC *rm;
139 + dfe_start_read(ri->rrdmetrics, rm) {
140 + if(unlikely(full_text_search_string(&ctl->q.fts, q, rm->id)) ||
141 + (rm->name != rm->id && full_text_search_string(&ctl->q.fts, q, rm->name))) {
142 + matched = FTS_MATCHED_DIMENSION;
143 + break;
144 + }
145 + }
146 + dfe_done(rm);
147 +
148 + size_t label_searches = 0;
149 + if(unlikely(ri->rrdlabels && dictionary_entries(ri->rrdlabels) &&
150 + rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, q, ':', &label_searches))) {
151 + ctl->q.fts.searches += label_searches;
152 + ctl->q.fts.char_searches += label_searches;
153 + matched = FTS_MATCHED_LABEL;
154 + break;
155 + }
156 + ctl->q.fts.searches += label_searches;
157 + ctl->q.fts.char_searches += label_searches;
158 +
159 + if(ri->rrdset) {
160 + RRDSET *st = ri->rrdset;
161 + netdata_rwlock_rdlock(&st->alerts.rwlock);
162 + for (RRDCALC *rcl = st->alerts.base; rcl; rcl = rcl->next) {
163 + if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->name))) {
164 + matched = FTS_MATCHED_ALERT;
165 + break;
166 + }
167 +
168 + if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->info))) {
169 + matched = FTS_MATCHED_ALERT_INFO;
170 + break;
171 + }
172 + }
173 + netdata_rwlock_unlock(&st->alerts.rwlock);
174 + }
175 + }
176 + dfe_done(ri);
177 + return matched;
178 +}
179 +
180 +static bool rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context __maybe_unused) {
181 + struct rrdcontext_to_json_v2_data *ctl = data;
182 +
183 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
184 +
185 + FTS_MATCH match = ctl->q.host_match;
186 + if((ctl->options & CONTEXTS_V2_SEARCH) && ctl->q.pattern) {
187 + match = rrdcontext_to_json_v2_full_text_search(ctl, rc, ctl->q.pattern);
188 +
189 + if(match == FTS_MATCHED_NONE)
190 + return false;
191 + }
192 +
193 + struct rrdcontext_to_json_v2_entry t = {
194 + .count = 0,
195 + .id = rc->id,
196 + .family = rc->family,
197 + .priority = rc->priority,
198 + .first_time_s = rc->first_time_s,
199 + .last_time_s = rc->last_time_s,
200 + .flags = rc->flags,
201 + .match = match,
202 + }, *z = dictionary_set(ctl->ctx, string2str(rc->id), &t, sizeof(t));
203 +
204 + if(!z->count) {
205 + // we just added this
206 + z->count = 1;
207 + }
208 + else {
209 + // it is already in there
210 + z->count++;
211 + z->flags |= rc->flags;
212 +
213 + if(z->priority > rc->priority)
214 + z->priority = rc->priority;
215 +
216 + if(z->first_time_s > rc->first_time_s)
217 + z->first_time_s = rc->first_time_s;
218 +
219 + if(z->last_time_s < rc->last_time_s)
220 + z->last_time_s = rc->last_time_s;
221 + }
222 +
223 + return true;
224 +}
225 +
226 +static bool rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) {
227 + if(!queryable_host || !host->rrdctx.contexts)
228 + // the host matches the 'scope_host' but does not match the 'host' patterns
229 + // or the host does not have any contexts
230 + return false;
231 +
232 + struct rrdcontext_to_json_v2_data *ctl = data;
233 + BUFFER *wb = ctl->wb;
234 +
235 + bool host_matched = (ctl->options & CONTEXTS_V2_NODES);
236 + bool do_contexts = (ctl->options & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH));
237 +
238 + ctl->q.host_match = FTS_MATCHED_NONE;
239 + if((ctl->options & CONTEXTS_V2_SEARCH)) {
240 + // check if we match the host itself
241 + if(ctl->q.pattern && (
242 + full_text_search_string(&ctl->q.fts, ctl->q.pattern, host->hostname) ||
243 + full_text_search_char(&ctl->q.fts, ctl->q.pattern, host->machine_guid) ||
244 + (ctl->q.pattern && full_text_search_char(&ctl->q.fts, ctl->q.pattern, ctl->q.host_uuid_buffer)))) {
245 + ctl->q.host_match = FTS_MATCHED_HOST;
246 + do_contexts = true;
247 + }
248 + }
249 +
250 + if(do_contexts) {
251 + // save it
252 + SIMPLE_PATTERN *old_q = ctl->q.pattern;
253 +
254 + if(ctl->q.host_match == FTS_MATCHED_HOST)
255 + // do not do pattern matching on contexts - we matched the host itself
256 + ctl->q.pattern = NULL;
257 +
258 + size_t added = query_scope_foreach_context(
259 + host, ctl->request->scope_contexts,
260 + ctl->contexts.scope_pattern, ctl->contexts.pattern,
261 + rrdcontext_to_json_v2_add_context, queryable_host, ctl);
262 +
263 + // restore it
264 + ctl->q.pattern = old_q;
265 +
266 + if(added)
267 + host_matched = true;
268 + }
269 +
270 + if(host_matched && (ctl->options & (CONTEXTS_V2_NODES | CONTEXTS_V2_DEBUG))) {
271 + buffer_json_add_array_item_object(wb);
272 + buffer_json_member_add_string(wb, "mg", host->machine_guid);
273 + buffer_json_member_add_uuid(wb, "nd", host->node_id);
274 + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
275 + buffer_json_object_close(wb);
276 + }
277 +
278 + return host_matched;
279 +}
280 +
281 +static void buffer_json_contexts_v2_options_to_array(BUFFER *wb, CONTEXTS_V2_OPTIONS options) {
282 + if(options & CONTEXTS_V2_DEBUG)
283 + buffer_json_add_array_item_string(wb, "debug");
284 +
285 + if(options & CONTEXTS_V2_NODES)
286 + buffer_json_add_array_item_string(wb, "nodes");
287 +
288 + if(options & CONTEXTS_V2_CONTEXTS)
289 + buffer_json_add_array_item_string(wb, "contexts");
290 +
291 + if(options & CONTEXTS_V2_SEARCH)
292 + buffer_json_add_array_item_string(wb, "search");
293 +}
294 +
295 +int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options) {
296 + req->timings.processing_ut = now_monotonic_usec();
297 +
298 + if(options & CONTEXTS_V2_SEARCH)
299 + options |= CONTEXTS_V2_CONTEXTS;
300 +
301 + struct rrdcontext_to_json_v2_data ctl = {
302 + .wb = wb,
303 + .request = req,
304 + .ctx = NULL,
305 + .options = options,
306 + .hard_hash = 0,
307 + .soft_hash = 0,
308 + .nodes.scope_pattern = string_to_simple_pattern(req->scope_nodes),
309 + .nodes.pattern = string_to_simple_pattern(req->nodes),
310 + .contexts.pattern = string_to_simple_pattern(req->contexts),
311 + .contexts.scope_pattern = string_to_simple_pattern(req->scope_contexts),
312 + .q.pattern = string_to_simple_pattern_nocase(req->q),
313 + };
314 +
315 + if(options & CONTEXTS_V2_CONTEXTS)
316 + ctl.ctx = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct rrdcontext_to_json_v2_entry));
317 +
318 + time_t now_s = now_realtime_sec();
319 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
320 +
321 + if(options & CONTEXTS_V2_DEBUG) {
322 + buffer_json_member_add_object(wb, "agent");
323 + buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
324 + buffer_json_member_add_uuid(wb, "nd", localhost->node_id);
325 + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost));
326 + if (req->q)
327 + buffer_json_member_add_string(wb, "q", req->q);
328 + buffer_json_member_add_time_t(wb, "now", now_s);
329 + buffer_json_object_close(wb);
330 +
331 + buffer_json_member_add_object(wb, "request");
332 +
333 + buffer_json_member_add_object(wb, "scope");
334 + buffer_json_member_add_string(wb, "scope_nodes", req->scope_nodes);
335 + buffer_json_member_add_string(wb, "scope_contexts", req->scope_contexts);
336 + buffer_json_object_close(wb);
337 +
338 + buffer_json_member_add_object(wb, "selectors");
339 + buffer_json_member_add_string(wb, "nodes", req->nodes);
340 + buffer_json_member_add_string(wb, "contexts", req->contexts);
341 + buffer_json_object_close(wb);
342 +
343 + buffer_json_member_add_string(wb, "q", req->q);
344 + buffer_json_member_add_array(wb, "options");
345 + buffer_json_contexts_v2_options_to_array(wb, options);
346 + buffer_json_array_close(wb);
347 +
348 + buffer_json_object_close(wb);
349 + }
350 +
351 + if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_DEBUG))
352 + buffer_json_member_add_array(wb, "nodes");
353 +
354 + query_scope_foreach_host(ctl.nodes.scope_pattern, ctl.nodes.pattern,
355 + rrdcontext_to_json_v2_add_host, &ctl, &ctl.hard_hash, &ctl.soft_hash,
356 + ctl.q.host_uuid_buffer);
357 + if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_DEBUG))
358 + buffer_json_array_close(wb);
359 +
360 + req->timings.output_ut = now_monotonic_usec();
361 + buffer_json_member_add_object(wb, "versions");
362 + buffer_json_member_add_uint64(wb, "contexts_hard_hash", ctl.hard_hash);
363 + buffer_json_member_add_uint64(wb, "contexts_soft_hash", ctl.soft_hash);
364 + buffer_json_object_close(wb);
365 +
366 + if(options & CONTEXTS_V2_CONTEXTS) {
367 + buffer_json_member_add_object(wb, "contexts");
368 + struct rrdcontext_to_json_v2_entry *z;
369 + dfe_start_read(ctl.ctx, z){
370 + bool collected = z->flags & RRD_FLAG_COLLECTED;
371 +
372 + buffer_json_member_add_object(wb, string2str(z->id));
373 + {
374 + buffer_json_member_add_string(wb, "family", string2str(z->family));
375 + buffer_json_member_add_uint64(wb, "priority", z->priority);
376 + buffer_json_member_add_time_t(wb, "first_entry", z->first_time_s);
377 + buffer_json_member_add_time_t(wb, "last_entry", collected ? now_s : z->last_time_s);
378 + buffer_json_member_add_boolean(wb, "live", collected);
379 + if (options & CONTEXTS_V2_SEARCH)
380 + buffer_json_member_add_string(wb, "match", fts_match_to_string(z->match));
381 + }
382 + buffer_json_object_close(wb);
383 + }
384 + dfe_done(z);
385 + buffer_json_object_close(wb); // contexts
386 + }
387 +
388 + if(options & CONTEXTS_V2_SEARCH) {
389 + buffer_json_member_add_object(wb, "searches");
390 + buffer_json_member_add_uint64(wb, "strings", ctl.q.fts.string_searches);
391 + buffer_json_member_add_uint64(wb, "char", ctl.q.fts.char_searches);
392 + buffer_json_member_add_uint64(wb, "total", ctl.q.fts.searches);
393 + buffer_json_object_close(wb);
394 + }
395 +
396 + req->timings.finished_ut = now_monotonic_usec();
397 + buffer_json_member_add_object(wb, "timings");
398 + buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(req->timings.processing_ut - req->timings.received_ut) / USEC_PER_MS);
399 + buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(req->timings.output_ut - req->timings.processing_ut) / USEC_PER_MS);
400 + buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(req->timings.finished_ut - req->timings.output_ut) / USEC_PER_MS);
401 + buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(req->timings.finished_ut - req->timings.received_ut) / USEC_PER_MS);
402 + buffer_json_object_close(wb);
403 + buffer_json_finalize(wb);
404 +
405 + dictionary_destroy(ctl.ctx);
406 + simple_pattern_free(ctl.nodes.scope_pattern);
407 + simple_pattern_free(ctl.nodes.pattern);
408 + simple_pattern_free(ctl.contexts.pattern);
409 + simple_pattern_free(ctl.contexts.scope_pattern);
410 + simple_pattern_free(ctl.q.pattern);
411 +
412 + return HTTP_RESP_OK;
413 +}
414 +
database/contexts/context.c new
+286
@@ -0,0 +1,286 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +inline const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca) {
6 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
7 + return string2str(rc->id);
8 +}
9 +
10 +inline bool rrdcontext_acquired_belongs_to_host(RRDCONTEXT_ACQUIRED *rca, RRDHOST *host) {
11 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
12 + return rc->rrdhost == host;
13 +}
14 +
15 +// ----------------------------------------------------------------------------
16 +// RRDCONTEXT
17 +
18 +static void rrdcontext_freez(RRDCONTEXT *rc) {
19 + string_freez(rc->id);
20 + string_freez(rc->title);
21 + string_freez(rc->units);
22 + string_freez(rc->family);
23 +}
24 +
25 +static void rrdcontext_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost) {
26 + RRDHOST *host = (RRDHOST *)rrdhost;
27 + RRDCONTEXT *rc = (RRDCONTEXT *)value;
28 +
29 + rc->rrdhost = host;
30 + rc->flags = rc->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics at constructor
31 +
32 + if(rc->hub.version) {
33 + // we are loading data from the SQL database
34 +
35 + if(rc->version)
36 + error("RRDCONTEXT: context '%s' is already initialized with version %"PRIu64", but it is loaded again from SQL with version %"PRIu64"", string2str(rc->id), rc->version, rc->hub.version);
37 +
38 + // IMPORTANT
39 + // replace all string pointers in rc->hub with our own versions
40 + // the originals are coming from a tmp allocation of sqlite
41 +
42 + string_freez(rc->id);
43 + rc->id = string_strdupz(rc->hub.id);
44 + rc->hub.id = string2str(rc->id);
45 +
46 + string_freez(rc->title);
47 + rc->title = string_strdupz(rc->hub.title);
48 + rc->hub.title = string2str(rc->title);
49 +
50 + string_freez(rc->units);
51 + rc->units = string_strdupz(rc->hub.units);
52 + rc->hub.units = string2str(rc->units);
53 +
54 + string_freez(rc->family);
55 + rc->family = string_strdupz(rc->hub.family);
56 + rc->hub.family = string2str(rc->family);
57 +
58 + rc->chart_type = rrdset_type_id(rc->hub.chart_type);
59 + rc->hub.chart_type = rrdset_type_name(rc->chart_type);
60 +
61 + rc->version = rc->hub.version;
62 + rc->priority = rc->hub.priority;
63 + rc->first_time_s = (time_t)rc->hub.first_time_s;
64 + rc->last_time_s = (time_t)rc->hub.last_time_s;
65 +
66 + if(rc->hub.deleted || !rc->hub.first_time_s)
67 + rrd_flag_set_deleted(rc, RRD_FLAG_NONE);
68 + else {
69 + if (rc->last_time_s == 0)
70 + rrd_flag_set_collected(rc);
71 + else
72 + rrd_flag_set_archived(rc);
73 + }
74 +
75 + rc->flags |= RRD_FLAG_UPDATE_REASON_LOAD_SQL; // no need for atomics at constructor
76 + }
77 + else {
78 + // we are adding this context now for the first time
79 + rc->version = now_realtime_sec();
80 + }
81 +
82 + rrdinstances_create_in_rrdcontext(rc);
83 + netdata_mutex_init(&rc->mutex);
84 +
85 + // signal the react callback to do the job
86 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
87 +}
88 +
89 +static void rrdcontext_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost __maybe_unused) {
90 +
91 + RRDCONTEXT *rc = (RRDCONTEXT *)value;
92 +
93 + rrdinstances_destroy_from_rrdcontext(rc);
94 + netdata_mutex_destroy(&rc->mutex);
95 + rrdcontext_freez(rc);
96 +}
97 +
98 +static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdhost __maybe_unused) {
99 + RRDCONTEXT *rc = (RRDCONTEXT *)old_value;
100 + RRDCONTEXT *rc_new = (RRDCONTEXT *)new_value;
101 +
102 + //current rc is not archived, new_rc is archived, don't merge
103 + if (!rrd_flag_is_archived(rc) && rrd_flag_is_archived(rc_new)) {
104 + rrdcontext_freez(rc_new);
105 + return false;
106 + }
107 +
108 + rrdcontext_lock(rc);
109 +
110 + if(rc->title != rc_new->title) {
111 + STRING *old_title = rc->title;
112 + if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new))
113 + rc->title = string_dup(rc_new->title);
114 + else
115 + rc->title = string_2way_merge(rc->title, rc_new->title);
116 + string_freez(old_title);
117 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
118 + }
119 +
120 + if(rc->units != rc_new->units) {
121 + STRING *old_units = rc->units;
122 + rc->units = string_dup(rc_new->units);
123 + string_freez(old_units);
124 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
125 + }
126 +
127 + if(rc->family != rc_new->family) {
128 + STRING *old_family = rc->family;
129 + if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new))
130 + rc->family = string_dup(rc_new->family);
131 + else
132 + rc->family = string_2way_merge(rc->family, rc_new->family);
133 + string_freez(old_family);
134 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
135 + }
136 +
137 + if(rc->chart_type != rc_new->chart_type) {
138 + rc->chart_type = rc_new->chart_type;
139 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
140 + }
141 +
142 + if(rc->priority != rc_new->priority) {
143 + rc->priority = rc_new->priority;
144 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
145 + }
146 +
147 + rrd_flag_set(rc, rc_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on rc_new
148 +
149 + if(rrd_flag_is_collected(rc) && rrd_flag_is_archived(rc))
150 + rrd_flag_set_collected(rc);
151 +
152 + if(rrd_flag_is_updated(rc))
153 + rrd_flag_set(rc, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
154 +
155 + rrdcontext_unlock(rc);
156 +
157 + // free the resources of the new one
158 + rrdcontext_freez(rc_new);
159 +
160 + // the react callback will continue from here
161 + return rrd_flag_is_updated(rc);
162 +}
163 +
164 +static void rrdcontext_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost __maybe_unused) {
165 + RRDCONTEXT *rc = (RRDCONTEXT *)value;
166 + rrdcontext_trigger_updates(rc, __FUNCTION__ );
167 +}
168 +
169 +void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function) {
170 + if(rrd_flag_is_updated(rc) || !rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
171 + rrdcontext_queue_for_post_processing(rc, function, rc->flags);
172 +}
173 +
174 +static void rrdcontext_hub_queue_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
175 + RRDCONTEXT *rc = context;
176 + rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
177 + rc->queue.queued_ut = now_realtime_usec();
178 + rc->queue.queued_flags = rrd_flags_get(rc);
179 +}
180 +
181 +static void rrdcontext_hub_queue_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
182 + RRDCONTEXT *rc = context;
183 + rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB);
184 +}
185 +
186 +static bool rrdcontext_hub_queue_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *new_context __maybe_unused, void *nothing __maybe_unused) {
187 + // context and new_context are the same
188 + // we just need to update the timings
189 + RRDCONTEXT *rc = context;
190 + rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
191 + rc->queue.queued_ut = now_realtime_usec();
192 + rc->queue.queued_flags |= rrd_flags_get(rc);
193 +
194 + return true;
195 +}
196 +
197 +static void rrdcontext_post_processing_queue_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
198 + RRDCONTEXT *rc = context;
199 + rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_PP);
200 + rc->pp.queued_flags = rc->flags;
201 + rc->pp.queued_ut = now_realtime_usec();
202 +}
203 +
204 +static void rrdcontext_post_processing_queue_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
205 + RRDCONTEXT *rc = context;
206 + rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_PP);
207 + rc->pp.dequeued_ut = now_realtime_usec();
208 +}
209 +
210 +static bool rrdcontext_post_processing_queue_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *new_context __maybe_unused, void *nothing __maybe_unused) {
211 + RRDCONTEXT *rc = context;
212 + bool changed = false;
213 +
214 + if(!(rc->flags & RRD_FLAG_QUEUED_FOR_PP)) {
215 + rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_PP);
216 + changed = true;
217 + }
218 +
219 + if(rc->pp.queued_flags != rc->flags) {
220 + rc->pp.queued_flags |= rc->flags;
221 + changed = true;
222 + }
223 +
224 + return changed;
225 +}
226 +
227 +
228 +void rrdhost_create_rrdcontexts(RRDHOST *host) {
229 + if(unlikely(!host)) return;
230 + if(likely(host->rrdctx.contexts)) return;
231 +
232 + host->rrdctx.contexts = dictionary_create_advanced(
233 + DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
234 + &dictionary_stats_category_rrdcontext, sizeof(RRDCONTEXT));
235 +
236 + dictionary_register_insert_callback(host->rrdctx.contexts, rrdcontext_insert_callback, host);
237 + dictionary_register_delete_callback(host->rrdctx.contexts, rrdcontext_delete_callback, host);
238 + dictionary_register_conflict_callback(host->rrdctx.contexts, rrdcontext_conflict_callback, host);
239 + dictionary_register_react_callback(host->rrdctx.contexts, rrdcontext_react_callback, host);
240 +
241 + host->rrdctx.hub_queue = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext, 0);
242 + dictionary_register_insert_callback(host->rrdctx.hub_queue, rrdcontext_hub_queue_insert_callback, NULL);
243 + dictionary_register_delete_callback(host->rrdctx.hub_queue, rrdcontext_hub_queue_delete_callback, NULL);
244 + dictionary_register_conflict_callback(host->rrdctx.hub_queue, rrdcontext_hub_queue_conflict_callback, NULL);
245 +
246 + host->rrdctx.pp_queue = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext, 0);
247 + dictionary_register_insert_callback(host->rrdctx.pp_queue, rrdcontext_post_processing_queue_insert_callback, NULL);
248 + dictionary_register_delete_callback(host->rrdctx.pp_queue, rrdcontext_post_processing_queue_delete_callback, NULL);
249 + dictionary_register_conflict_callback(host->rrdctx.pp_queue, rrdcontext_post_processing_queue_conflict_callback, NULL);
250 +}
251 +
252 +void rrdhost_destroy_rrdcontexts(RRDHOST *host) {
253 + if(unlikely(!host)) return;
254 + if(unlikely(!host->rrdctx.contexts)) return;
255 +
256 + DICTIONARY *old;
257 +
258 + if(host->rrdctx.hub_queue) {
259 + old = host->rrdctx.hub_queue;
260 + host->rrdctx.hub_queue = NULL;
261 +
262 + RRDCONTEXT *rc;
263 + dfe_start_write(old, rc) {
264 + dictionary_del(old, string2str(rc->id));
265 + }
266 + dfe_done(rc);
267 + dictionary_destroy(old);
268 + }
269 +
270 + if(host->rrdctx.pp_queue) {
271 + old = host->rrdctx.pp_queue;
272 + host->rrdctx.pp_queue = NULL;
273 +
274 + RRDCONTEXT *rc;
275 + dfe_start_write(old, rc) {
276 + dictionary_del(old, string2str(rc->id));
277 + }
278 + dfe_done(rc);
279 + dictionary_destroy(old);
280 + }
281 +
282 + old = host->rrdctx.contexts;
283 + host->rrdctx.contexts = NULL;
284 + dictionary_destroy(old);
285 +}
286 +
database/contexts/instance.c new
+519
@@ -0,0 +1,519 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +// helper one-liners for RRDINSTANCE
7 +
8 +bool rrdinstance_acquired_id_and_name_are_same(RRDINSTANCE_ACQUIRED *ria) {
9 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
10 + return ri->id == ri->name;
11 +}
12 +
13 +inline const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria) {
14 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
15 + return string2str(ri->id);
16 +}
17 +
18 +inline const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria) {
19 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
20 + return string2str(ri->name);
21 +}
22 +
23 +inline const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria) {
24 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
25 + return string2str(ri->units);
26 +}
27 +
28 +inline STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria) {
29 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
30 + return string_dup(ri->units);
31 +}
32 +
33 +inline DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) {
34 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
35 + return ri->rrdlabels;
36 +}
37 +
38 +inline DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria) {
39 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
40 + if(!ri->rrdset) return NULL;
41 + return ri->rrdset->functions_view;
42 +}
43 +
44 +inline RRDHOST *rrdinstance_acquired_rrdhost(RRDINSTANCE_ACQUIRED *ria) {
45 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
46 + return ri->rc->rrdhost;
47 +}
48 +
49 +inline bool rrdinstance_acquired_belongs_to_context(RRDINSTANCE_ACQUIRED *ria, RRDCONTEXT_ACQUIRED *rca) {
50 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
51 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
52 + return ri->rc == rc;
53 +}
54 +
55 +inline time_t rrdinstance_acquired_update_every(RRDINSTANCE_ACQUIRED *ria) {
56 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
57 + return ri->update_every_s;
58 +}
59 +
60 +// ----------------------------------------------------------------------------
61 +// RRDINSTANCE
62 +
63 +static void rrdinstance_free(RRDINSTANCE *ri) {
64 +
65 + if(rrd_flag_check(ri, RRD_FLAG_OWN_LABELS))
66 + dictionary_destroy(ri->rrdlabels);
67 +
68 + rrdmetrics_destroy_from_rrdinstance(ri);
69 + string_freez(ri->id);
70 + string_freez(ri->name);
71 + string_freez(ri->title);
72 + string_freez(ri->units);
73 + string_freez(ri->family);
74 +
75 + ri->id = NULL;
76 + ri->name = NULL;
77 + ri->title = NULL;
78 + ri->units = NULL;
79 + ri->family = NULL;
80 + ri->rc = NULL;
81 + ri->rrdlabels = NULL;
82 + ri->rrdmetrics = NULL;
83 + ri->rrdset = NULL;
84 +}
85 +
86 +static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext) {
87 + RRDINSTANCE *ri = value;
88 +
89 + // link it to its parent
90 + ri->rc = rrdcontext;
91 +
92 + ri->flags = ri->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics
93 +
94 + if(!ri->name)
95 + ri->name = string_dup(ri->id);
96 +
97 + if(ri->rrdset) {
98 + ri->rrdlabels = ri->rrdset->rrdlabels;
99 + ri->flags &= ~RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor
100 + }
101 + else {
102 + ri->rrdlabels = rrdlabels_create();
103 + ri->flags |= RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor
104 + }
105 +
106 + if(ri->rrdset) {
107 + if(unlikely(rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)))
108 + ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor
109 + else
110 + ri->flags &= ~RRD_FLAG_HIDDEN; // no need of atomics at the constructor
111 + }
112 +
113 + rrdmetrics_create_in_rrdinstance(ri);
114 +
115 + // signal the react callback to do the job
116 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
117 +}
118 +
119 +static void rrdinstance_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext __maybe_unused) {
120 + RRDINSTANCE *ri = (RRDINSTANCE *)value;
121 +
122 + internal_error(ri->rrdset, "RRDINSTANCE: '%s' is freed but there is a RRDSET linked to it.", string2str(ri->id));
123 +
124 + rrdinstance_free(ri);
125 +}
126 +
127 +static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdcontext __maybe_unused) {
128 + RRDINSTANCE *ri = (RRDINSTANCE *)old_value;
129 + RRDINSTANCE *ri_new = (RRDINSTANCE *)new_value;
130 +
131 + internal_error(ri->id != ri_new->id,
132 + "RRDINSTANCE: '%s' cannot change id to '%s'",
133 + string2str(ri->id), string2str(ri_new->id));
134 +
135 + if(uuid_compare(ri->uuid, ri_new->uuid) != 0) {
136 +#ifdef NETDATA_INTERNAL_CHECKS
137 + char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
138 + uuid_unparse(ri->uuid, uuid1);
139 + uuid_unparse(ri_new->uuid, uuid2);
140 + internal_error(true, "RRDINSTANCE: '%s' of host '%s' changed UUID from '%s' to '%s'",
141 + string2str(ri->id), rrdhost_hostname(ri->rc->rrdhost), uuid1, uuid2);
142 +#endif
143 +
144 + uuid_copy(ri->uuid, ri_new->uuid);
145 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
146 + }
147 +
148 + if(ri->rrdset && ri_new->rrdset && ri->rrdset != ri_new->rrdset) {
149 + ri->rrdset = ri_new->rrdset;
150 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
151 + }
152 +
153 +#ifdef NETDATA_INTERNAL_CHECKS
154 + if(ri->rrdset && uuid_compare(ri->uuid, ri->rrdset->chart_uuid) != 0) {
155 + char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
156 + uuid_unparse(ri->uuid, uuid1);
157 + uuid_unparse(ri->rrdset->chart_uuid, uuid2);
158 + internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), rrdset_id(ri->rrdset), uuid1, uuid2);
159 + }
160 +#endif
161 +
162 + if(ri->name != ri_new->name) {
163 + STRING *old = ri->name;
164 + ri->name = string_dup(ri_new->name);
165 + string_freez(old);
166 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
167 + }
168 +
169 + if(ri->title != ri_new->title) {
170 + STRING *old = ri->title;
171 + ri->title = string_dup(ri_new->title);
172 + string_freez(old);
173 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
174 + }
175 +
176 + if(ri->units != ri_new->units) {
177 + STRING *old = ri->units;
178 + ri->units = string_dup(ri_new->units);
179 + string_freez(old);
180 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
181 + }
182 +
183 + if(ri->family != ri_new->family) {
184 + STRING *old = ri->family;
185 + ri->family = string_dup(ri_new->family);
186 + string_freez(old);
187 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
188 + }
189 +
190 + if(ri->chart_type != ri_new->chart_type) {
191 + ri->chart_type = ri_new->chart_type;
192 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
193 + }
194 +
195 + if(ri->priority != ri_new->priority) {
196 + ri->priority = ri_new->priority;
197 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
198 + }
199 +
200 + if(ri->update_every_s != ri_new->update_every_s) {
201 + ri->update_every_s = ri_new->update_every_s;
202 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
203 + }
204 +
205 + if(ri->rrdset != ri_new->rrdset) {
206 + ri->rrdset = ri_new->rrdset;
207 +
208 + if(ri->rrdset && rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
209 + DICTIONARY *old = ri->rrdlabels;
210 + ri->rrdlabels = ri->rrdset->rrdlabels;
211 + rrd_flag_clear(ri, RRD_FLAG_OWN_LABELS);
212 + rrdlabels_destroy(old);
213 + }
214 + else if(!ri->rrdset && !rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
215 + ri->rrdlabels = rrdlabels_create();
216 + rrd_flag_set(ri, RRD_FLAG_OWN_LABELS);
217 + }
218 + }
219 +
220 + if(ri->rrdset) {
221 + if(unlikely(rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)))
222 + rrd_flag_set(ri, RRD_FLAG_HIDDEN);
223 + else
224 + rrd_flag_clear(ri, RRD_FLAG_HIDDEN);
225 + }
226 +
227 + rrd_flag_set(ri, ri_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on ri_new
228 +
229 + if(rrd_flag_is_collected(ri) && rrd_flag_is_archived(ri))
230 + rrd_flag_set_collected(ri);
231 +
232 + if(rrd_flag_is_updated(ri))
233 + rrd_flag_set(ri, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
234 +
235 + // free the new one
236 + rrdinstance_free(ri_new);
237 +
238 + // the react callback will continue from here
239 + return rrd_flag_is_updated(ri);
240 +}
241 +
242 +static void rrdinstance_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext __maybe_unused) {
243 + RRDINSTANCE *ri = value;
244 +
245 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
246 +}
247 +
248 +void rrdinstances_create_in_rrdcontext(RRDCONTEXT *rc) {
249 + if(unlikely(!rc || rc->rrdinstances)) return;
250 +
251 + rc->rrdinstances = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
252 + &dictionary_stats_category_rrdcontext, sizeof(RRDINSTANCE));
253 +
254 + dictionary_register_insert_callback(rc->rrdinstances, rrdinstance_insert_callback, rc);
255 + dictionary_register_delete_callback(rc->rrdinstances, rrdinstance_delete_callback, rc);
256 + dictionary_register_conflict_callback(rc->rrdinstances, rrdinstance_conflict_callback, rc);
257 + dictionary_register_react_callback(rc->rrdinstances, rrdinstance_react_callback, rc);
258 +}
259 +
260 +void rrdinstances_destroy_from_rrdcontext(RRDCONTEXT *rc) {
261 + if(unlikely(!rc || !rc->rrdinstances)) return;
262 +
263 + dictionary_destroy(rc->rrdinstances);
264 + rc->rrdinstances = NULL;
265 +}
266 +
267 +void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function) {
268 + RRDSET *st = ri->rrdset;
269 +
270 + if(likely(st)) {
271 + if(unlikely((unsigned int) st->priority != ri->priority)) {
272 + ri->priority = st->priority;
273 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
274 + }
275 + if(unlikely(st->update_every != ri->update_every_s)) {
276 + ri->update_every_s = st->update_every;
277 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
278 + }
279 + }
280 + else if(unlikely(rrd_flag_is_collected(ri))) {
281 + // there is no rrdset, but we have it as collected!
282 +
283 + rrd_flag_set_archived(ri);
284 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
285 + }
286 +
287 + if(rrd_flag_is_updated(ri) || !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)) {
288 + rrd_flag_set_updated(ri->rc, RRD_FLAG_UPDATE_REASON_TRIGGERED);
289 + rrdcontext_queue_for_post_processing(ri->rc, function, ri->flags);
290 + }
291 +}
292 +
293 +// ----------------------------------------------------------------------------
294 +// RRDINSTANCE HOOKS ON RRDSET
295 +
296 +inline void rrdinstance_from_rrdset(RRDSET *st) {
297 + RRDCONTEXT trc = {
298 + .id = string_dup(st->context),
299 + .title = string_dup(st->title),
300 + .units = string_dup(st->units),
301 + .family = string_dup(st->family),
302 + .priority = st->priority,
303 + .chart_type = st->chart_type,
304 + .flags = RRD_FLAG_NONE, // no need for atomics
305 + .rrdhost = st->rrdhost,
306 + };
307 +
308 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_set_and_acquire_item(st->rrdhost->rrdctx.contexts, string2str(trc.id), &trc, sizeof(trc));
309 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
310 +
311 + RRDINSTANCE tri = {
312 + .id = string_dup(st->id),
313 + .name = string_dup(st->name),
314 + .units = string_dup(st->units),
315 + .family = string_dup(st->family),
316 + .title = string_dup(st->title),
317 + .chart_type = st->chart_type,
318 + .priority = st->priority,
319 + .update_every_s = st->update_every,
320 + .flags = RRD_FLAG_NONE, // no need for atomics
321 + .rrdset = st,
322 + };
323 + uuid_copy(tri.uuid, st->chart_uuid);
324 +
325 + RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, string2str(tri.id), &tri, sizeof(tri));
326 +
327 + RRDCONTEXT_ACQUIRED *rca_old = st->rrdcontext;
328 + RRDINSTANCE_ACQUIRED *ria_old = st->rrdinstance;
329 +
330 + st->rrdcontext = rca;
331 + st->rrdinstance = ria;
332 +
333 + if(rca == rca_old) {
334 + rrdcontext_release(rca_old);
335 + rca_old = NULL;
336 + }
337 +
338 + if(ria == ria_old) {
339 + rrdinstance_release(ria_old);
340 + ria_old = NULL;
341 + }
342 +
343 + if(rca_old && ria_old) {
344 + // Oops! The chart changed context!
345 +
346 + // RRDCONTEXT *rc_old = rrdcontext_acquired_value(rca_old);
347 + RRDINSTANCE *ri_old = rrdinstance_acquired_value(ria_old);
348 +
349 + // migrate all dimensions to the new metrics
350 + RRDDIM *rd;
351 + rrddim_foreach_read(rd, st) {
352 + if (!rd->rrdmetric) continue;
353 +
354 + RRDMETRIC *rm_old = rrdmetric_acquired_value(rd->rrdmetric);
355 + rrd_flags_replace(rm_old, RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
356 + rm_old->rrddim = NULL;
357 + rm_old->first_time_s = 0;
358 + rm_old->last_time_s = 0;
359 +
360 + rrdmetric_release(rd->rrdmetric);
361 + rd->rrdmetric = NULL;
362 +
363 + rrdmetric_from_rrddim(rd);
364 + }
365 + rrddim_foreach_done(rd);
366 +
367 + // mark the old instance, ready to be deleted
368 + if(!rrd_flag_check(ri_old, RRD_FLAG_OWN_LABELS))
369 + ri_old->rrdlabels = rrdlabels_create();
370 +
371 + rrd_flags_replace(ri_old, RRD_FLAG_OWN_LABELS|RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
372 + ri_old->rrdset = NULL;
373 + ri_old->first_time_s = 0;
374 + ri_old->last_time_s = 0;
375 +
376 + rrdinstance_trigger_updates(ri_old, __FUNCTION__ );
377 + rrdinstance_release(ria_old);
378 +
379 + /*
380 + // trigger updates on the old context
381 + if(!dictionary_entries(rc_old->rrdinstances) && !dictionary_stats_referenced_items(rc_old->rrdinstances)) {
382 + rrdcontext_lock(rc_old);
383 + rc_old->flags = ((rc_old->flags & RRD_FLAG_QUEUED)?RRD_FLAG_QUEUED:RRD_FLAG_NONE)|RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION;
384 + rc_old->first_time_s = 0;
385 + rc_old->last_time_s = 0;
386 + rrdcontext_unlock(rc_old);
387 + rrdcontext_trigger_updates(rc_old, __FUNCTION__ );
388 + }
389 + else
390 + rrdcontext_trigger_updates(rc_old, __FUNCTION__ );
391 + */
392 +
393 + rrdcontext_release(rca_old);
394 + rca_old = NULL;
395 + ria_old = NULL;
396 + }
397 +
398 + if(rca_old || ria_old)
399 + fatal("RRDCONTEXT: cannot switch rrdcontext without switching rrdinstance too");
400 +}
401 +
402 +#define rrdset_get_rrdinstance(st) rrdset_get_rrdinstance_with_trace(st, __FUNCTION__);
403 +static inline RRDINSTANCE *rrdset_get_rrdinstance_with_trace(RRDSET *st, const char *function) {
404 + if(unlikely(!st->rrdinstance)) {
405 + error("RRDINSTANCE: RRDSET '%s' is not linked to an RRDINSTANCE at %s()", rrdset_id(st), function);
406 + return NULL;
407 + }
408 +
409 + RRDINSTANCE *ri = rrdinstance_acquired_value(st->rrdinstance);
410 + if(unlikely(!ri)) {
411 + error("RRDINSTANCE: RRDSET '%s' lost its link to an RRDINSTANCE at %s()", rrdset_id(st), function);
412 + return NULL;
413 + }
414 +
415 + if(unlikely(ri->rrdset != st))
416 + fatal("RRDINSTANCE: '%s' is not linked to RRDSET '%s' at %s()", string2str(ri->id), rrdset_id(st), function);
417 +
418 + return ri;
419 +}
420 +
421 +inline void rrdinstance_rrdset_is_freed(RRDSET *st) {
422 + RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
423 + if(unlikely(!ri)) return;
424 +
425 + rrd_flag_set_archived(ri);
426 +
427 + if(!rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
428 + ri->rrdlabels = rrdlabels_create();
429 + rrdlabels_copy(ri->rrdlabels, st->rrdlabels);
430 + rrd_flag_set(ri, RRD_FLAG_OWN_LABELS);
431 + }
432 +
433 + ri->rrdset = NULL;
434 +
435 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
436 +
437 + rrdinstance_release(st->rrdinstance);
438 + st->rrdinstance = NULL;
439 +
440 + rrdcontext_release(st->rrdcontext);
441 + st->rrdcontext = NULL;
442 +}
443 +
444 +inline void rrdinstance_rrdset_has_updated_retention(RRDSET *st) {
445 + RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
446 + if(unlikely(!ri)) return;
447 +
448 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION);
449 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
450 +}
451 +
452 +inline void rrdinstance_updated_rrdset_name(RRDSET *st) {
453 + // the chart may not be initialized when this is called
454 + if(unlikely(!st->rrdinstance)) return;
455 +
456 + RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
457 + if(unlikely(!ri)) return;
458 +
459 + if(st->name != ri->name) {
460 + STRING *old = ri->name;
461 + ri->name = string_dup(st->name);
462 + string_freez(old);
463 +
464 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
465 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
466 + }
467 +}
468 +
469 +inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, RRDSET *st) {
470 + if(unlikely(ri->rrdset != st))
471 + fatal("RRDCONTEXT: instance '%s' is not linked to chart '%s' on host '%s'",
472 + string2str(ri->id), rrdset_id(st), rrdhost_hostname(st->rrdhost));
473 +
474 + bool st_is_hidden = rrdset_flag_check(st, RRDSET_FLAG_HIDDEN);
475 + bool ri_is_hidden = rrd_flag_check(ri, RRD_FLAG_HIDDEN);
476 +
477 + if(unlikely(st_is_hidden != ri_is_hidden)) {
478 + if (unlikely(st_is_hidden && !ri_is_hidden))
479 + rrd_flag_set_updated(ri, RRD_FLAG_HIDDEN | RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
480 +
481 + else if (unlikely(!st_is_hidden && ri_is_hidden)) {
482 + rrd_flag_clear(ri, RRD_FLAG_HIDDEN);
483 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
484 + }
485 + }
486 +}
487 +
488 +inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
489 + RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
490 + if(unlikely(!ri)) return;
491 +
492 + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED|RRDSET_FLAG_OBSOLETE)))
493 + rrd_flag_set_archived(ri);
494 +
495 + rrdinstance_updated_rrdset_flags_no_action(ri, st);
496 +
497 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
498 +}
499 +
500 +inline void rrdinstance_collected_rrdset(RRDSET *st) {
501 + RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
502 + if(unlikely(!ri)) {
503 + rrdcontext_updated_rrdset(st);
504 + ri = rrdset_get_rrdinstance(st);
505 + if(unlikely(!ri))
506 + return;
507 + }
508 +
509 + rrdinstance_updated_rrdset_flags_no_action(ri, st);
510 +
511 + if(unlikely(ri->internal.collected_metrics_count && !rrd_flag_is_collected(ri)))
512 + rrd_flag_set_collected(ri);
513 +
514 + // we use this variable to detect BEGIN/END without SET
515 + ri->internal.collected_metrics_count = 0;
516 +
517 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
518 +}
519 +
database/contexts/internal.h new
+389
@@ -0,0 +1,389 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDCONTEXT_INTERNAL_H
4 +#define NETDATA_RRDCONTEXT_INTERNAL_H 1
5 +
6 +#include "rrdcontext.h"
7 +#include "../sqlite/sqlite_context.h"
8 +#include "../../aclk/schema-wrappers/context.h"
9 +#include "../../aclk/aclk_contexts_api.h"
10 +#include "../../aclk/aclk.h"
11 +#include "../storage_engine.h"
12 +
13 +#define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
14 +#define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
15 +#define RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC (1000 * USEC_PER_MS)
16 +#define RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY 10
17 +
18 +#define LOG_TRANSITIONS false
19 +
20 +#define WORKER_JOB_HOSTS 1
21 +#define WORKER_JOB_CHECK 2
22 +#define WORKER_JOB_SEND 3
23 +#define WORKER_JOB_DEQUEUE 4
24 +#define WORKER_JOB_RETENTION 5
25 +#define WORKER_JOB_QUEUED 6
26 +#define WORKER_JOB_CLEANUP 7
27 +#define WORKER_JOB_CLEANUP_DELETE 8
28 +#define WORKER_JOB_PP_METRIC 9 // post-processing metrics
29 +#define WORKER_JOB_PP_INSTANCE 10 // post-processing instances
30 +#define WORKER_JOB_PP_CONTEXT 11 // post-processing contexts
31 +#define WORKER_JOB_HUB_QUEUE_SIZE 12
32 +#define WORKER_JOB_PP_QUEUE_SIZE 13
33 +
34 +
35 +typedef enum __attribute__ ((__packed__)) {
36 + RRD_FLAG_NONE = 0,
37 + RRD_FLAG_DELETED = (1 << 0), // this is a deleted object (metrics, instances, contexts)
38 + RRD_FLAG_COLLECTED = (1 << 1), // this object is currently being collected
39 + RRD_FLAG_UPDATED = (1 << 2), // this object has updates to propagate
40 + RRD_FLAG_ARCHIVED = (1 << 3), // this object is not currently being collected
41 + RRD_FLAG_OWN_LABELS = (1 << 4), // this instance has its own labels - not linked to an RRDSET
42 + RRD_FLAG_LIVE_RETENTION = (1 << 5), // we have got live retention from the database
43 + RRD_FLAG_QUEUED_FOR_HUB = (1 << 6), // this context is currently queued to be dispatched to hub
44 + RRD_FLAG_QUEUED_FOR_PP = (1 << 7), // this context is currently queued to be post-processed
45 + RRD_FLAG_HIDDEN = (1 << 8), // don't expose this to the hub or the API
46 +
47 + RRD_FLAG_UPDATE_REASON_TRIGGERED = (1 << 9), // the update was triggered by the child object
48 + RRD_FLAG_UPDATE_REASON_LOAD_SQL = (1 << 10), // this object has just been loaded from SQL
49 + RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 11), // this object has just been created
50 + RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT = (1 << 12), // we received an update on this object
51 + RRD_FLAG_UPDATE_REASON_CHANGED_LINKING = (1 << 13), // an instance or a metric switched RRDSET or RRDDIM
52 + RRD_FLAG_UPDATE_REASON_CHANGED_METADATA = (1 << 14), // this context or instance changed uuid, name, units, title, family, chart type, priority, update every, rrd changed flags
53 + RRD_FLAG_UPDATE_REASON_ZERO_RETENTION = (1 << 15), // this object has no retention
54 + RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T = (1 << 16), // this object changed its oldest time in the db
55 + RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T = (1 << 17), // this object change its latest time in the db
56 + RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED = (1 << 18), // this object has stopped being collected
57 + RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED = (1 << 19), // this object has started being collected
58 + RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 20), // this context belongs to a host that just disconnected
59 + RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 21), // this context is not used anymore
60 + RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 22), // this context changed because of a db rotation
61 +
62 + // action to perform on an object
63 + RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION = (1 << 30), // this object has to update its retention from the db
64 +} RRD_FLAGS;
65 +
66 +struct rrdcontext_reason {
67 + RRD_FLAGS flag;
68 + const char *name;
69 + usec_t delay_ut;
70 +};
71 +
72 +extern struct rrdcontext_reason rrdcontext_reasons[];
73 +
74 +#define RRD_FLAG_ALL_UPDATE_REASONS ( \
75 + RRD_FLAG_UPDATE_REASON_TRIGGERED \
76 + |RRD_FLAG_UPDATE_REASON_LOAD_SQL \
77 + |RRD_FLAG_UPDATE_REASON_NEW_OBJECT \
78 + |RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT \
79 + |RRD_FLAG_UPDATE_REASON_CHANGED_LINKING \
80 + |RRD_FLAG_UPDATE_REASON_CHANGED_METADATA \
81 + |RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
82 + |RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T \
83 + |RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T \
84 + |RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED \
85 + |RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED \
86 + |RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \
87 + |RRD_FLAG_UPDATE_REASON_DB_ROTATION \
88 + |RRD_FLAG_UPDATE_REASON_UNUSED \
89 + )
90 +
91 +#define RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS ( \
92 + RRD_FLAG_ARCHIVED \
93 + |RRD_FLAG_HIDDEN \
94 + |RRD_FLAG_ALL_UPDATE_REASONS \
95 + )
96 +
97 +#define RRD_FLAGS_REQUIRED_FOR_DELETIONS ( \
98 + RRD_FLAG_DELETED \
99 + |RRD_FLAG_LIVE_RETENTION \
100 +)
101 +
102 +#define RRD_FLAGS_PREVENTING_DELETIONS ( \
103 + RRD_FLAG_QUEUED_FOR_HUB \
104 + |RRD_FLAG_COLLECTED \
105 + |RRD_FLAG_QUEUED_FOR_PP \
106 +)
107 +
108 +// get all the flags of an object
109 +#define rrd_flags_get(obj) __atomic_load_n(&((obj)->flags), __ATOMIC_SEQ_CST)
110 +
111 +// check if ANY of the given flags (bits) is set
112 +#define rrd_flag_check(obj, flag) (rrd_flags_get(obj) & (flag))
113 +
114 +// check if ALL the given flags (bits) are set
115 +#define rrd_flag_check_all(obj, flag) (rrd_flag_check(obj, flag) == (flag))
116 +
117 +// set one or more flags (bits)
118 +#define rrd_flag_set(obj, flag) __atomic_or_fetch(&((obj)->flags), flag, __ATOMIC_SEQ_CST)
119 +
120 +// clear one or more flags (bits)
121 +#define rrd_flag_clear(obj, flag) __atomic_and_fetch(&((obj)->flags), ~(flag), __ATOMIC_SEQ_CST)
122 +
123 +// replace the flags of an object, with the supplied ones
124 +#define rrd_flags_replace(obj, all_flags) __atomic_store_n(&((obj)->flags), all_flags, __ATOMIC_SEQ_CST)
125 +
126 +static inline void
127 +rrd_flag_add_remove_atomic(RRD_FLAGS *flags, RRD_FLAGS check, RRD_FLAGS conditionally_add, RRD_FLAGS always_remove) {
128 + RRD_FLAGS expected, desired;
129 +
130 + do {
131 + expected = *flags;
132 +
133 + desired = expected;
134 + desired &= ~(always_remove);
135 +
136 + if(!(expected & check))
137 + desired |= (check | conditionally_add);
138 +
139 + } while(!__atomic_compare_exchange_n(flags, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
140 +}
141 +
142 +#define rrd_flag_set_collected(obj) \
143 + rrd_flag_add_remove_atomic(&((obj)->flags) \
144 + /* check this flag */ \
145 + , RRD_FLAG_COLLECTED \
146 + \
147 + /* add these flags together with the above, if the above is not already set */ \
148 + , RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED | RRD_FLAG_UPDATED \
149 + \
150 + /* always remove these flags */ \
151 + , RRD_FLAG_ARCHIVED \
152 + | RRD_FLAG_DELETED \
153 + | RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED \
154 + | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
155 + | RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \
156 + )
157 +
158 +#define rrd_flag_set_archived(obj) \
159 + rrd_flag_add_remove_atomic(&((obj)->flags) \
160 + /* check this flag */ \
161 + , RRD_FLAG_ARCHIVED \
162 + \
163 + /* add these flags together with the above, if the above is not already set */ \
164 + , RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED | RRD_FLAG_UPDATED \
165 + \
166 + /* always remove these flags */ \
167 + , RRD_FLAG_COLLECTED \
168 + | RRD_FLAG_DELETED \
169 + | RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED \
170 + | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
171 + )
172 +
173 +#define rrd_flag_set_deleted(obj, reason) \
174 + rrd_flag_add_remove_atomic(&((obj)->flags) \
175 + /* check this flag */ \
176 + , RRD_FLAG_DELETED \
177 + \
178 + /* add these flags together with the above, if the above is not already set */ \
179 + , RRD_FLAG_UPDATE_REASON_ZERO_RETENTION | RRD_FLAG_UPDATED | (reason) \
180 + \
181 + /* always remove these flags */ \
182 + , RRD_FLAG_ARCHIVED \
183 + | RRD_FLAG_COLLECTED \
184 + )
185 +
186 +#define rrd_flag_is_collected(obj) rrd_flag_check(obj, RRD_FLAG_COLLECTED)
187 +#define rrd_flag_is_archived(obj) rrd_flag_check(obj, RRD_FLAG_ARCHIVED)
188 +#define rrd_flag_is_deleted(obj) rrd_flag_check(obj, RRD_FLAG_DELETED)
189 +#define rrd_flag_is_updated(obj) rrd_flag_check(obj, RRD_FLAG_UPDATED)
190 +
191 +// mark an object as updated, providing reasons (additional bits)
192 +#define rrd_flag_set_updated(obj, reason) rrd_flag_set(obj, RRD_FLAG_UPDATED | (reason))
193 +
194 +// clear an object as being updated, clearing also all the reasons
195 +#define rrd_flag_unset_updated(obj) rrd_flag_clear(obj, RRD_FLAG_UPDATED | RRD_FLAG_ALL_UPDATE_REASONS)
196 +
197 +
198 +typedef struct rrdmetric {
199 + uuid_t uuid;
200 +
201 + STRING *id;
202 + STRING *name;
203 +
204 + RRDDIM *rrddim;
205 +
206 + time_t first_time_s;
207 + time_t last_time_s;
208 + RRD_FLAGS flags;
209 +
210 + struct rrdinstance *ri;
211 +} RRDMETRIC;
212 +
213 +typedef struct rrdinstance {
214 + uuid_t uuid;
215 +
216 + STRING *id;
217 + STRING *name;
218 + STRING *title;
219 + STRING *units;
220 + STRING *family;
221 + uint32_t priority;
222 + RRDSET_TYPE chart_type;
223 +
224 + RRD_FLAGS flags; // flags related to this instance
225 + time_t first_time_s;
226 + time_t last_time_s;
227 +
228 + time_t update_every_s; // data collection frequency
229 + RRDSET *rrdset; // pointer to RRDSET when collected, or NULL
230 +
231 + DICTIONARY *rrdlabels; // linked to RRDSET->chart_labels or own version
232 +
233 + struct rrdcontext *rc;
234 + DICTIONARY *rrdmetrics;
235 +
236 + struct {
237 + uint32_t collected_metrics_count; // a temporary variable to detect BEGIN/END without SET
238 + // don't use it for other purposes
239 + // it goes up and then resets to zero, on every iteration
240 + } internal;
241 +} RRDINSTANCE;
242 +
243 +typedef struct rrdcontext {
244 + uint64_t version;
245 +
246 + STRING *id;
247 + STRING *title;
248 + STRING *units;
249 + STRING *family;
250 + uint32_t priority;
251 + RRDSET_TYPE chart_type;
252 +
253 + RRD_FLAGS flags;
254 + time_t first_time_s;
255 + time_t last_time_s;
256 +
257 + VERSIONED_CONTEXT_DATA hub;
258 +
259 + DICTIONARY *rrdinstances;
260 + RRDHOST *rrdhost;
261 +
262 + struct {
263 + RRD_FLAGS queued_flags; // the last flags that triggered the post-processing
264 + usec_t queued_ut; // the last time this was queued
265 + usec_t dequeued_ut; // the last time we sent (or deduplicated) this context
266 + size_t executions; // how many times this context has been processed
267 + } pp;
268 +
269 + struct {
270 + RRD_FLAGS queued_flags; // the last flags that triggered the queueing
271 + usec_t queued_ut; // the last time this was queued
272 + usec_t delay_calc_ut; // the last time we calculated the scheduled_dispatched_ut
273 + usec_t scheduled_dispatch_ut; // the time it was/is scheduled to be sent
274 + usec_t dequeued_ut; // the last time we sent (or deduplicated) this context
275 + size_t dispatches; // the number of times this has been dispatched to hub
276 + } queue;
277 +
278 + netdata_mutex_t mutex;
279 +} RRDCONTEXT;
280 +
281 +
282 +// ----------------------------------------------------------------------------
283 +// helper one-liners for RRDMETRIC
284 +
285 +bool rrdmetric_update_retention(RRDMETRIC *rm);
286 +
287 +static inline RRDMETRIC *rrdmetric_acquired_value(RRDMETRIC_ACQUIRED *rma) {
288 + return dictionary_acquired_item_value((DICTIONARY_ITEM *)rma);
289 +}
290 +
291 +static inline RRDMETRIC_ACQUIRED *rrdmetric_acquired_dup(RRDMETRIC_ACQUIRED *rma) {
292 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
293 + return (RRDMETRIC_ACQUIRED *)dictionary_acquired_item_dup(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma);
294 +}
295 +
296 +static inline void rrdmetric_release(RRDMETRIC_ACQUIRED *rma) {
297 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
298 + dictionary_acquired_item_release(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma);
299 +}
300 +
301 +void rrdmetric_rrddim_is_freed(RRDDIM *rd);
302 +void rrdmetric_updated_rrddim_flags(RRDDIM *rd);
303 +void rrdmetric_collected_rrddim(RRDDIM *rd);
304 +
305 +// ----------------------------------------------------------------------------
306 +// helper one-liners for RRDINSTANCE
307 +
308 +static inline RRDINSTANCE *rrdinstance_acquired_value(RRDINSTANCE_ACQUIRED *ria) {
309 + return dictionary_acquired_item_value((DICTIONARY_ITEM *)ria);
310 +}
311 +
312 +static inline RRDINSTANCE_ACQUIRED *rrdinstance_acquired_dup(RRDINSTANCE_ACQUIRED *ria) {
313 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
314 + return (RRDINSTANCE_ACQUIRED *)dictionary_acquired_item_dup(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria);
315 +}
316 +
317 +static inline void rrdinstance_release(RRDINSTANCE_ACQUIRED *ria) {
318 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
319 + dictionary_acquired_item_release(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria);
320 +}
321 +
322 +void rrdinstance_from_rrdset(RRDSET *st);
323 +void rrdinstance_rrdset_is_freed(RRDSET *st);
324 +void rrdinstance_rrdset_has_updated_retention(RRDSET *st);
325 +void rrdinstance_updated_rrdset_name(RRDSET *st);
326 +void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, RRDSET *st);
327 +void rrdinstance_updated_rrdset_flags(RRDSET *st);
328 +void rrdinstance_collected_rrdset(RRDSET *st);
329 +
330 +void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function, RRD_FLAGS flags);
331 +
332 +// ----------------------------------------------------------------------------
333 +// helper one-liners for RRDCONTEXT
334 +
335 +static inline RRDCONTEXT *rrdcontext_acquired_value(RRDCONTEXT_ACQUIRED *rca) {
336 + return dictionary_acquired_item_value((DICTIONARY_ITEM *)rca);
337 +}
338 +
339 +static inline RRDCONTEXT_ACQUIRED *rrdcontext_acquired_dup(RRDCONTEXT_ACQUIRED *rca) {
340 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
341 + return (RRDCONTEXT_ACQUIRED *)dictionary_acquired_item_dup(rc->rrdhost->rrdctx.contexts, (DICTIONARY_ITEM *)rca);
342 +}
343 +
344 +static inline void rrdcontext_release(RRDCONTEXT_ACQUIRED *rca) {
345 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
346 + dictionary_acquired_item_release(rc->rrdhost->rrdctx.contexts, (DICTIONARY_ITEM *)rca);
347 +}
348 +
349 +// ----------------------------------------------------------------------------
350 +// Forward definitions
351 +
352 +void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs);
353 +void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs);
354 +
355 +#define rrdcontext_lock(rc) netdata_mutex_lock(&((rc)->mutex))
356 +#define rrdcontext_unlock(rc) netdata_mutex_unlock(&((rc)->mutex))
357 +
358 +void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function);
359 +void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function);
360 +
361 +void rrdinstances_create_in_rrdcontext(RRDCONTEXT *rc);
362 +void rrdinstances_destroy_from_rrdcontext(RRDCONTEXT *rc);
363 +
364 +void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri);
365 +void rrdmetrics_create_in_rrdinstance(RRDINSTANCE *ri);
366 +
367 +void rrdmetric_from_rrddim(RRDDIM *rd);
368 +
369 +void rrd_reasons_to_buffer_json_array_items(RRD_FLAGS flags, BUFFER *wb);
370 +
371 +#define rrdcontext_version_hash(host) rrdcontext_version_hash_with_callback(host, NULL, false, NULL)
372 +uint64_t rrdcontext_version_hash_with_callback(
373 + RRDHOST *host,
374 + void (*callback)(RRDCONTEXT *, bool, void *),
375 + bool snapshot,
376 + void *bundle);
377 +
378 +void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused);
379 +
380 +// ----------------------------------------------------------------------------
381 +// scope
382 +
383 +typedef bool (*foreach_host_cb_t)(void *data, RRDHOST *host, bool queryable);
384 +uint64_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN *hosts_sp, foreach_host_cb_t cb, void *data, uint64_t *hard_hash, uint64_t *soft_hash, char *host_uuid_buffer);
385 +
386 +typedef bool (*foreach_context_cb_t)(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context);
387 +size_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp, SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data);
388 +
389 +#endif //NETDATA_RRDCONTEXT_INTERNAL_H
database/contexts/metric.c new
+314
@@ -0,0 +1,314 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
6 +
7 +inline const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma) {
8 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
9 + return string2str(rm->id);
10 +}
11 +
12 +inline const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma) {
13 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
14 + return string2str(rm->name);
15 +}
16 +
17 +inline STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma) {
18 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
19 + return string_dup(rm->id);
20 +}
21 +
22 +inline STRING *rrdmetric_acquired_name_dup(RRDMETRIC_ACQUIRED *rma) {
23 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
24 + return string_dup(rm->name);
25 +}
26 +
27 +inline NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma) {
28 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
29 +
30 + if(rm->rrddim)
31 + return rm->rrddim->last_stored_value;
32 +
33 + return NAN;
34 +}
35 +
36 +inline bool rrdmetric_acquired_belongs_to_instance(RRDMETRIC_ACQUIRED *rma, RRDINSTANCE_ACQUIRED *ria) {
37 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
38 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
39 + return rm->ri == ri;
40 +}
41 +
42 +inline time_t rrdmetric_acquired_first_entry(RRDMETRIC_ACQUIRED *rma) {
43 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
44 + return rm->first_time_s;
45 +}
46 +
47 +inline time_t rrdmetric_acquired_last_entry(RRDMETRIC_ACQUIRED *rma) {
48 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
49 +
50 + if(rrd_flag_check(rm, RRD_FLAG_COLLECTED))
51 + return 0;
52 +
53 + return rm->last_time_s;
54 +}
55 +
56 +// ----------------------------------------------------------------------------
57 +// RRDMETRIC
58 +
59 +// free the contents of RRDMETRIC.
60 +// RRDMETRIC itself is managed by DICTIONARY - no need to free it here.
61 +static void rrdmetric_free(RRDMETRIC *rm) {
62 + string_freez(rm->id);
63 + string_freez(rm->name);
64 +
65 + rm->id = NULL;
66 + rm->name = NULL;
67 + rm->ri = NULL;
68 +}
69 +
70 +// called when this rrdmetric is inserted to the rrdmetrics dictionary of a rrdinstance
71 +// the constructor of the rrdmetric object
72 +static void rrdmetric_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance) {
73 + RRDMETRIC *rm = value;
74 +
75 + // link it to its parent
76 + rm->ri = rrdinstance;
77 +
78 + // remove flags that we need to figure out at runtime
79 + rm->flags = rm->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics
80 +
81 + // signal the react callback to do the job
82 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
83 +}
84 +
85 +// called when this rrdmetric is deleted from the rrdmetrics dictionary of a rrdinstance
86 +// the destructor of the rrdmetric object
87 +static void rrdmetric_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) {
88 + RRDMETRIC *rm = value;
89 +
90 + internal_error(rm->rrddim, "RRDMETRIC: '%s' is freed but there is a RRDDIM linked to it.", string2str(rm->id));
91 +
92 + // free the resources
93 + rrdmetric_free(rm);
94 +}
95 +
96 +// called when the same rrdmetric is inserted again to the rrdmetrics dictionary of a rrdinstance
97 +// while this is called, the dictionary is write locked, but there may be other users of the object
98 +static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdinstance __maybe_unused) {
99 + RRDMETRIC *rm = old_value;
100 + RRDMETRIC *rm_new = new_value;
101 +
102 + internal_error(rm->id != rm_new->id,
103 + "RRDMETRIC: '%s' cannot change id to '%s'",
104 + string2str(rm->id), string2str(rm_new->id));
105 +
106 + if(uuid_compare(rm->uuid, rm_new->uuid) != 0) {
107 +#ifdef NETDATA_INTERNAL_CHECKS
108 + char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
109 + uuid_unparse(rm->uuid, uuid1);
110 + uuid_unparse(rm_new->uuid, uuid2);
111 +
112 + time_t old_first_time_s = 0;
113 + time_t old_last_time_s = 0;
114 + if(rrdmetric_update_retention(rm)) {
115 + old_first_time_s = rm->first_time_s;
116 + old_last_time_s = rm->last_time_s;
117 + }
118 +
119 + uuid_copy(rm->uuid, rm_new->uuid);
120 +
121 + time_t new_first_time_s = 0;
122 + time_t new_last_time_s = 0;
123 + if(rrdmetric_update_retention(rm)) {
124 + new_first_time_s = rm->first_time_s;
125 + new_last_time_s = rm->last_time_s;
126 + }
127 +
128 + internal_error(true,
129 + "RRDMETRIC: '%s' of instance '%s' of host '%s' changed UUID from '%s' (retention %ld to %ld, %ld secs) to '%s' (retention %ld to %ld, %ld secs)"
130 + , string2str(rm->id)
131 + , string2str(rm->ri->id)
132 + , rrdhost_hostname(rm->ri->rc->rrdhost)
133 + , uuid1, old_first_time_s, old_last_time_s, old_last_time_s - old_first_time_s
134 + , uuid2, new_first_time_s, new_last_time_s, new_last_time_s - new_first_time_s
135 + );
136 +#else
137 + uuid_copy(rm->uuid, rm_new->uuid);
138 +#endif
139 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
140 + }
141 +
142 + if(rm->rrddim && rm_new->rrddim && rm->rrddim != rm_new->rrddim) {
143 + rm->rrddim = rm_new->rrddim;
144 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
145 + }
146 +
147 +#ifdef NETDATA_INTERNAL_CHECKS
148 + if(rm->rrddim && uuid_compare(rm->uuid, rm->rrddim->metric_uuid) != 0) {
149 + char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
150 + uuid_unparse(rm->uuid, uuid1);
151 + uuid_unparse(rm_new->uuid, uuid2);
152 + internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rrddim_id(rm->rrddim), uuid1, uuid2);
153 + }
154 +#endif
155 +
156 + if(rm->rrddim != rm_new->rrddim)
157 + rm->rrddim = rm_new->rrddim;
158 +
159 + if(rm->name != rm_new->name) {
160 + STRING *old = rm->name;
161 + rm->name = string_dup(rm_new->name);
162 + string_freez(old);
163 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
164 + }
165 +
166 + if(!rm->first_time_s || (rm_new->first_time_s && rm_new->first_time_s < rm->first_time_s)) {
167 + rm->first_time_s = rm_new->first_time_s;
168 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
169 + }
170 +
171 + if(!rm->last_time_s || (rm_new->last_time_s && rm_new->last_time_s > rm->last_time_s)) {
172 + rm->last_time_s = rm_new->last_time_s;
173 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
174 + }
175 +
176 + rrd_flag_set(rm, rm_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no needs for atomics on rm_new
177 +
178 + if(rrd_flag_is_collected(rm) && rrd_flag_is_archived(rm))
179 + rrd_flag_set_collected(rm);
180 +
181 + if(rrd_flag_check(rm, RRD_FLAG_UPDATED))
182 + rrd_flag_set(rm, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
183 +
184 + rrdmetric_free(rm_new);
185 +
186 + // the react callback will continue from here
187 + return rrd_flag_is_updated(rm);
188 +}
189 +
190 +// this is called after the insert or the conflict callbacks,
191 +// but the dictionary is now unlocked
192 +static void rrdmetric_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) {
193 + RRDMETRIC *rm = value;
194 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
195 +}
196 +
197 +void rrdmetrics_create_in_rrdinstance(RRDINSTANCE *ri) {
198 + if(unlikely(!ri)) return;
199 + if(likely(ri->rrdmetrics)) return;
200 +
201 + ri->rrdmetrics = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
202 + &dictionary_stats_category_rrdcontext, sizeof(RRDMETRIC));
203 +
204 + dictionary_register_insert_callback(ri->rrdmetrics, rrdmetric_insert_callback, ri);
205 + dictionary_register_delete_callback(ri->rrdmetrics, rrdmetric_delete_callback, ri);
206 + dictionary_register_conflict_callback(ri->rrdmetrics, rrdmetric_conflict_callback, ri);
207 + dictionary_register_react_callback(ri->rrdmetrics, rrdmetric_react_callback, ri);
208 +}
209 +
210 +void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) {
211 + if(unlikely(!ri || !ri->rrdmetrics)) return;
212 + dictionary_destroy(ri->rrdmetrics);
213 + ri->rrdmetrics = NULL;
214 +}
215 +
216 +// trigger post-processing of the rrdmetric, escalating changes to the rrdinstance it belongs
217 +static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) {
218 + if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD)))
219 + rrd_flag_set_archived(rm);
220 +
221 + if(rrd_flag_is_updated(rm) || !rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)) {
222 + rrd_flag_set_updated(rm->ri, RRD_FLAG_UPDATE_REASON_TRIGGERED);
223 + rrdcontext_queue_for_post_processing(rm->ri->rc, function, rm->flags);
224 + }
225 +}
226 +
227 +// ----------------------------------------------------------------------------
228 +// RRDMETRIC HOOKS ON RRDDIM
229 +
230 +void rrdmetric_from_rrddim(RRDDIM *rd) {
231 + if(unlikely(!rd->rrdset))
232 + fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rrddim_id(rd));
233 +
234 + if(unlikely(!rd->rrdset->rrdhost))
235 + fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rrdset_id(rd->rrdset));
236 +
237 + if(unlikely(!rd->rrdset->rrdinstance))
238 + fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rrdset_id(rd->rrdset));
239 +
240 + RRDINSTANCE *ri = rrdinstance_acquired_value(rd->rrdset->rrdinstance);
241 +
242 + RRDMETRIC trm = {
243 + .id = string_dup(rd->id),
244 + .name = string_dup(rd->name),
245 + .flags = RRD_FLAG_NONE, // no need for atomics
246 + .rrddim = rd,
247 + };
248 + uuid_copy(trm.uuid, rd->metric_uuid);
249 +
250 + RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)dictionary_set_and_acquire_item(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
251 +
252 + if(rd->rrdmetric)
253 + rrdmetric_release(rd->rrdmetric);
254 +
255 + rd->rrdmetric = rma;
256 +}
257 +
258 +#define rrddim_get_rrdmetric(rd) rrddim_get_rrdmetric_with_trace(rd, __FUNCTION__)
259 +static inline RRDMETRIC *rrddim_get_rrdmetric_with_trace(RRDDIM *rd, const char *function) {
260 + if(unlikely(!rd->rrdmetric)) {
261 + error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rrddim_id(rd), function);
262 + return NULL;
263 + }
264 +
265 + RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdmetric);
266 + if(unlikely(!rm)) {
267 + error("RRDMETRIC: RRDDIM '%s' lost the link to its RRDMETRIC at %s()", rrddim_id(rd), function);
268 + return NULL;
269 + }
270 +
271 + if(unlikely(rm->rrddim != rd))
272 + fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rrddim_id(rd), function);
273 +
274 + return rm;
275 +}
276 +
277 +inline void rrdmetric_rrddim_is_freed(RRDDIM *rd) {
278 + RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
279 + if(unlikely(!rm)) return;
280 +
281 + if(unlikely(rrd_flag_is_collected(rm)))
282 + rrd_flag_set_archived(rm);
283 +
284 + rm->rrddim = NULL;
285 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
286 + rrdmetric_release(rd->rrdmetric);
287 + rd->rrdmetric = NULL;
288 +}
289 +
290 +inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) {
291 + RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
292 + if(unlikely(!rm)) return;
293 +
294 + if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED|RRDDIM_FLAG_OBSOLETE))) {
295 + if(unlikely(rrd_flag_is_collected(rm)))
296 + rrd_flag_set_archived(rm);
297 + }
298 +
299 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
300 +}
301 +
302 +inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
303 + RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
304 + if(unlikely(!rm)) return;
305 +
306 + if(unlikely(!rrd_flag_is_collected(rm)))
307 + rrd_flag_set_collected(rm);
308 +
309 + // we use this variable to detect BEGIN/END without SET
310 + rm->ri->internal.collected_metrics_count++;
311 +
312 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
313 +}
314 +
database/contexts/query_scope.c new
+84
@@ -0,0 +1,84 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +uint64_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN *hosts_sp, foreach_host_cb_t cb, void *data, uint64_t *hard_hash, uint64_t *soft_hash, char *host_uuid_buffer) {
6 + char uuid[UUID_STR_LEN];
7 + if(!host_uuid_buffer) host_uuid_buffer = uuid;
8 + host_uuid_buffer[0] = '\0';
9 +
10 + RRDHOST *host;
11 + size_t count = 0;
12 + uint64_t v_hash = 0;
13 + uint64_t h_hash = 0;
14 +
15 + dfe_start_reentrant(rrdhost_root_index, host) {
16 + if(host->node_id)
17 + uuid_unparse_lower(*host->node_id, host_uuid_buffer);
18 +
19 + if(!scope_hosts_sp ||
20 + simple_pattern_matches_string(scope_hosts_sp, host->hostname) ||
21 + simple_pattern_matches(scope_hosts_sp, host->machine_guid) ||
22 + (*host_uuid_buffer && simple_pattern_matches(scope_hosts_sp, host_uuid_buffer))) {
23 +
24 + bool queryable_host = false;
25 + if(!hosts_sp ||
26 + simple_pattern_matches_string(hosts_sp, host->hostname) ||
27 + simple_pattern_matches(hosts_sp, host->machine_guid) ||
28 + (*host_uuid_buffer && simple_pattern_matches(hosts_sp, host_uuid_buffer)))
29 + queryable_host = true;
30 +
31 + count++;
32 + v_hash += dictionary_version(host->rrdctx.contexts);
33 + h_hash += dictionary_version(host->rrdctx.hub_queue);
34 + cb(data, host, queryable_host);
35 + }
36 + }
37 + dfe_done(host);
38 +
39 + if(hard_hash)
40 + *hard_hash = v_hash;
41 +
42 + if(soft_hash)
43 + *soft_hash = h_hash;
44 +
45 + return count;
46 +}
47 +
48 +size_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp, SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data) {
49 + size_t added = 0;
50 +
51 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, scope_contexts);
52 + if(likely(rca)) {
53 + // we found it!
54 +
55 + bool queryable_context = queryable_host;
56 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
57 + if(queryable_context && contexts_sp && !simple_pattern_matches_string(contexts_sp, rc->id))
58 + queryable_context = false;
59 +
60 + if(cb(data, rca, queryable_context))
61 + added++;
62 +
63 + rrdcontext_release(rca);
64 + }
65 + else {
66 + // Probably it is a pattern, we need to search for it...
67 + RRDCONTEXT *rc;
68 + dfe_start_read(host->rrdctx.contexts, rc) {
69 + if(scope_contexts_sp && !simple_pattern_matches_string(scope_contexts_sp, rc->id))
70 + continue;
71 +
72 + bool queryable_context = queryable_host;
73 + if(queryable_context && contexts_sp && !simple_pattern_matches_string(contexts_sp, rc->id))
74 + queryable_context = false;
75 +
76 + if(cb(data, (RRDCONTEXT_ACQUIRED *)rc_dfe.item, queryable_context))
77 + added++;
78 + }
79 + dfe_done(rc);
80 + }
81 +
82 + return added;
83 +}
84 +
database/contexts/query_target.c new
+1026
@@ -0,0 +1,1026 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +static void query_metric_release(QUERY_METRIC *qm);
6 +static void query_dimension_release(QUERY_DIMENSION *qd);
7 +static void query_instance_release(QUERY_INSTANCE *qi);
8 +static void query_context_release(QUERY_CONTEXT *qc);
9 +static void query_node_release(QUERY_NODE *qn);
10 +
11 +static __thread QUERY_TARGET thread_query_target = {};
12 +
13 +// ----------------------------------------------------------------------------
14 +// query API
15 +
16 +typedef struct query_target_locals {
17 + time_t start_s;
18 +
19 + QUERY_TARGET *qt;
20 +
21 + RRDSET *st;
22 +
23 + const char *scope_nodes;
24 + const char *scope_contexts;
25 +
26 + const char *nodes;
27 + const char *contexts;
28 + const char *charts;
29 + const char *dimensions;
30 + const char *chart_label_key;
31 + const char *labels;
32 + const char *alerts;
33 +
34 + long long after;
35 + long long before;
36 + bool match_ids;
37 + bool match_names;
38 +
39 + size_t metrics_skipped_due_to_not_matching_timeframe;
40 +
41 + char host_uuid_buffer[UUID_STR_LEN];
42 + QUERY_NODE *qn; // temp to pass on callbacks, ignore otherwise - no need to free
43 +} QUERY_TARGET_LOCALS;
44 +
45 +void query_target_release(QUERY_TARGET *qt) {
46 + if(unlikely(!qt || !qt->used)) return;
47 +
48 + simple_pattern_free(qt->nodes.scope_pattern);
49 + qt->nodes.scope_pattern = NULL;
50 +
51 + simple_pattern_free(qt->nodes.pattern);
52 + qt->nodes.pattern = NULL;
53 +
54 + simple_pattern_free(qt->contexts.scope_pattern);
55 + qt->contexts.scope_pattern = NULL;
56 +
57 + simple_pattern_free(qt->contexts.pattern);
58 + qt->contexts.pattern = NULL;
59 +
60 + simple_pattern_free(qt->instances.pattern);
61 + qt->instances.pattern = NULL;
62 +
63 + simple_pattern_free(qt->instances.chart_label_key_pattern);
64 + qt->instances.chart_label_key_pattern = NULL;
65 +
66 + simple_pattern_free(qt->instances.labels_pattern);
67 + qt->instances.labels_pattern = NULL;
68 +
69 + simple_pattern_free(qt->query.pattern);
70 + qt->query.pattern = NULL;
71 +
72 + // release the query
73 + for(size_t i = 0, used = qt->query.used; i < used ;i++) {
74 + QUERY_METRIC *qm = query_metric(qt, i);
75 + query_metric_release(qm);
76 + }
77 + qt->query.used = 0;
78 +
79 + // release the dimensions
80 + for(size_t i = 0, used = qt->dimensions.used; i < used ; i++) {
81 + QUERY_DIMENSION *qd = query_dimension(qt, i);
82 + query_dimension_release(qd);
83 + }
84 + qt->dimensions.used = 0;
85 +
86 + // release the instances
87 + for(size_t i = 0, used = qt->instances.used; i < used ;i++) {
88 + QUERY_INSTANCE *qi = query_instance(qt, i);
89 + query_instance_release(qi);
90 + }
91 + qt->instances.used = 0;
92 +
93 + // release the contexts
94 + for(size_t i = 0, used = qt->contexts.used; i < used ;i++) {
95 + QUERY_CONTEXT *qc = query_context(qt, i);
96 + rrdcontext_release(qc->rca);
97 + qc->rca = NULL;
98 + }
99 + qt->contexts.used = 0;
100 +
101 + // release the nodes
102 + for(size_t i = 0, used = qt->nodes.used; i < used ; i++) {
103 + QUERY_NODE *qn = query_node(qt, i);
104 + query_node_release(qn);
105 + }
106 + qt->nodes.used = 0;
107 +
108 + qt->db.minimum_latest_update_every_s = 0;
109 + qt->db.first_time_s = 0;
110 + qt->db.last_time_s = 0;
111 +
112 + qt->group_by.used = 0;
113 +
114 + qt->id[0] = '\0';
115 +
116 + qt->used = false;
117 +}
118 +void query_target_free(void) {
119 + QUERY_TARGET *qt = &thread_query_target;
120 +
121 + if(qt->used)
122 + query_target_release(qt);
123 +
124 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->query.size * sizeof(QUERY_METRIC), __ATOMIC_RELAXED);
125 + freez(qt->query.array);
126 + qt->query.array = NULL;
127 + qt->query.size = 0;
128 +
129 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->dimensions.size * sizeof(RRDMETRIC_ACQUIRED *), __ATOMIC_RELAXED);
130 + freez(qt->dimensions.array);
131 + qt->dimensions.array = NULL;
132 + qt->dimensions.size = 0;
133 +
134 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->instances.size * sizeof(RRDINSTANCE_ACQUIRED *), __ATOMIC_RELAXED);
135 + freez(qt->instances.array);
136 + qt->instances.array = NULL;
137 + qt->instances.size = 0;
138 +
139 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->contexts.size * sizeof(RRDCONTEXT_ACQUIRED *), __ATOMIC_RELAXED);
140 + freez(qt->contexts.array);
141 + qt->contexts.array = NULL;
142 + qt->contexts.size = 0;
143 +
144 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->nodes.size * sizeof(RRDHOST *), __ATOMIC_RELAXED);
145 + freez(qt->nodes.array);
146 + qt->nodes.array = NULL;
147 + qt->nodes.size = 0;
148 +}
149 +
150 +#define query_target_retention_matches_query(qt, first_entry_s, last_entry_s, update_every_s) \
151 + (((first_entry_s) - ((update_every_s) * 2) <= (qt)->window.before) && \
152 + ((last_entry_s) + ((update_every_s) * 2) >= (qt)->window.after))
153 +
154 +
155 +static inline void query_metric_release(QUERY_METRIC *qm) {
156 + // reset the plans
157 + for(size_t p = 0; p < qm->plan.used; p++) {
158 + internal_fatal(qm->plan.array[p].initialized &&
159 + !qm->plan.array[p].finalized,
160 + "QUERY: left-over initialized plan");
161 +
162 + qm->plan.array[p].initialized = false;
163 + qm->plan.array[p].finalized = false;
164 + }
165 + qm->plan.used = 0;
166 +
167 + // reset the tiers
168 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
169 + if(qm->tiers[tier].db_metric_handle) {
170 + STORAGE_ENGINE *eng = qm->tiers[tier].eng;
171 + eng->api.metric_release(qm->tiers[tier].db_metric_handle);
172 + qm->tiers[tier].db_metric_handle = NULL;
173 + qm->tiers[tier].eng = NULL;
174 + }
175 + }
176 +}
177 +
178 +static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CONTEXT *qc,
179 + QUERY_INSTANCE *qi, size_t qd_slot, RRDMETRIC *rm, RRDR_DIMENSION_FLAGS options) {
180 + QUERY_TARGET *qt = qtl->qt;
181 + RRDINSTANCE *ri = rm->ri;
182 +
183 + time_t common_first_time_s = 0;
184 + time_t common_last_time_s = 0;
185 + time_t common_update_every_s = 0;
186 + size_t tiers_added = 0;
187 +
188 + struct {
189 + STORAGE_ENGINE *eng;
190 + STORAGE_METRIC_HANDLE *db_metric_handle;
191 + time_t db_first_time_s;
192 + time_t db_last_time_s;
193 + time_t db_update_every_s;
194 + } tier_retention[storage_tiers];
195 +
196 + for (size_t tier = 0; tier < storage_tiers; tier++) {
197 + STORAGE_ENGINE *eng = qn->rrdhost->db[tier].eng;
198 + tier_retention[tier].eng = eng;
199 + tier_retention[tier].db_update_every_s = (time_t) (qn->rrdhost->db[tier].tier_grouping * ri->update_every_s);
200 +
201 + if(rm->rrddim && rm->rrddim->tiers[tier].db_metric_handle)
202 + tier_retention[tier].db_metric_handle = eng->api.metric_dup(rm->rrddim->tiers[tier].db_metric_handle);
203 + else
204 + tier_retention[tier].db_metric_handle = eng->api.metric_get(qn->rrdhost->db[tier].instance, &rm->uuid);
205 +
206 + if(tier_retention[tier].db_metric_handle) {
207 + tier_retention[tier].db_first_time_s = tier_retention[tier].eng->api.query_ops.oldest_time_s(tier_retention[tier].db_metric_handle);
208 + tier_retention[tier].db_last_time_s = tier_retention[tier].eng->api.query_ops.latest_time_s(tier_retention[tier].db_metric_handle);
209 +
210 + if(!common_first_time_s)
211 + common_first_time_s = tier_retention[tier].db_first_time_s;
212 + else if(tier_retention[tier].db_first_time_s)
213 + common_first_time_s = MIN(common_first_time_s, tier_retention[tier].db_first_time_s);
214 +
215 + if(!common_last_time_s)
216 + common_last_time_s = tier_retention[tier].db_last_time_s;
217 + else
218 + common_last_time_s = MAX(common_last_time_s, tier_retention[tier].db_last_time_s);
219 +
220 + if(!common_update_every_s)
221 + common_update_every_s = tier_retention[tier].db_update_every_s;
222 + else if(tier_retention[tier].db_update_every_s)
223 + common_update_every_s = MIN(common_update_every_s, tier_retention[tier].db_update_every_s);
224 +
225 + tiers_added++;
226 + }
227 + else {
228 + tier_retention[tier].db_first_time_s = 0;
229 + tier_retention[tier].db_last_time_s = 0;
230 + tier_retention[tier].db_update_every_s = 0;
231 + }
232 + }
233 +
234 + for (size_t tier = 0; tier < storage_tiers; tier++) {
235 + if(!qt->db.tiers[tier].update_every || (tier_retention[tier].db_update_every_s && tier_retention[tier].db_update_every_s < qt->db.tiers[tier].update_every))
236 + qt->db.tiers[tier].update_every = tier_retention[tier].db_update_every_s;
237 +
238 + if(!qt->db.tiers[tier].retention.first_time_s || (tier_retention[tier].db_first_time_s && tier_retention[tier].db_first_time_s < qt->db.tiers[tier].retention.first_time_s))
239 + qt->db.tiers[tier].retention.first_time_s = tier_retention[tier].db_first_time_s;
240 +
241 + if(!qt->db.tiers[tier].retention.last_time_s || (tier_retention[tier].db_last_time_s && tier_retention[tier].db_last_time_s > qt->db.tiers[tier].retention.last_time_s))
242 + qt->db.tiers[tier].retention.last_time_s = tier_retention[tier].db_last_time_s;
243 + }
244 +
245 + bool timeframe_matches =
246 + (tiers_added &&
247 + query_target_retention_matches_query(qt, common_first_time_s, common_last_time_s, common_update_every_s))
248 + ? true : false;
249 +
250 + if(timeframe_matches) {
251 + if(ri->rrdset)
252 + ri->rrdset->last_accessed_time_s = qtl->start_s;
253 +
254 + if (qt->query.used == qt->query.size) {
255 + size_t old_mem = qt->query.size * sizeof(*qt->query.array);
256 + qt->query.size = (qt->query.size) ? qt->query.size * 2 : 1;
257 + size_t new_mem = qt->query.size * sizeof(*qt->query.array);
258 + qt->query.array = reallocz(qt->query.array, new_mem);
259 +
260 + __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
261 + }
262 + QUERY_METRIC *qm = &qt->query.array[qt->query.used++];
263 + memset(qm, 0, sizeof(*qm));
264 +
265 + qm->status = options;
266 +
267 + qm->link.query_host_id = qn->slot;
268 + qm->link.query_context_id = qc->slot;
269 + qm->link.query_instance_id = qi->slot;
270 + qm->link.query_dimension_id = qd_slot;
271 +
272 + if (!qt->db.first_time_s || common_first_time_s < qt->db.first_time_s)
273 + qt->db.first_time_s = common_first_time_s;
274 +
275 + if (!qt->db.last_time_s || common_last_time_s > qt->db.last_time_s)
276 + qt->db.last_time_s = common_last_time_s;
277 +
278 + for (size_t tier = 0; tier < storage_tiers; tier++) {
279 + qm->tiers[tier].eng = tier_retention[tier].eng;
280 + qm->tiers[tier].db_metric_handle = tier_retention[tier].db_metric_handle;
281 + qm->tiers[tier].db_first_time_s = tier_retention[tier].db_first_time_s;
282 + qm->tiers[tier].db_last_time_s = tier_retention[tier].db_last_time_s;
283 + qm->tiers[tier].db_update_every_s = tier_retention[tier].db_update_every_s;
284 + }
285 +
286 + return true;
287 + }
288 + else {
289 + // cleanup anything we allocated to the retention we will not use
290 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
291 + if (tier_retention[tier].db_metric_handle)
292 + tier_retention[tier].eng->api.metric_release(tier_retention[tier].db_metric_handle);
293 + }
294 +
295 + return false;
296 + }
297 +}
298 +
299 +static inline bool rrdmetric_retention_matches_query(QUERY_TARGET *qt, RRDMETRIC *rm, time_t now_s) {
300 + time_t first_time_s = rm->first_time_s;
301 + time_t last_time_s = rrd_flag_is_collected(rm) ? now_s : rm->last_time_s;
302 + time_t update_every_s = rm->ri->update_every_s;
303 + return query_target_retention_matches_query(qt, first_time_s, last_time_s, update_every_s);
304 +}
305 +
306 +static inline void query_dimension_release(QUERY_DIMENSION *qd) {
307 + rrdmetric_release(qd->rma);
308 + qd->rma = NULL;
309 +}
310 +
311 +static QUERY_DIMENSION *query_dimension_allocate(QUERY_TARGET *qt, RRDMETRIC_ACQUIRED *rma, QUERY_STATUS status) {
312 + if(qt->dimensions.used == qt->dimensions.size) {
313 + size_t old_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
314 + qt->dimensions.size = (qt->dimensions.size) ? qt->dimensions.size * 2 : 1;
315 + size_t new_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
316 + qt->dimensions.array = reallocz(qt->dimensions.array, new_mem);
317 +
318 + __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
319 + }
320 + QUERY_DIMENSION *qd = &qt->dimensions.array[qt->dimensions.used];
321 + memset(qd, 0, sizeof(*qd));
322 +
323 + qd->slot = qt->dimensions.used++;
324 + qd->rma = rrdmetric_acquired_dup(rma);
325 + qd->status = status;
326 +
327 + return qd;
328 +}
329 +
330 +static bool query_dimension_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi,
331 + RRDMETRIC_ACQUIRED *rma, bool queryable_instance, size_t *metrics_added) {
332 + QUERY_TARGET *qt = qtl->qt;
333 +
334 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
335 + if(rrd_flag_is_deleted(rm))
336 + return false;
337 +
338 + QUERY_STATUS status = QUERY_STATUS_NONE;
339 +
340 + bool undo = false;
341 + if(!queryable_instance) {
342 + if(rrdmetric_retention_matches_query(qt, rm, qtl->start_s)) {
343 + qi->metrics.excluded++;
344 + qc->metrics.excluded++;
345 + qn->metrics.excluded++;
346 + status |= QUERY_STATUS_EXCLUDED;
347 + }
348 + else
349 + undo = true;
350 + }
351 + else {
352 + RRDR_DIMENSION_FLAGS options = RRDR_DIMENSION_DEFAULT;
353 + bool needed = false;
354 +
355 + if (qt->query.pattern) {
356 + // the user asked for specific dimensions
357 +
358 + if ((qtl->match_ids && simple_pattern_matches_string(qt->query.pattern, rm->id)) ||
359 + (qtl->match_names && rm->name != rm->id &&
360 + simple_pattern_matches_string(qt->query.pattern, rm->name))) {
361 + // the user asked for this dimension
362 + needed = true;
363 + options |= RRDR_DIMENSION_SELECTED | RRDR_DIMENSION_NONZERO;
364 + }
365 + else {
366 + // the user selection does not match this dimension
367 + // but, we may still need to query it
368 +
369 + if (qt->request.options & RRDR_OPTION_PERCENTAGE) {
370 + // this is percentage calculation
371 + // so, we need this dimension to calculate the percentage
372 + needed = true;
373 + options |= RRDR_DIMENSION_HIDDEN;
374 + }
375 + else {
376 + // the user did not select this dimension
377 + // and the calculation is not percentage
378 + // so, no need to query it
379 + ;
380 + }
381 + }
382 + }
383 + else {
384 + // we don't have a dimensions pattern
385 + // so this is a selected dimension
386 + // if it is not hidden
387 +
388 + if(rrd_flag_check(rm, RRD_FLAG_HIDDEN) || (rm->rrddim && rrddim_option_check(rm->rrddim, RRDDIM_OPTION_HIDDEN))) {
389 + // this is a hidden dimension
390 + // we don't need to query it
391 + status |= QUERY_STATUS_DIMENSION_HIDDEN;
392 + options |= RRDR_DIMENSION_HIDDEN;
393 +
394 + if (qt->request.options & RRDR_OPTION_PERCENTAGE) {
395 + // this is percentage calculation
396 + // so, we need this dimension to calculate the percentage
397 + needed = true;
398 + }
399 + }
400 + else {
401 + // this is a not hidden dimension
402 + // and the user did not provide any selection for dimensions
403 + // so, we need to query it
404 + needed = true;
405 + options |= RRDR_DIMENSION_SELECTED;
406 + }
407 + }
408 +
409 + if (needed) {
410 + if(query_metric_add(qtl, qn, qc, qi, qt->dimensions.used, rm, options)) {
411 + (*metrics_added)++;
412 +
413 + qi->metrics.selected++;
414 + qc->metrics.selected++;
415 + qn->metrics.selected++;
416 + }
417 + else {
418 + undo = true;
419 + qtl->metrics_skipped_due_to_not_matching_timeframe++;
420 + }
421 + }
422 + else if(rrdmetric_retention_matches_query(qt, rm, qtl->start_s)) {
423 + qi->metrics.excluded++;
424 + qc->metrics.excluded++;
425 + qn->metrics.excluded++;
426 + status |= QUERY_STATUS_EXCLUDED;
427 + }
428 + else
429 + undo = true;
430 + }
431 +
432 + if(undo)
433 + return false;
434 +
435 + query_dimension_allocate(qt, rma, status);
436 + return true;
437 +}
438 +
439 +static inline STRING *rrdinstance_create_id_fqdn_v1(RRDINSTANCE_ACQUIRED *ria) {
440 + if(unlikely(!ria))
441 + return NULL;
442 +
443 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
444 + return string_dup(ri->id);
445 +}
446 +
447 +static inline STRING *rrdinstance_create_name_fqdn_v1(RRDINSTANCE_ACQUIRED *ria) {
448 + if(unlikely(!ria))
449 + return NULL;
450 +
451 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
452 + return string_dup(ri->name);
453 +}
454 +
455 +static inline STRING *rrdinstance_create_id_fqdn_v2(RRDINSTANCE_ACQUIRED *ria) {
456 + if(unlikely(!ria))
457 + return NULL;
458 +
459 + char buffer[RRD_ID_LENGTH_MAX + 1];
460 +
461 + RRDHOST *host = rrdinstance_acquired_rrdhost(ria);
462 + snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s@%s", rrdinstance_acquired_id(ria), host->machine_guid);
463 + return string_strdupz(buffer);
464 +}
465 +
466 +static inline STRING *rrdinstance_create_name_fqdn_v2(RRDINSTANCE_ACQUIRED *ria) {
467 + if(unlikely(!ria))
468 + return NULL;
469 +
470 + char buffer[RRD_ID_LENGTH_MAX + 1];
471 +
472 + RRDHOST *host = rrdinstance_acquired_rrdhost(ria);
473 + snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s@%s", rrdinstance_acquired_name(ria), rrdhost_hostname(host));
474 + return string_strdupz(buffer);
475 +}
476 +
477 +inline STRING *query_instance_id_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi) {
478 + if(!qi->id_fqdn) {
479 + if (qt->request.version <= 1)
480 + qi->id_fqdn = rrdinstance_create_id_fqdn_v1(qi->ria);
481 + else
482 + qi->id_fqdn = rrdinstance_create_id_fqdn_v2(qi->ria);
483 + }
484 +
485 + return qi->id_fqdn;
486 +}
487 +
488 +inline STRING *query_instance_name_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi) {
489 + if(!qi->name_fqdn) {
490 + if (qt->request.version <= 1)
491 + qi->name_fqdn = rrdinstance_create_name_fqdn_v1(qi->ria);
492 + else
493 + qi->name_fqdn = rrdinstance_create_name_fqdn_v2(qi->ria);
494 + }
495 +
496 + return qi->name_fqdn;
497 +}
498 +
499 +RRDSET *rrdinstance_acquired_rrdset(RRDINSTANCE_ACQUIRED *ria) {
500 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
501 + return ri->rrdset;
502 +}
503 +
504 +const char *rrdcontext_acquired_units(RRDCONTEXT_ACQUIRED *rca) {
505 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
506 + return string2str(rc->units);
507 +}
508 +
509 +RRDSET_TYPE rrdcontext_acquired_chart_type(RRDCONTEXT_ACQUIRED *rca) {
510 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
511 + return rc->chart_type;
512 +}
513 +
514 +const char *rrdcontext_acquired_title(RRDCONTEXT_ACQUIRED *rca) {
515 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
516 + return string2str(rc->title);
517 +}
518 +
519 +static void query_target_eval_instance_rrdcalc(QUERY_TARGET_LOCALS *qtl __maybe_unused,
520 + QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi) {
521 + RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
522 + if (st) {
523 + netdata_rwlock_rdlock(&st->alerts.rwlock);
524 + for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
525 + switch(rc->status) {
526 + case RRDCALC_STATUS_CLEAR:
527 + qi->alerts.clear++;
528 + qc->alerts.clear++;
529 + qn->alerts.clear++;
530 + break;
531 +
532 + case RRDCALC_STATUS_WARNING:
533 + qi->alerts.warning++;
534 + qc->alerts.warning++;
535 + qn->alerts.warning++;
536 + break;
537 +
538 + case RRDCALC_STATUS_CRITICAL:
539 + qi->alerts.critical++;
540 + qc->alerts.critical++;
541 + qn->alerts.critical++;
542 + break;
543 +
544 + default:
545 + case RRDCALC_STATUS_UNINITIALIZED:
546 + case RRDCALC_STATUS_UNDEFINED:
547 + case RRDCALC_STATUS_REMOVED:
548 + qi->alerts.other++;
549 + qc->alerts.other++;
550 + qn->alerts.other++;
551 + break;
552 + }
553 + }
554 + netdata_rwlock_unlock(&st->alerts.rwlock);
555 + }
556 +}
557 +
558 +static bool query_target_match_alert_pattern(QUERY_INSTANCE *qi, SIMPLE_PATTERN *pattern) {
559 + if(!pattern)
560 + return true;
561 +
562 + RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
563 + if (!st)
564 + return false;
565 +
566 + BUFFER *wb = NULL;
567 + bool matched = false;
568 + netdata_rwlock_rdlock(&st->alerts.rwlock);
569 + if (st->alerts.base) {
570 + for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
571 + if(simple_pattern_matches_string(pattern, rc->name)) {
572 + matched = true;
573 + break;
574 + }
575 +
576 + if(!wb)
577 + wb = buffer_create(0, NULL);
578 + else
579 + buffer_flush(wb);
580 +
581 + buffer_fast_strcat(wb, string2str(rc->name), string_strlen(rc->name));
582 + buffer_fast_strcat(wb, ":", 1);
583 + buffer_strcat(wb, rrdcalc_status2string(rc->status));
584 +
585 + if(simple_pattern_matches_buffer(pattern, wb)) {
586 + matched = true;
587 + break;
588 + }
589 + }
590 + }
591 + netdata_rwlock_unlock(&st->alerts.rwlock);
592 +
593 + buffer_free(wb);
594 + return matched;
595 +}
596 +
597 +static inline void query_instance_release(QUERY_INSTANCE *qi) {
598 + rrdinstance_release(qi->ria);
599 + qi->ria = NULL;
600 +
601 + string_freez(qi->id_fqdn);
602 + qi->id_fqdn = NULL;
603 +
604 + string_freez(qi->name_fqdn);
605 + qi->name_fqdn = NULL;
606 +}
607 +
608 +static inline QUERY_INSTANCE *query_instance_allocate(QUERY_TARGET *qt, RRDINSTANCE_ACQUIRED *ria, size_t qn_slot) {
609 + if(qt->instances.used == qt->instances.size) {
610 + size_t old_mem = qt->instances.size * sizeof(*qt->instances.array);
611 + qt->instances.size = (qt->instances.size) ? qt->instances.size * 2 : 1;
612 + size_t new_mem = qt->instances.size * sizeof(*qt->instances.array);
613 + qt->instances.array = reallocz(qt->instances.array, new_mem);
614 +
615 + __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
616 + }
617 + QUERY_INSTANCE *qi = &qt->instances.array[qt->instances.used];
618 + memset(qi, 0, sizeof(*qi));
619 +
620 + qi->slot = qt->instances.used;
621 + qt->instances.used++;
622 + qi->ria = rrdinstance_acquired_dup(ria);
623 + qi->query_host_id = qn_slot;
624 +
625 + return qi;
626 +}
627 +
628 +static bool query_instance_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CONTEXT *qc,
629 + RRDINSTANCE_ACQUIRED *ria, bool queryable_instance, bool filter_instances) {
630 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
631 + if(rrd_flag_is_deleted(ri))
632 + return false;
633 +
634 + QUERY_TARGET *qt = qtl->qt;
635 + QUERY_INSTANCE *qi = query_instance_allocate(qt, ria, qn->slot);
636 +
637 + if(qt->db.minimum_latest_update_every_s == 0 || ri->update_every_s < qt->db.minimum_latest_update_every_s)
638 + qt->db.minimum_latest_update_every_s = ri->update_every_s;
639 +
640 + if(queryable_instance && filter_instances) {
641 + queryable_instance = false;
642 + if(!qt->instances.pattern
643 + || (qtl->match_ids && simple_pattern_matches_string(qt->instances.pattern, ri->id))
644 + || (qtl->match_names && ri->name != ri->id && simple_pattern_matches_string(qt->instances.pattern, ri->name))
645 + || (qtl->match_ids && simple_pattern_matches_string(qt->instances.pattern, query_instance_id_fqdn(qt, qi)))
646 + || (qtl->match_names && simple_pattern_matches_string(qt->instances.pattern, query_instance_name_fqdn(qt, qi)))
647 + )
648 + queryable_instance = true;
649 + }
650 +
651 + if(queryable_instance) {
652 + if ((qt->instances.chart_label_key_pattern && !rrdlabels_match_simple_pattern_parsed(
653 + ri->rrdlabels, qt->instances.chart_label_key_pattern,'\0', NULL)) ||
654 + (qt->instances.labels_pattern && !rrdlabels_match_simple_pattern_parsed(
655 + ri->rrdlabels, qt->instances.labels_pattern, ':', NULL)))
656 + queryable_instance = false;
657 + }
658 +
659 + if(queryable_instance) {
660 + if(qt->instances.alerts_pattern && !query_target_match_alert_pattern(qi, qt->instances.alerts_pattern))
661 + queryable_instance = false;
662 + }
663 +
664 + if(queryable_instance && qt->request.version >= 2)
665 + query_target_eval_instance_rrdcalc(qtl, qn, qc, qi);
666 +
667 + size_t dimensions_added = 0, metrics_added = 0;
668 +
669 + if(unlikely(qt->request.rma)) {
670 + if(query_dimension_add(qtl, qn, qc, qi, qt->request.rma, queryable_instance, &metrics_added))
671 + dimensions_added++;
672 + }
673 + else {
674 + RRDMETRIC *rm;
675 + dfe_start_read(ri->rrdmetrics, rm) {
676 + if(query_dimension_add(qtl, qn, qc, qi, (RRDMETRIC_ACQUIRED *) rm_dfe.item,
677 + queryable_instance, &metrics_added))
678 + dimensions_added++;
679 + }
680 + dfe_done(rm);
681 + }
682 +
683 + if(!dimensions_added) {
684 + qt->instances.used--;
685 + query_instance_release(qi);
686 + return false;
687 + }
688 + else {
689 + if(metrics_added) {
690 + qc->instances.selected++;
691 + qn->instances.selected++;
692 + }
693 + else {
694 + qc->instances.excluded++;
695 + qn->instances.excluded++;
696 + }
697 + }
698 +
699 + return true;
700 +}
701 +
702 +static inline void query_context_release(QUERY_CONTEXT *qc) {
703 + rrdcontext_release(qc->rca);
704 + qc->rca = NULL;
705 +}
706 +
707 +static inline QUERY_CONTEXT *query_context_allocate(QUERY_TARGET *qt, RRDCONTEXT_ACQUIRED *rca) {
708 + if(qt->contexts.used == qt->contexts.size) {
709 + size_t old_mem = qt->contexts.size * sizeof(*qt->contexts.array);
710 + qt->contexts.size = (qt->contexts.size) ? qt->contexts.size * 2 : 1;
711 + size_t new_mem = qt->contexts.size * sizeof(*qt->contexts.array);
712 + qt->contexts.array = reallocz(qt->contexts.array, new_mem);
713 +
714 + __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
715 + }
716 + QUERY_CONTEXT *qc = &qt->contexts.array[qt->contexts.used];
717 + memset(qc, 0, sizeof(*qc));
718 + qc->slot = qt->contexts.used++;
719 + qc->rca = rrdcontext_acquired_dup(rca);
720 +
721 + return qc;
722 +}
723 +
724 +static bool query_context_add(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context) {
725 + QUERY_TARGET_LOCALS *qtl = data;
726 +
727 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
728 + if(rrd_flag_is_deleted(rc))
729 + return false;
730 +
731 + QUERY_NODE *qn = qtl->qn;
732 + QUERY_TARGET *qt = qtl->qt;
733 + QUERY_CONTEXT *qc = query_context_allocate(qt, rca);
734 +
735 + size_t added = 0;
736 + if(unlikely(qt->request.ria)) {
737 + if(query_instance_add(qtl, qn, qc, qt->request.ria, queryable_context, false))
738 + added++;
739 + }
740 + else if(unlikely(qtl->st && qtl->st->rrdcontext == rca && qtl->st->rrdinstance)) {
741 + if(query_instance_add(qtl, qn, qc, qtl->st->rrdinstance, queryable_context, false))
742 + added++;
743 + }
744 + else {
745 + RRDINSTANCE *ri;
746 + dfe_start_read(rc->rrdinstances, ri) {
747 + if(query_instance_add(qtl, qn, qc, (RRDINSTANCE_ACQUIRED *) ri_dfe.item, queryable_context, true))
748 + added++;
749 + }
750 + dfe_done(ri);
751 + }
752 +
753 + if(!added) {
754 + query_context_release(qc);
755 + qt->contexts.used--;
756 + return false;
757 + }
758 +
759 + return true;
760 +}
761 +
762 +static inline void query_node_release(QUERY_NODE *qn) {
763 + qn->rrdhost = NULL;
764 +}
765 +
766 +static inline QUERY_NODE *query_node_allocate(QUERY_TARGET *qt, RRDHOST *host) {
767 + if(qt->nodes.used == qt->nodes.size) {
768 + size_t old_mem = qt->nodes.size * sizeof(*qt->nodes.array);
769 + qt->nodes.size = (qt->nodes.size) ? qt->nodes.size * 2 : 1;
770 + size_t new_mem = qt->nodes.size * sizeof(*qt->nodes.array);
771 + qt->nodes.array = reallocz(qt->nodes.array, new_mem);
772 +
773 + __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
774 + }
775 + QUERY_NODE *qn = &qt->nodes.array[qt->nodes.used];
776 + memset(qn, 0, sizeof(*qn));
777 +
778 + qn->slot = qt->nodes.used++;
779 + qn->rrdhost = host;
780 +
781 + return qn;
782 +}
783 +
784 +static bool query_node_add(void *data, RRDHOST *host, bool queryable_host) {
785 + QUERY_TARGET_LOCALS *qtl = data;
786 + QUERY_TARGET *qt = qtl->qt;
787 + QUERY_NODE *qn = query_node_allocate(qt, host);
788 +
789 + if(host->node_id) {
790 + if(!qtl->host_uuid_buffer[0])
791 + uuid_unparse_lower(*host->node_id, qn->node_id);
792 + else
793 + memcpy(qn->node_id, qtl->host_uuid_buffer, sizeof(qn->node_id));
794 + }
795 + else
796 + qn->node_id[0] = '\0';
797 +
798 + // is the chart given valid?
799 + if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
800 + error("QUERY TARGET: RRDSET '%s' given, but it is not linked to rrdcontext structures. Linking it now.", rrdset_name(qtl->st));
801 + rrdinstance_from_rrdset(qtl->st);
802 +
803 + if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
804 + error("QUERY TARGET: RRDSET '%s' given, but failed to be linked to rrdcontext structures. Switching to context query.",
805 + rrdset_name(qtl->st));
806 +
807 + if (!is_valid_sp(qtl->charts))
808 + qtl->charts = rrdset_name(qtl->st);
809 +
810 + qtl->st = NULL;
811 + }
812 + }
813 +
814 + qtl->qn = qn;
815 +
816 + size_t added = 0;
817 + if(unlikely(qt->request.rca)) {
818 + if(query_context_add(qtl, qt->request.rca, true))
819 + added++;
820 + }
821 + else if(unlikely(qtl->st)) {
822 + // single chart data queries
823 + if(query_context_add(qtl, qtl->st->rrdcontext, true))
824 + added++;
825 + }
826 + else {
827 + // context pattern queries
828 + added = query_scope_foreach_context(
829 + host, qtl->scope_contexts,
830 + qt->contexts.scope_pattern, qt->contexts.pattern,
831 + query_context_add, queryable_host, qtl);
832 + }
833 +
834 + qtl->qn = NULL;
835 +
836 + if(!added) {
837 + query_node_release(qn);
838 + qt->nodes.used--;
839 + return false;
840 + }
841 +
842 + return true;
843 +}
844 +
845 +void query_target_generate_name(QUERY_TARGET *qt) {
846 + char options_buffer[100 + 1];
847 + web_client_api_request_v1_data_options_to_string(options_buffer, 100, qt->request.options);
848 +
849 + char resampling_buffer[20 + 1] = "";
850 + if(qt->request.resampling_time > 1)
851 + snprintfz(resampling_buffer, 20, "/resampling:%lld", (long long)qt->request.resampling_time);
852 +
853 + char tier_buffer[20 + 1] = "";
854 + if(qt->request.options & RRDR_OPTION_SELECTED_TIER)
855 + snprintfz(tier_buffer, 20, "/tier:%zu", qt->request.tier);
856 +
857 + if(qt->request.st)
858 + snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "chart://hosts:%s/instance:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
859 + , rrdhost_hostname(qt->request.st->rrdhost)
860 + , rrdset_name(qt->request.st)
861 + , (qt->request.dimensions) ? qt->request.dimensions : "*"
862 + , (long long)qt->request.after
863 + , (long long)qt->request.before
864 + , qt->request.points
865 + , time_grouping_tostring(qt->request.time_group_method)
866 + , qt->request.time_group_options ? qt->request.time_group_options : ""
867 + , options_buffer
868 + , resampling_buffer
869 + , tier_buffer
870 + );
871 + else if(qt->request.host && qt->request.rca && qt->request.ria && qt->request.rma)
872 + snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "metric://hosts:%s/context:%s/instance:%s/dimension:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
873 + , rrdhost_hostname(qt->request.host)
874 + , rrdcontext_acquired_id(qt->request.rca)
875 + , rrdinstance_acquired_id(qt->request.ria)
876 + , rrdmetric_acquired_id(qt->request.rma)
877 + , (long long)qt->request.after
878 + , (long long)qt->request.before
879 + , qt->request.points
880 + , time_grouping_tostring(qt->request.time_group_method)
881 + , qt->request.time_group_options ? qt->request.time_group_options : ""
882 + , options_buffer
883 + , resampling_buffer
884 + , tier_buffer
885 + );
886 + else if(qt->request.version >= 2)
887 + snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "data_v2://scope_nodes:%s/scope_contexts:%s/nodes:%s/contexts:%s/instances:%s/labels:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/time_group:%s%s/options:%s%s%s"
888 + , qt->request.scope_nodes ? qt->request.scope_nodes : "*"
889 + , qt->request.scope_contexts ? qt->request.scope_contexts : "*"
890 + , qt->request.nodes ? qt->request.nodes : "*"
891 + , (qt->request.contexts) ? qt->request.contexts : "*"
892 + , (qt->request.instances) ? qt->request.instances : "*"
893 + , (qt->request.labels) ? qt->request.labels : "*"
894 + , (qt->request.dimensions) ? qt->request.dimensions : "*"
895 + , (long long)qt->request.after
896 + , (long long)qt->request.before
897 + , qt->request.points
898 + , time_grouping_tostring(qt->request.time_group_method)
899 + , qt->request.time_group_options ? qt->request.time_group_options : ""
900 + , options_buffer
901 + , resampling_buffer
902 + , tier_buffer
903 + );
904 + else
905 + snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "context://hosts:%s/contexts:%s/instances:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
906 + , (qt->request.host) ? rrdhost_hostname(qt->request.host) : ((qt->request.nodes) ? qt->request.nodes : "*")
907 + , (qt->request.contexts) ? qt->request.contexts : "*"
908 + , (qt->request.instances) ? qt->request.instances : "*"
909 + , (qt->request.dimensions) ? qt->request.dimensions : "*"
910 + , (long long)qt->request.after
911 + , (long long)qt->request.before
912 + , qt->request.points
913 + , time_grouping_tostring(qt->request.time_group_method)
914 + , qt->request.time_group_options ? qt->request.time_group_options : ""
915 + , options_buffer
916 + , resampling_buffer
917 + , tier_buffer
918 + );
919 +
920 + json_fix_string(qt->id);
921 +}
922 +
923 +QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
924 + if(!service_running(ABILITY_DATA_QUERIES))
925 + return NULL;
926 +
927 + QUERY_TARGET *qt = &thread_query_target;
928 +
929 + if(qt->used)
930 + fatal("QUERY TARGET: this query target is already used (%zu queries made with this QUERY_TARGET so far).", qt->queries);
931 +
932 + qt->used = true;
933 + qt->queries++;
934 +
935 + if(!qtr->received_ut)
936 + qtr->received_ut = now_monotonic_usec();
937 +
938 + qt->timings.received_ut = qtr->received_ut;
939 +
940 + if(qtr->nodes && !qtr->scope_nodes)
941 + qtr->scope_nodes = qtr->nodes;
942 +
943 + if(qtr->contexts && !qtr->scope_contexts)
944 + qtr->scope_contexts = qtr->contexts;
945 +
946 + memset(&qt->db, 0, sizeof(qt->db));
947 + memset(&qt->query_stats, 0, sizeof(qt->query_stats));
948 +
949 + // copy the request into query_thread_target
950 + qt->request = *qtr;
951 +
952 + query_target_generate_name(qt);
953 + qt->window.after = qt->request.after;
954 + qt->window.before = qt->request.before;
955 + rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before);
956 +
957 + // prepare our local variables - we need these across all these functions
958 + QUERY_TARGET_LOCALS qtl = {
959 + .qt = qt,
960 + .start_s = now_realtime_sec(),
961 + .st = qt->request.st,
962 + .scope_nodes = qt->request.scope_nodes,
963 + .scope_contexts = qt->request.scope_contexts,
964 + .nodes = qt->request.nodes,
965 + .contexts = qt->request.contexts,
966 + .charts = qt->request.instances,
967 + .dimensions = qt->request.dimensions,
968 + .chart_label_key = qt->request.chart_label_key,
969 + .labels = qt->request.labels,
970 + .alerts = qt->request.alerts,
971 + };
972 +
973 + RRDHOST *host = qt->request.host;
974 +
975 + // prepare all the patterns
976 + qt->nodes.scope_pattern = string_to_simple_pattern(qtl.scope_nodes);
977 + qt->nodes.pattern = string_to_simple_pattern(qtl.nodes);
978 +
979 + qt->contexts.pattern = string_to_simple_pattern(qtl.contexts);
980 + qt->contexts.scope_pattern = string_to_simple_pattern(qtl.scope_contexts);
981 +
982 + qt->instances.pattern = string_to_simple_pattern(qtl.charts);
983 + qt->query.pattern = string_to_simple_pattern(qtl.dimensions);
984 + qt->instances.chart_label_key_pattern = string_to_simple_pattern(qtl.chart_label_key);
985 + qt->instances.labels_pattern = string_to_simple_pattern(qtl.labels);
986 + qt->instances.alerts_pattern = string_to_simple_pattern(qtl.alerts);
987 +
988 + qtl.match_ids = qt->request.options & RRDR_OPTION_MATCH_IDS;
989 + qtl.match_names = qt->request.options & RRDR_OPTION_MATCH_NAMES;
990 + if(likely(!qtl.match_ids && !qtl.match_names))
991 + qtl.match_ids = qtl.match_names = true;
992 +
993 + // verify that the chart belongs to the host we are interested
994 + if(qtl.st) {
995 + if (!host) {
996 + // It is NULL, set it ourselves.
997 + host = qtl.st->rrdhost;
998 + }
999 + else if (unlikely(host != qtl.st->rrdhost)) {
1000 + // Oops! A different host!
1001 + error("QUERY TARGET: RRDSET '%s' given does not belong to host '%s'. Switching query host to '%s'",
1002 + rrdset_name(qtl.st), rrdhost_hostname(host), rrdhost_hostname(qtl.st->rrdhost));
1003 + host = qtl.st->rrdhost;
1004 + }
1005 + }
1006 +
1007 + if(host) {
1008 + // single host query
1009 + qt->versions.contexts_hard_hash = dictionary_version(host->rrdctx.contexts);
1010 + qt->versions.contexts_soft_hash = dictionary_version(host->rrdctx.hub_queue);
1011 + query_node_add(&qtl, host, true);
1012 + qtl.nodes = rrdhost_hostname(host);
1013 + }
1014 + else
1015 + query_scope_foreach_host(qt->nodes.scope_pattern, qt->nodes.pattern, query_node_add, &qtl,
1016 + &qt->versions.contexts_hard_hash, &qt->versions.contexts_soft_hash,
1017 + qtl.host_uuid_buffer);
1018 +
1019 + // we need the available db retention for this call
1020 + // so it has to be done last
1021 + query_target_calculate_window(qt);
1022 +
1023 + qt->timings.preprocessed_ut = now_monotonic_usec();
1024 +
1025 + return qt;
1026 +}
database/contexts/rrdcontext.c new
+379
@@ -0,0 +1,379 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +// visualizing flags
7 +
8 +struct rrdcontext_reason rrdcontext_reasons[] = {
9 + // context related
10 + {RRD_FLAG_UPDATE_REASON_TRIGGERED, "triggered transition", 65 * USEC_PER_SEC },
11 + {RRD_FLAG_UPDATE_REASON_NEW_OBJECT, "object created", 65 * USEC_PER_SEC },
12 + {RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT, "object updated", 65 * USEC_PER_SEC },
13 + {RRD_FLAG_UPDATE_REASON_LOAD_SQL, "loaded from sql", 65 * USEC_PER_SEC },
14 + {RRD_FLAG_UPDATE_REASON_CHANGED_METADATA, "changed metadata", 65 * USEC_PER_SEC },
15 + {RRD_FLAG_UPDATE_REASON_ZERO_RETENTION, "has no retention", 65 * USEC_PER_SEC },
16 + {RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T, "updated first_time_t", 65 * USEC_PER_SEC },
17 + {RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T, "updated last_time_t", 65 * USEC_PER_SEC },
18 + {RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED, "stopped collected", 65 * USEC_PER_SEC },
19 + {RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED, "started collected", 5 * USEC_PER_SEC },
20 + {RRD_FLAG_UPDATE_REASON_UNUSED, "unused", 5 * USEC_PER_SEC },
21 +
22 + // not context related
23 + {RRD_FLAG_UPDATE_REASON_CHANGED_LINKING, "changed rrd link", 65 * USEC_PER_SEC },
24 + {RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, "child disconnected", 65 * USEC_PER_SEC },
25 + {RRD_FLAG_UPDATE_REASON_DB_ROTATION, "db rotation", 65 * USEC_PER_SEC },
26 + {RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION, "updated retention", 65 * USEC_PER_SEC },
27 +
28 + // terminator
29 + {0, NULL, 0 },
30 +};
31 +
32 +void rrd_reasons_to_buffer_json_array_items(RRD_FLAGS flags, BUFFER *wb) {
33 + for(int i = 0, added = 0; rrdcontext_reasons[i].name ; i++) {
34 + if (flags & rrdcontext_reasons[i].flag) {
35 + buffer_json_add_array_item_string(wb, rrdcontext_reasons[i].name);
36 + added++;
37 + }
38 + }
39 +}
40 +// ----------------------------------------------------------------------------
41 +// public API
42 +
43 +void rrdcontext_updated_rrddim(RRDDIM *rd) {
44 + rrdmetric_from_rrddim(rd);
45 +}
46 +
47 +void rrdcontext_removed_rrddim(RRDDIM *rd) {
48 + rrdmetric_rrddim_is_freed(rd);
49 +}
50 +
51 +void rrdcontext_updated_rrddim_algorithm(RRDDIM *rd) {
52 + rrdmetric_updated_rrddim_flags(rd);
53 +}
54 +
55 +void rrdcontext_updated_rrddim_multiplier(RRDDIM *rd) {
56 + rrdmetric_updated_rrddim_flags(rd);
57 +}
58 +
59 +void rrdcontext_updated_rrddim_divisor(RRDDIM *rd) {
60 + rrdmetric_updated_rrddim_flags(rd);
61 +}
62 +
63 +void rrdcontext_updated_rrddim_flags(RRDDIM *rd) {
64 + rrdmetric_updated_rrddim_flags(rd);
65 +}
66 +
67 +void rrdcontext_collected_rrddim(RRDDIM *rd) {
68 + rrdmetric_collected_rrddim(rd);
69 +}
70 +
71 +void rrdcontext_updated_rrdset(RRDSET *st) {
72 + rrdinstance_from_rrdset(st);
73 +}
74 +
75 +void rrdcontext_removed_rrdset(RRDSET *st) {
76 + rrdinstance_rrdset_is_freed(st);
77 +}
78 +
79 +void rrdcontext_updated_retention_rrdset(RRDSET *st) {
80 + rrdinstance_rrdset_has_updated_retention(st);
81 +}
82 +
83 +void rrdcontext_updated_rrdset_name(RRDSET *st) {
84 + rrdinstance_updated_rrdset_name(st);
85 +}
86 +
87 +void rrdcontext_updated_rrdset_flags(RRDSET *st) {
88 + rrdinstance_updated_rrdset_flags(st);
89 +}
90 +
91 +void rrdcontext_collected_rrdset(RRDSET *st) {
92 + rrdinstance_collected_rrdset(st);
93 +}
94 +
95 +void rrdcontext_host_child_connected(RRDHOST *host) {
96 + (void)host;
97 +
98 + // no need to do anything here
99 + ;
100 +}
101 +
102 +usec_t rrdcontext_next_db_rotation_ut = 0;
103 +void rrdcontext_db_rotation(void) {
104 + // called when the db rotates its database
105 + rrdcontext_next_db_rotation_ut = now_realtime_usec() + FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS * USEC_PER_SEC;
106 +}
107 +
108 +int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, uuid_t *store_uuid) {
109 + if(!st->rrdhost) return 1;
110 + if(!st->context) return 2;
111 +
112 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(st->rrdhost->rrdctx.contexts, string2str(st->context));
113 + if(!rca) return 3;
114 +
115 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
116 +
117 + RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_get_and_acquire_item(rc->rrdinstances, string2str(st->id));
118 + if(!ria) {
119 + rrdcontext_release(rca);
120 + return 4;
121 + }
122 +
123 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
124 +
125 + RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)dictionary_get_and_acquire_item(ri->rrdmetrics, id);
126 + if(!rma) {
127 + rrdinstance_release(ria);
128 + rrdcontext_release(rca);
129 + return 5;
130 + }
131 +
132 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
133 +
134 + uuid_copy(*store_uuid, rm->uuid);
135 +
136 + rrdmetric_release(rma);
137 + rrdinstance_release(ria);
138 + rrdcontext_release(rca);
139 + return 0;
140 +}
141 +
142 +int rrdcontext_find_chart_uuid(RRDSET *st, uuid_t *store_uuid) {
143 + if(!st->rrdhost) return 1;
144 + if(!st->context) return 2;
145 +
146 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(st->rrdhost->rrdctx.contexts, string2str(st->context));
147 + if(!rca) return 3;
148 +
149 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
150 +
151 + RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_get_and_acquire_item(rc->rrdinstances, string2str(st->id));
152 + if(!ria) {
153 + rrdcontext_release(rca);
154 + return 4;
155 + }
156 +
157 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
158 + uuid_copy(*store_uuid, ri->uuid);
159 +
160 + rrdinstance_release(ria);
161 + rrdcontext_release(rca);
162 + return 0;
163 +}
164 +
165 +void rrdcontext_host_child_disconnected(RRDHOST *host) {
166 + rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, false);
167 +}
168 +
169 +int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char *context, int (*callback)(RRDSET *st, void *data), void *data) {
170 + if(unlikely(!host || !context || !*context || !callback))
171 + return -1;
172 +
173 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, context);
174 + if(unlikely(!rca)) return -1;
175 +
176 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
177 + if(unlikely(!rc)) return -1;
178 +
179 + int ret = 0;
180 + RRDINSTANCE *ri;
181 + dfe_start_read(rc->rrdinstances, ri) {
182 + if(ri->rrdset) {
183 + int r = callback(ri->rrdset, data);
184 + if(r >= 0) ret += r;
185 + else {
186 + ret = r;
187 + break;
188 + }
189 + }
190 + }
191 + dfe_done(ri);
192 +
193 + rrdcontext_release(rca);
194 +
195 + return ret;
196 +}
197 +
198 +// ----------------------------------------------------------------------------
199 +// ACLK interface
200 +
201 +static bool rrdhost_check_our_claim_id(const char *claim_id) {
202 + if(!localhost->aclk_state.claimed_id) return false;
203 + return (strcasecmp(claim_id, localhost->aclk_state.claimed_id) == 0) ? true : false;
204 +}
205 +
206 +static RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
207 + uuid_t uuid;
208 + if (uuid_parse(node_id, uuid))
209 + return NULL;
210 +
211 + RRDHOST *host = NULL;
212 + dfe_start_read(rrdhost_root_index, host) {
213 + if(!host->node_id) continue;
214 +
215 + if(uuid_compare(uuid, *host->node_id) == 0)
216 + break;
217 + }
218 + dfe_done(host);
219 +
220 + return host;
221 +}
222 +
223 +void rrdcontext_hub_checkpoint_command(void *ptr) {
224 + struct ctxs_checkpoint *cmd = ptr;
225 +
226 + if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
227 + error("RRDCONTEXT: received checkpoint command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
228 + cmd->claim_id, cmd->node_id,
229 + localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
230 + cmd->claim_id);
231 +
232 + return;
233 + }
234 +
235 + RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id);
236 + if(!host) {
237 + error("RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', but there is no node with such node id here. Ignoring command.",
238 + cmd->claim_id, cmd->node_id);
239 +
240 + return;
241 + }
242 +
243 + if(rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
244 + info("RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', while node '%s' has an active context streaming.",
245 + cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
246 +
247 + // disable it temporarily, so that our worker will not attempt to send messages in parallel
248 + rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
249 + }
250 +
251 + uint64_t our_version_hash = rrdcontext_version_hash(host);
252 +
253 + if(cmd->version_hash != our_version_hash) {
254 + error("RRDCONTEXT: received version hash %"PRIu64" for host '%s', does not match our version hash %"PRIu64". Sending snapshot of all contexts.",
255 + cmd->version_hash, rrdhost_hostname(host), our_version_hash);
256 +
257 +#ifdef ENABLE_ACLK
258 + // prepare the snapshot
259 + char uuid[UUID_STR_LEN];
260 + uuid_unparse_lower(*host->node_id, uuid);
261 + contexts_snapshot_t bundle = contexts_snapshot_new(cmd->claim_id, uuid, our_version_hash);
262 +
263 + // do a deep scan on every metric of the host to make sure all our data are updated
264 + rrdcontext_recalculate_host_retention(host, RRD_FLAG_NONE, false);
265 +
266 + // calculate version hash and pack all the messages together in one go
267 + our_version_hash = rrdcontext_version_hash_with_callback(host, rrdcontext_message_send_unsafe, true, bundle);
268 +
269 + // update the version
270 + contexts_snapshot_set_version(bundle, our_version_hash);
271 +
272 + // send it
273 + aclk_send_contexts_snapshot(bundle);
274 +#endif
275 + }
276 +
277 + internal_error(true, "RRDCONTEXT: host '%s' enabling streaming of contexts", rrdhost_hostname(host));
278 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
279 + char node_str[UUID_STR_LEN];
280 + uuid_unparse_lower(*host->node_id, node_str);
281 + log_access("ACLK REQ [%s (%s)]: STREAM CONTEXTS ENABLED", node_str, rrdhost_hostname(host));
282 +}
283 +
284 +void rrdcontext_hub_stop_streaming_command(void *ptr) {
285 + struct stop_streaming_ctxs *cmd = ptr;
286 +
287 + if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
288 + error("RRDCONTEXT: received stop streaming command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
289 + cmd->claim_id, cmd->node_id,
290 + localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
291 + cmd->claim_id);
292 +
293 + return;
294 + }
295 +
296 + RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id);
297 + if(!host) {
298 + error("RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', but there is no node with such node id here. Ignoring command.",
299 + cmd->claim_id, cmd->node_id);
300 +
301 + return;
302 + }
303 +
304 + if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
305 + error("RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', but node '%s' does not have active context streaming. Ignoring command.",
306 + cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
307 +
308 + return;
309 + }
310 +
311 + internal_error(true, "RRDCONTEXT: host '%s' disabling streaming of contexts", rrdhost_hostname(host));
312 + rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
313 +}
314 +
315 +// ----------------------------------------------------------------------------
316 +// weights API
317 +
318 +static void metric_entry_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
319 + struct metric_entry *t = value;
320 + t->rca = rrdcontext_acquired_dup(t->rca);
321 + t->ria = rrdinstance_acquired_dup(t->ria);
322 + t->rma = rrdmetric_acquired_dup(t->rma);
323 +}
324 +static void metric_entry_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
325 + struct metric_entry *t = value;
326 + rrdcontext_release(t->rca);
327 + rrdinstance_release(t->ria);
328 + rrdmetric_release(t->rma);
329 +}
330 +static bool metric_entry_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value __maybe_unused, void *new_value __maybe_unused, void *data __maybe_unused) {
331 + internal_fatal("RRDCONTEXT: %s() detected a conflict on a metric pointer!", __FUNCTION__);
332 + return false;
333 +}
334 +
335 +DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contexts) {
336 + if(!host || !host->rrdctx.contexts)
337 + return NULL;
338 +
339 + DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext, 0);
340 + dictionary_register_insert_callback(dict, metric_entry_insert_callback, NULL);
341 + dictionary_register_delete_callback(dict, metric_entry_delete_callback, NULL);
342 + dictionary_register_conflict_callback(dict, metric_entry_conflict_callback, NULL);
343 +
344 + RRDCONTEXT *rc;
345 + dfe_start_reentrant(host->rrdctx.contexts, rc) {
346 + if(rrd_flag_is_deleted(rc))
347 + continue;
348 +
349 + if(contexts && !simple_pattern_matches_string(contexts, rc->id))
350 + continue;
351 +
352 + RRDINSTANCE *ri;
353 + dfe_start_read(rc->rrdinstances, ri) {
354 + if(rrd_flag_is_deleted(ri))
355 + continue;
356 +
357 + RRDMETRIC *rm;
358 + dfe_start_read(ri->rrdmetrics, rm) {
359 + if(rrd_flag_is_deleted(rm))
360 + continue;
361 +
362 + struct metric_entry tmp = {
363 + .rca = (RRDCONTEXT_ACQUIRED *)rc_dfe.item,
364 + .ria = (RRDINSTANCE_ACQUIRED *)ri_dfe.item,
365 + .rma = (RRDMETRIC_ACQUIRED *)rm_dfe.item,
366 + };
367 +
368 + char buffer[20 + 1];
369 + ssize_t len = snprintfz(buffer, 20, "%p", rm);
370 + dictionary_set_advanced(dict, buffer, len + 1, &tmp, sizeof(struct metric_entry), NULL);
371 + }
372 + dfe_done(rm);
373 + }
374 + dfe_done(ri);
375 + }
376 + dfe_done(rc);
377 +
378 + return dict;
379 +}
database/contexts/rrdcontext.h renamed
+76 -28
@@ -16,13 +16,13 @@ typedef struct rrdinstance_acquired RRDINSTANCE_ACQUIRED;
16 // ----------------------------------------------------------------------------
17 // RRDCONTEXT
18
19 -typedef struct rrdcontexts_dictionary RRDCONTEXTS;
19 typedef struct rrdcontext_acquired RRDCONTEXT_ACQUIRED;
20
21 // ----------------------------------------------------------------------------
22
24 -#include "rrd.h"
23 +#include "../rrd.h"
24
25 +bool rrdinstance_acquired_id_and_name_are_same(RRDINSTANCE_ACQUIRED *ria);
26 const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma);
27 const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma);
28
@@ -140,10 +140,9 @@ DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contex
140 typedef enum __attribute__ ((__packed__)) {
141 QUERY_STATUS_NONE = 0,
142 QUERY_STATUS_QUERIED = (1 << 0),
143 - QUERY_STATUS_DIMENSION_NODATA = (1 << 1),
144 - QUERY_STATUS_DIMENSION_HIDDEN = (1 << 2),
145 - QUERY_STATUS_EXCLUDED = (1 << 3),
146 - QUERY_STATUS_FAILED = (1 << 4),
143 + QUERY_STATUS_DIMENSION_HIDDEN = (1 << 1),
144 + QUERY_STATUS_EXCLUDED = (1 << 2),
145 + QUERY_STATUS_FAILED = (1 << 3),
146 } QUERY_STATUS;
147
148 typedef struct query_plan_entry {
@@ -162,12 +161,14 @@ typedef struct query_plan_entry {
161
162 #define QUERY_PLANS_MAX (RRD_STORAGE_TIERS * 2)
163
165 -struct query_instances_counts {
164 +struct query_metrics_counts {
165 size_t selected;
166 size_t excluded;
167 + size_t queried;
168 + size_t failed;
169 };
170
170 -struct query_metrics_counts {
171 +struct query_instances_counts {
172 size_t selected;
173 size_t excluded;
174 size_t queried;
@@ -181,24 +182,25 @@ struct query_alerts_counts {
182 size_t other;
183 };
184
184 -struct query_data_statistics {
185 - size_t count;
186 - NETDATA_DOUBLE min;
187 - NETDATA_DOUBLE max;
188 - NETDATA_DOUBLE sum;
189 - NETDATA_DOUBLE volume;
185 +struct query_data_statistics { // time-aggregated (group points) statistics
186 + size_t group_points; // the number of group points the query generated
187 + NETDATA_DOUBLE min; // the min value of the group points
188 + NETDATA_DOUBLE max; // the max value of the group points
189 + NETDATA_DOUBLE sum; // the sum of the group points
190 + NETDATA_DOUBLE volume; // the volume of the group points
191 + NETDATA_DOUBLE anomaly_sum; // the anomaly sum of the group points
192 };
193
192 -typedef struct query_host {
194 +typedef struct query_node {
195 uint32_t slot;
194 - RRDHOST *host;
196 + RRDHOST *rrdhost;
197 char node_id[UUID_STR_LEN];
198
199 struct query_data_statistics query_stats;
200 struct query_instances_counts instances;
201 struct query_metrics_counts metrics;
202 struct query_alerts_counts alerts;
201 -} QUERY_HOST;
203 +} QUERY_NODE;
204
205 typedef struct query_context {
206 uint32_t slot;
@@ -214,8 +216,8 @@ typedef struct query_instance {
216 uint32_t slot;
217 uint32_t query_host_id;
218 RRDINSTANCE_ACQUIRED *ria;
217 - STRING *id_fqdn;
218 - STRING *name_fqdn;
219 + STRING *id_fqdn; // never access this directly - it is created on demand via query_instance_id_fqdn()
220 + STRING *name_fqdn; // never access this directly - it is created on demand via query_instance_name_fqdn()
221
222 struct query_data_statistics query_stats;
223 struct query_metrics_counts metrics;
@@ -268,7 +270,7 @@ typedef struct query_metric {
270 typedef struct query_target_request {
271 size_t version;
272
271 - const char *scope_hosts;
273 + const char *scope_nodes;
274 const char *scope_contexts;
275
276 // selecting / filtering metrics to be queried
@@ -277,9 +279,9 @@ typedef struct query_target_request {
279 RRDINSTANCE_ACQUIRED *ria; // the instance to be queried (can be NULL)
280 RRDMETRIC_ACQUIRED *rma; // the metric to be queried (can be NULL)
281 RRDSET *st; // the chart to be queried (NULL, for context queries)
280 - const char *hosts; // hosts simple pattern
282 + const char *nodes; // hosts simple pattern
283 const char *contexts; // contexts simple pattern (context queries)
282 - const char *charts; // charts simple pattern (for context queries)
284 + const char *instances; // charts simple pattern (for context queries)
285 const char *dimensions; // dimensions simple pattern
286 const char *chart_label_key; // select only the chart having this label key
287 const char *labels; // select only the charts having this combo of label key:value
@@ -314,12 +316,22 @@ typedef struct query_target_request {
316
317 #define GROUP_BY_MAX_LABEL_KEYS 10
318
319 +struct query_tier_statistics {
320 + size_t queries;
321 + size_t points;
322 + time_t update_every;
323 + struct {
324 + time_t first_time_s;
325 + time_t last_time_s;
326 + } retention;
327 +};
328 +
329 typedef struct query_target {
330 char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging)
331 QUERY_TARGET_REQUEST request;
332
333 bool used; // when true, this query is currently being used
322 - size_t queries; // how many query we have done so far
334 + size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries
335
336 struct {
337 bool relative; // true when the request made with relative timestamps, true if it was absolute
@@ -338,9 +350,11 @@ typedef struct query_target {
350 } window;
351
352 struct {
353 + size_t queries[RRD_STORAGE_TIERS];
354 time_t first_time_s; // the combined first_time_t of all metrics in the query, across all tiers
355 time_t last_time_s; // the combined last_time_T of all metrics in the query, across all tiers
356 time_t minimum_latest_update_every_s; // the min update every of the metrics in the query
357 + struct query_tier_statistics tiers[RRD_STORAGE_TIERS];
358 } db;
359
360 struct {
@@ -375,12 +389,12 @@ typedef struct query_target {
389 } contexts;
390
391 struct {
378 - QUERY_HOST *array;
392 + QUERY_NODE *array;
393 uint32_t used; // how many items of the array are used
394 uint32_t size; // the size of the array
395 SIMPLE_PATTERN *pattern;
396 SIMPLE_PATTERN *scope_pattern;
383 - } hosts;
397 + } nodes;
398
399 struct {
400 size_t used;
@@ -389,6 +403,11 @@ typedef struct query_target {
403
404 struct query_data_statistics query_stats;
405
406 + struct {
407 + uint64_t contexts_hard_hash;
408 + uint64_t contexts_soft_hash;
409 + } versions;
410 +
411 struct {
412 usec_t received_ut;
413 usec_t preprocessed_ut;
@@ -398,9 +417,9 @@ typedef struct query_target {
417 } timings;
418 } QUERY_TARGET;
419
401 -static inline NEVERNULL QUERY_HOST *query_host(QUERY_TARGET *qt, size_t id) {
402 - internal_fatal(id >= qt->hosts.used, "QUERY: invalid query host id");
403 - return &qt->hosts.array[id];
420 +static inline NEVERNULL QUERY_NODE *query_node(QUERY_TARGET *qt, size_t id) {
421 + internal_fatal(id >= qt->nodes.used, "QUERY: invalid query host id");
422 + return &qt->nodes.array[id];
423 }
424
425 static inline NEVERNULL QUERY_CONTEXT *query_context(QUERY_TARGET *qt, size_t query_context_id) {
@@ -433,10 +452,39 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)
452 return rrdmetric_acquired_name(qd->rma);
453 }
454
455 +STRING *query_instance_id_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);
456 +STRING *query_instance_name_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);
457 +
458 void query_target_free(void);
459 void query_target_release(QUERY_TARGET *qt);
460
461 QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr);
462
463 +struct api_v2_contexts_request {
464 + char *scope_nodes;
465 + char *scope_contexts;
466 + char *nodes;
467 + char *contexts;
468 + char *q;
469 +
470 + struct {
471 + usec_t received_ut;
472 + usec_t processing_ut;
473 + usec_t output_ut;
474 + usec_t finished_ut;
475 + } timings;
476 +};
477 +
478 +typedef enum __attribute__ ((__packed__)) {
479 + CONTEXTS_V2_DEBUG = (1 << 0),
480 + CONTEXTS_V2_SEARCH = (1 << 1),
481 + CONTEXTS_V2_NODES = (1 << 2),
482 + CONTEXTS_V2_CONTEXTS = (1 << 3),
483 +} CONTEXTS_V2_OPTIONS;
484 +
485 +int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options);
486 +
487 +RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o);
488 +
489 #endif // NETDATA_RRDCONTEXT_H
490
database/contexts/worker.c new
+1090
@@ -0,0 +1,1090 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "internal.h"
4 +
5 +static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc);
6 +
7 +static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused);
8 +
9 +static void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc);
10 +
11 +static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc);
12 +static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs);
13 +
14 +static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs);
15 +static void rrdcontext_garbage_collect_for_all_hosts(void);
16 +
17 +extern usec_t rrdcontext_next_db_rotation_ut;
18 +
19 +// ----------------------------------------------------------------------------
20 +// load from SQL
21 +
22 +static void rrdinstance_load_clabel(SQL_CLABEL_DATA *sld, void *data) {
23 + RRDINSTANCE *ri = data;
24 + rrdlabels_add(ri->rrdlabels, sld->label_key, sld->label_value, sld->label_source);
25 +}
26 +
27 +static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
28 + RRDINSTANCE *ri = data;
29 +
30 + RRDMETRIC trm = {
31 + .id = string_strdupz(sd->id),
32 + .name = string_strdupz(sd->name),
33 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomic
34 + };
35 + if(sd->hidden) trm.flags |= RRD_FLAG_HIDDEN;
36 +
37 + uuid_copy(trm.uuid, sd->dim_id);
38 +
39 + dictionary_set(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
40 +}
41 +
42 +static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
43 + RRDHOST *host = data;
44 +
45 + RRDCONTEXT tc = {
46 + .id = string_strdupz(sc->context),
47 + .title = string_strdupz(sc->title),
48 + .units = string_strdupz(sc->units),
49 + .family = string_strdupz(sc->family),
50 + .priority = sc->priority,
51 + .chart_type = sc->chart_type,
52 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
53 + .rrdhost = host,
54 + };
55 +
56 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_set_and_acquire_item(host->rrdctx.contexts, string2str(tc.id), &tc, sizeof(tc));
57 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
58 +
59 + RRDINSTANCE tri = {
60 + .id = string_strdupz(sc->id),
61 + .name = string_strdupz(sc->name),
62 + .title = string_strdupz(sc->title),
63 + .units = string_strdupz(sc->units),
64 + .family = string_strdupz(sc->family),
65 + .chart_type = sc->chart_type,
66 + .priority = sc->priority,
67 + .update_every_s = sc->update_every,
68 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
69 + };
70 + uuid_copy(tri.uuid, sc->chart_id);
71 +
72 + RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, sc->id, &tri, sizeof(tri));
73 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
74 +
75 + ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri);
76 + ctx_get_label_list(&ri->uuid, rrdinstance_load_clabel, ri);
77 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
78 + rrdinstance_release(ria);
79 + rrdcontext_release(rca);
80 +}
81 +
82 +static void rrdcontext_load_context_callback(VERSIONED_CONTEXT_DATA *ctx_data, void *data) {
83 + RRDHOST *host = data;
84 + (void)host;
85 +
86 + RRDCONTEXT trc = {
87 + .id = string_strdupz(ctx_data->id),
88 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
89 +
90 + // no need to set more data here
91 + // we only need the hub data
92 +
93 + .hub = *ctx_data,
94 + };
95 + dictionary_set(host->rrdctx.contexts, string2str(trc.id), &trc, sizeof(trc));
96 +}
97 +
98 +void rrdhost_load_rrdcontext_data(RRDHOST *host) {
99 + if(host->rrdctx.contexts) return;
100 +
101 + rrdhost_create_rrdcontexts(host);
102 + ctx_get_context_list(&host->host_uuid, rrdcontext_load_context_callback, host);
103 + ctx_get_chart_list(&host->host_uuid, rrdinstance_load_chart_callback, host);
104 +
105 + RRDCONTEXT *rc;
106 + dfe_start_read(host->rrdctx.contexts, rc) {
107 + rrdcontext_trigger_updates(rc, __FUNCTION__ );
108 + }
109 + dfe_done(rc);
110 +
111 + rrdcontext_garbage_collect_single_host(host, false);
112 +}
113 +
114 +// ----------------------------------------------------------------------------
115 +// version hash calculation
116 +
117 +uint64_t rrdcontext_version_hash_with_callback(
118 + RRDHOST *host,
119 + void (*callback)(RRDCONTEXT *, bool, void *),
120 + bool snapshot,
121 + void *bundle) {
122 +
123 + if(unlikely(!host || !host->rrdctx.contexts)) return 0;
124 +
125 + RRDCONTEXT *rc;
126 + uint64_t hash = 0;
127 +
128 + // loop through all contexts of the host
129 + dfe_start_read(host->rrdctx.contexts, rc) {
130 +
131 + rrdcontext_lock(rc);
132 +
133 + if(unlikely(rrd_flag_check(rc, RRD_FLAG_HIDDEN))) {
134 + rrdcontext_unlock(rc);
135 + continue;
136 + }
137 +
138 + if(unlikely(callback))
139 + callback(rc, snapshot, bundle);
140 +
141 + // skip any deleted contexts
142 + if(unlikely(rrd_flag_is_deleted(rc))) {
143 + rrdcontext_unlock(rc);
144 + continue;
145 + }
146 +
147 + // we use rc->hub.* which has the latest
148 + // metadata we have sent to the hub
149 +
150 + // if a context is currently queued, rc->hub.* does NOT
151 + // reflect the queued changes. rc->hub.* is updated with
152 + // their metadata, after messages are dispatched to hub.
153 +
154 + // when the context is being collected,
155 + // rc->hub.last_time_t is already zero
156 +
157 + hash += rc->hub.version + rc->hub.last_time_s - rc->hub.first_time_s;
158 +
159 + rrdcontext_unlock(rc);
160 +
161 + }
162 + dfe_done(rc);
163 +
164 + return hash;
165 +}
166 +
167 +// ----------------------------------------------------------------------------
168 +// retention recalculation
169 +
170 +void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs) {
171 + rrdcontext_post_process_updates(rc, true, reason, worker_jobs);
172 +}
173 +
174 +void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs) {
175 + if(unlikely(!host || !host->rrdctx.contexts)) return;
176 +
177 + RRDCONTEXT *rc;
178 + dfe_start_read(host->rrdctx.contexts, rc) {
179 + rrdcontext_recalculate_context_retention(rc, reason, worker_jobs);
180 + }
181 + dfe_done(rc);
182 +}
183 +
184 +static void rrdcontext_recalculate_retention_all_hosts(void) {
185 + rrdcontext_next_db_rotation_ut = 0;
186 + RRDHOST *host;
187 + dfe_start_reentrant(rrdhost_root_index, host) {
188 + worker_is_busy(WORKER_JOB_RETENTION);
189 + rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DB_ROTATION, true);
190 + }
191 + dfe_done(host);
192 +}
193 +
194 +// ----------------------------------------------------------------------------
195 +// garbage collector
196 +
197 +bool rrdmetric_update_retention(RRDMETRIC *rm) {
198 + time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
199 +
200 + if(rm->rrddim) {
201 + min_first_time_t = rrddim_first_entry_s(rm->rrddim);
202 + max_last_time_t = rrddim_last_entry_s(rm->rrddim);
203 + }
204 + else {
205 + RRDHOST *rrdhost = rm->ri->rc->rrdhost;
206 + for (size_t tier = 0; tier < storage_tiers; tier++) {
207 + STORAGE_ENGINE *eng = rrdhost->db[tier].eng;
208 +
209 + time_t first_time_t, last_time_t;
210 + if (eng->api.metric_retention_by_uuid(rrdhost->db[tier].instance, &rm->uuid, &first_time_t, &last_time_t)) {
211 + if (first_time_t < min_first_time_t)
212 + min_first_time_t = first_time_t;
213 +
214 + if (last_time_t > max_last_time_t)
215 + max_last_time_t = last_time_t;
216 + }
217 + }
218 + }
219 +
220 + if((min_first_time_t == LONG_MAX || min_first_time_t == 0) && max_last_time_t == 0)
221 + return false;
222 +
223 + if(min_first_time_t == LONG_MAX)
224 + min_first_time_t = 0;
225 +
226 + if(min_first_time_t > max_last_time_t) {
227 + internal_error(true, "RRDMETRIC: retention of '%s' is flipped, first_time_t = %ld, last_time_t = %ld", string2str(rm->id), min_first_time_t, max_last_time_t);
228 + time_t tmp = min_first_time_t;
229 + min_first_time_t = max_last_time_t;
230 + max_last_time_t = tmp;
231 + }
232 +
233 + // check if retention changed
234 +
235 + if (min_first_time_t != rm->first_time_s) {
236 + rm->first_time_s = min_first_time_t;
237 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
238 + }
239 +
240 + if (max_last_time_t != rm->last_time_s) {
241 + rm->last_time_s = max_last_time_t;
242 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
243 + }
244 +
245 + if(unlikely(!rm->first_time_s && !rm->last_time_s))
246 + rrd_flag_set_deleted(rm, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
247 +
248 + rrd_flag_set(rm, RRD_FLAG_LIVE_RETENTION);
249 +
250 + return true;
251 +}
252 +
253 +static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) {
254 + if(likely(!rrd_flag_check(rm, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
255 + return false;
256 +
257 + if(likely(rrd_flag_check(rm, RRD_FLAGS_PREVENTING_DELETIONS)))
258 + return false;
259 +
260 + if(likely(rm->rrddim))
261 + return false;
262 +
263 + rrdmetric_update_retention(rm);
264 + if(rm->first_time_s || rm->last_time_s)
265 + return false;
266 +
267 + return true;
268 +}
269 +
270 +static inline bool rrdinstance_should_be_deleted(RRDINSTANCE *ri) {
271 + if(likely(!rrd_flag_check(ri, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
272 + return false;
273 +
274 + if(likely(rrd_flag_check(ri, RRD_FLAGS_PREVENTING_DELETIONS)))
275 + return false;
276 +
277 + if(likely(ri->rrdset))
278 + return false;
279 +
280 + if(unlikely(dictionary_referenced_items(ri->rrdmetrics) != 0))
281 + return false;
282 +
283 + if(unlikely(dictionary_entries(ri->rrdmetrics) != 0))
284 + return false;
285 +
286 + if(ri->first_time_s || ri->last_time_s)
287 + return false;
288 +
289 + return true;
290 +}
291 +
292 +static inline bool rrdcontext_should_be_deleted(RRDCONTEXT *rc) {
293 + if(likely(!rrd_flag_check(rc, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
294 + return false;
295 +
296 + if(likely(rrd_flag_check(rc, RRD_FLAGS_PREVENTING_DELETIONS)))
297 + return false;
298 +
299 + if(unlikely(dictionary_referenced_items(rc->rrdinstances) != 0))
300 + return false;
301 +
302 + if(unlikely(dictionary_entries(rc->rrdinstances) != 0))
303 + return false;
304 +
305 + if(unlikely(rc->first_time_s || rc->last_time_s))
306 + return false;
307 +
308 + return true;
309 +}
310 +
311 +void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) {
312 + // we need to refresh the string pointers in rc->hub
313 + // in case the context changed values
314 + rc->hub.id = string2str(rc->id);
315 + rc->hub.title = string2str(rc->title);
316 + rc->hub.units = string2str(rc->units);
317 + rc->hub.family = string2str(rc->family);
318 +
319 + // delete it from SQL
320 + if(ctx_delete_context(&rc->rrdhost->host_uuid, &rc->hub) != 0)
321 + error("RRDCONTEXT: failed to delete context '%s' version %"PRIu64" from SQL.", rc->hub.id, rc->hub.version);
322 +}
323 +
324 +static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs) {
325 +
326 + internal_error(true, "RRDCONTEXT: garbage collecting context structures of host '%s'", rrdhost_hostname(host));
327 +
328 + RRDCONTEXT *rc;
329 + dfe_start_reentrant(host->rrdctx.contexts, rc) {
330 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
331 +
332 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP);
333 +
334 + rrdcontext_lock(rc);
335 +
336 + RRDINSTANCE *ri;
337 + dfe_start_reentrant(rc->rrdinstances, ri) {
338 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
339 +
340 + RRDMETRIC *rm;
341 + dfe_start_write(ri->rrdmetrics, rm) {
342 + if(rrdmetric_should_be_deleted(rm)) {
343 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
344 + if(!dictionary_del(ri->rrdmetrics, string2str(rm->id)))
345 + error("RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
346 + string2str(rm->id),
347 + string2str(ri->id),
348 + string2str(rc->id),
349 + rrdhost_hostname(host));
350 + else
351 + internal_error(
352 + true,
353 + "RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.",
354 + string2str(rm->id),
355 + string2str(ri->id),
356 + string2str(rc->id),
357 + rrdhost_hostname(host));
358 + }
359 + }
360 + dfe_done(rm);
361 +
362 + if(rrdinstance_should_be_deleted(ri)) {
363 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
364 + if(!dictionary_del(rc->rrdinstances, string2str(ri->id)))
365 + error("RRDCONTEXT: instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
366 + string2str(ri->id),
367 + string2str(rc->id),
368 + rrdhost_hostname(host));
369 + else
370 + internal_error(
371 + true,
372 + "RRDCONTEXT: instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.",
373 + string2str(ri->id),
374 + string2str(rc->id),
375 + rrdhost_hostname(host));
376 + }
377 + }
378 + dfe_done(ri);
379 +
380 + if(unlikely(rrdcontext_should_be_deleted(rc))) {
381 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
382 + rrdcontext_dequeue_from_post_processing(rc);
383 + rrdcontext_delete_from_sql_unsafe(rc);
384 +
385 + if(!dictionary_del(host->rrdctx.contexts, string2str(rc->id)))
386 + error("RRDCONTEXT: context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
387 + string2str(rc->id),
388 + rrdhost_hostname(host));
389 + else
390 + internal_error(
391 + true,
392 + "RRDCONTEXT: context '%s' of host '%s', deleted from rrdmetrics dictionary.",
393 + string2str(rc->id),
394 + rrdhost_hostname(host));
395 + }
396 +
397 + // the item is referenced in the dictionary
398 + // so, it is still here to unlock, even if we have deleted it
399 + rrdcontext_unlock(rc);
400 + }
401 + dfe_done(rc);
402 +}
403 +
404 +static void rrdcontext_garbage_collect_for_all_hosts(void) {
405 + RRDHOST *host;
406 + dfe_start_reentrant(rrdhost_root_index, host) {
407 + rrdcontext_garbage_collect_single_host(host, true);
408 + }
409 + dfe_done(host);
410 +}
411 +
412 +// ----------------------------------------------------------------------------
413 +// post processing
414 +
415 +static void rrdmetric_process_updates(RRDMETRIC *rm, bool force, RRD_FLAGS reason, bool worker_jobs) {
416 + if(reason != RRD_FLAG_NONE)
417 + rrd_flag_set_updated(rm, reason);
418 +
419 + if(!force && !rrd_flag_is_updated(rm) && rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION) && !rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
420 + return;
421 +
422 + if(worker_jobs)
423 + worker_is_busy(WORKER_JOB_PP_METRIC);
424 +
425 + if(reason & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD) {
426 + rrd_flag_set_archived(rm);
427 + rrd_flag_set(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD);
428 + }
429 + if(rrd_flag_is_deleted(rm) && (reason & RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
430 + rrd_flag_set_archived(rm);
431 +
432 + rrdmetric_update_retention(rm);
433 +
434 + rrd_flag_unset_updated(rm);
435 +}
436 +
437 +static void rrdinstance_post_process_updates(RRDINSTANCE *ri, bool force, RRD_FLAGS reason, bool worker_jobs) {
438 + if(reason != RRD_FLAG_NONE)
439 + rrd_flag_set_updated(ri, reason);
440 +
441 + if(!force && !rrd_flag_is_updated(ri) && rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION))
442 + return;
443 +
444 + if(worker_jobs)
445 + worker_is_busy(WORKER_JOB_PP_INSTANCE);
446 +
447 + time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
448 + size_t metrics_active = 0, metrics_deleted = 0;
449 + bool live_retention = true, currently_collected = false;
450 + if(dictionary_entries(ri->rrdmetrics) > 0) {
451 + RRDMETRIC *rm;
452 + dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) {
453 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
454 +
455 + RRD_FLAGS reason_to_pass = reason;
456 + if(rrd_flag_check(ri, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
457 + reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION;
458 +
459 + rrdmetric_process_updates(rm, force, reason_to_pass, worker_jobs);
460 +
461 + if(unlikely(!rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)))
462 + live_retention = false;
463 +
464 + if (unlikely((rrdmetric_should_be_deleted(rm)))) {
465 + metrics_deleted++;
466 + continue;
467 + }
468 +
469 + if(!currently_collected && rrd_flag_check(rm, RRD_FLAG_COLLECTED) && rm->first_time_s)
470 + currently_collected = true;
471 +
472 + metrics_active++;
473 +
474 + if (rm->first_time_s && rm->first_time_s < min_first_time_t)
475 + min_first_time_t = rm->first_time_s;
476 +
477 + if (rm->last_time_s && rm->last_time_s > max_last_time_t)
478 + max_last_time_t = rm->last_time_s;
479 + }
480 + dfe_done(rm);
481 + }
482 +
483 + if(unlikely(live_retention && !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
484 + rrd_flag_set(ri, RRD_FLAG_LIVE_RETENTION);
485 + else if(unlikely(!live_retention && rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
486 + rrd_flag_clear(ri, RRD_FLAG_LIVE_RETENTION);
487 +
488 + if(unlikely(!metrics_active)) {
489 + // no metrics available
490 +
491 + if(ri->first_time_s) {
492 + ri->first_time_s = 0;
493 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
494 + }
495 +
496 + if(ri->last_time_s) {
497 + ri->last_time_s = 0;
498 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
499 + }
500 +
501 + rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
502 + }
503 + else {
504 + // we have active metrics...
505 +
506 + if (unlikely(min_first_time_t == LONG_MAX))
507 + min_first_time_t = 0;
508 +
509 + if (unlikely(min_first_time_t == 0 || max_last_time_t == 0)) {
510 + if(ri->first_time_s) {
511 + ri->first_time_s = 0;
512 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
513 + }
514 +
515 + if(ri->last_time_s) {
516 + ri->last_time_s = 0;
517 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
518 + }
519 +
520 + if(likely(live_retention))
521 + rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
522 + }
523 + else {
524 + rrd_flag_clear(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
525 +
526 + if (unlikely(ri->first_time_s != min_first_time_t)) {
527 + ri->first_time_s = min_first_time_t;
528 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
529 + }
530 +
531 + if (unlikely(ri->last_time_s != max_last_time_t)) {
532 + ri->last_time_s = max_last_time_t;
533 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
534 + }
535 +
536 + if(likely(currently_collected))
537 + rrd_flag_set_collected(ri);
538 + else
539 + rrd_flag_set_archived(ri);
540 + }
541 + }
542 +
543 + rrd_flag_unset_updated(ri);
544 +}
545 +
546 +static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs) {
547 + if(reason != RRD_FLAG_NONE)
548 + rrd_flag_set_updated(rc, reason);
549 +
550 + if(worker_jobs)
551 + worker_is_busy(WORKER_JOB_PP_CONTEXT);
552 +
553 + size_t min_priority_collected = LONG_MAX;
554 + size_t min_priority_not_collected = LONG_MAX;
555 + size_t min_priority = LONG_MAX;
556 + time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
557 + size_t instances_active = 0, instances_deleted = 0;
558 + bool live_retention = true, currently_collected = false, hidden = true;
559 + if(dictionary_entries(rc->rrdinstances) > 0) {
560 + RRDINSTANCE *ri;
561 + dfe_start_reentrant(rc->rrdinstances, ri) {
562 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
563 +
564 + RRD_FLAGS reason_to_pass = reason;
565 + if(rrd_flag_check(rc, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
566 + reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION;
567 +
568 + rrdinstance_post_process_updates(ri, force, reason_to_pass, worker_jobs);
569 +
570 + if(unlikely(hidden && !rrd_flag_check(ri, RRD_FLAG_HIDDEN)))
571 + hidden = false;
572 +
573 + if(unlikely(live_retention && !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
574 + live_retention = false;
575 +
576 + if (unlikely(rrdinstance_should_be_deleted(ri))) {
577 + instances_deleted++;
578 + continue;
579 + }
580 +
581 + if(unlikely(!currently_collected && rrd_flag_is_collected(ri) && ri->first_time_s))
582 + currently_collected = true;
583 +
584 + internal_error(rc->units != ri->units,
585 + "RRDCONTEXT: '%s' rrdinstance '%s' has different units, context '%s', instance '%s'",
586 + string2str(rc->id), string2str(ri->id),
587 + string2str(rc->units), string2str(ri->units));
588 +
589 + instances_active++;
590 +
591 + if (ri->priority >= RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY) {
592 + if(rrd_flag_check(ri, RRD_FLAG_COLLECTED)) {
593 + if(ri->priority < min_priority_collected)
594 + min_priority_collected = ri->priority;
595 + }
596 + else {
597 + if(ri->priority < min_priority_not_collected)
598 + min_priority_not_collected = ri->priority;
599 + }
600 + }
601 +
602 + if (ri->first_time_s && ri->first_time_s < min_first_time_t)
603 + min_first_time_t = ri->first_time_s;
604 +
605 + if (ri->last_time_s && ri->last_time_s > max_last_time_t)
606 + max_last_time_t = ri->last_time_s;
607 + }
608 + dfe_done(ri);
609 +
610 + if(min_priority_collected != LONG_MAX)
611 + // use the collected priority
612 + min_priority = min_priority_collected;
613 + else
614 + // use the non-collected priority
615 + min_priority = min_priority_not_collected;
616 + }
617 +
618 + {
619 + bool previous_hidden = rrd_flag_check(rc, RRD_FLAG_HIDDEN);
620 + if (hidden != previous_hidden) {
621 + if (hidden && !rrd_flag_check(rc, RRD_FLAG_HIDDEN))
622 + rrd_flag_set(rc, RRD_FLAG_HIDDEN);
623 + else if (!hidden && rrd_flag_check(rc, RRD_FLAG_HIDDEN))
624 + rrd_flag_clear(rc, RRD_FLAG_HIDDEN);
625 + }
626 +
627 + bool previous_live_retention = rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION);
628 + if (live_retention != previous_live_retention) {
629 + if (live_retention && !rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
630 + rrd_flag_set(rc, RRD_FLAG_LIVE_RETENTION);
631 + else if (!live_retention && rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
632 + rrd_flag_clear(rc, RRD_FLAG_LIVE_RETENTION);
633 + }
634 + }
635 +
636 + rrdcontext_lock(rc);
637 + rc->pp.executions++;
638 +
639 + if(unlikely(!instances_active)) {
640 + // we had some instances, but they are gone now...
641 +
642 + if(rc->first_time_s) {
643 + rc->first_time_s = 0;
644 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
645 + }
646 +
647 + if(rc->last_time_s) {
648 + rc->last_time_s = 0;
649 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
650 + }
651 +
652 + rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
653 + }
654 + else {
655 + // we have some active instances...
656 +
657 + if (unlikely(min_first_time_t == LONG_MAX))
658 + min_first_time_t = 0;
659 +
660 + if (unlikely(min_first_time_t == 0 && max_last_time_t == 0)) {
661 + if(rc->first_time_s) {
662 + rc->first_time_s = 0;
663 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
664 + }
665 +
666 + if(rc->last_time_s) {
667 + rc->last_time_s = 0;
668 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
669 + }
670 +
671 + rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
672 + }
673 + else {
674 + rrd_flag_clear(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
675 +
676 + if (unlikely(rc->first_time_s != min_first_time_t)) {
677 + rc->first_time_s = min_first_time_t;
678 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
679 + }
680 +
681 + if (rc->last_time_s != max_last_time_t) {
682 + rc->last_time_s = max_last_time_t;
683 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
684 + }
685 +
686 + if(likely(currently_collected))
687 + rrd_flag_set_collected(rc);
688 + else
689 + rrd_flag_set_archived(rc);
690 + }
691 +
692 + if (min_priority != LONG_MAX && rc->priority != min_priority) {
693 + rc->priority = min_priority;
694 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
695 + }
696 + }
697 +
698 + if(unlikely(rrd_flag_is_updated(rc) && rc->rrdhost->rrdctx.hub_queue)) {
699 + if(check_if_cloud_version_changed_unsafe(rc, false)) {
700 + rc->version = rrdcontext_get_next_version(rc);
701 + dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx.hub_queue,
702 + string2str(rc->id), rc, sizeof(*rc));
703 + }
704 + }
705 +
706 + rrd_flag_unset_updated(rc);
707 + rrdcontext_unlock(rc);
708 +}
709 +
710 +void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function __maybe_unused, RRD_FLAGS flags __maybe_unused) {
711 + if(unlikely(!rc->rrdhost->rrdctx.pp_queue)) return;
712 +
713 + if(!rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_PP)) {
714 + dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx.pp_queue,
715 + string2str(rc->id),
716 + rc,
717 + sizeof(*rc));
718 +
719 +#if(defined(NETDATA_INTERNAL_CHECKS) && defined(LOG_POST_PROCESSING_QUEUE_INSERTIONS))
720 + {
721 + BUFFER *wb_flags = buffer_create(1000);
722 + rrd_flags_to_buffer(flags, wb_flags);
723 +
724 + BUFFER *wb_reasons = buffer_create(1000);
725 + rrd_reasons_to_buffer(flags, wb_reasons);
726 +
727 + internal_error(true, "RRDCONTEXT: '%s' update triggered by function %s(), due to flags: %s, reasons: %s",
728 + string2str(rc->id), function,
729 + buffer_tostring(wb_flags),
730 + buffer_tostring(wb_reasons));
731 +
732 + buffer_free(wb_reasons);
733 + buffer_free(wb_flags);
734 + }
735 +#endif
736 + }
737 +}
738 +
739 +static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc) {
740 + if(unlikely(!rc->rrdhost->rrdctx.pp_queue)) return;
741 + dictionary_del(rc->rrdhost->rrdctx.pp_queue, string2str(rc->id));
742 +}
743 +
744 +static void rrdcontext_post_process_queued_contexts(RRDHOST *host) {
745 + if(unlikely(!host->rrdctx.pp_queue)) return;
746 +
747 + RRDCONTEXT *rc;
748 + dfe_start_reentrant(host->rrdctx.pp_queue, rc) {
749 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
750 +
751 + rrdcontext_dequeue_from_post_processing(rc);
752 + rrdcontext_post_process_updates(rc, false, RRD_FLAG_NONE, true);
753 + }
754 + dfe_done(rc);
755 +}
756 +
757 +// ----------------------------------------------------------------------------
758 +// dispatching contexts to cloud
759 +
760 +static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc) {
761 + time_t now = now_realtime_sec();
762 + uint64_t version = MAX(rc->version, rc->hub.version);
763 + version = MAX((uint64_t)now, version);
764 + version++;
765 + return version;
766 +}
767 +
768 +void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused) {
769 +
770 + // save it, so that we know the last version we sent to hub
771 + rc->version = rc->hub.version = rrdcontext_get_next_version(rc);
772 + rc->hub.id = string2str(rc->id);
773 + rc->hub.title = string2str(rc->title);
774 + rc->hub.units = string2str(rc->units);
775 + rc->hub.family = string2str(rc->family);
776 + rc->hub.chart_type = rrdset_type_name(rc->chart_type);
777 + rc->hub.priority = rc->priority;
778 + rc->hub.first_time_s = rc->first_time_s;
779 + rc->hub.last_time_s = rrd_flag_is_collected(rc) ? 0 : rc->last_time_s;
780 + rc->hub.deleted = rrd_flag_is_deleted(rc) ? true : false;
781 +
782 +#ifdef ENABLE_ACLK
783 + struct context_updated message = {
784 + .id = rc->hub.id,
785 + .version = rc->hub.version,
786 + .title = rc->hub.title,
787 + .units = rc->hub.units,
788 + .family = rc->hub.family,
789 + .chart_type = rc->hub.chart_type,
790 + .priority = rc->hub.priority,
791 + .first_entry = rc->hub.first_time_s,
792 + .last_entry = rc->hub.last_time_s,
793 + .deleted = rc->hub.deleted,
794 + };
795 +
796 + if(likely(!rrd_flag_check(rc, RRD_FLAG_HIDDEN))) {
797 + if (snapshot) {
798 + if (!rc->hub.deleted)
799 + contexts_snapshot_add_ctx_update(bundle, &message);
800 + }
801 + else
802 + contexts_updated_add_ctx_update(bundle, &message);
803 + }
804 +#endif
805 +
806 + // store it to SQL
807 +
808 + if(rrd_flag_is_deleted(rc))
809 + rrdcontext_delete_from_sql_unsafe(rc);
810 +
811 + else if (ctx_store_context(&rc->rrdhost->host_uuid, &rc->hub) != 0)
812 + error("RRDCONTEXT: failed to save context '%s' version %"PRIu64" to SQL.", rc->hub.id, rc->hub.version);
813 +}
814 +
815 +static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused) {
816 + bool id_changed = false,
817 + title_changed = false,
818 + units_changed = false,
819 + family_changed = false,
820 + chart_type_changed = false,
821 + priority_changed = false,
822 + first_time_changed = false,
823 + last_time_changed = false,
824 + deleted_changed = false;
825 +
826 + RRD_FLAGS flags = rrd_flags_get(rc);
827 +
828 + if(unlikely(string2str(rc->id) != rc->hub.id))
829 + id_changed = true;
830 +
831 + if(unlikely(string2str(rc->title) != rc->hub.title))
832 + title_changed = true;
833 +
834 + if(unlikely(string2str(rc->units) != rc->hub.units))
835 + units_changed = true;
836 +
837 + if(unlikely(string2str(rc->family) != rc->hub.family))
838 + family_changed = true;
839 +
840 + if(unlikely(rrdset_type_name(rc->chart_type) != rc->hub.chart_type))
841 + chart_type_changed = true;
842 +
843 + if(unlikely(rc->priority != rc->hub.priority))
844 + priority_changed = true;
845 +
846 + if(unlikely((uint64_t)rc->first_time_s != rc->hub.first_time_s))
847 + first_time_changed = true;
848 +
849 + if(unlikely((uint64_t)((flags & RRD_FLAG_COLLECTED) ? 0 : rc->last_time_s) != rc->hub.last_time_s))
850 + last_time_changed = true;
851 +
852 + if(unlikely(((flags & RRD_FLAG_DELETED) ? true : false) != rc->hub.deleted))
853 + deleted_changed = true;
854 +
855 + if(unlikely(id_changed || title_changed || units_changed || family_changed || chart_type_changed || priority_changed || first_time_changed || last_time_changed || deleted_changed)) {
856 +
857 + internal_error(LOG_TRANSITIONS,
858 + "RRDCONTEXT: %s NEW VERSION '%s'%s of host '%s', version %"PRIu64", title '%s'%s, units '%s'%s, family '%s'%s, chart type '%s'%s, priority %u%s, first_time_t %ld%s, last_time_t %ld%s, deleted '%s'%s, (queued for %llu ms, expected %llu ms)",
859 + sending?"SENDING":"QUEUE",
860 + string2str(rc->id), id_changed ? " (CHANGED)" : "",
861 + rrdhost_hostname(rc->rrdhost),
862 + rc->version,
863 + string2str(rc->title), title_changed ? " (CHANGED)" : "",
864 + string2str(rc->units), units_changed ? " (CHANGED)" : "",
865 + string2str(rc->family), family_changed ? " (CHANGED)" : "",
866 + rrdset_type_name(rc->chart_type), chart_type_changed ? " (CHANGED)" : "",
867 + rc->priority, priority_changed ? " (CHANGED)" : "",
868 + rc->first_time_s, first_time_changed ? " (CHANGED)" : "",
869 + (flags & RRD_FLAG_COLLECTED) ? 0 : rc->last_time_s, last_time_changed ? " (CHANGED)" : "",
870 + (flags & RRD_FLAG_DELETED) ? "true" : "false", deleted_changed ? " (CHANGED)" : "",
871 + sending ? (now_realtime_usec() - rc->queue.queued_ut) / USEC_PER_MS : 0,
872 + sending ? (rc->queue.scheduled_dispatch_ut - rc->queue.queued_ut) / USEC_PER_MS : 0
873 + );
874 +
875 + return true;
876 + }
877 +
878 + return false;
879 +}
880 +
881 +static inline usec_t rrdcontext_calculate_queued_dispatch_time_ut(RRDCONTEXT *rc, usec_t now_ut) {
882 +
883 + if(likely(rc->queue.delay_calc_ut >= rc->queue.queued_ut))
884 + return rc->queue.scheduled_dispatch_ut;
885 +
886 + RRD_FLAGS flags = rc->queue.queued_flags;
887 +
888 + usec_t delay = LONG_MAX;
889 + int i;
890 + struct rrdcontext_reason *reason;
891 + for(i = 0, reason = &rrdcontext_reasons[i]; reason->name ; reason = &rrdcontext_reasons[++i]) {
892 + if(unlikely(flags & reason->flag)) {
893 + if(reason->delay_ut < delay)
894 + delay = reason->delay_ut;
895 + }
896 + }
897 +
898 + if(unlikely(delay == LONG_MAX)) {
899 + internal_error(true, "RRDCONTEXT: '%s', cannot find minimum delay of flags %x", string2str(rc->id), (unsigned int)flags);
900 + delay = 60 * USEC_PER_SEC;
901 + }
902 +
903 + rc->queue.delay_calc_ut = now_ut;
904 + usec_t dispatch_ut = rc->queue.scheduled_dispatch_ut = rc->queue.queued_ut + delay;
905 + return dispatch_ut;
906 +}
907 +
908 +static void rrdcontext_dequeue_from_hub_queue(RRDCONTEXT *rc) {
909 + dictionary_del(rc->rrdhost->rrdctx.hub_queue, string2str(rc->id));
910 +}
911 +
912 +static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now_ut) {
913 +
914 + // check if we have received a streaming command for this host
915 + if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS) || !aclk_connected || !host->rrdctx.hub_queue)
916 + return;
917 +
918 + // check if there are queued items to send
919 + if(!dictionary_entries(host->rrdctx.hub_queue))
920 + return;
921 +
922 + if(!host->node_id)
923 + return;
924 +
925 + size_t messages_added = 0;
926 + contexts_updated_t bundle = NULL;
927 +
928 + RRDCONTEXT *rc;
929 + dfe_start_reentrant(host->rrdctx.hub_queue, rc) {
930 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
931 +
932 + if(unlikely(messages_added >= MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST))
933 + break;
934 +
935 + worker_is_busy(WORKER_JOB_QUEUED);
936 + usec_t dispatch_ut = rrdcontext_calculate_queued_dispatch_time_ut(rc, now_ut);
937 + char *claim_id = get_agent_claimid();
938 +
939 + if(unlikely(now_ut >= dispatch_ut) && claim_id) {
940 + worker_is_busy(WORKER_JOB_CHECK);
941 +
942 + rrdcontext_lock(rc);
943 +
944 + if(check_if_cloud_version_changed_unsafe(rc, true)) {
945 + worker_is_busy(WORKER_JOB_SEND);
946 +
947 +#ifdef ENABLE_ACLK
948 + if(!bundle) {
949 + // prepare the bundle to send the messages
950 + char uuid[UUID_STR_LEN];
951 + uuid_unparse_lower(*host->node_id, uuid);
952 +
953 + bundle = contexts_updated_new(claim_id, uuid, 0, now_ut);
954 + }
955 +#endif
956 + // update the hub data of the context, give a new version, pack the message
957 + // and save an update to SQL
958 + rrdcontext_message_send_unsafe(rc, false, bundle);
959 + messages_added++;
960 +
961 + rc->queue.dispatches++;
962 + rc->queue.dequeued_ut = now_ut;
963 + }
964 + else
965 + rc->version = rc->hub.version;
966 +
967 + // remove it from the queue
968 + worker_is_busy(WORKER_JOB_DEQUEUE);
969 + rrdcontext_dequeue_from_hub_queue(rc);
970 +
971 + if(unlikely(rrdcontext_should_be_deleted(rc))) {
972 + // this is a deleted context - delete it forever...
973 +
974 + worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
975 +
976 + rrdcontext_dequeue_from_post_processing(rc);
977 + rrdcontext_delete_from_sql_unsafe(rc);
978 +
979 + STRING *id = string_dup(rc->id);
980 + rrdcontext_unlock(rc);
981 +
982 + // delete it from the master dictionary
983 + if(!dictionary_del(host->rrdctx.contexts, string2str(rc->id)))
984 + error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.",
985 + string2str(id), rrdhost_hostname(host));
986 +
987 + string_freez(id);
988 + }
989 + else
990 + rrdcontext_unlock(rc);
991 + }
992 + freez(claim_id);
993 + }
994 + dfe_done(rc);
995 +
996 +#ifdef ENABLE_ACLK
997 + if(service_running(SERVICE_CONTEXT) && bundle) {
998 + // we have a bundle to send messages
999 +
1000 + // update the version hash
1001 + contexts_updated_update_version_hash(bundle, rrdcontext_version_hash(host));
1002 +
1003 + // send it
1004 + aclk_send_contexts_updated(bundle);
1005 + }
1006 + else if(bundle)
1007 + contexts_updated_delete(bundle);
1008 +#endif
1009 +
1010 +}
1011 +
1012 +// ----------------------------------------------------------------------------
1013 +// worker thread
1014 +
1015 +static void rrdcontext_main_cleanup(void *ptr) {
1016 + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
1017 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
1018 +
1019 + // custom code
1020 + worker_unregister();
1021 +
1022 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
1023 +}
1024 +
1025 +void *rrdcontext_main(void *ptr) {
1026 + netdata_thread_cleanup_push(rrdcontext_main_cleanup, ptr);
1027 +
1028 + worker_register("RRDCONTEXT");
1029 + worker_register_job_name(WORKER_JOB_HOSTS, "hosts");
1030 + worker_register_job_name(WORKER_JOB_CHECK, "dedup checks");
1031 + worker_register_job_name(WORKER_JOB_SEND, "sent contexts");
1032 + worker_register_job_name(WORKER_JOB_DEQUEUE, "deduplicated contexts");
1033 + worker_register_job_name(WORKER_JOB_RETENTION, "metrics retention");
1034 + worker_register_job_name(WORKER_JOB_QUEUED, "queued contexts");
1035 + worker_register_job_name(WORKER_JOB_CLEANUP, "cleanups");
1036 + worker_register_job_name(WORKER_JOB_CLEANUP_DELETE, "deletes");
1037 + worker_register_job_name(WORKER_JOB_PP_METRIC, "check metrics");
1038 + worker_register_job_name(WORKER_JOB_PP_INSTANCE, "check instances");
1039 + worker_register_job_name(WORKER_JOB_PP_CONTEXT, "check contexts");
1040 +
1041 + worker_register_job_custom_metric(WORKER_JOB_HUB_QUEUE_SIZE, "hub queue size", "contexts", WORKER_METRIC_ABSOLUTE);
1042 + worker_register_job_custom_metric(WORKER_JOB_PP_QUEUE_SIZE, "post processing queue size", "contexts", WORKER_METRIC_ABSOLUTE);
1043 +
1044 + heartbeat_t hb;
1045 + heartbeat_init(&hb);
1046 + usec_t step = RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC;
1047 +
1048 + while (service_running(SERVICE_CONTEXT)) {
1049 + worker_is_idle();
1050 + heartbeat_next(&hb, step);
1051 +
1052 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
1053 +
1054 + usec_t now_ut = now_realtime_usec();
1055 +
1056 + if(rrdcontext_next_db_rotation_ut && now_ut > rrdcontext_next_db_rotation_ut) {
1057 + rrdcontext_recalculate_retention_all_hosts();
1058 + rrdcontext_garbage_collect_for_all_hosts();
1059 + rrdcontext_next_db_rotation_ut = 0;
1060 + }
1061 +
1062 + size_t hub_queued_contexts_for_all_hosts = 0;
1063 + size_t pp_queued_contexts_for_all_hosts = 0;
1064 +
1065 + RRDHOST *host;
1066 + dfe_start_reentrant(rrdhost_root_index, host) {
1067 + if(unlikely(!service_running(SERVICE_CONTEXT))) break;
1068 +
1069 + worker_is_busy(WORKER_JOB_HOSTS);
1070 +
1071 + if(host->rrdctx.pp_queue) {
1072 + pp_queued_contexts_for_all_hosts +=
1073 + dictionary_entries(host->rrdctx.pp_queue);
1074 + rrdcontext_post_process_queued_contexts(host);
1075 + }
1076 +
1077 + if(host->rrdctx.hub_queue) {
1078 + hub_queued_contexts_for_all_hosts += dictionary_entries(host->rrdctx.hub_queue);
1079 + rrdcontext_dispatch_queued_contexts_to_hub(host, now_ut);
1080 + }
1081 + }
1082 + dfe_done(host);
1083 +
1084 + worker_set_metric(WORKER_JOB_HUB_QUEUE_SIZE, (NETDATA_DOUBLE)hub_queued_contexts_for_all_hosts);
1085 + worker_set_metric(WORKER_JOB_PP_QUEUE_SIZE, (NETDATA_DOUBLE)pp_queued_contexts_for_all_hosts);
1086 + }
1087 +
1088 + netdata_thread_cleanup_pop(1);
1089 + return NULL;
1090 +}
database/engine/cache.c
+1 -1
@@ -2092,7 +2092,7 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2092
2093 struct section_pages *sp = *section_pages_pptr;
2094 if(!netdata_spinlock_trylock(&sp->migration_to_v2_spinlock)) {
2095 - internal_fatal(true, "DBENGINE: migration to journal v2 is already running for this section");
2095 + info("DBENGINE: migration to journal v2 for datafile %u is postponed, another jv2 indexer is already running for this section", datafile_fileno);
2096 pgc_ll_unlock(cache, &cache->hot);
2097 return;
2098 }
database/engine/rrdengine.c
+14 -7
@@ -1442,21 +1442,28 @@ static void *journal_v2_indexing_tp_worker(struct rrdengine_instance *ctx __mayb
1442 worker_is_busy(UV_EVENT_DBENGINE_JOURNAL_INDEX);
1443 count = 0;
1444 while (datafile && datafile->fileno != ctx_last_fileno_get(ctx) && datafile->fileno != ctx_last_flush_fileno_get(ctx)) {
1445 + if(journalfile_v2_data_available(datafile->journalfile)) {
1446 + // journal file v2 is already there for this datafile
1447 + datafile = datafile->next;
1448 + continue;
1449 + }
1450
1451 netdata_spinlock_lock(&datafile->writers.spinlock);
1452 bool available = (datafile->writers.running || datafile->writers.flushed_to_open_running) ? false : true;
1453 netdata_spinlock_unlock(&datafile->writers.spinlock);
1454
1450 - if(!available)
1455 + if(!available) {
1456 + info("DBENGINE: journal file %u needs to be indexed, but it has writers working on it - skipping it for now", datafile->fileno);
1457 + datafile = datafile->next;
1458 continue;
1452 -
1453 - if (unlikely(!journalfile_v2_data_available(datafile->journalfile))) {
1454 - info("DBENGINE: journal file %u is ready to be indexed", datafile->fileno);
1455 - pgc_open_cache_to_journal_v2(open_cache, (Word_t) ctx, (int) datafile->fileno, ctx->config.page_type,
1456 - journalfile_migrate_to_v2_callback, (void *) datafile->journalfile);
1457 - count++;
1459 }
1460
1461 + info("DBENGINE: journal file %u is ready to be indexed", datafile->fileno);
1462 + pgc_open_cache_to_journal_v2(open_cache, (Word_t) ctx, (int) datafile->fileno, ctx->config.page_type,
1463 + journalfile_migrate_to_v2_callback, (void *) datafile->journalfile);
1464 +
1465 + count++;
1466 +
1467 datafile = datafile->next;
1468
1469 if (unlikely(!ctx_is_available_for_queries(ctx)))
database/rrd.h
+8 -5
@@ -148,7 +148,7 @@ typedef enum __attribute__ ((__packed__)) rrdset_type {
148 RRDSET_TYPE rrdset_type_id(const char *name);
149 const char *rrdset_type_name(RRDSET_TYPE chart_type);
150
151 -#include "rrdcontext.h"
151 +#include "contexts/rrdcontext.h"
152
153 extern bool unittest_running;
154 extern bool dbengine_enabled;
@@ -297,7 +297,8 @@ int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const
297
298 void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
299 bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
300 -bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal);
300 +
301 +bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches);
302 int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
303 void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb);
304
@@ -1086,9 +1087,11 @@ struct rrdhost {
1087 DICTIONARY *rrdvars; // the host's chart variables index
1088 // this includes custom host variables
1089
1089 - RRDCONTEXTS *rrdctx_hub_queue;
1090 - RRDCONTEXTS *rrdctx_post_processing_queue;
1091 - RRDCONTEXTS *rrdctx;
1090 + struct {
1091 + DICTIONARY *contexts;
1092 + DICTIONARY *hub_queue;
1093 + DICTIONARY *pp_queue;
1094 + } rrdctx;
1095
1096 uuid_t host_uuid; // Global GUID for this host
1097 uuid_t *node_id; // Cloud node_id
database/rrdcalc.c
+7 -9
@@ -359,13 +359,14 @@ static inline bool rrdcalc_check_if_it_matches_rrdset(RRDCALC *rc, RRDSET *st) {
359 && (rc->chart != st->name))
360 return false;
361
362 - if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st)))
362 + if (rc->module_pattern && !simple_pattern_matches_string(rc->module_pattern, st->module_name))
363 return false;
364
365 - if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st)))
365 + if (rc->plugin_pattern && !simple_pattern_matches_string(rc->plugin_pattern, st->module_name))
366 return false;
367
368 - if (st->rrdhost->rrdlabels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->rrdlabels, rc->host_labels_pattern, '='))
368 + if (st->rrdhost->rrdlabels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(
369 + st->rrdhost->rrdlabels, rc->host_labels_pattern, '=', NULL))
370 return false;
371
372 return true;
@@ -741,7 +742,7 @@ void rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(RRDHOST *host
742 if (!rc->host_labels)
743 continue;
744
744 - if(!rrdlabels_match_simple_pattern_parsed(host->rrdlabels, rc->host_labels_pattern, '=')) {
745 + if(!rrdlabels_match_simple_pattern_parsed(host->rrdlabels, rc->host_labels_pattern, '=', NULL)) {
746 log_health("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
747 rrdcalc_name(rc),
748 rrdhost_hostname(host),
@@ -754,18 +755,15 @@ void rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(RRDHOST *host
755 }
756
757 void rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts() {
757 - rrd_rdlock();
758 -
758 RRDHOST *host;
760 - rrdhost_foreach_read(host) {
759 + dfe_start_reentrant(rrdhost_root_index, host) {
760 if (unlikely(!host->health.health_enabled))
761 continue;
762
763 if (host->rrdlabels)
764 rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
765 }
767 -
768 - rrd_unlock();
766 + dfe_done(host);
767 }
768
769 void rrdcalc_unlink_all_rrdset_alerts(RRDSET *st) {
database/rrdcalc.h
+1 -1
@@ -31,7 +31,7 @@ typedef enum {
31 typedef enum {
32 // This list uses several other options from RRDR_OPTIONS for db lookups.
33 // To add an item here, you need to reserve a bit in RRDR_OPTIONS.
34 - RRDCALC_OPTION_NO_CLEAR_NOTIFICATION = 0x80000000,
34 + RRDCALC_OPTION_NO_CLEAR_NOTIFICATION = RRDR_OPTION_HEALTH_RSRVD1,
35 } RRDCALC_OPTIONS;
36
37 #define RRDCALC_ALL_OPTIONS_EXCLUDING_THE_RRDR_ONES (RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)
database/rrdcalctemplate.c
+9 -6
@@ -34,26 +34,29 @@ bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RR
34 if(rt->foreach_dimension_pattern && !rrdset_number_of_dimensions(st))
35 return false;
36
37 - if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, rrdset_name(st)) && !simple_pattern_matches(rt->charts_pattern, rrdset_id(st)))
37 + if (rt->charts_pattern && !simple_pattern_matches_string(rt->charts_pattern, st->name) && !simple_pattern_matches_string(rt->charts_pattern, st->id))
38 return false;
39
40 - if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, rrdset_family(st)))
40 + if (rt->family_pattern && !simple_pattern_matches_string(rt->family_pattern, st->family))
41 return false;
42
43 - if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, rrdset_module_name(st)))
43 + if (rt->module_pattern && !simple_pattern_matches_string(rt->module_pattern, st->module_name))
44 return false;
45
46 - if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, rrdset_plugin_name(st)))
46 + if (rt->plugin_pattern && !simple_pattern_matches_string(rt->plugin_pattern, st->plugin_name))
47 return false;
48
49 - if(host->rrdlabels && rt->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(host->rrdlabels, rt->host_labels_pattern, '='))
49 + if(host->rrdlabels && rt->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(host->rrdlabels,
50 + rt->host_labels_pattern,
51 + '=', NULL))
52 return false;
53
54 return true;
55 }
56
57 void rrdcalctemplate_check_rrddim_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDDIM *rd, RRDHOST *host) {
56 - if (simple_pattern_matches(rt->foreach_dimension_pattern, rrddim_id(rd)) || simple_pattern_matches(rt->foreach_dimension_pattern, rrddim_name(rd))) {
58 + if (simple_pattern_matches_string(rt->foreach_dimension_pattern, rd->id) ||
59 + simple_pattern_matches_string(rt->foreach_dimension_pattern, rd->name)) {
60 char *overwrite_alert_name = rrdcalc_alert_name_with_dimension(
61 rrdcalctemplate_name(rt), string_strlen(rt->name), rrddim_name(rd), string_strlen(rd->name));
62 rrdcalc_add_from_rrdcalctemplate(host, rt, st, overwrite_alert_name, rrddim_name(rd));
database/rrdcontext.c deleted
-4249
@@ -1,4249 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "rrdcontext.h"
4 -#include "sqlite/sqlite_context.h"
5 -#include "aclk/schema-wrappers/context.h"
6 -#include "aclk/aclk_contexts_api.h"
7 -#include "aclk/aclk.h"
8 -#include "storage_engine.h"
9 -
10 -#define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
11 -#define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
12 -#define RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC (1000 * USEC_PER_MS)
13 -#define RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY 10
14 -
15 -#define LOG_TRANSITIONS false
16 -
17 -#define WORKER_JOB_HOSTS 1
18 -#define WORKER_JOB_CHECK 2
19 -#define WORKER_JOB_SEND 3
20 -#define WORKER_JOB_DEQUEUE 4
21 -#define WORKER_JOB_RETENTION 5
22 -#define WORKER_JOB_QUEUED 6
23 -#define WORKER_JOB_CLEANUP 7
24 -#define WORKER_JOB_CLEANUP_DELETE 8
25 -#define WORKER_JOB_PP_METRIC 9 // post-processing metrics
26 -#define WORKER_JOB_PP_INSTANCE 10 // post-processing instances
27 -#define WORKER_JOB_PP_CONTEXT 11 // post-processing contexts
28 -#define WORKER_JOB_HUB_QUEUE_SIZE 12
29 -#define WORKER_JOB_PP_QUEUE_SIZE 13
30 -
31 -
32 -typedef enum __attribute__ ((__packed__)) {
33 - RRD_FLAG_NONE = 0,
34 - RRD_FLAG_DELETED = (1 << 0), // this is a deleted object (metrics, instances, contexts)
35 - RRD_FLAG_COLLECTED = (1 << 1), // this object is currently being collected
36 - RRD_FLAG_UPDATED = (1 << 2), // this object has updates to propagate
37 - RRD_FLAG_ARCHIVED = (1 << 3), // this object is not currently being collected
38 - RRD_FLAG_OWN_LABELS = (1 << 4), // this instance has its own labels - not linked to an RRDSET
39 - RRD_FLAG_LIVE_RETENTION = (1 << 5), // we have got live retention from the database
40 - RRD_FLAG_QUEUED_FOR_HUB = (1 << 6), // this context is currently queued to be dispatched to hub
41 - RRD_FLAG_QUEUED_FOR_PP = (1 << 7), // this context is currently queued to be post-processed
42 - RRD_FLAG_HIDDEN = (1 << 8), // don't expose this to the hub or the API
43 -
44 - RRD_FLAG_UPDATE_REASON_TRIGGERED = (1 << 9), // the update was triggered by the child object
45 - RRD_FLAG_UPDATE_REASON_LOAD_SQL = (1 << 10), // this object has just been loaded from SQL
46 - RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 11), // this object has just been created
47 - RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT = (1 << 12), // we received an update on this object
48 - RRD_FLAG_UPDATE_REASON_CHANGED_LINKING = (1 << 13), // an instance or a metric switched RRDSET or RRDDIM
49 - RRD_FLAG_UPDATE_REASON_CHANGED_METADATA = (1 << 14), // this context or instance changed uuid, name, units, title, family, chart type, priority, update every, rrd changed flags
50 - RRD_FLAG_UPDATE_REASON_ZERO_RETENTION = (1 << 15), // this object has no retention
51 - RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T = (1 << 16), // this object changed its oldest time in the db
52 - RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T = (1 << 17), // this object change its latest time in the db
53 - RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED = (1 << 18), // this object has stopped being collected
54 - RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED = (1 << 19), // this object has started being collected
55 - RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 20), // this context belongs to a host that just disconnected
56 - RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 21), // this context is not used anymore
57 - RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 22), // this context changed because of a db rotation
58 -
59 - // action to perform on an object
60 - RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION = (1 << 30), // this object has to update its retention from the db
61 -} RRD_FLAGS;
62 -
63 -#define RRD_FLAG_ALL_UPDATE_REASONS ( \
64 - RRD_FLAG_UPDATE_REASON_TRIGGERED \
65 - |RRD_FLAG_UPDATE_REASON_LOAD_SQL \
66 - |RRD_FLAG_UPDATE_REASON_NEW_OBJECT \
67 - |RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT \
68 - |RRD_FLAG_UPDATE_REASON_CHANGED_LINKING \
69 - |RRD_FLAG_UPDATE_REASON_CHANGED_METADATA \
70 - |RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
71 - |RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T \
72 - |RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T \
73 - |RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED \
74 - |RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED \
75 - |RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \
76 - |RRD_FLAG_UPDATE_REASON_DB_ROTATION \
77 - |RRD_FLAG_UPDATE_REASON_UNUSED \
78 - )
79 -
80 -#define RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS ( \
81 - RRD_FLAG_ARCHIVED \
82 - |RRD_FLAG_HIDDEN \
83 - |RRD_FLAG_ALL_UPDATE_REASONS \
84 - )
85 -
86 -#define RRD_FLAGS_REQUIRED_FOR_DELETIONS ( \
87 - RRD_FLAG_DELETED \
88 - |RRD_FLAG_LIVE_RETENTION \
89 -)
90 -
91 -#define RRD_FLAGS_PREVENTING_DELETIONS ( \
92 - RRD_FLAG_QUEUED_FOR_HUB \
93 - |RRD_FLAG_COLLECTED \
94 - |RRD_FLAG_QUEUED_FOR_PP \
95 -)
96 -
97 -// get all the flags of an object
98 -#define rrd_flags_get(obj) __atomic_load_n(&((obj)->flags), __ATOMIC_SEQ_CST)
99 -
100 -// check if ANY of the given flags (bits) is set
101 -#define rrd_flag_check(obj, flag) (rrd_flags_get(obj) & (flag))
102 -
103 -// check if ALL the given flags (bits) are set
104 -#define rrd_flag_check_all(obj, flag) (rrd_flag_check(obj, flag) == (flag))
105 -
106 -// set one or more flags (bits)
107 -#define rrd_flag_set(obj, flag) __atomic_or_fetch(&((obj)->flags), flag, __ATOMIC_SEQ_CST)
108 -
109 -// clear one or more flags (bits)
110 -#define rrd_flag_clear(obj, flag) __atomic_and_fetch(&((obj)->flags), ~(flag), __ATOMIC_SEQ_CST)
111 -
112 -// replace the flags of an object, with the supplied ones
113 -#define rrd_flags_replace(obj, all_flags) __atomic_store_n(&((obj)->flags), all_flags, __ATOMIC_SEQ_CST)
114 -
115 -static inline void
116 -rrd_flag_add_remove_atomic(RRD_FLAGS *flags, RRD_FLAGS check, RRD_FLAGS conditionally_add, RRD_FLAGS always_remove) {
117 - RRD_FLAGS expected, desired;
118 -
119 - do {
120 - expected = *flags;
121 -
122 - desired = expected;
123 - desired &= ~(always_remove);
124 -
125 - if(!(expected & check))
126 - desired |= (check | conditionally_add);
127 -
128 - } while(!__atomic_compare_exchange_n(flags, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
129 -}
130 -
131 -#define rrd_flag_set_collected(obj) \
132 - rrd_flag_add_remove_atomic(&((obj)->flags) \
133 - /* check this flag */ \
134 - , RRD_FLAG_COLLECTED \
135 - \
136 - /* add these flags together with the above, if the above is not already set */ \
137 - , RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED | RRD_FLAG_UPDATED \
138 - \
139 - /* always remove these flags */ \
140 - , RRD_FLAG_ARCHIVED \
141 - | RRD_FLAG_DELETED \
142 - | RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED \
143 - | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
144 - | RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \
145 - )
146 -
147 -#define rrd_flag_set_archived(obj) \
148 - rrd_flag_add_remove_atomic(&((obj)->flags) \
149 - /* check this flag */ \
150 - , RRD_FLAG_ARCHIVED \
151 - \
152 - /* add these flags together with the above, if the above is not already set */ \
153 - , RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED | RRD_FLAG_UPDATED \
154 - \
155 - /* always remove these flags */ \
156 - , RRD_FLAG_COLLECTED \
157 - | RRD_FLAG_DELETED \
158 - | RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED \
159 - | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
160 - )
161 -
162 -#define rrd_flag_set_deleted(obj, reason) \
163 - rrd_flag_add_remove_atomic(&((obj)->flags) \
164 - /* check this flag */ \
165 - , RRD_FLAG_DELETED \
166 - \
167 - /* add these flags together with the above, if the above is not already set */ \
168 - , RRD_FLAG_UPDATE_REASON_ZERO_RETENTION | RRD_FLAG_UPDATED | (reason) \
169 - \
170 - /* always remove these flags */ \
171 - , RRD_FLAG_ARCHIVED \
172 - | RRD_FLAG_COLLECTED \
173 - )
174 -
175 -#define rrd_flag_is_collected(obj) rrd_flag_check(obj, RRD_FLAG_COLLECTED)
176 -#define rrd_flag_is_archived(obj) rrd_flag_check(obj, RRD_FLAG_ARCHIVED)
177 -#define rrd_flag_is_deleted(obj) rrd_flag_check(obj, RRD_FLAG_DELETED)
178 -#define rrd_flag_is_updated(obj) rrd_flag_check(obj, RRD_FLAG_UPDATED)
179 -
180 -// mark an object as updated, providing reasons (additional bits)
181 -#define rrd_flag_set_updated(obj, reason) rrd_flag_set(obj, RRD_FLAG_UPDATED | (reason))
182 -
183 -// clear an object as being updated, clearing also all the reasons
184 -#define rrd_flag_unset_updated(obj) rrd_flag_clear(obj, RRD_FLAG_UPDATED | RRD_FLAG_ALL_UPDATE_REASONS)
185 -
186 -
187 -static struct rrdcontext_reason {
188 - RRD_FLAGS flag;
189 - const char *name;
190 - usec_t delay_ut;
191 -} rrdcontext_reasons[] = {
192 - // context related
193 - {RRD_FLAG_UPDATE_REASON_TRIGGERED, "triggered transition", 65 * USEC_PER_SEC },
194 - {RRD_FLAG_UPDATE_REASON_NEW_OBJECT, "object created", 65 * USEC_PER_SEC },
195 - {RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT, "object updated", 65 * USEC_PER_SEC },
196 - {RRD_FLAG_UPDATE_REASON_LOAD_SQL, "loaded from sql", 65 * USEC_PER_SEC },
197 - {RRD_FLAG_UPDATE_REASON_CHANGED_METADATA, "changed metadata", 65 * USEC_PER_SEC },
198 - {RRD_FLAG_UPDATE_REASON_ZERO_RETENTION, "has no retention", 65 * USEC_PER_SEC },
199 - {RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T, "updated first_time_t", 65 * USEC_PER_SEC },
200 - {RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T, "updated last_time_t", 65 * USEC_PER_SEC },
201 - {RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED, "stopped collected", 65 * USEC_PER_SEC },
202 - {RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED, "started collected", 5 * USEC_PER_SEC },
203 - {RRD_FLAG_UPDATE_REASON_UNUSED, "unused", 5 * USEC_PER_SEC },
204 -
205 - // not context related
206 - {RRD_FLAG_UPDATE_REASON_CHANGED_LINKING, "changed rrd link", 65 * USEC_PER_SEC },
207 - {RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, "child disconnected", 65 * USEC_PER_SEC },
208 - {RRD_FLAG_UPDATE_REASON_DB_ROTATION, "db rotation", 65 * USEC_PER_SEC },
209 - {RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION, "updated retention", 65 * USEC_PER_SEC },
210 -
211 - // terminator
212 - {0, NULL, 0 },
213 -};
214 -
215 -
216 -typedef struct rrdmetric {
217 - uuid_t uuid;
218 -
219 - STRING *id;
220 - STRING *name;
221 -
222 - RRDDIM *rrddim;
223 -
224 - time_t first_time_s;
225 - time_t last_time_s;
226 - RRD_FLAGS flags;
227 -
228 - struct rrdinstance *ri;
229 -} RRDMETRIC;
230 -
231 -typedef struct rrdinstance {
232 - uuid_t uuid;
233 -
234 - STRING *id;
235 - STRING *name;
236 - STRING *title;
237 - STRING *units;
238 - STRING *family;
239 - uint32_t priority;
240 - RRDSET_TYPE chart_type;
241 -
242 - RRD_FLAGS flags; // flags related to this instance
243 - time_t first_time_s;
244 - time_t last_time_s;
245 -
246 - time_t update_every_s; // data collection frequency
247 - RRDSET *rrdset; // pointer to RRDSET when collected, or NULL
248 -
249 - DICTIONARY *rrdlabels; // linked to RRDSET->chart_labels or own version
250 -
251 - struct rrdcontext *rc;
252 - DICTIONARY *rrdmetrics;
253 -
254 - struct {
255 - uint32_t collected_metrics_count; // a temporary variable to detect BEGIN/END without SET
256 - // don't use it for other purposes
257 - // it goes up and then resets to zero, on every iteration
258 - } internal;
259 -} RRDINSTANCE;
260 -
261 -typedef struct rrdcontext {
262 - uint64_t version;
263 -
264 - STRING *id;
265 - STRING *title;
266 - STRING *units;
267 - STRING *family;
268 - uint32_t priority;
269 - RRDSET_TYPE chart_type;
270 -
271 - RRD_FLAGS flags;
272 - time_t first_time_s;
273 - time_t last_time_s;
274 -
275 - VERSIONED_CONTEXT_DATA hub;
276 -
277 - DICTIONARY *rrdinstances;
278 - RRDHOST *rrdhost;
279 -
280 - struct {
281 - RRD_FLAGS queued_flags; // the last flags that triggered the post-processing
282 - usec_t queued_ut; // the last time this was queued
283 - usec_t dequeued_ut; // the last time we sent (or deduplicated) this context
284 - size_t executions; // how many times this context has been processed
285 - } pp;
286 -
287 - struct {
288 - RRD_FLAGS queued_flags; // the last flags that triggered the queueing
289 - usec_t queued_ut; // the last time this was queued
290 - usec_t delay_calc_ut; // the last time we calculated the scheduled_dispatched_ut
291 - usec_t scheduled_dispatch_ut; // the time it was/is scheduled to be sent
292 - usec_t dequeued_ut; // the last time we sent (or deduplicated) this context
293 - size_t dispatches; // the number of times this has been dispatched to hub
294 - } queue;
295 -
296 - netdata_mutex_t mutex;
297 -} RRDCONTEXT;
298 -
299 -// ----------------------------------------------------------------------------
300 -// helper one-liners for RRDMETRIC
301 -
302 -static bool rrdmetric_update_retention(RRDMETRIC *rm);
303 -
304 -static inline RRDMETRIC *rrdmetric_acquired_value(RRDMETRIC_ACQUIRED *rma) {
305 - return dictionary_acquired_item_value((DICTIONARY_ITEM *)rma);
306 -}
307 -
308 -static inline RRDMETRIC_ACQUIRED *rrdmetric_acquired_dup(RRDMETRIC_ACQUIRED *rma) {
309 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
310 - return (RRDMETRIC_ACQUIRED *)dictionary_acquired_item_dup(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma);
311 -}
312 -
313 -static inline void rrdmetric_release(RRDMETRIC_ACQUIRED *rma) {
314 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
315 - dictionary_acquired_item_release(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma);
316 -}
317 -
318 -const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma) {
319 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
320 - return string2str(rm->id);
321 -}
322 -
323 -const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma) {
324 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
325 - return string2str(rm->name);
326 -}
327 -
328 -STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma) {
329 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
330 - return string_dup(rm->id);
331 -}
332 -
333 -STRING *rrdmetric_acquired_name_dup(RRDMETRIC_ACQUIRED *rma) {
334 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
335 - return string_dup(rm->name);
336 -}
337 -
338 -NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma) {
339 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
340 -
341 - if(rm->rrddim)
342 - return rm->rrddim->last_stored_value;
343 -
344 - return NAN;
345 -}
346 -
347 -// ----------------------------------------------------------------------------
348 -// helper one-liners for RRDINSTANCE
349 -
350 -static inline RRDINSTANCE *rrdinstance_acquired_value(RRDINSTANCE_ACQUIRED *ria) {
351 - return dictionary_acquired_item_value((DICTIONARY_ITEM *)ria);
352 -}
353 -
354 -static inline RRDINSTANCE_ACQUIRED *rrdinstance_acquired_dup(RRDINSTANCE_ACQUIRED *ria) {
355 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
356 - return (RRDINSTANCE_ACQUIRED *)dictionary_acquired_item_dup(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria);
357 -}
358 -
359 -static inline void rrdinstance_release(RRDINSTANCE_ACQUIRED *ria) {
360 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
361 - dictionary_acquired_item_release(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria);
362 -}
363 -
364 -const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria) {
365 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
366 - return string2str(ri->id);
367 -}
368 -
369 -const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria) {
370 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
371 - return string2str(ri->name);
372 -}
373 -
374 -const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria) {
375 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
376 - return string2str(ri->units);
377 -}
378 -
379 -STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria) {
380 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
381 - return string_dup(ri->units);
382 -}
383 -
384 -DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) {
385 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
386 - return ri->rrdlabels;
387 -}
388 -
389 -DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria) {
390 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
391 - if(!ri->rrdset) return NULL;
392 - return ri->rrdset->functions_view;
393 -}
394 -
395 -RRDHOST *rrdinstance_acquired_rrdhost(RRDINSTANCE_ACQUIRED *ria) {
396 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
397 - return ri->rc->rrdhost;
398 -}
399 -
400 -// ----------------------------------------------------------------------------
401 -// helper one-liners for RRDCONTEXT
402 -
403 -static inline RRDCONTEXT *rrdcontext_acquired_value(RRDCONTEXT_ACQUIRED *rca) {
404 - return dictionary_acquired_item_value((DICTIONARY_ITEM *)rca);
405 -}
406 -
407 -const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca) {
408 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
409 - return string2str(rc->id);
410 -}
411 -
412 -bool rrdcontext_acquired_belongs_to_host(RRDCONTEXT_ACQUIRED *rca, RRDHOST *host) {
413 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
414 - return rc->rrdhost == host;
415 -}
416 -
417 -bool rrdinstance_acquired_belongs_to_context(RRDINSTANCE_ACQUIRED *ria, RRDCONTEXT_ACQUIRED *rca) {
418 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
419 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
420 - return ri->rc == rc;
421 -}
422 -
423 -bool rrdmetric_acquired_belongs_to_instance(RRDMETRIC_ACQUIRED *rma, RRDINSTANCE_ACQUIRED *ria) {
424 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
425 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
426 - return rm->ri == ri;
427 -}
428 -
429 -time_t rrdinstance_acquired_update_every(RRDINSTANCE_ACQUIRED *ria) {
430 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
431 - return ri->update_every_s;
432 -}
433 -time_t rrdmetric_acquired_first_entry(RRDMETRIC_ACQUIRED *rma) {
434 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
435 - return rm->first_time_s;
436 -}
437 -time_t rrdmetric_acquired_last_entry(RRDMETRIC_ACQUIRED *rma) {
438 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
439 -
440 - if(rrd_flag_check(rm, RRD_FLAG_COLLECTED))
441 - return 0;
442 -
443 - return rm->last_time_s;
444 -}
445 -
446 -static inline RRDCONTEXT_ACQUIRED *rrdcontext_acquired_dup(RRDCONTEXT_ACQUIRED *rca) {
447 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
448 - return (RRDCONTEXT_ACQUIRED *)dictionary_acquired_item_dup((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca);
449 -}
450 -
451 -static inline void rrdcontext_release(RRDCONTEXT_ACQUIRED *rca) {
452 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
453 - dictionary_acquired_item_release((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca);
454 -}
455 -
456 -static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs);
457 -static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs);
458 -
459 -#define rrdcontext_version_hash(host) rrdcontext_version_hash_with_callback(host, NULL, false, NULL)
460 -static uint64_t rrdcontext_version_hash_with_callback(RRDHOST *host, void (*callback)(RRDCONTEXT *, bool, void *), bool snapshot, void *bundle);
461 -
462 -static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs);
463 -static void rrdcontext_garbage_collect_for_all_hosts(void);
464 -
465 -#define rrdcontext_lock(rc) netdata_mutex_lock(&((rc)->mutex))
466 -#define rrdcontext_unlock(rc) netdata_mutex_unlock(&((rc)->mutex))
467 -
468 -// ----------------------------------------------------------------------------
469 -// Forward definitions
470 -
471 -static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc);
472 -static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused);
473 -static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused);
474 -
475 -static void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc);
476 -
477 -static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc);
478 -static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function, RRD_FLAGS flags);
479 -static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs);
480 -
481 -static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
482 -static void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function);
483 -static void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function);
484 -
485 -// ----------------------------------------------------------------------------
486 -// visualizing flags
487 -
488 -static void rrd_flags_to_buffer_json_array_items(RRD_FLAGS flags, BUFFER *wb) {
489 - if(flags & RRD_FLAG_QUEUED_FOR_HUB)
490 - buffer_json_add_array_item_string(wb, "QUEUED");
491 -
492 - if(flags & RRD_FLAG_DELETED)
493 - buffer_json_add_array_item_string(wb, "DELETED");
494 -
495 - if(flags & RRD_FLAG_COLLECTED)
496 - buffer_json_add_array_item_string(wb, "COLLECTED");
497 -
498 - if(flags & RRD_FLAG_UPDATED)
499 - buffer_json_add_array_item_string(wb, "UPDATED");
500 -
501 - if(flags & RRD_FLAG_ARCHIVED)
502 - buffer_json_add_array_item_string(wb, "ARCHIVED");
503 -
504 - if(flags & RRD_FLAG_OWN_LABELS)
505 - buffer_json_add_array_item_string(wb, "OWN_LABELS");
506 -
507 - if(flags & RRD_FLAG_LIVE_RETENTION)
508 - buffer_json_add_array_item_string(wb, "LIVE_RETENTION");
509 -
510 - if(flags & RRD_FLAG_HIDDEN)
511 - buffer_json_add_array_item_string(wb, "HIDDEN");
512 -
513 - if(flags & RRD_FLAG_QUEUED_FOR_PP)
514 - buffer_json_add_array_item_string(wb, "PENDING_UPDATES");
515 -}
516 -
517 -static void rrd_reasons_to_buffer_json_array_items(RRD_FLAGS flags, BUFFER *wb) {
518 - for(int i = 0, added = 0; rrdcontext_reasons[i].name ; i++) {
519 - if (flags & rrdcontext_reasons[i].flag) {
520 - buffer_json_add_array_item_string(wb, rrdcontext_reasons[i].name);
521 - added++;
522 - }
523 - }
524 -}
525 -
526 -// ----------------------------------------------------------------------------
527 -// RRDMETRIC
528 -
529 -// free the contents of RRDMETRIC.
530 -// RRDMETRIC itself is managed by DICTIONARY - no need to free it here.
531 -static void rrdmetric_free(RRDMETRIC *rm) {
532 - string_freez(rm->id);
533 - string_freez(rm->name);
534 -
535 - rm->id = NULL;
536 - rm->name = NULL;
537 - rm->ri = NULL;
538 -}
539 -
540 -// called when this rrdmetric is inserted to the rrdmetrics dictionary of a rrdinstance
541 -// the constructor of the rrdmetric object
542 -static void rrdmetric_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance) {
543 - RRDMETRIC *rm = value;
544 -
545 - // link it to its parent
546 - rm->ri = rrdinstance;
547 -
548 - // remove flags that we need to figure out at runtime
549 - rm->flags = rm->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics
550 -
551 - // signal the react callback to do the job
552 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
553 -}
554 -
555 -// called when this rrdmetric is deleted from the rrdmetrics dictionary of a rrdinstance
556 -// the destructor of the rrdmetric object
557 -static void rrdmetric_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) {
558 - RRDMETRIC *rm = value;
559 -
560 - internal_error(rm->rrddim, "RRDMETRIC: '%s' is freed but there is a RRDDIM linked to it.", string2str(rm->id));
561 -
562 - // free the resources
563 - rrdmetric_free(rm);
564 -}
565 -
566 -// called when the same rrdmetric is inserted again to the rrdmetrics dictionary of a rrdinstance
567 -// while this is called, the dictionary is write locked, but there may be other users of the object
568 -static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdinstance __maybe_unused) {
569 - RRDMETRIC *rm = old_value;
570 - RRDMETRIC *rm_new = new_value;
571 -
572 - internal_error(rm->id != rm_new->id,
573 - "RRDMETRIC: '%s' cannot change id to '%s'",
574 - string2str(rm->id), string2str(rm_new->id));
575 -
576 - if(uuid_compare(rm->uuid, rm_new->uuid) != 0) {
577 -#ifdef NETDATA_INTERNAL_CHECKS
578 - char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
579 - uuid_unparse(rm->uuid, uuid1);
580 - uuid_unparse(rm_new->uuid, uuid2);
581 -
582 - time_t old_first_time_s = 0;
583 - time_t old_last_time_s = 0;
584 - if(rrdmetric_update_retention(rm)) {
585 - old_first_time_s = rm->first_time_s;
586 - old_last_time_s = rm->last_time_s;
587 - }
588 -
589 - uuid_copy(rm->uuid, rm_new->uuid);
590 -
591 - time_t new_first_time_s = 0;
592 - time_t new_last_time_s = 0;
593 - if(rrdmetric_update_retention(rm)) {
594 - new_first_time_s = rm->first_time_s;
595 - new_last_time_s = rm->last_time_s;
596 - }
597 -
598 - internal_error(true,
599 - "RRDMETRIC: '%s' of instance '%s' of host '%s' changed UUID from '%s' (retention %ld to %ld, %ld secs) to '%s' (retention %ld to %ld, %ld secs)"
600 - , string2str(rm->id)
601 - , string2str(rm->ri->id)
602 - , rrdhost_hostname(rm->ri->rc->rrdhost)
603 - , uuid1, old_first_time_s, old_last_time_s, old_last_time_s - old_first_time_s
604 - , uuid2, new_first_time_s, new_last_time_s, new_last_time_s - new_first_time_s
605 - );
606 -#else
607 - uuid_copy(rm->uuid, rm_new->uuid);
608 -#endif
609 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
610 - }
611 -
612 - if(rm->rrddim && rm_new->rrddim && rm->rrddim != rm_new->rrddim) {
613 - rm->rrddim = rm_new->rrddim;
614 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
615 - }
616 -
617 -#ifdef NETDATA_INTERNAL_CHECKS
618 - if(rm->rrddim && uuid_compare(rm->uuid, rm->rrddim->metric_uuid) != 0) {
619 - char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
620 - uuid_unparse(rm->uuid, uuid1);
621 - uuid_unparse(rm_new->uuid, uuid2);
622 - internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rrddim_id(rm->rrddim), uuid1, uuid2);
623 - }
624 -#endif
625 -
626 - if(rm->rrddim != rm_new->rrddim)
627 - rm->rrddim = rm_new->rrddim;
628 -
629 - if(rm->name != rm_new->name) {
630 - STRING *old = rm->name;
631 - rm->name = string_dup(rm_new->name);
632 - string_freez(old);
633 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
634 - }
635 -
636 - if(!rm->first_time_s || (rm_new->first_time_s && rm_new->first_time_s < rm->first_time_s)) {
637 - rm->first_time_s = rm_new->first_time_s;
638 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
639 - }
640 -
641 - if(!rm->last_time_s || (rm_new->last_time_s && rm_new->last_time_s > rm->last_time_s)) {
642 - rm->last_time_s = rm_new->last_time_s;
643 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
644 - }
645 -
646 - rrd_flag_set(rm, rm_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no needs for atomics on rm_new
647 -
648 - if(rrd_flag_is_collected(rm) && rrd_flag_is_archived(rm))
649 - rrd_flag_set_collected(rm);
650 -
651 - if(rrd_flag_check(rm, RRD_FLAG_UPDATED))
652 - rrd_flag_set(rm, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
653 -
654 - rrdmetric_free(rm_new);
655 -
656 - // the react callback will continue from here
657 - return rrd_flag_is_updated(rm);
658 -}
659 -
660 -// this is called after the insert or the conflict callbacks,
661 -// but the dictionary is now unlocked
662 -static void rrdmetric_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) {
663 - RRDMETRIC *rm = value;
664 - rrdmetric_trigger_updates(rm, __FUNCTION__ );
665 -}
666 -
667 -static void rrdmetrics_create_in_rrdinstance(RRDINSTANCE *ri) {
668 - if(unlikely(!ri)) return;
669 - if(likely(ri->rrdmetrics)) return;
670 -
671 - ri->rrdmetrics = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
672 - &dictionary_stats_category_rrdcontext, sizeof(RRDMETRIC));
673 -
674 - dictionary_register_insert_callback(ri->rrdmetrics, rrdmetric_insert_callback, ri);
675 - dictionary_register_delete_callback(ri->rrdmetrics, rrdmetric_delete_callback, ri);
676 - dictionary_register_conflict_callback(ri->rrdmetrics, rrdmetric_conflict_callback, ri);
677 - dictionary_register_react_callback(ri->rrdmetrics, rrdmetric_react_callback, ri);
678 -}
679 -
680 -static void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) {
681 - if(unlikely(!ri || !ri->rrdmetrics)) return;
682 - dictionary_destroy(ri->rrdmetrics);
683 - ri->rrdmetrics = NULL;
684 -}
685 -
686 -// trigger post-processing of the rrdmetric, escalating changes to the rrdinstance it belongs
687 -static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) {
688 - if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD)))
689 - rrd_flag_set_archived(rm);
690 -
691 - if(rrd_flag_is_updated(rm) || !rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)) {
692 - rrd_flag_set_updated(rm->ri, RRD_FLAG_UPDATE_REASON_TRIGGERED);
693 - rrdcontext_queue_for_post_processing(rm->ri->rc, function, rm->flags);
694 - }
695 -}
696 -
697 -// ----------------------------------------------------------------------------
698 -// RRDMETRIC HOOKS ON RRDDIM
699 -
700 -static inline void rrdmetric_from_rrddim(RRDDIM *rd) {
701 - if(unlikely(!rd->rrdset))
702 - fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rrddim_id(rd));
703 -
704 - if(unlikely(!rd->rrdset->rrdhost))
705 - fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rrdset_id(rd->rrdset));
706 -
707 - if(unlikely(!rd->rrdset->rrdinstance))
708 - fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rrdset_id(rd->rrdset));
709 -
710 - RRDINSTANCE *ri = rrdinstance_acquired_value(rd->rrdset->rrdinstance);
711 -
712 - RRDMETRIC trm = {
713 - .id = string_dup(rd->id),
714 - .name = string_dup(rd->name),
715 - .flags = RRD_FLAG_NONE, // no need for atomics
716 - .rrddim = rd,
717 - };
718 - uuid_copy(trm.uuid, rd->metric_uuid);
719 -
720 - RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)dictionary_set_and_acquire_item(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
721 -
722 - if(rd->rrdmetric)
723 - rrdmetric_release(rd->rrdmetric);
724 -
725 - rd->rrdmetric = rma;
726 -}
727 -
728 -#define rrddim_get_rrdmetric(rd) rrddim_get_rrdmetric_with_trace(rd, __FUNCTION__)
729 -static inline RRDMETRIC *rrddim_get_rrdmetric_with_trace(RRDDIM *rd, const char *function) {
730 - if(unlikely(!rd->rrdmetric)) {
731 - error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rrddim_id(rd), function);
732 - return NULL;
733 - }
734 -
735 - RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdmetric);
736 - if(unlikely(!rm)) {
737 - error("RRDMETRIC: RRDDIM '%s' lost the link to its RRDMETRIC at %s()", rrddim_id(rd), function);
738 - return NULL;
739 - }
740 -
741 - if(unlikely(rm->rrddim != rd))
742 - fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rrddim_id(rd), function);
743 -
744 - return rm;
745 -}
746 -
747 -static inline void rrdmetric_rrddim_is_freed(RRDDIM *rd) {
748 - RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
749 - if(unlikely(!rm)) return;
750 -
751 - if(unlikely(rrd_flag_is_collected(rm)))
752 - rrd_flag_set_archived(rm);
753 -
754 - rm->rrddim = NULL;
755 - rrdmetric_trigger_updates(rm, __FUNCTION__ );
756 - rrdmetric_release(rd->rrdmetric);
757 - rd->rrdmetric = NULL;
758 -}
759 -
760 -static inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) {
761 - RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
762 - if(unlikely(!rm)) return;
763 -
764 - if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED|RRDDIM_FLAG_OBSOLETE))) {
765 - if(unlikely(rrd_flag_is_collected(rm)))
766 - rrd_flag_set_archived(rm);
767 - }
768 -
769 - rrdmetric_trigger_updates(rm, __FUNCTION__ );
770 -}
771 -
772 -static inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
773 - RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
774 - if(unlikely(!rm)) return;
775 -
776 - if(unlikely(!rrd_flag_is_collected(rm)))
777 - rrd_flag_set_collected(rm);
778 -
779 - // we use this variable to detect BEGIN/END without SET
780 - rm->ri->internal.collected_metrics_count++;
781 -
782 - rrdmetric_trigger_updates(rm, __FUNCTION__ );
783 -}
784 -
785 -// ----------------------------------------------------------------------------
786 -// RRDINSTANCE
787 -
788 -static void rrdinstance_free(RRDINSTANCE *ri) {
789 -
790 - if(rrd_flag_check(ri, RRD_FLAG_OWN_LABELS))
791 - dictionary_destroy(ri->rrdlabels);
792 -
793 - rrdmetrics_destroy_from_rrdinstance(ri);
794 - string_freez(ri->id);
795 - string_freez(ri->name);
796 - string_freez(ri->title);
797 - string_freez(ri->units);
798 - string_freez(ri->family);
799 -
800 - ri->id = NULL;
801 - ri->name = NULL;
802 - ri->title = NULL;
803 - ri->units = NULL;
804 - ri->family = NULL;
805 - ri->rc = NULL;
806 - ri->rrdlabels = NULL;
807 - ri->rrdmetrics = NULL;
808 - ri->rrdset = NULL;
809 -}
810 -
811 -static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext) {
812 - RRDINSTANCE *ri = value;
813 -
814 - // link it to its parent
815 - ri->rc = rrdcontext;
816 -
817 - ri->flags = ri->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics
818 -
819 - if(!ri->name)
820 - ri->name = string_dup(ri->id);
821 -
822 - if(ri->rrdset) {
823 - ri->rrdlabels = ri->rrdset->rrdlabels;
824 - ri->flags &= ~RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor
825 - }
826 - else {
827 - ri->rrdlabels = rrdlabels_create();
828 - ri->flags |= RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor
829 - }
830 -
831 - if(ri->rrdset) {
832 - if(unlikely(rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)))
833 - ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor
834 - else
835 - ri->flags &= ~RRD_FLAG_HIDDEN; // no need of atomics at the constructor
836 - }
837 -
838 - rrdmetrics_create_in_rrdinstance(ri);
839 -
840 - // signal the react callback to do the job
841 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
842 -}
843 -
844 -static void rrdinstance_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext __maybe_unused) {
845 - RRDINSTANCE *ri = (RRDINSTANCE *)value;
846 -
847 - internal_error(ri->rrdset, "RRDINSTANCE: '%s' is freed but there is a RRDSET linked to it.", string2str(ri->id));
848 -
849 - rrdinstance_free(ri);
850 -}
851 -
852 -static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdcontext __maybe_unused) {
853 - RRDINSTANCE *ri = (RRDINSTANCE *)old_value;
854 - RRDINSTANCE *ri_new = (RRDINSTANCE *)new_value;
855 -
856 - internal_error(ri->id != ri_new->id,
857 - "RRDINSTANCE: '%s' cannot change id to '%s'",
858 - string2str(ri->id), string2str(ri_new->id));
859 -
860 - if(uuid_compare(ri->uuid, ri_new->uuid) != 0) {
861 -#ifdef NETDATA_INTERNAL_CHECKS
862 - char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
863 - uuid_unparse(ri->uuid, uuid1);
864 - uuid_unparse(ri_new->uuid, uuid2);
865 - internal_error(true, "RRDINSTANCE: '%s' of host '%s' changed UUID from '%s' to '%s'",
866 - string2str(ri->id), rrdhost_hostname(ri->rc->rrdhost), uuid1, uuid2);
867 -#endif
868 -
869 - uuid_copy(ri->uuid, ri_new->uuid);
870 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
871 - }
872 -
873 - if(ri->rrdset && ri_new->rrdset && ri->rrdset != ri_new->rrdset) {
874 - ri->rrdset = ri_new->rrdset;
875 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
876 - }
877 -
878 -#ifdef NETDATA_INTERNAL_CHECKS
879 - if(ri->rrdset && uuid_compare(ri->uuid, ri->rrdset->chart_uuid) != 0) {
880 - char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
881 - uuid_unparse(ri->uuid, uuid1);
882 - uuid_unparse(ri->rrdset->chart_uuid, uuid2);
883 - internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), rrdset_id(ri->rrdset), uuid1, uuid2);
884 - }
885 -#endif
886 -
887 - if(ri->name != ri_new->name) {
888 - STRING *old = ri->name;
889 - ri->name = string_dup(ri_new->name);
890 - string_freez(old);
891 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
892 - }
893 -
894 - if(ri->title != ri_new->title) {
895 - STRING *old = ri->title;
896 - ri->title = string_dup(ri_new->title);
897 - string_freez(old);
898 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
899 - }
900 -
901 - if(ri->units != ri_new->units) {
902 - STRING *old = ri->units;
903 - ri->units = string_dup(ri_new->units);
904 - string_freez(old);
905 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
906 - }
907 -
908 - if(ri->family != ri_new->family) {
909 - STRING *old = ri->family;
910 - ri->family = string_dup(ri_new->family);
911 - string_freez(old);
912 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
913 - }
914 -
915 - if(ri->chart_type != ri_new->chart_type) {
916 - ri->chart_type = ri_new->chart_type;
917 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
918 - }
919 -
920 - if(ri->priority != ri_new->priority) {
921 - ri->priority = ri_new->priority;
922 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
923 - }
924 -
925 - if(ri->update_every_s != ri_new->update_every_s) {
926 - ri->update_every_s = ri_new->update_every_s;
927 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
928 - }
929 -
930 - if(ri->rrdset != ri_new->rrdset) {
931 - ri->rrdset = ri_new->rrdset;
932 -
933 - if(ri->rrdset && rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
934 - DICTIONARY *old = ri->rrdlabels;
935 - ri->rrdlabels = ri->rrdset->rrdlabels;
936 - rrd_flag_clear(ri, RRD_FLAG_OWN_LABELS);
937 - rrdlabels_destroy(old);
938 - }
939 - else if(!ri->rrdset && !rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
940 - ri->rrdlabels = rrdlabels_create();
941 - rrd_flag_set(ri, RRD_FLAG_OWN_LABELS);
942 - }
943 - }
944 -
945 - if(ri->rrdset) {
946 - if(unlikely(rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)))
947 - rrd_flag_set(ri, RRD_FLAG_HIDDEN);
948 - else
949 - rrd_flag_clear(ri, RRD_FLAG_HIDDEN);
950 - }
951 -
952 - rrd_flag_set(ri, ri_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on ri_new
953 -
954 - if(rrd_flag_is_collected(ri) && rrd_flag_is_archived(ri))
955 - rrd_flag_set_collected(ri);
956 -
957 - if(rrd_flag_is_updated(ri))
958 - rrd_flag_set(ri, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
959 -
960 - // free the new one
961 - rrdinstance_free(ri_new);
962 -
963 - // the react callback will continue from here
964 - return rrd_flag_is_updated(ri);
965 -}
966 -
967 -static void rrdinstance_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext __maybe_unused) {
968 - RRDINSTANCE *ri = value;
969 -
970 - rrdinstance_trigger_updates(ri, __FUNCTION__ );
971 -}
972 -
973 -void rrdinstances_create_in_rrdcontext(RRDCONTEXT *rc) {
974 - if(unlikely(!rc || rc->rrdinstances)) return;
975 -
976 - rc->rrdinstances = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
977 - &dictionary_stats_category_rrdcontext, sizeof(RRDINSTANCE));
978 -
979 - dictionary_register_insert_callback(rc->rrdinstances, rrdinstance_insert_callback, rc);
980 - dictionary_register_delete_callback(rc->rrdinstances, rrdinstance_delete_callback, rc);
981 - dictionary_register_conflict_callback(rc->rrdinstances, rrdinstance_conflict_callback, rc);
982 - dictionary_register_react_callback(rc->rrdinstances, rrdinstance_react_callback, rc);
983 -}
984 -
985 -void rrdinstances_destroy_from_rrdcontext(RRDCONTEXT *rc) {
986 - if(unlikely(!rc || !rc->rrdinstances)) return;
987 -
988 - dictionary_destroy(rc->rrdinstances);
989 - rc->rrdinstances = NULL;
990 -}
991 -
992 -static void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function) {
993 - RRDSET *st = ri->rrdset;
994 -
995 - if(likely(st)) {
996 - if(unlikely((unsigned int) st->priority != ri->priority)) {
997 - ri->priority = st->priority;
998 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
999 - }
1000 - if(unlikely(st->update_every != ri->update_every_s)) {
1001 - ri->update_every_s = st->update_every;
1002 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1003 - }
1004 - }
1005 - else if(unlikely(rrd_flag_is_collected(ri))) {
1006 - // there is no rrdset, but we have it as collected!
1007 -
1008 - rrd_flag_set_archived(ri);
1009 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
1010 - }
1011 -
1012 - if(rrd_flag_is_updated(ri) || !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)) {
1013 - rrd_flag_set_updated(ri->rc, RRD_FLAG_UPDATE_REASON_TRIGGERED);
1014 - rrdcontext_queue_for_post_processing(ri->rc, function, ri->flags);
1015 - }
1016 -}
1017 -
1018 -// ----------------------------------------------------------------------------
1019 -// RRDINSTANCE HOOKS ON RRDSET
1020 -
1021 -static inline void rrdinstance_from_rrdset(RRDSET *st) {
1022 - RRDCONTEXT trc = {
1023 - .id = string_dup(st->context),
1024 - .title = string_dup(st->title),
1025 - .units = string_dup(st->units),
1026 - .family = string_dup(st->family),
1027 - .priority = st->priority,
1028 - .chart_type = st->chart_type,
1029 - .flags = RRD_FLAG_NONE, // no need for atomics
1030 - .rrdhost = st->rrdhost,
1031 - };
1032 -
1033 - RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_set_and_acquire_item((DICTIONARY *)st->rrdhost->rrdctx, string2str(trc.id), &trc, sizeof(trc));
1034 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
1035 -
1036 - RRDINSTANCE tri = {
1037 - .id = string_dup(st->id),
1038 - .name = string_dup(st->name),
1039 - .units = string_dup(st->units),
1040 - .family = string_dup(st->family),
1041 - .title = string_dup(st->title),
1042 - .chart_type = st->chart_type,
1043 - .priority = st->priority,
1044 - .update_every_s = st->update_every,
1045 - .flags = RRD_FLAG_NONE, // no need for atomics
1046 - .rrdset = st,
1047 - };
1048 - uuid_copy(tri.uuid, st->chart_uuid);
1049 -
1050 - RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, string2str(tri.id), &tri, sizeof(tri));
1051 -
1052 - RRDCONTEXT_ACQUIRED *rca_old = st->rrdcontext;
1053 - RRDINSTANCE_ACQUIRED *ria_old = st->rrdinstance;
1054 -
1055 - st->rrdcontext = rca;
1056 - st->rrdinstance = ria;
1057 -
1058 - if(rca == rca_old) {
1059 - rrdcontext_release(rca_old);
1060 - rca_old = NULL;
1061 - }
1062 -
1063 - if(ria == ria_old) {
1064 - rrdinstance_release(ria_old);
1065 - ria_old = NULL;
1066 - }
1067 -
1068 - if(rca_old && ria_old) {
1069 - // Oops! The chart changed context!
1070 -
1071 - // RRDCONTEXT *rc_old = rrdcontext_acquired_value(rca_old);
1072 - RRDINSTANCE *ri_old = rrdinstance_acquired_value(ria_old);
1073 -
1074 - // migrate all dimensions to the new metrics
1075 - RRDDIM *rd;
1076 - rrddim_foreach_read(rd, st) {
1077 - if (!rd->rrdmetric) continue;
1078 -
1079 - RRDMETRIC *rm_old = rrdmetric_acquired_value(rd->rrdmetric);
1080 - rrd_flags_replace(rm_old, RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
1081 - rm_old->rrddim = NULL;
1082 - rm_old->first_time_s = 0;
1083 - rm_old->last_time_s = 0;
1084 -
1085 - rrdmetric_release(rd->rrdmetric);
1086 - rd->rrdmetric = NULL;
1087 -
1088 - rrdmetric_from_rrddim(rd);
1089 - }
1090 - rrddim_foreach_done(rd);
1091 -
1092 - // mark the old instance, ready to be deleted
1093 - if(!rrd_flag_check(ri_old, RRD_FLAG_OWN_LABELS))
1094 - ri_old->rrdlabels = rrdlabels_create();
1095 -
1096 - rrd_flags_replace(ri_old, RRD_FLAG_OWN_LABELS|RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
1097 - ri_old->rrdset = NULL;
1098 - ri_old->first_time_s = 0;
1099 - ri_old->last_time_s = 0;
1100 -
1101 - rrdinstance_trigger_updates(ri_old, __FUNCTION__ );
1102 - rrdinstance_release(ria_old);
1103 -
1104 - /*
1105 - // trigger updates on the old context
1106 - if(!dictionary_entries(rc_old->rrdinstances) && !dictionary_stats_referenced_items(rc_old->rrdinstances)) {
1107 - rrdcontext_lock(rc_old);
1108 - rc_old->flags = ((rc_old->flags & RRD_FLAG_QUEUED)?RRD_FLAG_QUEUED:RRD_FLAG_NONE)|RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION;
1109 - rc_old->first_time_s = 0;
1110 - rc_old->last_time_s = 0;
1111 - rrdcontext_unlock(rc_old);
1112 - rrdcontext_trigger_updates(rc_old, __FUNCTION__ );
1113 - }
1114 - else
1115 - rrdcontext_trigger_updates(rc_old, __FUNCTION__ );
1116 - */
1117 -
1118 - rrdcontext_release(rca_old);
1119 - rca_old = NULL;
1120 - ria_old = NULL;
1121 - }
1122 -
1123 - if(rca_old || ria_old)
1124 - fatal("RRDCONTEXT: cannot switch rrdcontext without switching rrdinstance too");
1125 -}
1126 -
1127 -#define rrdset_get_rrdinstance(st) rrdset_get_rrdinstance_with_trace(st, __FUNCTION__);
1128 -static inline RRDINSTANCE *rrdset_get_rrdinstance_with_trace(RRDSET *st, const char *function) {
1129 - if(unlikely(!st->rrdinstance)) {
1130 - error("RRDINSTANCE: RRDSET '%s' is not linked to an RRDINSTANCE at %s()", rrdset_id(st), function);
1131 - return NULL;
1132 - }
1133 -
1134 - RRDINSTANCE *ri = rrdinstance_acquired_value(st->rrdinstance);
1135 - if(unlikely(!ri)) {
1136 - error("RRDINSTANCE: RRDSET '%s' lost its link to an RRDINSTANCE at %s()", rrdset_id(st), function);
1137 - return NULL;
1138 - }
1139 -
1140 - if(unlikely(ri->rrdset != st))
1141 - fatal("RRDINSTANCE: '%s' is not linked to RRDSET '%s' at %s()", string2str(ri->id), rrdset_id(st), function);
1142 -
1143 - return ri;
1144 -}
1145 -
1146 -static inline void rrdinstance_rrdset_is_freed(RRDSET *st) {
1147 - RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
1148 - if(unlikely(!ri)) return;
1149 -
1150 - rrd_flag_set_archived(ri);
1151 -
1152 - if(!rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
1153 - ri->rrdlabels = rrdlabels_create();
1154 - rrdlabels_copy(ri->rrdlabels, st->rrdlabels);
1155 - rrd_flag_set(ri, RRD_FLAG_OWN_LABELS);
1156 - }
1157 -
1158 - ri->rrdset = NULL;
1159 -
1160 - rrdinstance_trigger_updates(ri, __FUNCTION__ );
1161 -
1162 - rrdinstance_release(st->rrdinstance);
1163 - st->rrdinstance = NULL;
1164 -
1165 - rrdcontext_release(st->rrdcontext);
1166 - st->rrdcontext = NULL;
1167 -}
1168 -
1169 -static inline void rrdinstance_rrdset_has_updated_retention(RRDSET *st) {
1170 - RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
1171 - if(unlikely(!ri)) return;
1172 -
1173 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION);
1174 - rrdinstance_trigger_updates(ri, __FUNCTION__ );
1175 -}
1176 -
1177 -static inline void rrdinstance_updated_rrdset_name(RRDSET *st) {
1178 - // the chart may not be initialized when this is called
1179 - if(unlikely(!st->rrdinstance)) return;
1180 -
1181 - RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
1182 - if(unlikely(!ri)) return;
1183 -
1184 - if(st->name != ri->name) {
1185 - STRING *old = ri->name;
1186 - ri->name = string_dup(st->name);
1187 - string_freez(old);
1188 -
1189 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1190 - rrdinstance_trigger_updates(ri, __FUNCTION__ );
1191 - }
1192 -}
1193 -
1194 -static inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, RRDSET *st) {
1195 - if(unlikely(ri->rrdset != st))
1196 - fatal("RRDCONTEXT: instance '%s' is not linked to chart '%s' on host '%s'",
1197 - string2str(ri->id), rrdset_id(st), rrdhost_hostname(st->rrdhost));
1198 -
1199 - bool st_is_hidden = rrdset_flag_check(st, RRDSET_FLAG_HIDDEN);
1200 - bool ri_is_hidden = rrd_flag_check(ri, RRD_FLAG_HIDDEN);
1201 -
1202 - if(unlikely(st_is_hidden != ri_is_hidden)) {
1203 - if (unlikely(st_is_hidden && !ri_is_hidden))
1204 - rrd_flag_set_updated(ri, RRD_FLAG_HIDDEN | RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1205 -
1206 - else if (unlikely(!st_is_hidden && ri_is_hidden)) {
1207 - rrd_flag_clear(ri, RRD_FLAG_HIDDEN);
1208 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1209 - }
1210 - }
1211 -}
1212 -
1213 -static inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
1214 - RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
1215 - if(unlikely(!ri)) return;
1216 -
1217 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED|RRDSET_FLAG_OBSOLETE)))
1218 - rrd_flag_set_archived(ri);
1219 -
1220 - rrdinstance_updated_rrdset_flags_no_action(ri, st);
1221 -
1222 - rrdinstance_trigger_updates(ri, __FUNCTION__ );
1223 -}
1224 -
1225 -static inline void rrdinstance_collected_rrdset(RRDSET *st) {
1226 - RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
1227 - if(unlikely(!ri)) return;
1228 -
1229 - rrdinstance_updated_rrdset_flags_no_action(ri, st);
1230 -
1231 - if(unlikely(ri->internal.collected_metrics_count && !rrd_flag_is_collected(ri)))
1232 - rrd_flag_set_collected(ri);
1233 -
1234 - // we use this variable to detect BEGIN/END without SET
1235 - ri->internal.collected_metrics_count = 0;
1236 -
1237 - rrdinstance_trigger_updates(ri, __FUNCTION__ );
1238 -}
1239 -
1240 -// ----------------------------------------------------------------------------
1241 -// RRDCONTEXT
1242 -
1243 -static void rrdcontext_freez(RRDCONTEXT *rc) {
1244 - string_freez(rc->id);
1245 - string_freez(rc->title);
1246 - string_freez(rc->units);
1247 - string_freez(rc->family);
1248 -}
1249 -
1250 -static void rrdcontext_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost) {
1251 - RRDHOST *host = (RRDHOST *)rrdhost;
1252 - RRDCONTEXT *rc = (RRDCONTEXT *)value;
1253 -
1254 - rc->rrdhost = host;
1255 - rc->flags = rc->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics at constructor
1256 -
1257 - if(rc->hub.version) {
1258 - // we are loading data from the SQL database
1259 -
1260 - if(rc->version)
1261 - error("RRDCONTEXT: context '%s' is already initialized with version %"PRIu64", but it is loaded again from SQL with version %"PRIu64"", string2str(rc->id), rc->version, rc->hub.version);
1262 -
1263 - // IMPORTANT
1264 - // replace all string pointers in rc->hub with our own versions
1265 - // the originals are coming from a tmp allocation of sqlite
1266 -
1267 - string_freez(rc->id);
1268 - rc->id = string_strdupz(rc->hub.id);
1269 - rc->hub.id = string2str(rc->id);
1270 -
1271 - string_freez(rc->title);
1272 - rc->title = string_strdupz(rc->hub.title);
1273 - rc->hub.title = string2str(rc->title);
1274 -
1275 - string_freez(rc->units);
1276 - rc->units = string_strdupz(rc->hub.units);
1277 - rc->hub.units = string2str(rc->units);
1278 -
1279 - string_freez(rc->family);
1280 - rc->family = string_strdupz(rc->hub.family);
1281 - rc->hub.family = string2str(rc->family);
1282 -
1283 - rc->chart_type = rrdset_type_id(rc->hub.chart_type);
1284 - rc->hub.chart_type = rrdset_type_name(rc->chart_type);
1285 -
1286 - rc->version = rc->hub.version;
1287 - rc->priority = rc->hub.priority;
1288 - rc->first_time_s = (time_t)rc->hub.first_time_s;
1289 - rc->last_time_s = (time_t)rc->hub.last_time_s;
1290 -
1291 - if(rc->hub.deleted || !rc->hub.first_time_s)
1292 - rrd_flag_set_deleted(rc, RRD_FLAG_NONE);
1293 - else {
1294 - if (rc->last_time_s == 0)
1295 - rrd_flag_set_collected(rc);
1296 - else
1297 - rrd_flag_set_archived(rc);
1298 - }
1299 -
1300 - rc->flags |= RRD_FLAG_UPDATE_REASON_LOAD_SQL; // no need for atomics at constructor
1301 - }
1302 - else {
1303 - // we are adding this context now for the first time
1304 - rc->version = now_realtime_sec();
1305 - }
1306 -
1307 - rrdinstances_create_in_rrdcontext(rc);
1308 - netdata_mutex_init(&rc->mutex);
1309 -
1310 - // signal the react callback to do the job
1311 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
1312 -}
1313 -
1314 -static void rrdcontext_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost __maybe_unused) {
1315 -
1316 - RRDCONTEXT *rc = (RRDCONTEXT *)value;
1317 -
1318 - rrdinstances_destroy_from_rrdcontext(rc);
1319 - netdata_mutex_destroy(&rc->mutex);
1320 - rrdcontext_freez(rc);
1321 -}
1322 -
1323 -static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdhost __maybe_unused) {
1324 - RRDCONTEXT *rc = (RRDCONTEXT *)old_value;
1325 - RRDCONTEXT *rc_new = (RRDCONTEXT *)new_value;
1326 -
1327 - //current rc is not archived, new_rc is archived, don't merge
1328 - if (!rrd_flag_is_archived(rc) && rrd_flag_is_archived(rc_new)) {
1329 - rrdcontext_freez(rc_new);
1330 - return false;
1331 - }
1332 -
1333 - rrdcontext_lock(rc);
1334 -
1335 - if(rc->title != rc_new->title) {
1336 - STRING *old_title = rc->title;
1337 - if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new))
1338 - rc->title = string_dup(rc_new->title);
1339 - else
1340 - rc->title = string_2way_merge(rc->title, rc_new->title);
1341 - string_freez(old_title);
1342 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1343 - }
1344 -
1345 - if(rc->units != rc_new->units) {
1346 - STRING *old_units = rc->units;
1347 - rc->units = string_dup(rc_new->units);
1348 - string_freez(old_units);
1349 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1350 - }
1351 -
1352 - if(rc->family != rc_new->family) {
1353 - STRING *old_family = rc->family;
1354 - if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new))
1355 - rc->family = string_dup(rc_new->family);
1356 - else
1357 - rc->family = string_2way_merge(rc->family, rc_new->family);
1358 - string_freez(old_family);
1359 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1360 - }
1361 -
1362 - if(rc->chart_type != rc_new->chart_type) {
1363 - rc->chart_type = rc_new->chart_type;
1364 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1365 - }
1366 -
1367 - if(rc->priority != rc_new->priority) {
1368 - rc->priority = rc_new->priority;
1369 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1370 - }
1371 -
1372 - rrd_flag_set(rc, rc_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on rc_new
1373 -
1374 - if(rrd_flag_is_collected(rc) && rrd_flag_is_archived(rc))
1375 - rrd_flag_set_collected(rc);
1376 -
1377 - if(rrd_flag_is_updated(rc))
1378 - rrd_flag_set(rc, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
1379 -
1380 - rrdcontext_unlock(rc);
1381 -
1382 - // free the resources of the new one
1383 - rrdcontext_freez(rc_new);
1384 -
1385 - // the react callback will continue from here
1386 - return rrd_flag_is_updated(rc);
1387 -}
1388 -
1389 -static void rrdcontext_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost __maybe_unused) {
1390 - RRDCONTEXT *rc = (RRDCONTEXT *)value;
1391 - rrdcontext_trigger_updates(rc, __FUNCTION__ );
1392 -}
1393 -
1394 -static void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function) {
1395 - if(rrd_flag_is_updated(rc) || !rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
1396 - rrdcontext_queue_for_post_processing(rc, function, rc->flags);
1397 -}
1398 -
1399 -static void rrdcontext_hub_queue_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
1400 - RRDCONTEXT *rc = context;
1401 - rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
1402 - rc->queue.queued_ut = now_realtime_usec();
1403 - rc->queue.queued_flags = rrd_flags_get(rc);
1404 -}
1405 -
1406 -static void rrdcontext_hub_queue_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
1407 - RRDCONTEXT *rc = context;
1408 - rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB);
1409 -}
1410 -
1411 -static bool rrdcontext_hub_queue_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *new_context __maybe_unused, void *nothing __maybe_unused) {
1412 - // context and new_context are the same
1413 - // we just need to update the timings
1414 - RRDCONTEXT *rc = context;
1415 - rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
1416 - rc->queue.queued_ut = now_realtime_usec();
1417 - rc->queue.queued_flags |= rrd_flags_get(rc);
1418 -
1419 - return true;
1420 -}
1421 -
1422 -static void rrdcontext_post_processing_queue_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
1423 - RRDCONTEXT *rc = context;
1424 - rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_PP);
1425 - rc->pp.queued_flags = rc->flags;
1426 - rc->pp.queued_ut = now_realtime_usec();
1427 -}
1428 -
1429 -static void rrdcontext_post_processing_queue_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
1430 - RRDCONTEXT *rc = context;
1431 - rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_PP);
1432 - rc->pp.dequeued_ut = now_realtime_usec();
1433 -}
1434 -
1435 -static bool rrdcontext_post_processing_queue_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *new_context __maybe_unused, void *nothing __maybe_unused) {
1436 - RRDCONTEXT *rc = context;
1437 - bool changed = false;
1438 -
1439 - if(!(rc->flags & RRD_FLAG_QUEUED_FOR_PP)) {
1440 - rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_PP);
1441 - changed = true;
1442 - }
1443 -
1444 - if(rc->pp.queued_flags != rc->flags) {
1445 - rc->pp.queued_flags |= rc->flags;
1446 - changed = true;
1447 - }
1448 -
1449 - return changed;
1450 -}
1451 -
1452 -void rrdhost_create_rrdcontexts(RRDHOST *host) {
1453 - if(unlikely(!host)) return;
1454 - if(likely(host->rrdctx)) return;
1455 -
1456 - host->rrdctx = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
1457 - &dictionary_stats_category_rrdcontext, sizeof(RRDCONTEXT));
1458 -
1459 - dictionary_register_insert_callback((DICTIONARY *)host->rrdctx, rrdcontext_insert_callback, host);
1460 - dictionary_register_delete_callback((DICTIONARY *)host->rrdctx, rrdcontext_delete_callback, host);
1461 - dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx, rrdcontext_conflict_callback, host);
1462 - dictionary_register_react_callback((DICTIONARY *)host->rrdctx, rrdcontext_react_callback, host);
1463 -
1464 - host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext, 0);
1465 - dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_insert_callback, NULL);
1466 - dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_delete_callback, NULL);
1467 - dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_conflict_callback, NULL);
1468 -
1469 - host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE, &dictionary_stats_category_rrdcontext, 0);
1470 - dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_post_processing_queue, rrdcontext_post_processing_queue_insert_callback, NULL);
1471 - dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_post_processing_queue, rrdcontext_post_processing_queue_delete_callback, NULL);
1472 - dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx_post_processing_queue, rrdcontext_post_processing_queue_conflict_callback, NULL);
1473 -}
1474 -
1475 -void rrdhost_destroy_rrdcontexts(RRDHOST *host) {
1476 - if(unlikely(!host)) return;
1477 - if(unlikely(!host->rrdctx)) return;
1478 -
1479 - DICTIONARY *old;
1480 -
1481 - if(host->rrdctx_hub_queue) {
1482 - old = (DICTIONARY *)host->rrdctx_hub_queue;
1483 - host->rrdctx_hub_queue = NULL;
1484 -
1485 - RRDCONTEXT *rc;
1486 - dfe_start_write(old, rc) {
1487 - dictionary_del(old, string2str(rc->id));
1488 - }
1489 - dfe_done(rc);
1490 - dictionary_destroy(old);
1491 - }
1492 -
1493 - if(host->rrdctx_post_processing_queue) {
1494 - old = (DICTIONARY *)host->rrdctx_post_processing_queue;
1495 - host->rrdctx_post_processing_queue = NULL;
1496 -
1497 - RRDCONTEXT *rc;
1498 - dfe_start_write(old, rc) {
1499 - dictionary_del(old, string2str(rc->id));
1500 - }
1501 - dfe_done(rc);
1502 - dictionary_destroy(old);
1503 - }
1504 -
1505 - old = (DICTIONARY *)host->rrdctx;
1506 - host->rrdctx = NULL;
1507 - dictionary_destroy(old);
1508 -}
1509 -
1510 -// ----------------------------------------------------------------------------
1511 -// public API
1512 -
1513 -void rrdcontext_updated_rrddim(RRDDIM *rd) {
1514 - rrdmetric_from_rrddim(rd);
1515 -}
1516 -
1517 -void rrdcontext_removed_rrddim(RRDDIM *rd) {
1518 - rrdmetric_rrddim_is_freed(rd);
1519 -}
1520 -
1521 -void rrdcontext_updated_rrddim_algorithm(RRDDIM *rd) {
1522 - rrdmetric_updated_rrddim_flags(rd);
1523 -}
1524 -
1525 -void rrdcontext_updated_rrddim_multiplier(RRDDIM *rd) {
1526 - rrdmetric_updated_rrddim_flags(rd);
1527 -}
1528 -
1529 -void rrdcontext_updated_rrddim_divisor(RRDDIM *rd) {
1530 - rrdmetric_updated_rrddim_flags(rd);
1531 -}
1532 -
1533 -void rrdcontext_updated_rrddim_flags(RRDDIM *rd) {
1534 - rrdmetric_updated_rrddim_flags(rd);
1535 -}
1536 -
1537 -void rrdcontext_collected_rrddim(RRDDIM *rd) {
1538 - rrdmetric_collected_rrddim(rd);
1539 -}
1540 -
1541 -void rrdcontext_updated_rrdset(RRDSET *st) {
1542 - rrdinstance_from_rrdset(st);
1543 -}
1544 -
1545 -void rrdcontext_removed_rrdset(RRDSET *st) {
1546 - rrdinstance_rrdset_is_freed(st);
1547 -}
1548 -
1549 -void rrdcontext_updated_retention_rrdset(RRDSET *st) {
1550 - rrdinstance_rrdset_has_updated_retention(st);
1551 -}
1552 -
1553 -void rrdcontext_updated_rrdset_name(RRDSET *st) {
1554 - rrdinstance_updated_rrdset_name(st);
1555 -}
1556 -
1557 -void rrdcontext_updated_rrdset_flags(RRDSET *st) {
1558 - rrdinstance_updated_rrdset_flags(st);
1559 -}
1560 -
1561 -void rrdcontext_collected_rrdset(RRDSET *st) {
1562 - rrdinstance_collected_rrdset(st);
1563 -}
1564 -
1565 -void rrdcontext_host_child_connected(RRDHOST *host) {
1566 - (void)host;
1567 -
1568 - // no need to do anything here
1569 - ;
1570 -}
1571 -
1572 -int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, uuid_t *store_uuid) {
1573 - if(!st->rrdhost) return 1;
1574 - if(!st->context) return 2;
1575 -
1576 - RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)st->rrdhost->rrdctx, string2str(st->context));
1577 - if(!rca) return 3;
1578 -
1579 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
1580 -
1581 - RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_get_and_acquire_item(rc->rrdinstances, string2str(st->id));
1582 - if(!ria) {
1583 - rrdcontext_release(rca);
1584 - return 4;
1585 - }
1586 -
1587 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
1588 -
1589 - RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)dictionary_get_and_acquire_item(ri->rrdmetrics, id);
1590 - if(!rma) {
1591 - rrdinstance_release(ria);
1592 - rrdcontext_release(rca);
1593 - return 5;
1594 - }
1595 -
1596 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
1597 -
1598 - uuid_copy(*store_uuid, rm->uuid);
1599 -
1600 - rrdmetric_release(rma);
1601 - rrdinstance_release(ria);
1602 - rrdcontext_release(rca);
1603 - return 0;
1604 -}
1605 -
1606 -int rrdcontext_find_chart_uuid(RRDSET *st, uuid_t *store_uuid) {
1607 - if(!st->rrdhost) return 1;
1608 - if(!st->context) return 2;
1609 -
1610 - RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)st->rrdhost->rrdctx, string2str(st->context));
1611 - if(!rca) return 3;
1612 -
1613 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
1614 -
1615 - RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_get_and_acquire_item(rc->rrdinstances, string2str(st->id));
1616 - if(!ria) {
1617 - rrdcontext_release(rca);
1618 - return 4;
1619 - }
1620 -
1621 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
1622 - uuid_copy(*store_uuid, ri->uuid);
1623 -
1624 - rrdinstance_release(ria);
1625 - rrdcontext_release(rca);
1626 - return 0;
1627 -}
1628 -
1629 -void rrdcontext_host_child_disconnected(RRDHOST *host) {
1630 - rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, false);
1631 -}
1632 -
1633 -static usec_t rrdcontext_next_db_rotation_ut = 0;
1634 -void rrdcontext_db_rotation(void) {
1635 - // called when the db rotates its database
1636 - rrdcontext_next_db_rotation_ut = now_realtime_usec() + FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS * USEC_PER_SEC;
1637 -}
1638 -
1639 -int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char *context, int (*callback)(RRDSET *st, void *data), void *data) {
1640 - if(unlikely(!host || !context || !*context || !callback))
1641 - return -1;
1642 -
1643 - RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)host->rrdctx, context);
1644 - if(unlikely(!rca)) return -1;
1645 -
1646 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
1647 - if(unlikely(!rc)) return -1;
1648 -
1649 - int ret = 0;
1650 - RRDINSTANCE *ri;
1651 - dfe_start_read(rc->rrdinstances, ri) {
1652 - if(ri->rrdset) {
1653 - int r = callback(ri->rrdset, data);
1654 - if(r >= 0) ret += r;
1655 - else {
1656 - ret = r;
1657 - break;
1658 - }
1659 - }
1660 - }
1661 - dfe_done(ri);
1662 -
1663 - rrdcontext_release(rca);
1664 -
1665 - return ret;
1666 -}
1667 -
1668 -// ----------------------------------------------------------------------------
1669 -// ACLK interface
1670 -
1671 -static bool rrdhost_check_our_claim_id(const char *claim_id) {
1672 - if(!localhost->aclk_state.claimed_id) return false;
1673 - return (strcasecmp(claim_id, localhost->aclk_state.claimed_id) == 0) ? true : false;
1674 -}
1675 -
1676 -static RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
1677 - uuid_t uuid;
1678 - if (uuid_parse(node_id, uuid))
1679 - return NULL;
1680 -
1681 - RRDHOST *host = NULL;
1682 -
1683 - rrd_rdlock();
1684 - rrdhost_foreach_read(host) {
1685 - if(!host->node_id) continue;
1686 -
1687 - if(uuid_compare(uuid, *host->node_id) == 0)
1688 - break;
1689 - }
1690 - rrd_unlock();
1691 -
1692 - return host;
1693 -}
1694 -
1695 -void rrdcontext_hub_checkpoint_command(void *ptr) {
1696 - struct ctxs_checkpoint *cmd = ptr;
1697 -
1698 - if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
1699 - error("RRDCONTEXT: received checkpoint command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
1700 - cmd->claim_id, cmd->node_id,
1701 - localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
1702 - cmd->claim_id);
1703 -
1704 - return;
1705 - }
1706 -
1707 - RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id);
1708 - if(!host) {
1709 - error("RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', but there is no node with such node id here. Ignoring command.",
1710 - cmd->claim_id, cmd->node_id);
1711 -
1712 - return;
1713 - }
1714 -
1715 - if(rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
1716 - info("RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', while node '%s' has an active context streaming.",
1717 - cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
1718 -
1719 - // disable it temporarily, so that our worker will not attempt to send messages in parallel
1720 - rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
1721 - }
1722 -
1723 - uint64_t our_version_hash = rrdcontext_version_hash(host);
1724 -
1725 - if(cmd->version_hash != our_version_hash) {
1726 - error("RRDCONTEXT: received version hash %"PRIu64" for host '%s', does not match our version hash %"PRIu64". Sending snapshot of all contexts.",
1727 - cmd->version_hash, rrdhost_hostname(host), our_version_hash);
1728 -
1729 -#ifdef ENABLE_ACLK
1730 - // prepare the snapshot
1731 - char uuid[UUID_STR_LEN];
1732 - uuid_unparse_lower(*host->node_id, uuid);
1733 - contexts_snapshot_t bundle = contexts_snapshot_new(cmd->claim_id, uuid, our_version_hash);
1734 -
1735 - // do a deep scan on every metric of the host to make sure all our data are updated
1736 - rrdcontext_recalculate_host_retention(host, RRD_FLAG_NONE, false);
1737 -
1738 - // calculate version hash and pack all the messages together in one go
1739 - our_version_hash = rrdcontext_version_hash_with_callback(host, rrdcontext_message_send_unsafe, true, bundle);
1740 -
1741 - // update the version
1742 - contexts_snapshot_set_version(bundle, our_version_hash);
1743 -
1744 - // send it
1745 - aclk_send_contexts_snapshot(bundle);
1746 -#endif
1747 - }
1748 -
1749 - internal_error(true, "RRDCONTEXT: host '%s' enabling streaming of contexts", rrdhost_hostname(host));
1750 - rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
1751 - char node_str[UUID_STR_LEN];
1752 - uuid_unparse_lower(*host->node_id, node_str);
1753 - log_access("ACLK REQ [%s (%s)]: STREAM CONTEXTS ENABLED", node_str, rrdhost_hostname(host));
1754 -}
1755 -
1756 -void rrdcontext_hub_stop_streaming_command(void *ptr) {
1757 - struct stop_streaming_ctxs *cmd = ptr;
1758 -
1759 - if(!rrdhost_check_our_claim_id(cmd->claim_id)) {
1760 - error("RRDCONTEXT: received stop streaming command for claim_id '%s', node id '%s', but this is not our claim id. Ours '%s', received '%s'. Ignoring command.",
1761 - cmd->claim_id, cmd->node_id,
1762 - localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET",
1763 - cmd->claim_id);
1764 -
1765 - return;
1766 - }
1767 -
1768 - RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id);
1769 - if(!host) {
1770 - error("RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', but there is no node with such node id here. Ignoring command.",
1771 - cmd->claim_id, cmd->node_id);
1772 -
1773 - return;
1774 - }
1775 -
1776 - if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
1777 - error("RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', but node '%s' does not have active context streaming. Ignoring command.",
1778 - cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
1779 -
1780 - return;
1781 - }
1782 -
1783 - internal_error(true, "RRDCONTEXT: host '%s' disabling streaming of contexts", rrdhost_hostname(host));
1784 - rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
1785 -}
1786 -
1787 -// ----------------------------------------------------------------------------
1788 -// web API
1789 -
1790 -struct rrdcontext_to_json {
1791 - BUFFER *wb;
1792 - RRDCONTEXT_TO_JSON_OPTIONS options;
1793 - time_t after;
1794 - time_t before;
1795 - SIMPLE_PATTERN *chart_label_key;
1796 - SIMPLE_PATTERN *chart_labels_filter;
1797 - SIMPLE_PATTERN *chart_dimensions;
1798 - size_t written;
1799 - time_t now;
1800 - time_t combined_first_time_s;
1801 - time_t combined_last_time_s;
1802 - RRD_FLAGS combined_flags;
1803 -};
1804 -
1805 -static inline int rrdmetric_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
1806 - const char *id = dictionary_acquired_item_name(item);
1807 - struct rrdcontext_to_json * t = data;
1808 - RRDMETRIC *rm = value;
1809 - BUFFER *wb = t->wb;
1810 - RRDCONTEXT_TO_JSON_OPTIONS options = t->options;
1811 - time_t after = t->after;
1812 - time_t before = t->before;
1813 -
1814 - if(unlikely(rrd_flag_is_deleted(rm) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
1815 - return 0;
1816 -
1817 - if(after && (!rm->last_time_s || after > rm->last_time_s))
1818 - return 0;
1819 -
1820 - if(before && (!rm->first_time_s || before < rm->first_time_s))
1821 - return 0;
1822 -
1823 - if(t->chart_dimensions
1824 - && !simple_pattern_matches(t->chart_dimensions, string2str(rm->id))
1825 - && !simple_pattern_matches(t->chart_dimensions, string2str(rm->name)))
1826 - return 0;
1827 -
1828 - if(t->written) {
1829 - t->combined_first_time_s = MIN(t->combined_first_time_s, rm->first_time_s);
1830 - t->combined_last_time_s = MAX(t->combined_last_time_s, rm->last_time_s);
1831 - t->combined_flags |= rrd_flags_get(rm);
1832 - }
1833 - else {
1834 - t->combined_first_time_s = rm->first_time_s;
1835 - t->combined_last_time_s = rm->last_time_s;
1836 - t->combined_flags = rrd_flags_get(rm);
1837 - }
1838 -
1839 - buffer_json_member_add_object(wb, id);
1840 -
1841 - if(options & RRDCONTEXT_OPTION_SHOW_UUIDS) {
1842 - char uuid[UUID_STR_LEN];
1843 - uuid_unparse(rm->uuid, uuid);
1844 - buffer_json_member_add_string(wb, "uuid", uuid);
1845 - }
1846 -
1847 - buffer_json_member_add_string(wb, "name", string2str(rm->name));
1848 - buffer_json_member_add_time_t(wb, "first_time_t", rm->first_time_s);
1849 - buffer_json_member_add_time_t(wb, "last_time_t", rrd_flag_is_collected(rm) ? (long long)t->now : (long long)rm->last_time_s);
1850 - buffer_json_member_add_boolean(wb, "collected", rrd_flag_is_collected(rm));
1851 -
1852 - if(options & RRDCONTEXT_OPTION_SHOW_DELETED)
1853 - buffer_json_member_add_boolean(wb, "deleted", rrd_flag_is_deleted(rm));
1854 -
1855 - if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
1856 - buffer_json_member_add_array(wb, "flags");
1857 - rrd_flags_to_buffer_json_array_items(rrd_flags_get(rm), wb);
1858 - buffer_json_array_close(wb);
1859 - }
1860 -
1861 - buffer_json_object_close(wb);
1862 - t->written++;
1863 - return 1;
1864 -}
1865 -
1866 -static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
1867 - const char *id = dictionary_acquired_item_name(item);
1868 -
1869 - struct rrdcontext_to_json *t_parent = data;
1870 - RRDINSTANCE *ri = value;
1871 - BUFFER *wb = t_parent->wb;
1872 - RRDCONTEXT_TO_JSON_OPTIONS options = t_parent->options;
1873 - time_t after = t_parent->after;
1874 - time_t before = t_parent->before;
1875 - bool has_filter = t_parent->chart_label_key || t_parent->chart_labels_filter || t_parent->chart_dimensions;
1876 -
1877 - if(unlikely(rrd_flag_is_deleted(ri) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
1878 - return 0;
1879 -
1880 - if(after && (!ri->last_time_s || after > ri->last_time_s))
1881 - return 0;
1882 -
1883 - if(before && (!ri->first_time_s || before < ri->first_time_s))
1884 - return 0;
1885 -
1886 - if(t_parent->chart_label_key && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, t_parent->chart_label_key, '\0'))
1887 - return 0;
1888 -
1889 - if(t_parent->chart_labels_filter && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, t_parent->chart_labels_filter, ':'))
1890 - return 0;
1891 -
1892 - time_t first_time_s = ri->first_time_s;
1893 - time_t last_time_s = ri->last_time_s;
1894 - RRD_FLAGS flags = rrd_flags_get(ri);
1895 -
1896 - BUFFER *wb_metrics = NULL;
1897 - if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) {
1898 -
1899 - wb_metrics = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
1900 - buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false);
1901 -
1902 - struct rrdcontext_to_json t_metrics = {
1903 - .wb = wb_metrics,
1904 - .options = options,
1905 - .chart_label_key = t_parent->chart_label_key,
1906 - .chart_labels_filter = t_parent->chart_labels_filter,
1907 - .chart_dimensions = t_parent->chart_dimensions,
1908 - .after = after,
1909 - .before = before,
1910 - .written = 0,
1911 - .now = t_parent->now,
1912 - };
1913 - dictionary_walkthrough_read(ri->rrdmetrics, rrdmetric_to_json_callback, &t_metrics);
1914 -
1915 - if(has_filter && !t_metrics.written) {
1916 - buffer_free(wb_metrics);
1917 - return 0;
1918 - }
1919 -
1920 - first_time_s = t_metrics.combined_first_time_s;
1921 - last_time_s = t_metrics.combined_last_time_s;
1922 - flags = t_metrics.combined_flags;
1923 - }
1924 -
1925 - if(t_parent->written) {
1926 - t_parent->combined_first_time_s = MIN(t_parent->combined_first_time_s, first_time_s);
1927 - t_parent->combined_last_time_s = MAX(t_parent->combined_last_time_s, last_time_s);
1928 - t_parent->combined_flags |= flags;
1929 - }
1930 - else {
1931 - t_parent->combined_first_time_s = first_time_s;
1932 - t_parent->combined_last_time_s = last_time_s;
1933 - t_parent->combined_flags = flags;
1934 - }
1935 -
1936 - buffer_json_member_add_object(wb, id);
1937 -
1938 - if(options & RRDCONTEXT_OPTION_SHOW_UUIDS) {
1939 - char uuid[UUID_STR_LEN];
1940 - uuid_unparse(ri->uuid, uuid);
1941 - buffer_json_member_add_string(wb, "uuid", uuid);
1942 - }
1943 -
1944 - buffer_json_member_add_string(wb, "name", string2str(ri->name));
1945 - buffer_json_member_add_string(wb, "context", string2str(ri->rc->id));
1946 - buffer_json_member_add_string(wb, "title", string2str(ri->title));
1947 - buffer_json_member_add_string(wb, "units", string2str(ri->units));
1948 - buffer_json_member_add_string(wb, "family", string2str(ri->family));
1949 - buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(ri->chart_type));
1950 - buffer_json_member_add_uint64(wb, "priority", ri->priority);
1951 - buffer_json_member_add_time_t(wb, "update_every", ri->update_every_s);
1952 - buffer_json_member_add_time_t(wb, "first_time_t", first_time_s);
1953 - buffer_json_member_add_time_t(wb, "last_time_t", (flags & RRD_FLAG_COLLECTED) ? (long long)t_parent->now : (long long)last_time_s);
1954 - buffer_json_member_add_boolean(wb, "collected", flags & RRD_FLAG_COLLECTED);
1955 -
1956 - if(options & RRDCONTEXT_OPTION_SHOW_DELETED)
1957 - buffer_json_member_add_boolean(wb, "deleted", rrd_flag_is_deleted(ri));
1958 -
1959 - if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
1960 - buffer_json_member_add_array(wb, "flags");
1961 - rrd_flags_to_buffer_json_array_items(rrd_flags_get(ri), wb);
1962 - buffer_json_array_close(wb);
1963 - }
1964 -
1965 - if(options & RRDCONTEXT_OPTION_SHOW_LABELS && ri->rrdlabels && dictionary_entries(ri->rrdlabels)) {
1966 - buffer_json_member_add_object(wb, "labels");
1967 - rrdlabels_to_buffer_json_members(ri->rrdlabels, wb);
1968 - buffer_json_object_close(wb);
1969 - }
1970 -
1971 - if(wb_metrics) {
1972 - buffer_json_member_add_object(wb, "dimensions");
1973 - buffer_fast_strcat(wb, buffer_tostring(wb_metrics), buffer_strlen(wb_metrics));
1974 - buffer_json_object_close(wb);
1975 -
1976 - buffer_free(wb_metrics);
1977 - }
1978 -
1979 - buffer_json_object_close(wb);
1980 - t_parent->written++;
1981 - return 1;
1982 -}
1983 -
1984 -static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
1985 - const char *id = dictionary_acquired_item_name(item);
1986 - struct rrdcontext_to_json *t_parent = data;
1987 - RRDCONTEXT *rc = value;
1988 - BUFFER *wb = t_parent->wb;
1989 - RRDCONTEXT_TO_JSON_OPTIONS options = t_parent->options;
1990 - time_t after = t_parent->after;
1991 - time_t before = t_parent->before;
1992 - bool has_filter = t_parent->chart_label_key || t_parent->chart_labels_filter || t_parent->chart_dimensions;
1993 -
1994 - if(unlikely(rrd_flag_check(rc, RRD_FLAG_HIDDEN) && !(options & RRDCONTEXT_OPTION_SHOW_HIDDEN)))
1995 - return 0;
1996 -
1997 - if(unlikely(rrd_flag_is_deleted(rc) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
1998 - return 0;
1999 -
2000 - if(options & RRDCONTEXT_OPTION_DEEPSCAN)
2001 - rrdcontext_recalculate_context_retention(rc, RRD_FLAG_NONE, false);
2002 -
2003 - if(after && (!rc->last_time_s || after > rc->last_time_s))
2004 - return 0;
2005 -
2006 - if(before && (!rc->first_time_s || before < rc->first_time_s))
2007 - return 0;
2008 -
2009 - time_t first_time_s = rc->first_time_s;
2010 - time_t last_time_s = rc->last_time_s;
2011 - RRD_FLAGS flags = rrd_flags_get(rc);
2012 -
2013 - BUFFER *wb_instances = NULL;
2014 - if((options & (RRDCONTEXT_OPTION_SHOW_LABELS|RRDCONTEXT_OPTION_SHOW_INSTANCES|RRDCONTEXT_OPTION_SHOW_METRICS))
2015 - || t_parent->chart_label_key
2016 - || t_parent->chart_labels_filter
2017 - || t_parent->chart_dimensions) {
2018 -
2019 - wb_instances = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
2020 - buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false);
2021 -
2022 - struct rrdcontext_to_json t_instances = {
2023 - .wb = wb_instances,
2024 - .options = options,
2025 - .chart_label_key = t_parent->chart_label_key,
2026 - .chart_labels_filter = t_parent->chart_labels_filter,
2027 - .chart_dimensions = t_parent->chart_dimensions,
2028 - .after = after,
2029 - .before = before,
2030 - .written = 0,
2031 - .now = t_parent->now,
2032 - };
2033 - dictionary_walkthrough_read(rc->rrdinstances, rrdinstance_to_json_callback, &t_instances);
2034 -
2035 - if(has_filter && !t_instances.written) {
2036 - buffer_free(wb_instances);
2037 - return 0;
2038 - }
2039 -
2040 - first_time_s = t_instances.combined_first_time_s;
2041 - last_time_s = t_instances.combined_last_time_s;
2042 - flags = t_instances.combined_flags;
2043 - }
2044 -
2045 - if(!(options & RRDCONTEXT_OPTION_SKIP_ID))
2046 - buffer_json_member_add_object(wb, id);
2047 -
2048 - rrdcontext_lock(rc);
2049 -
2050 - buffer_json_member_add_string(wb, "title", string2str(rc->title));
2051 - buffer_json_member_add_string(wb, "units", string2str(rc->units));
2052 - buffer_json_member_add_string(wb, "family", string2str(rc->family));
2053 - buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rc->chart_type));
2054 - buffer_json_member_add_uint64(wb, "priority", rc->priority);
2055 - buffer_json_member_add_time_t(wb, "first_time_t", first_time_s);
2056 - buffer_json_member_add_time_t(wb, "last_time_t", (flags & RRD_FLAG_COLLECTED) ? (long long)t_parent->now : (long long)last_time_s);
2057 - buffer_json_member_add_boolean(wb, "collected", (flags & RRD_FLAG_COLLECTED));
2058 -
2059 - if(options & RRDCONTEXT_OPTION_SHOW_DELETED)
2060 - buffer_json_member_add_boolean(wb, "deleted", rrd_flag_is_deleted(rc));
2061 -
2062 - if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
2063 - buffer_json_member_add_array(wb, "flags");
2064 - rrd_flags_to_buffer_json_array_items(rrd_flags_get(rc), wb);
2065 - buffer_json_array_close(wb);
2066 - }
2067 -
2068 - if(options & RRDCONTEXT_OPTION_SHOW_QUEUED) {
2069 - buffer_json_member_add_array(wb, "queued_reasons");
2070 - rrd_reasons_to_buffer_json_array_items(rc->queue.queued_flags, wb);
2071 - buffer_json_array_close(wb);
2072 -
2073 - buffer_json_member_add_time_t(wb, "last_queued", (time_t)(rc->queue.queued_ut / USEC_PER_SEC));
2074 - buffer_json_member_add_time_t(wb, "scheduled_dispatch", (time_t)(rc->queue.scheduled_dispatch_ut / USEC_PER_SEC));
2075 - buffer_json_member_add_time_t(wb, "last_dequeued", (time_t)(rc->queue.dequeued_ut / USEC_PER_SEC));
2076 - buffer_json_member_add_uint64(wb, "dispatches", rc->queue.dispatches);
2077 - buffer_json_member_add_uint64(wb, "hub_version", rc->hub.version);
2078 - buffer_json_member_add_uint64(wb, "version", rc->version);
2079 -
2080 - buffer_json_member_add_array(wb, "pp_reasons");
2081 - rrd_reasons_to_buffer_json_array_items(rc->pp.queued_flags, wb);
2082 - buffer_json_array_close(wb);
2083 -
2084 - buffer_json_member_add_time_t(wb, "pp_last_queued", (time_t)(rc->pp.queued_ut / USEC_PER_SEC));
2085 - buffer_json_member_add_time_t(wb, "pp_last_dequeued", (time_t)(rc->pp.dequeued_ut / USEC_PER_SEC));
2086 - buffer_json_member_add_uint64(wb, "pp_executed", rc->pp.executions);
2087 - }
2088 -
2089 - rrdcontext_unlock(rc);
2090 -
2091 - if(wb_instances) {
2092 - buffer_json_member_add_object(wb, "charts");
2093 - buffer_fast_strcat(wb, buffer_tostring(wb_instances), buffer_strlen(wb_instances));
2094 - buffer_json_object_close(wb);
2095 -
2096 - buffer_free(wb_instances);
2097 - }
2098 -
2099 - if(!(options & RRDCONTEXT_OPTION_SKIP_ID))
2100 - buffer_json_object_close(wb);
2101 -
2102 - t_parent->written++;
2103 - return 1;
2104 -}
2105 -
2106 -int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) {
2107 - if(!host->rrdctx) {
2108 - error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host));
2109 - return HTTP_RESP_NOT_FOUND;
2110 - }
2111 -
2112 - RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)host->rrdctx, context);
2113 - if(!rca) return HTTP_RESP_NOT_FOUND;
2114 -
2115 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2116 -
2117 - if(after != 0 && before != 0)
2118 - rrdr_relative_window_to_absolute(&after, &before);
2119 -
2120 - buffer_json_initialize(wb, "\"", "\"", 0, true);
2121 - struct rrdcontext_to_json t_contexts = {
2122 - .wb = wb,
2123 - .options = options|RRDCONTEXT_OPTION_SKIP_ID,
2124 - .chart_label_key = chart_label_key,
2125 - .chart_labels_filter = chart_labels_filter,
2126 - .chart_dimensions = chart_dimensions,
2127 - .after = after,
2128 - .before = before,
2129 - .written = 0,
2130 - .now = now_realtime_sec(),
2131 - };
2132 - rrdcontext_to_json_callback((DICTIONARY_ITEM *)rca, rc, &t_contexts);
2133 - buffer_json_finalize(wb);
2134 -
2135 - rrdcontext_release(rca);
2136 -
2137 - if(!t_contexts.written)
2138 - return HTTP_RESP_NOT_FOUND;
2139 -
2140 - return HTTP_RESP_OK;
2141 -}
2142 -
2143 -int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) {
2144 - if(!host->rrdctx) {
2145 - error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host));
2146 - return HTTP_RESP_NOT_FOUND;
2147 - }
2148 -
2149 - char node_uuid[UUID_STR_LEN] = "";
2150 -
2151 - if(host->node_id)
2152 - uuid_unparse(*host->node_id, node_uuid);
2153 -
2154 - if(after != 0 && before != 0)
2155 - rrdr_relative_window_to_absolute(&after, &before);
2156 -
2157 - buffer_json_initialize(wb, "\"", "\"", 0, true);
2158 - buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
2159 - buffer_json_member_add_string(wb, "machine_guid", host->machine_guid);
2160 - buffer_json_member_add_string(wb, "node_id", node_uuid);
2161 - buffer_json_member_add_string(wb, "claim_id", host->aclk_state.claimed_id ? host->aclk_state.claimed_id : "");
2162 -
2163 - if(options & RRDCONTEXT_OPTION_SHOW_LABELS) {
2164 - buffer_json_member_add_object(wb, "host_labels");
2165 - rrdlabels_to_buffer_json_members(host->rrdlabels, wb);
2166 - buffer_json_object_close(wb);
2167 - }
2168 -
2169 - buffer_json_member_add_object(wb, "contexts");
2170 - struct rrdcontext_to_json t_contexts = {
2171 - .wb = wb,
2172 - .options = options,
2173 - .chart_label_key = chart_label_key,
2174 - .chart_labels_filter = chart_labels_filter,
2175 - .chart_dimensions = chart_dimensions,
2176 - .after = after,
2177 - .before = before,
2178 - .written = 0,
2179 - .now = now_realtime_sec(),
2180 - };
2181 - dictionary_walkthrough_read((DICTIONARY *)host->rrdctx, rrdcontext_to_json_callback, &t_contexts);
2182 - buffer_json_object_close(wb);
2183 -
2184 - buffer_json_finalize(wb);
2185 -
2186 - return HTTP_RESP_OK;
2187 -}
2188 -
2189 -// ----------------------------------------------------------------------------
2190 -// weights API
2191 -
2192 -static void metric_entry_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
2193 - struct metric_entry *t = value;
2194 - t->rca = rrdcontext_acquired_dup(t->rca);
2195 - t->ria = rrdinstance_acquired_dup(t->ria);
2196 - t->rma = rrdmetric_acquired_dup(t->rma);
2197 -}
2198 -static void metric_entry_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
2199 - struct metric_entry *t = value;
2200 - rrdcontext_release(t->rca);
2201 - rrdinstance_release(t->ria);
2202 - rrdmetric_release(t->rma);
2203 -}
2204 -static bool metric_entry_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value __maybe_unused, void *new_value __maybe_unused, void *data __maybe_unused) {
2205 - internal_fatal("RRDCONTEXT: %s() detected a conflict on a metric pointer!", __FUNCTION__);
2206 - return false;
2207 -}
2208 -
2209 -DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contexts) {
2210 - if(!host || !host->rrdctx)
2211 - return NULL;
2212 -
2213 - DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_rrdcontext, 0);
2214 - dictionary_register_insert_callback(dict, metric_entry_insert_callback, NULL);
2215 - dictionary_register_delete_callback(dict, metric_entry_delete_callback, NULL);
2216 - dictionary_register_conflict_callback(dict, metric_entry_conflict_callback, NULL);
2217 -
2218 - RRDCONTEXT *rc;
2219 - dfe_start_reentrant((DICTIONARY *)host->rrdctx, rc) {
2220 - if(rrd_flag_is_deleted(rc))
2221 - continue;
2222 -
2223 - if(contexts && !simple_pattern_matches(contexts, string2str(rc->id)))
2224 - continue;
2225 -
2226 - RRDINSTANCE *ri;
2227 - dfe_start_read(rc->rrdinstances, ri) {
2228 - if(rrd_flag_is_deleted(ri))
2229 - continue;
2230 -
2231 - RRDMETRIC *rm;
2232 - dfe_start_read(ri->rrdmetrics, rm) {
2233 - if(rrd_flag_is_deleted(rm))
2234 - continue;
2235 -
2236 - struct metric_entry tmp = {
2237 - .rca = (RRDCONTEXT_ACQUIRED *)rc_dfe.item,
2238 - .ria = (RRDINSTANCE_ACQUIRED *)ri_dfe.item,
2239 - .rma = (RRDMETRIC_ACQUIRED *)rm_dfe.item,
2240 - };
2241 -
2242 - char buffer[20 + 1];
2243 - ssize_t len = snprintfz(buffer, 20, "%p", rm);
2244 - dictionary_set_advanced(dict, buffer, len + 1, &tmp, sizeof(struct metric_entry), NULL);
2245 - }
2246 - dfe_done(rm);
2247 - }
2248 - dfe_done(ri);
2249 - }
2250 - dfe_done(rc);
2251 -
2252 - return dict;
2253 -}
2254 -
2255 -// ----------------------------------------------------------------------------
2256 -// query API
2257 -
2258 -typedef struct query_target_locals {
2259 - time_t start_s;
2260 -
2261 - QUERY_TARGET *qt;
2262 -
2263 - RRDSET *st;
2264 -
2265 - const char *scope_hosts;
2266 - const char *scope_contexts;
2267 -
2268 - const char *hosts;
2269 - const char *contexts;
2270 - const char *charts;
2271 - const char *dimensions;
2272 - const char *chart_label_key;
2273 - const char *labels;
2274 - const char *alerts;
2275 -
2276 - long long after;
2277 - long long before;
2278 - bool match_ids;
2279 - bool match_names;
2280 -
2281 - size_t metrics_skipped_due_to_not_matching_timeframe;
2282 -} QUERY_TARGET_LOCALS;
2283 -
2284 -static __thread QUERY_TARGET thread_query_target = {};
2285 -void query_target_release(QUERY_TARGET *qt) {
2286 - if(unlikely(!qt || !qt->used)) return;
2287 -
2288 - simple_pattern_free(qt->hosts.scope_pattern);
2289 - qt->hosts.scope_pattern = NULL;
2290 -
2291 - simple_pattern_free(qt->hosts.pattern);
2292 - qt->hosts.pattern = NULL;
2293 -
2294 - simple_pattern_free(qt->contexts.scope_pattern);
2295 - qt->contexts.scope_pattern = NULL;
2296 -
2297 - simple_pattern_free(qt->contexts.pattern);
2298 - qt->contexts.pattern = NULL;
2299 -
2300 - simple_pattern_free(qt->instances.pattern);
2301 - qt->instances.pattern = NULL;
2302 -
2303 - simple_pattern_free(qt->instances.chart_label_key_pattern);
2304 - qt->instances.chart_label_key_pattern = NULL;
2305 -
2306 - simple_pattern_free(qt->instances.labels_pattern);
2307 - qt->instances.labels_pattern = NULL;
2308 -
2309 - simple_pattern_free(qt->query.pattern);
2310 - qt->query.pattern = NULL;
2311 -
2312 - // release the query
2313 - for(size_t i = 0, used = qt->query.used; i < used ;i++) {
2314 - QUERY_METRIC *qm = query_metric(qt, i);
2315 -
2316 - // reset the plans
2317 - for(size_t p = 0; p < qm->plan.used; p++) {
2318 - internal_fatal(qm->plan.array[p].initialized &&
2319 - !qm->plan.array[p].finalized,
2320 - "QUERY: left-over initialized plan");
2321 -
2322 - qm->plan.array[p].initialized = false;
2323 - qm->plan.array[p].finalized = false;
2324 - }
2325 - qm->plan.used = 0;
2326 -
2327 - // reset the tiers
2328 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
2329 - if(qm->tiers[tier].db_metric_handle) {
2330 - STORAGE_ENGINE *eng = qm->tiers[tier].eng;
2331 - eng->api.metric_release(qm->tiers[tier].db_metric_handle);
2332 - qm->tiers[tier].db_metric_handle = NULL;
2333 - qm->tiers[tier].eng = NULL;
2334 - }
2335 - }
2336 - }
2337 - qt->query.used = 0;
2338 -
2339 - // release the dimensions
2340 - for(size_t i = 0, used = qt->dimensions.used; i < used ; i++) {
2341 - QUERY_DIMENSION *qd = query_dimension(qt, i);
2342 - rrdmetric_release(qd->rma);
2343 - qd->rma = NULL;
2344 - }
2345 - qt->dimensions.used = 0;
2346 -
2347 - // release the instances
2348 - for(size_t i = 0, used = qt->instances.used; i < used ;i++) {
2349 - QUERY_INSTANCE *qi = &qt->instances.array[i];
2350 -
2351 - rrdinstance_release(qi->ria);
2352 - qi->ria = NULL;
2353 -
2354 - string_freez(qi->id_fqdn);
2355 - qi->id_fqdn = NULL;
2356 -
2357 - string_freez(qi->name_fqdn);
2358 - qi->name_fqdn = NULL;
2359 - }
2360 - qt->instances.used = 0;
2361 -
2362 - // release the contexts
2363 - for(size_t i = 0, used = qt->contexts.used; i < used ;i++) {
2364 - QUERY_CONTEXT *qc = query_context(qt, i);
2365 - rrdcontext_release(qc->rca);
2366 - qc->rca = NULL;
2367 - }
2368 - qt->contexts.used = 0;
2369 -
2370 - // release the hosts
2371 - for(size_t i = 0, used = qt->hosts.used; i < used ;i++) {
2372 - QUERY_HOST *qh = query_host(qt, i);
2373 - qh->host = NULL;
2374 - }
2375 - qt->hosts.used = 0;
2376 -
2377 - qt->db.minimum_latest_update_every_s = 0;
2378 - qt->db.first_time_s = 0;
2379 - qt->db.last_time_s = 0;
2380 -
2381 - qt->group_by.used = 0;
2382 -
2383 - qt->id[0] = '\0';
2384 -
2385 - qt->used = false;
2386 -}
2387 -void query_target_free(void) {
2388 - QUERY_TARGET *qt = &thread_query_target;
2389 -
2390 - if(qt->used)
2391 - query_target_release(qt);
2392 -
2393 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->query.size * sizeof(QUERY_METRIC), __ATOMIC_RELAXED);
2394 - freez(qt->query.array);
2395 - qt->query.array = NULL;
2396 - qt->query.size = 0;
2397 -
2398 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->dimensions.size * sizeof(RRDMETRIC_ACQUIRED *), __ATOMIC_RELAXED);
2399 - freez(qt->dimensions.array);
2400 - qt->dimensions.array = NULL;
2401 - qt->dimensions.size = 0;
2402 -
2403 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->instances.size * sizeof(RRDINSTANCE_ACQUIRED *), __ATOMIC_RELAXED);
2404 - freez(qt->instances.array);
2405 - qt->instances.array = NULL;
2406 - qt->instances.size = 0;
2407 -
2408 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->contexts.size * sizeof(RRDCONTEXT_ACQUIRED *), __ATOMIC_RELAXED);
2409 - freez(qt->contexts.array);
2410 - qt->contexts.array = NULL;
2411 - qt->contexts.size = 0;
2412 -
2413 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->hosts.size * sizeof(RRDHOST *), __ATOMIC_RELAXED);
2414 - freez(qt->hosts.array);
2415 - qt->hosts.array = NULL;
2416 - qt->hosts.size = 0;
2417 -}
2418 -
2419 -static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, QUERY_CONTEXT *qc,
2420 - QUERY_INSTANCE *qi, QUERY_DIMENSION *qd) {
2421 - QUERY_TARGET *qt = qtl->qt;
2422 - RRDMETRIC *rm = rrdmetric_acquired_value(qd->rma);
2423 - RRDINSTANCE *ri = rm->ri;
2424 -
2425 - time_t common_first_time_s = 0;
2426 - time_t common_last_time_s = 0;
2427 - time_t common_update_every_s = 0;
2428 - size_t tiers_added = 0;
2429 -
2430 - struct {
2431 - STORAGE_ENGINE *eng;
2432 - STORAGE_METRIC_HANDLE *db_metric_handle;
2433 - time_t db_first_time_s;
2434 - time_t db_last_time_s;
2435 - time_t db_update_every_s;
2436 - } tier_retention[storage_tiers];
2437 -
2438 - for (size_t tier = 0; tier < storage_tiers; tier++) {
2439 - STORAGE_ENGINE *eng = qh->host->db[tier].eng;
2440 - tier_retention[tier].eng = eng;
2441 - tier_retention[tier].db_update_every_s = (time_t) (qh->host->db[tier].tier_grouping * ri->update_every_s);
2442 -
2443 - if(rm->rrddim && rm->rrddim->tiers[tier].db_metric_handle)
2444 - tier_retention[tier].db_metric_handle = eng->api.metric_dup(rm->rrddim->tiers[tier].db_metric_handle);
2445 - else
2446 - tier_retention[tier].db_metric_handle = eng->api.metric_get(qh->host->db[tier].instance, &rm->uuid);
2447 -
2448 - if(tier_retention[tier].db_metric_handle) {
2449 - tier_retention[tier].db_first_time_s = tier_retention[tier].eng->api.query_ops.oldest_time_s(tier_retention[tier].db_metric_handle);
2450 - tier_retention[tier].db_last_time_s = tier_retention[tier].eng->api.query_ops.latest_time_s(tier_retention[tier].db_metric_handle);
2451 -
2452 - if(!common_first_time_s)
2453 - common_first_time_s = tier_retention[tier].db_first_time_s;
2454 - else if(tier_retention[tier].db_first_time_s)
2455 - common_first_time_s = MIN(common_first_time_s, tier_retention[tier].db_first_time_s);
2456 -
2457 - if(!common_last_time_s)
2458 - common_last_time_s = tier_retention[tier].db_last_time_s;
2459 - else
2460 - common_last_time_s = MAX(common_last_time_s, tier_retention[tier].db_last_time_s);
2461 -
2462 - if(!common_update_every_s)
2463 - common_update_every_s = tier_retention[tier].db_update_every_s;
2464 - else if(tier_retention[tier].db_update_every_s)
2465 - common_update_every_s = MIN(common_update_every_s, tier_retention[tier].db_update_every_s);
2466 -
2467 - tiers_added++;
2468 - }
2469 - else {
2470 - tier_retention[tier].db_first_time_s = 0;
2471 - tier_retention[tier].db_last_time_s = 0;
2472 - tier_retention[tier].db_update_every_s = 0;
2473 - }
2474 - }
2475 -
2476 - bool release_retention = true;
2477 - bool timeframe_matches =
2478 - (tiers_added
2479 - && (common_first_time_s - common_update_every_s * 2) <= qt->window.before
2480 - && (common_last_time_s + common_update_every_s * 2) >= qt->window.after
2481 - ) ? true : false;
2482 -
2483 - if(timeframe_matches) {
2484 - RRDR_DIMENSION_FLAGS options = RRDR_DIMENSION_DEFAULT;
2485 -
2486 - if (rrd_flag_check(rm, RRD_FLAG_HIDDEN)
2487 - || (rm->rrddim && rrddim_option_check(rm->rrddim, RRDDIM_OPTION_HIDDEN))) {
2488 - options |= RRDR_DIMENSION_HIDDEN;
2489 - options &= ~RRDR_DIMENSION_SELECTED;
2490 - }
2491 -
2492 - if (qt->query.pattern) {
2493 - // we have a dimensions pattern
2494 - // lets see if this dimension is selected
2495 -
2496 - if ((qtl->match_ids && simple_pattern_matches(qt->query.pattern, string2str(rm->id)))
2497 - || (qtl->match_names && simple_pattern_matches(qt->query.pattern, string2str(rm->name)))
2498 - ) {
2499 - // it matches the pattern
2500 - options |= (RRDR_DIMENSION_SELECTED | RRDR_DIMENSION_NONZERO);
2501 - options &= ~RRDR_DIMENSION_HIDDEN;
2502 - }
2503 - else {
2504 - // it does not match the pattern
2505 - options |= RRDR_DIMENSION_HIDDEN;
2506 - options &= ~RRDR_DIMENSION_SELECTED;
2507 - }
2508 - }
2509 - else {
2510 - // we don't have a dimensions pattern
2511 - // so this is a selected dimension
2512 - // if it is not hidden
2513 - if(!(options & RRDR_DIMENSION_HIDDEN))
2514 - options |= RRDR_DIMENSION_SELECTED;
2515 - }
2516 -
2517 - if((options & RRDR_DIMENSION_HIDDEN) && (options & RRDR_DIMENSION_SELECTED))
2518 - options &= ~RRDR_DIMENSION_HIDDEN;
2519 -
2520 - if(!(options & RRDR_DIMENSION_HIDDEN) || (qt->request.options & RRDR_OPTION_PERCENTAGE)) {
2521 - // we have a non-hidden dimension
2522 - // let's add it to the query metrics
2523 -
2524 - if(ri->rrdset)
2525 - ri->rrdset->last_accessed_time_s = qtl->start_s;
2526 -
2527 - if (qt->query.used == qt->query.size) {
2528 - size_t old_mem = qt->query.size * sizeof(*qt->query.array);
2529 - qt->query.size = (qt->query.size) ? qt->query.size * 2 : 1;
2530 - size_t new_mem = qt->query.size * sizeof(*qt->query.array);
2531 - qt->query.array = reallocz(qt->query.array, new_mem);
2532 -
2533 - __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2534 - }
2535 - QUERY_METRIC *qm = &qt->query.array[qt->query.used++];
2536 - memset(qm, 0, sizeof(*qm));
2537 -
2538 - qm->status = options;
2539 -
2540 - qm->link.query_host_id = qh->slot;
2541 - qm->link.query_context_id = qc->slot;
2542 - qm->link.query_instance_id = qi->slot;
2543 - qm->link.query_dimension_id = qd->slot;
2544 -
2545 - if (!qt->db.first_time_s || common_first_time_s < qt->db.first_time_s)
2546 - qt->db.first_time_s = common_first_time_s;
2547 -
2548 - if (!qt->db.last_time_s || common_last_time_s > qt->db.last_time_s)
2549 - qt->db.last_time_s = common_last_time_s;
2550 -
2551 - for (size_t tier = 0; tier < storage_tiers; tier++) {
2552 - qm->tiers[tier].eng = tier_retention[tier].eng;
2553 - qm->tiers[tier].db_metric_handle = tier_retention[tier].db_metric_handle;
2554 - qm->tiers[tier].db_first_time_s = tier_retention[tier].db_first_time_s;
2555 - qm->tiers[tier].db_last_time_s = tier_retention[tier].db_last_time_s;
2556 - qm->tiers[tier].db_update_every_s = tier_retention[tier].db_update_every_s;
2557 - }
2558 -
2559 - release_retention = false;
2560 -
2561 - qi->metrics.selected++;
2562 - qc->metrics.selected++;
2563 - qh->metrics.selected++;
2564 - }
2565 - else {
2566 - qi->metrics.excluded++;
2567 - qc->metrics.excluded++;
2568 - qh->metrics.excluded++;
2569 -
2570 - qd->status |= QUERY_STATUS_DIMENSION_HIDDEN;
2571 - }
2572 - }
2573 - else {
2574 - qi->metrics.excluded++;
2575 - qc->metrics.excluded++;
2576 - qh->metrics.excluded++;
2577 -
2578 - qd->status |= QUERY_STATUS_DIMENSION_NODATA;
2579 - qtl->metrics_skipped_due_to_not_matching_timeframe++;
2580 - }
2581 -
2582 - if(release_retention) {
2583 - // cleanup anything we allocated to the retention we will not use
2584 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
2585 - if (tier_retention[tier].db_metric_handle)
2586 - tier_retention[tier].eng->api.metric_release(tier_retention[tier].db_metric_handle);
2587 - }
2588 - }
2589 -}
2590 -
2591 -static void query_target_add_dimension(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi,
2592 - RRDMETRIC_ACQUIRED *rma, bool queryable_instance) {
2593 - QUERY_TARGET *qt = qtl->qt;
2594 -
2595 - RRDMETRIC *rm = rrdmetric_acquired_value(rma);
2596 - if(rrd_flag_is_deleted(rm))
2597 - return;
2598 -
2599 - if(qt->dimensions.used == qt->dimensions.size) {
2600 - size_t old_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
2601 - qt->dimensions.size = (qt->dimensions.size) ? qt->dimensions.size * 2 : 1;
2602 - size_t new_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
2603 - qt->dimensions.array = reallocz(qt->dimensions.array, new_mem);
2604 -
2605 - __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2606 - }
2607 - QUERY_DIMENSION *qd = &qt->dimensions.array[qt->dimensions.used];
2608 - memset(qd, 0, sizeof(*qd));
2609 -
2610 - qd->slot = qt->dimensions.used++;
2611 - qd->rma = rrdmetric_acquired_dup(rma);
2612 - qd->status = QUERY_STATUS_NONE;
2613 -
2614 - if(!queryable_instance) {
2615 - qi->metrics.excluded++;
2616 - qc->metrics.excluded++;
2617 - qh->metrics.excluded++;
2618 - qd->status |= QUERY_STATUS_EXCLUDED;
2619 - return;
2620 - }
2621 -
2622 - query_target_add_metric(qtl, qh, qc, qi, qd);
2623 -}
2624 -
2625 -static inline STRING *rrdinstance_id_fqdn_v1(RRDINSTANCE_ACQUIRED *ria) {
2626 - if(unlikely(!ria))
2627 - return NULL;
2628 -
2629 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2630 - return string_dup(ri->id);
2631 -}
2632 -
2633 -static inline STRING *rrdinstance_name_fqdn_v1(RRDINSTANCE_ACQUIRED *ria) {
2634 - if(unlikely(!ria))
2635 - return NULL;
2636 -
2637 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2638 - return string_dup(ri->name);
2639 -}
2640 -
2641 -static inline STRING *rrdinstance_id_fqdn_v2(RRDINSTANCE_ACQUIRED *ria) {
2642 - if(unlikely(!ria))
2643 - return NULL;
2644 -
2645 - char buffer[RRD_ID_LENGTH_MAX + 1];
2646 -
2647 - RRDHOST *host = rrdinstance_acquired_rrdhost(ria);
2648 - snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s@%s", rrdinstance_acquired_id(ria), host->machine_guid);
2649 - return string_strdupz(buffer);
2650 -}
2651 -
2652 -static inline STRING *rrdinstance_name_fqdn_v2(RRDINSTANCE_ACQUIRED *ria) {
2653 - if(unlikely(!ria))
2654 - return NULL;
2655 -
2656 - char buffer[RRD_ID_LENGTH_MAX + 1];
2657 -
2658 - RRDHOST *host = rrdinstance_acquired_rrdhost(ria);
2659 - snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s@%s", rrdinstance_acquired_name(ria), rrdhost_hostname(host));
2660 - return string_strdupz(buffer);
2661 -}
2662 -
2663 -RRDSET *rrdinstance_acquired_rrdset(RRDINSTANCE_ACQUIRED *ria) {
2664 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2665 - return ri->rrdset;
2666 -}
2667 -
2668 -const char *rrdcontext_acquired_units(RRDCONTEXT_ACQUIRED *rca) {
2669 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2670 - return string2str(rc->units);
2671 -}
2672 -
2673 -RRDSET_TYPE rrdcontext_acquired_chart_type(RRDCONTEXT_ACQUIRED *rca) {
2674 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2675 - return rc->chart_type;
2676 -}
2677 -
2678 -const char *rrdcontext_acquired_title(RRDCONTEXT_ACQUIRED *rca) {
2679 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2680 - return string2str(rc->title);
2681 -}
2682 -
2683 -static void query_target_eval_instance_rrdcalc(QUERY_TARGET_LOCALS *qtl __maybe_unused,
2684 - QUERY_HOST *qh, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi) {
2685 - RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
2686 - if (st) {
2687 - netdata_rwlock_rdlock(&st->alerts.rwlock);
2688 - if (st->alerts.base) {
2689 - for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
2690 - switch(rc->status) {
2691 - case RRDCALC_STATUS_CLEAR:
2692 - qi->alerts.clear++;
2693 - qc->alerts.clear++;
2694 - qh->alerts.clear++;
2695 - break;
2696 -
2697 - case RRDCALC_STATUS_WARNING:
2698 - qi->alerts.warning++;
2699 - qc->alerts.warning++;
2700 - qh->alerts.warning++;
2701 - break;
2702 -
2703 - case RRDCALC_STATUS_CRITICAL:
2704 - qi->alerts.critical++;
2705 - qc->alerts.critical++;
2706 - qh->alerts.critical++;
2707 - break;
2708 -
2709 - default:
2710 - case RRDCALC_STATUS_UNINITIALIZED:
2711 - case RRDCALC_STATUS_UNDEFINED:
2712 - case RRDCALC_STATUS_REMOVED:
2713 - qi->alerts.other++;
2714 - qc->alerts.other++;
2715 - qh->alerts.other++;
2716 - break;
2717 - }
2718 - }
2719 - }
2720 - netdata_rwlock_unlock(&st->alerts.rwlock);
2721 - }
2722 -}
2723 -
2724 -static bool query_target_match_alert_pattern(QUERY_INSTANCE *qi, SIMPLE_PATTERN *pattern) {
2725 - if(!pattern)
2726 - return true;
2727 -
2728 - RRDSET *st = rrdinstance_acquired_rrdset(qi->ria);
2729 - if (!st)
2730 - return false;
2731 -
2732 - BUFFER *wb = NULL;
2733 - bool matched = false;
2734 - netdata_rwlock_rdlock(&st->alerts.rwlock);
2735 - if (st->alerts.base) {
2736 - for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
2737 - if(simple_pattern_matches(pattern, string2str(rc->name))) {
2738 - matched = true;
2739 - break;
2740 - }
2741 -
2742 - if(!wb)
2743 - wb = buffer_create(0, NULL);
2744 - else
2745 - buffer_flush(wb);
2746 -
2747 - buffer_fast_strcat(wb, string2str(rc->name), string_strlen(rc->name));
2748 - buffer_fast_strcat(wb, ":", 1);
2749 - buffer_strcat(wb, rrdcalc_status2string(rc->status));
2750 -
2751 - if(simple_pattern_matches(pattern, buffer_tostring(wb))) {
2752 - matched = true;
2753 - break;
2754 - }
2755 - }
2756 - }
2757 - netdata_rwlock_unlock(&st->alerts.rwlock);
2758 -
2759 - buffer_free(wb);
2760 - return matched;
2761 -}
2762 -
2763 -static void query_target_add_instance(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, QUERY_CONTEXT *qc,
2764 - RRDINSTANCE_ACQUIRED *ria, bool queryable_instance, bool filter_instances) {
2765 - QUERY_TARGET *qt = qtl->qt;
2766 -
2767 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2768 - if(rrd_flag_is_deleted(ri))
2769 - return;
2770 -
2771 - if(qt->instances.used == qt->instances.size) {
2772 - size_t old_mem = qt->instances.size * sizeof(*qt->instances.array);
2773 - qt->instances.size = (qt->instances.size) ? qt->instances.size * 2 : 1;
2774 - size_t new_mem = qt->instances.size * sizeof(*qt->instances.array);
2775 - qt->instances.array = reallocz(qt->instances.array, new_mem);
2776 -
2777 - __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2778 - }
2779 - QUERY_INSTANCE *qi = &qt->instances.array[qt->instances.used];
2780 - memset(qi, 0, sizeof(*qi));
2781 -
2782 - qi->slot = qt->instances.used;
2783 - qt->instances.used++;
2784 - qi->ria = rrdinstance_acquired_dup(ria);
2785 - qi->query_host_id = qh->slot;
2786 -
2787 - if(qt->request.version <= 1) {
2788 - qi->id_fqdn = rrdinstance_id_fqdn_v1(ria);
2789 - qi->name_fqdn = rrdinstance_name_fqdn_v1(ria);
2790 - }
2791 - else {
2792 - qi->id_fqdn = rrdinstance_id_fqdn_v2(ria);
2793 - qi->name_fqdn = rrdinstance_name_fqdn_v2(ria);
2794 - }
2795 -
2796 - if(qt->db.minimum_latest_update_every_s == 0 || ri->update_every_s < qt->db.minimum_latest_update_every_s)
2797 - qt->db.minimum_latest_update_every_s = ri->update_every_s;
2798 -
2799 - if(queryable_instance && filter_instances) {
2800 - queryable_instance = false;
2801 - if(!qt->instances.pattern
2802 - || (qtl->match_ids && simple_pattern_matches(qt->instances.pattern, string2str(ri->id)))
2803 - || (qtl->match_ids && simple_pattern_matches(qt->instances.pattern, string2str(qi->id_fqdn)))
2804 - || (qtl->match_names && simple_pattern_matches(qt->instances.pattern, string2str(ri->name)))
2805 - || (qtl->match_names && simple_pattern_matches(qt->instances.pattern, string2str(qi->name_fqdn)))
2806 - )
2807 - queryable_instance = true;
2808 - }
2809 -
2810 - if(queryable_instance) {
2811 - if ((qt->instances.chart_label_key_pattern && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, qt->instances.chart_label_key_pattern, ':')) ||
2812 - (qt->instances.labels_pattern && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, qt->instances.labels_pattern, ':')))
2813 - queryable_instance = false;
2814 - }
2815 -
2816 - if(queryable_instance) {
2817 - if(qt->instances.alerts_pattern && !query_target_match_alert_pattern(qi, qt->instances.alerts_pattern))
2818 - queryable_instance = false;
2819 - }
2820 -
2821 - if(queryable_instance && qt->request.version >= 2)
2822 - query_target_eval_instance_rrdcalc(qtl, qh, qc, qi);
2823 -
2824 - size_t added = 0;
2825 -
2826 - if(unlikely(qt->request.rma)) {
2827 - query_target_add_dimension(qtl, qh, qc, qi, qt->request.rma, queryable_instance);
2828 - added++;
2829 - }
2830 - else {
2831 - RRDMETRIC *rm;
2832 - dfe_start_read(ri->rrdmetrics, rm) {
2833 - query_target_add_dimension(qtl, qh, qc, qi, (RRDMETRIC_ACQUIRED *) rm_dfe.item, queryable_instance);
2834 - added++;
2835 - }
2836 - dfe_done(rm);
2837 - }
2838 -
2839 - if(!added) {
2840 - qc->instances.excluded++;
2841 - qh->instances.excluded++;
2842 -
2843 - qt->instances.used--;
2844 - rrdinstance_release(ria);
2845 - qi->ria = NULL;
2846 -
2847 - string_freez(qi->id_fqdn);
2848 - qi->id_fqdn = NULL;
2849 -
2850 - string_freez(qi->name_fqdn);
2851 - qi->name_fqdn = NULL;
2852 - }
2853 - else {
2854 - qc->instances.selected++;
2855 - qh->instances.selected++;
2856 - }
2857 -}
2858 -
2859 -static void query_target_add_context(QUERY_TARGET_LOCALS *qtl, QUERY_HOST *qh, RRDCONTEXT_ACQUIRED *rca, bool queryable_context) {
2860 - QUERY_TARGET *qt = qtl->qt;
2861 -
2862 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2863 - if(rrd_flag_is_deleted(rc))
2864 - return;
2865 -
2866 - if(qt->contexts.used == qt->contexts.size) {
2867 - size_t old_mem = qt->contexts.size * sizeof(*qt->contexts.array);
2868 - qt->contexts.size = (qt->contexts.size) ? qt->contexts.size * 2 : 1;
2869 - size_t new_mem = qt->contexts.size * sizeof(*qt->contexts.array);
2870 - qt->contexts.array = reallocz(qt->contexts.array, new_mem);
2871 -
2872 - __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2873 - }
2874 - QUERY_CONTEXT *qc = &qt->contexts.array[qt->contexts.used];
2875 - memset(qc, 0, sizeof(*qc));
2876 - qc->slot = qt->contexts.used++;
2877 - qc->rca = rrdcontext_acquired_dup(rca);
2878 -
2879 - size_t added = 0;
2880 - if(unlikely(qt->request.ria)) {
2881 - query_target_add_instance(qtl, qh, qc, qt->request.ria, queryable_context, false);
2882 - added++;
2883 - }
2884 - else if(unlikely(qtl->st && qtl->st->rrdcontext == rca && qtl->st->rrdinstance)) {
2885 - query_target_add_instance(qtl, qh, qc, qtl->st->rrdinstance, queryable_context, false);
2886 - added++;
2887 - }
2888 - else {
2889 - RRDINSTANCE *ri;
2890 - dfe_start_read(rc->rrdinstances, ri) {
2891 - query_target_add_instance(qtl, qh, qc, (RRDINSTANCE_ACQUIRED *)ri_dfe.item, queryable_context, true);
2892 - added++;
2893 - }
2894 - dfe_done(ri);
2895 - }
2896 -
2897 - if(!added) {
2898 - qt->contexts.used--;
2899 - rrdcontext_release(rca);
2900 - }
2901 -}
2902 -
2903 -static void query_target_add_host(QUERY_TARGET_LOCALS *qtl, RRDHOST *host, bool queryable_host) {
2904 - QUERY_TARGET *qt = qtl->qt;
2905 -
2906 - if(qt->hosts.used == qt->hosts.size) {
2907 - size_t old_mem = qt->hosts.size * sizeof(*qt->hosts.array);
2908 - qt->hosts.size = (qt->hosts.size) ? qt->hosts.size * 2 : 1;
2909 - size_t new_mem = qt->hosts.size * sizeof(*qt->hosts.array);
2910 - qt->hosts.array = reallocz(qt->hosts.array, new_mem);
2911 -
2912 - __atomic_add_fetch(&netdata_buffers_statistics.query_targets_size, new_mem - old_mem, __ATOMIC_RELAXED);
2913 - }
2914 - QUERY_HOST *qh = &qt->hosts.array[qt->hosts.used];
2915 - memset(qh, 0, sizeof(*qh));
2916 - qh->slot = qt->hosts.used++;
2917 - qh->host = host;
2918 -
2919 - if(host->node_id)
2920 - uuid_unparse_lower(*host->node_id, qh->node_id);
2921 - else
2922 - qh->node_id[0] = '\0';
2923 -
2924 - // is the chart given valid?
2925 - if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
2926 - error("QUERY TARGET: RRDSET '%s' given, but it is not linked to rrdcontext structures. Linking it now.", rrdset_name(qtl->st));
2927 - rrdinstance_from_rrdset(qtl->st);
2928 -
2929 - if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
2930 - error("QUERY TARGET: RRDSET '%s' given, but failed to be linked to rrdcontext structures. Switching to context query.",
2931 - rrdset_name(qtl->st));
2932 -
2933 - if (!is_valid_sp(qtl->charts))
2934 - qtl->charts = rrdset_name(qtl->st);
2935 -
2936 - qtl->st = NULL;
2937 - }
2938 - }
2939 -
2940 - size_t added = 0;
2941 - if(unlikely(qt->request.rca)) {
2942 - query_target_add_context(qtl, qh, qt->request.rca, true);
2943 - added++;
2944 - }
2945 - else if(unlikely(qtl->st)) {
2946 - // single chart data queries
2947 - query_target_add_context(qtl, qh, qtl->st->rrdcontext, true);
2948 - added++;
2949 - }
2950 - else {
2951 - // context pattern queries
2952 - RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)host->rrdctx, qtl->scope_contexts);
2953 - if(likely(rca)) {
2954 - // we found it!
2955 -
2956 - bool queryable_context = queryable_host;
2957 - if(queryable_context && qt->contexts.pattern && !simple_pattern_matches(qt->contexts.pattern, rrdcontext_acquired_id(rca)))
2958 - queryable_context = false;
2959 -
2960 - query_target_add_context(qtl, qh, rca, queryable_context);
2961 - rrdcontext_release(rca);
2962 - added++;
2963 - }
2964 - else {
2965 - // Probably it is a pattern, we need to search for it...
2966 - RRDCONTEXT *rc;
2967 - dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
2968 - if(qt->contexts.scope_pattern && !simple_pattern_matches(qt->contexts.scope_pattern, string2str(rc->id)))
2969 - continue;
2970 -
2971 - bool queryable_context = queryable_host;
2972 - if(queryable_context && qt->contexts.pattern && !simple_pattern_matches(qt->contexts.pattern, string2str(rc->id)))
2973 - queryable_context = false;
2974 -
2975 - query_target_add_context(qtl, qh, (RRDCONTEXT_ACQUIRED *)rc_dfe.item, queryable_context);
2976 - added++;
2977 - }
2978 - dfe_done(rc);
2979 - }
2980 - }
2981 -
2982 - if(!added) {
2983 - qt->hosts.used--;
2984 - }
2985 -}
2986 -
2987 -void query_target_generate_name(QUERY_TARGET *qt) {
2988 - char options_buffer[100 + 1];
2989 - web_client_api_request_v1_data_options_to_string(options_buffer, 100, qt->request.options);
2990 -
2991 - char resampling_buffer[20 + 1] = "";
2992 - if(qt->request.resampling_time > 1)
2993 - snprintfz(resampling_buffer, 20, "/resampling:%lld", (long long)qt->request.resampling_time);
2994 -
2995 - char tier_buffer[20 + 1] = "";
2996 - if(qt->request.options & RRDR_OPTION_SELECTED_TIER)
2997 - snprintfz(tier_buffer, 20, "/tier:%zu", qt->request.tier);
2998 -
2999 - if(qt->request.st)
3000 - snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "chart://hosts:%s/instance:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3001 - , rrdhost_hostname(qt->request.st->rrdhost)
3002 - , rrdset_name(qt->request.st)
3003 - , (qt->request.dimensions) ? qt->request.dimensions : "*"
3004 - , (long long)qt->request.after
3005 - , (long long)qt->request.before
3006 - , qt->request.points
3007 - , time_grouping_tostring(qt->request.time_group_method)
3008 - , qt->request.time_group_options ? qt->request.time_group_options : ""
3009 - , options_buffer
3010 - , resampling_buffer
3011 - , tier_buffer
3012 - );
3013 - else if(qt->request.host && qt->request.rca && qt->request.ria && qt->request.rma)
3014 - snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "metric://hosts:%s/context:%s/instance:%s/dimension:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3015 - , rrdhost_hostname(qt->request.host)
3016 - , rrdcontext_acquired_id(qt->request.rca)
3017 - , rrdinstance_acquired_id(qt->request.ria)
3018 - , rrdmetric_acquired_id(qt->request.rma)
3019 - , (long long)qt->request.after
3020 - , (long long)qt->request.before
3021 - , qt->request.points
3022 - , time_grouping_tostring(qt->request.time_group_method)
3023 - , qt->request.time_group_options ? qt->request.time_group_options : ""
3024 - , options_buffer
3025 - , resampling_buffer
3026 - , tier_buffer
3027 - );
3028 - else
3029 - snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "context://hosts:%s/contexts:%s/instances:%s/dimensions:%s/after:%lld/before:%lld/points:%zu/group:%s%s/options:%s%s%s"
3030 - , (qt->request.host) ? rrdhost_hostname(qt->request.host) : ((qt->request.hosts) ? qt->request.hosts : "*")
3031 - , (qt->request.contexts) ? qt->request.contexts : "*"
3032 - , (qt->request.charts) ? qt->request.charts : "*"
3033 - , (qt->request.dimensions) ? qt->request.dimensions : "*"
3034 - , (long long)qt->request.after
3035 - , (long long)qt->request.before
3036 - , qt->request.points
3037 - , time_grouping_tostring(qt->request.time_group_method)
3038 - , qt->request.time_group_options ? qt->request.time_group_options : ""
3039 - , options_buffer
3040 - , resampling_buffer
3041 - , tier_buffer
3042 - );
3043 -
3044 - json_fix_string(qt->id);
3045 -}
3046 -
3047 -QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
3048 - if(!service_running(ABILITY_DATA_QUERIES))
3049 - return NULL;
3050 -
3051 - QUERY_TARGET *qt = &thread_query_target;
3052 -
3053 - if(qt->used)
3054 - fatal("QUERY TARGET: this query target is already used (%zu queries made with this QUERY_TARGET so far).", qt->queries);
3055 -
3056 - qt->used = true;
3057 - qt->queries++;
3058 -
3059 - if(!qtr->received_ut)
3060 - qtr->received_ut = now_monotonic_usec();
3061 -
3062 - qt->timings.received_ut = qtr->received_ut;
3063 -
3064 - if(qtr->hosts && !qtr->scope_hosts)
3065 - qtr->scope_hosts = qtr->hosts;
3066 -
3067 - if(qtr->contexts && !qtr->scope_contexts)
3068 - qtr->scope_contexts = qtr->contexts;
3069 -
3070 - memset(&qt->query_stats, 0, sizeof(qt->query_stats));
3071 -
3072 - // copy the request into query_thread_target
3073 - qt->request = *qtr;
3074 -
3075 - query_target_generate_name(qt);
3076 - qt->window.after = qt->request.after;
3077 - qt->window.before = qt->request.before;
3078 - rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before);
3079 -
3080 - // prepare our local variables - we need these across all these functions
3081 - QUERY_TARGET_LOCALS qtl = {
3082 - .qt = qt,
3083 - .start_s = now_realtime_sec(),
3084 - .st = qt->request.st,
3085 - .scope_hosts = qt->request.scope_hosts,
3086 - .scope_contexts = qt->request.scope_contexts,
3087 - .hosts = qt->request.hosts,
3088 - .contexts = qt->request.contexts,
3089 - .charts = qt->request.charts,
3090 - .dimensions = qt->request.dimensions,
3091 - .chart_label_key = qt->request.chart_label_key,
3092 - .labels = qt->request.labels,
3093 - .alerts = qt->request.alerts,
3094 - };
3095 -
3096 - RRDHOST *host = qt->request.host;
3097 -
3098 - qt->db.minimum_latest_update_every_s = 0; // it will be updated by query_target_add_query()
3099 -
3100 - // prepare all the patterns
3101 - qt->hosts.pattern = is_valid_sp(qtl.hosts) ? simple_pattern_create(qtl.hosts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3102 - qt->hosts.scope_pattern = is_valid_sp(qtl.scope_hosts) ? simple_pattern_create(qtl.scope_hosts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3103 -
3104 - qt->contexts.pattern = is_valid_sp(qtl.contexts) ? simple_pattern_create(qtl.contexts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3105 - qt->contexts.scope_pattern = is_valid_sp(qtl.scope_contexts) ? simple_pattern_create(qtl.scope_contexts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3106 -
3107 - qt->instances.pattern = is_valid_sp(qtl.charts) ? simple_pattern_create(qtl.charts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3108 - qt->query.pattern = is_valid_sp(qtl.dimensions) ? simple_pattern_create(qtl.dimensions, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3109 - qt->instances.chart_label_key_pattern = is_valid_sp(qtl.chart_label_key) ? simple_pattern_create(qtl.chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3110 - qt->instances.labels_pattern = is_valid_sp(qtl.labels) ? simple_pattern_create(qtl.labels, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3111 - qt->instances.alerts_pattern = is_valid_sp(qtl.alerts) ? simple_pattern_create(qtl.alerts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
3112 -
3113 - qtl.match_ids = qt->request.options & RRDR_OPTION_MATCH_IDS;
3114 - qtl.match_names = qt->request.options & RRDR_OPTION_MATCH_NAMES;
3115 - if(likely(!qtl.match_ids && !qtl.match_names))
3116 - qtl.match_ids = qtl.match_names = true;
3117 -
3118 - // verify that the chart belongs to the host we are interested
3119 - if(qtl.st) {
3120 - if (!host) {
3121 - // It is NULL, set it ourselves.
3122 - host = qtl.st->rrdhost;
3123 - }
3124 - else if (unlikely(host != qtl.st->rrdhost)) {
3125 - // Oops! A different host!
3126 - error("QUERY TARGET: RRDSET '%s' given does not belong to host '%s'. Switching query host to '%s'",
3127 - rrdset_name(qtl.st), rrdhost_hostname(host), rrdhost_hostname(qtl.st->rrdhost));
3128 - host = qtl.st->rrdhost;
3129 - }
3130 - }
3131 -
3132 - if(host) {
3133 - // single host query
3134 - query_target_add_host(&qtl, host, true);
3135 - qtl.hosts = rrdhost_hostname(host);
3136 - }
3137 - else {
3138 - char uuid[UUID_STR_LEN];
3139 -
3140 - // multi host query
3141 - rrd_rdlock();
3142 - rrdhost_foreach_read(host) {
3143 -
3144 - if(!host->node_id)
3145 - uuid_unparse_lower(*host->node_id, uuid);
3146 - else
3147 - uuid[0] = '\0';
3148 -
3149 - if(!qt->hosts.scope_pattern ||
3150 - simple_pattern_matches(qt->hosts.scope_pattern, rrdhost_hostname(host)) ||
3151 - simple_pattern_matches(qt->hosts.scope_pattern, host->machine_guid) ||
3152 - (*uuid && simple_pattern_matches(qt->hosts.scope_pattern, uuid))) {
3153 -
3154 - bool queryable_host = false;
3155 - if(!qt->hosts.pattern ||
3156 - simple_pattern_matches(qt->hosts.pattern, rrdhost_hostname(host)) ||
3157 - simple_pattern_matches(qt->hosts.pattern, host->machine_guid) ||
3158 - simple_pattern_matches(qt->hosts.pattern, uuid))
3159 - queryable_host = true;
3160 -
3161 - query_target_add_host(&qtl, host, queryable_host);
3162 - }
3163 - }
3164 - rrd_unlock();
3165 - }
3166 -
3167 - query_target_calculate_window(qt);
3168 -
3169 - qt->timings.preprocessed_ut = now_monotonic_usec();
3170 -
3171 - return qt;
3172 -}
3173 -
3174 -
3175 -// ----------------------------------------------------------------------------
3176 -// load from SQL
3177 -
3178 -static void rrdinstance_load_clabel(SQL_CLABEL_DATA *sld, void *data) {
3179 - RRDINSTANCE *ri = data;
3180 - rrdlabels_add(ri->rrdlabels, sld->label_key, sld->label_value, sld->label_source);
3181 -}
3182 -
3183 -static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
3184 - RRDINSTANCE *ri = data;
3185 -
3186 - RRDMETRIC trm = {
3187 - .id = string_strdupz(sd->id),
3188 - .name = string_strdupz(sd->name),
3189 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomic
3190 - };
3191 - if(sd->hidden) trm.flags |= RRD_FLAG_HIDDEN;
3192 -
3193 - uuid_copy(trm.uuid, sd->dim_id);
3194 -
3195 - dictionary_set(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
3196 -}
3197 -
3198 -static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
3199 - RRDHOST *host = data;
3200 -
3201 - RRDCONTEXT tc = {
3202 - .id = string_strdupz(sc->context),
3203 - .title = string_strdupz(sc->title),
3204 - .units = string_strdupz(sc->units),
3205 - .family = string_strdupz(sc->family),
3206 - .priority = sc->priority,
3207 - .chart_type = sc->chart_type,
3208 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
3209 - .rrdhost = host,
3210 - };
3211 -
3212 - RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_set_and_acquire_item((DICTIONARY *)host->rrdctx, string2str(tc.id), &tc, sizeof(tc));
3213 - RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
3214 -
3215 - RRDINSTANCE tri = {
3216 - .id = string_strdupz(sc->id),
3217 - .name = string_strdupz(sc->name),
3218 - .title = string_strdupz(sc->title),
3219 - .units = string_strdupz(sc->units),
3220 - .family = string_strdupz(sc->family),
3221 - .chart_type = sc->chart_type,
3222 - .priority = sc->priority,
3223 - .update_every_s = sc->update_every,
3224 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
3225 - };
3226 - uuid_copy(tri.uuid, sc->chart_id);
3227 -
3228 - RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, sc->id, &tri, sizeof(tri));
3229 - RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
3230 -
3231 - ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri);
3232 - ctx_get_label_list(&ri->uuid, rrdinstance_load_clabel, ri);
3233 - rrdinstance_trigger_updates(ri, __FUNCTION__ );
3234 - rrdinstance_release(ria);
3235 - rrdcontext_release(rca);
3236 -}
3237 -
3238 -static void rrdcontext_load_context_callback(VERSIONED_CONTEXT_DATA *ctx_data, void *data) {
3239 - RRDHOST *host = data;
3240 - (void)host;
3241 -
3242 - RRDCONTEXT trc = {
3243 - .id = string_strdupz(ctx_data->id),
3244 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
3245 -
3246 - // no need to set more data here
3247 - // we only need the hub data
3248 -
3249 - .hub = *ctx_data,
3250 - };
3251 - dictionary_set((DICTIONARY *)host->rrdctx, string2str(trc.id), &trc, sizeof(trc));
3252 -}
3253 -
3254 -void rrdhost_load_rrdcontext_data(RRDHOST *host) {
3255 - if(host->rrdctx) return;
3256 -
3257 - rrdhost_create_rrdcontexts(host);
3258 - ctx_get_context_list(&host->host_uuid, rrdcontext_load_context_callback, host);
3259 - ctx_get_chart_list(&host->host_uuid, rrdinstance_load_chart_callback, host);
3260 -
3261 - RRDCONTEXT *rc;
3262 - dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
3263 - rrdcontext_trigger_updates(rc, __FUNCTION__ );
3264 - }
3265 - dfe_done(rc);
3266 -
3267 - rrdcontext_garbage_collect_single_host(host, false);
3268 -}
3269 -
3270 -// ----------------------------------------------------------------------------
3271 -// version hash calculation
3272 -
3273 -static uint64_t rrdcontext_version_hash_with_callback(
3274 - RRDHOST *host,
3275 - void (*callback)(RRDCONTEXT *, bool, void *),
3276 - bool snapshot,
3277 - void *bundle) {
3278 -
3279 - if(unlikely(!host || !host->rrdctx)) return 0;
3280 -
3281 - RRDCONTEXT *rc;
3282 - uint64_t hash = 0;
3283 -
3284 - // loop through all contexts of the host
3285 - dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
3286 -
3287 - rrdcontext_lock(rc);
3288 -
3289 - if(unlikely(rrd_flag_check(rc, RRD_FLAG_HIDDEN))) {
3290 - rrdcontext_unlock(rc);
3291 - continue;
3292 - }
3293 -
3294 - if(unlikely(callback))
3295 - callback(rc, snapshot, bundle);
3296 -
3297 - // skip any deleted contexts
3298 - if(unlikely(rrd_flag_is_deleted(rc))) {
3299 - rrdcontext_unlock(rc);
3300 - continue;
3301 - }
3302 -
3303 - // we use rc->hub.* which has the latest
3304 - // metadata we have sent to the hub
3305 -
3306 - // if a context is currently queued, rc->hub.* does NOT
3307 - // reflect the queued changes. rc->hub.* is updated with
3308 - // their metadata, after messages are dispatched to hub.
3309 -
3310 - // when the context is being collected,
3311 - // rc->hub.last_time_t is already zero
3312 -
3313 - hash += rc->hub.version + rc->hub.last_time_s - rc->hub.first_time_s;
3314 -
3315 - rrdcontext_unlock(rc);
3316 -
3317 - }
3318 - dfe_done(rc);
3319 -
3320 - return hash;
3321 -}
3322 -
3323 -// ----------------------------------------------------------------------------
3324 -// retention recalculation
3325 -
3326 -static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs) {
3327 - rrdcontext_post_process_updates(rc, true, reason, worker_jobs);
3328 -}
3329 -
3330 -static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs) {
3331 - if(unlikely(!host || !host->rrdctx)) return;
3332 -
3333 - RRDCONTEXT *rc;
3334 - dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
3335 - rrdcontext_recalculate_context_retention(rc, reason, worker_jobs);
3336 - }
3337 - dfe_done(rc);
3338 -}
3339 -
3340 -static void rrdcontext_recalculate_retention_all_hosts(void) {
3341 - rrdcontext_next_db_rotation_ut = 0;
3342 - rrd_rdlock();
3343 - RRDHOST *host;
3344 - rrdhost_foreach_read(host) {
3345 - worker_is_busy(WORKER_JOB_RETENTION);
3346 - rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DB_ROTATION, true);
3347 - }
3348 - rrd_unlock();
3349 -}
3350 -
3351 -// ----------------------------------------------------------------------------
3352 -// garbage collector
3353 -
3354 -static bool rrdmetric_update_retention(RRDMETRIC *rm) {
3355 - time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
3356 -
3357 - if(rm->rrddim) {
3358 - min_first_time_t = rrddim_first_entry_s(rm->rrddim);
3359 - max_last_time_t = rrddim_last_entry_s(rm->rrddim);
3360 - }
3361 - else {
3362 - RRDHOST *rrdhost = rm->ri->rc->rrdhost;
3363 - for (size_t tier = 0; tier < storage_tiers; tier++) {
3364 - STORAGE_ENGINE *eng = rrdhost->db[tier].eng;
3365 -
3366 - time_t first_time_t, last_time_t;
3367 - if (eng->api.metric_retention_by_uuid(rrdhost->db[tier].instance, &rm->uuid, &first_time_t, &last_time_t)) {
3368 - if (first_time_t < min_first_time_t)
3369 - min_first_time_t = first_time_t;
3370 -
3371 - if (last_time_t > max_last_time_t)
3372 - max_last_time_t = last_time_t;
3373 - }
3374 - }
3375 - }
3376 -
3377 - if((min_first_time_t == LONG_MAX || min_first_time_t == 0) && max_last_time_t == 0)
3378 - return false;
3379 -
3380 - if(min_first_time_t == LONG_MAX)
3381 - min_first_time_t = 0;
3382 -
3383 - if(min_first_time_t > max_last_time_t) {
3384 - internal_error(true, "RRDMETRIC: retention of '%s' is flipped, first_time_t = %ld, last_time_t = %ld", string2str(rm->id), min_first_time_t, max_last_time_t);
3385 - time_t tmp = min_first_time_t;
3386 - min_first_time_t = max_last_time_t;
3387 - max_last_time_t = tmp;
3388 - }
3389 -
3390 - // check if retention changed
3391 -
3392 - if (min_first_time_t != rm->first_time_s) {
3393 - rm->first_time_s = min_first_time_t;
3394 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
3395 - }
3396 -
3397 - if (max_last_time_t != rm->last_time_s) {
3398 - rm->last_time_s = max_last_time_t;
3399 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
3400 - }
3401 -
3402 - if(unlikely(!rm->first_time_s && !rm->last_time_s))
3403 - rrd_flag_set_deleted(rm, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3404 -
3405 - rrd_flag_set(rm, RRD_FLAG_LIVE_RETENTION);
3406 -
3407 - return true;
3408 -}
3409 -
3410 -static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) {
3411 - if(likely(!rrd_flag_check(rm, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
3412 - return false;
3413 -
3414 - if(likely(rrd_flag_check(rm, RRD_FLAGS_PREVENTING_DELETIONS)))
3415 - return false;
3416 -
3417 - if(likely(rm->rrddim))
3418 - return false;
3419 -
3420 - rrdmetric_update_retention(rm);
3421 - if(rm->first_time_s || rm->last_time_s)
3422 - return false;
3423 -
3424 - return true;
3425 -}
3426 -
3427 -static inline bool rrdinstance_should_be_deleted(RRDINSTANCE *ri) {
3428 - if(likely(!rrd_flag_check(ri, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
3429 - return false;
3430 -
3431 - if(likely(rrd_flag_check(ri, RRD_FLAGS_PREVENTING_DELETIONS)))
3432 - return false;
3433 -
3434 - if(likely(ri->rrdset))
3435 - return false;
3436 -
3437 - if(unlikely(dictionary_referenced_items(ri->rrdmetrics) != 0))
3438 - return false;
3439 -
3440 - if(unlikely(dictionary_entries(ri->rrdmetrics) != 0))
3441 - return false;
3442 -
3443 - if(ri->first_time_s || ri->last_time_s)
3444 - return false;
3445 -
3446 - return true;
3447 -}
3448 -
3449 -static inline bool rrdcontext_should_be_deleted(RRDCONTEXT *rc) {
3450 - if(likely(!rrd_flag_check(rc, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
3451 - return false;
3452 -
3453 - if(likely(rrd_flag_check(rc, RRD_FLAGS_PREVENTING_DELETIONS)))
3454 - return false;
3455 -
3456 - if(unlikely(dictionary_referenced_items(rc->rrdinstances) != 0))
3457 - return false;
3458 -
3459 - if(unlikely(dictionary_entries(rc->rrdinstances) != 0))
3460 - return false;
3461 -
3462 - if(unlikely(rc->first_time_s || rc->last_time_s))
3463 - return false;
3464 -
3465 - return true;
3466 -}
3467 -
3468 -void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) {
3469 - // we need to refresh the string pointers in rc->hub
3470 - // in case the context changed values
3471 - rc->hub.id = string2str(rc->id);
3472 - rc->hub.title = string2str(rc->title);
3473 - rc->hub.units = string2str(rc->units);
3474 - rc->hub.family = string2str(rc->family);
3475 -
3476 - // delete it from SQL
3477 - if(ctx_delete_context(&rc->rrdhost->host_uuid, &rc->hub) != 0)
3478 - error("RRDCONTEXT: failed to delete context '%s' version %"PRIu64" from SQL.", rc->hub.id, rc->hub.version);
3479 -}
3480 -
3481 -static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs) {
3482 -
3483 - internal_error(true, "RRDCONTEXT: garbage collecting context structures of host '%s'", rrdhost_hostname(host));
3484 -
3485 - RRDCONTEXT *rc;
3486 - dfe_start_reentrant((DICTIONARY *)host->rrdctx, rc) {
3487 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
3488 -
3489 - if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP);
3490 -
3491 - rrdcontext_lock(rc);
3492 -
3493 - RRDINSTANCE *ri;
3494 - dfe_start_reentrant(rc->rrdinstances, ri) {
3495 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
3496 -
3497 - RRDMETRIC *rm;
3498 - dfe_start_write(ri->rrdmetrics, rm) {
3499 - if(rrdmetric_should_be_deleted(rm)) {
3500 - if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
3501 - if(!dictionary_del(ri->rrdmetrics, string2str(rm->id)))
3502 - error("RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
3503 - string2str(rm->id),
3504 - string2str(ri->id),
3505 - string2str(rc->id),
3506 - rrdhost_hostname(host));
3507 - else
3508 - internal_error(
3509 - true,
3510 - "RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.",
3511 - string2str(rm->id),
3512 - string2str(ri->id),
3513 - string2str(rc->id),
3514 - rrdhost_hostname(host));
3515 - }
3516 - }
3517 - dfe_done(rm);
3518 -
3519 - if(rrdinstance_should_be_deleted(ri)) {
3520 - if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
3521 - if(!dictionary_del(rc->rrdinstances, string2str(ri->id)))
3522 - error("RRDCONTEXT: instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
3523 - string2str(ri->id),
3524 - string2str(rc->id),
3525 - rrdhost_hostname(host));
3526 - else
3527 - internal_error(
3528 - true,
3529 - "RRDCONTEXT: instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.",
3530 - string2str(ri->id),
3531 - string2str(rc->id),
3532 - rrdhost_hostname(host));
3533 - }
3534 - }
3535 - dfe_done(ri);
3536 -
3537 - if(unlikely(rrdcontext_should_be_deleted(rc))) {
3538 - if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
3539 - rrdcontext_dequeue_from_post_processing(rc);
3540 - rrdcontext_delete_from_sql_unsafe(rc);
3541 -
3542 - if(!dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)))
3543 - error("RRDCONTEXT: context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
3544 - string2str(rc->id),
3545 - rrdhost_hostname(host));
3546 - else
3547 - internal_error(
3548 - true,
3549 - "RRDCONTEXT: context '%s' of host '%s', deleted from rrdmetrics dictionary.",
3550 - string2str(rc->id),
3551 - rrdhost_hostname(host));
3552 - }
3553 -
3554 - // the item is referenced in the dictionary
3555 - // so, it is still here to unlock, even if we have deleted it
3556 - rrdcontext_unlock(rc);
3557 - }
3558 - dfe_done(rc);
3559 -}
3560 -
3561 -static void rrdcontext_garbage_collect_for_all_hosts(void) {
3562 - rrd_rdlock();
3563 - RRDHOST *host;
3564 - rrdhost_foreach_read(host) {
3565 - rrdcontext_garbage_collect_single_host(host, true);
3566 - }
3567 - rrd_unlock();
3568 -}
3569 -
3570 -// ----------------------------------------------------------------------------
3571 -// post processing
3572 -
3573 -static void rrdmetric_process_updates(RRDMETRIC *rm, bool force, RRD_FLAGS reason, bool worker_jobs) {
3574 - if(reason != RRD_FLAG_NONE)
3575 - rrd_flag_set_updated(rm, reason);
3576 -
3577 - if(!force && !rrd_flag_is_updated(rm) && rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION) && !rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3578 - return;
3579 -
3580 - if(worker_jobs)
3581 - worker_is_busy(WORKER_JOB_PP_METRIC);
3582 -
3583 - if(reason & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD) {
3584 - rrd_flag_set_archived(rm);
3585 - rrd_flag_set(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD);
3586 - }
3587 - if(rrd_flag_is_deleted(rm) && (reason & RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3588 - rrd_flag_set_archived(rm);
3589 -
3590 - rrdmetric_update_retention(rm);
3591 -
3592 - rrd_flag_unset_updated(rm);
3593 -}
3594 -
3595 -static void rrdinstance_post_process_updates(RRDINSTANCE *ri, bool force, RRD_FLAGS reason, bool worker_jobs) {
3596 - if(reason != RRD_FLAG_NONE)
3597 - rrd_flag_set_updated(ri, reason);
3598 -
3599 - if(!force && !rrd_flag_is_updated(ri) && rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION))
3600 - return;
3601 -
3602 - if(worker_jobs)
3603 - worker_is_busy(WORKER_JOB_PP_INSTANCE);
3604 -
3605 - time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
3606 - size_t metrics_active = 0, metrics_deleted = 0;
3607 - bool live_retention = true, currently_collected = false;
3608 - if(dictionary_entries(ri->rrdmetrics) > 0) {
3609 - RRDMETRIC *rm;
3610 - dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) {
3611 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
3612 -
3613 - RRD_FLAGS reason_to_pass = reason;
3614 - if(rrd_flag_check(ri, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3615 - reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION;
3616 -
3617 - rrdmetric_process_updates(rm, force, reason_to_pass, worker_jobs);
3618 -
3619 - if(unlikely(!rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)))
3620 - live_retention = false;
3621 -
3622 - if (unlikely((rrdmetric_should_be_deleted(rm)))) {
3623 - metrics_deleted++;
3624 - continue;
3625 - }
3626 -
3627 - if(!currently_collected && rrd_flag_check(rm, RRD_FLAG_COLLECTED) && rm->first_time_s)
3628 - currently_collected = true;
3629 -
3630 - metrics_active++;
3631 -
3632 - if (rm->first_time_s && rm->first_time_s < min_first_time_t)
3633 - min_first_time_t = rm->first_time_s;
3634 -
3635 - if (rm->last_time_s && rm->last_time_s > max_last_time_t)
3636 - max_last_time_t = rm->last_time_s;
3637 - }
3638 - dfe_done(rm);
3639 - }
3640 -
3641 - if(unlikely(live_retention && !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
3642 - rrd_flag_set(ri, RRD_FLAG_LIVE_RETENTION);
3643 - else if(unlikely(!live_retention && rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
3644 - rrd_flag_clear(ri, RRD_FLAG_LIVE_RETENTION);
3645 -
3646 - if(unlikely(!metrics_active)) {
3647 - // no metrics available
3648 -
3649 - if(ri->first_time_s) {
3650 - ri->first_time_s = 0;
3651 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
3652 - }
3653 -
3654 - if(ri->last_time_s) {
3655 - ri->last_time_s = 0;
3656 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
3657 - }
3658 -
3659 - rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3660 - }
3661 - else {
3662 - // we have active metrics...
3663 -
3664 - if (unlikely(min_first_time_t == LONG_MAX))
3665 - min_first_time_t = 0;
3666 -
3667 - if (unlikely(min_first_time_t == 0 || max_last_time_t == 0)) {
3668 - if(ri->first_time_s) {
3669 - ri->first_time_s = 0;
3670 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
3671 - }
3672 -
3673 - if(ri->last_time_s) {
3674 - ri->last_time_s = 0;
3675 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
3676 - }
3677 -
3678 - if(likely(live_retention))
3679 - rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3680 - }
3681 - else {
3682 - rrd_flag_clear(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3683 -
3684 - if (unlikely(ri->first_time_s != min_first_time_t)) {
3685 - ri->first_time_s = min_first_time_t;
3686 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
3687 - }
3688 -
3689 - if (unlikely(ri->last_time_s != max_last_time_t)) {
3690 - ri->last_time_s = max_last_time_t;
3691 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
3692 - }
3693 -
3694 - if(likely(currently_collected))
3695 - rrd_flag_set_collected(ri);
3696 - else
3697 - rrd_flag_set_archived(ri);
3698 - }
3699 - }
3700 -
3701 - rrd_flag_unset_updated(ri);
3702 -}
3703 -
3704 -static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs) {
3705 - if(reason != RRD_FLAG_NONE)
3706 - rrd_flag_set_updated(rc, reason);
3707 -
3708 - if(worker_jobs)
3709 - worker_is_busy(WORKER_JOB_PP_CONTEXT);
3710 -
3711 - size_t min_priority_collected = LONG_MAX;
3712 - size_t min_priority_not_collected = LONG_MAX;
3713 - size_t min_priority = LONG_MAX;
3714 - time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
3715 - size_t instances_active = 0, instances_deleted = 0;
3716 - bool live_retention = true, currently_collected = false, hidden = true;
3717 - if(dictionary_entries(rc->rrdinstances) > 0) {
3718 - RRDINSTANCE *ri;
3719 - dfe_start_reentrant(rc->rrdinstances, ri) {
3720 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
3721 -
3722 - RRD_FLAGS reason_to_pass = reason;
3723 - if(rrd_flag_check(rc, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3724 - reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION;
3725 -
3726 - rrdinstance_post_process_updates(ri, force, reason_to_pass, worker_jobs);
3727 -
3728 - if(unlikely(hidden && !rrd_flag_check(ri, RRD_FLAG_HIDDEN)))
3729 - hidden = false;
3730 -
3731 - if(unlikely(live_retention && !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
3732 - live_retention = false;
3733 -
3734 - if (unlikely(rrdinstance_should_be_deleted(ri))) {
3735 - instances_deleted++;
3736 - continue;
3737 - }
3738 -
3739 - if(unlikely(!currently_collected && rrd_flag_is_collected(ri) && ri->first_time_s))
3740 - currently_collected = true;
3741 -
3742 - internal_error(rc->units != ri->units,
3743 - "RRDCONTEXT: '%s' rrdinstance '%s' has different units, context '%s', instance '%s'",
3744 - string2str(rc->id), string2str(ri->id),
3745 - string2str(rc->units), string2str(ri->units));
3746 -
3747 - instances_active++;
3748 -
3749 - if (ri->priority >= RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY) {
3750 - if(rrd_flag_check(ri, RRD_FLAG_COLLECTED)) {
3751 - if(ri->priority < min_priority_collected)
3752 - min_priority_collected = ri->priority;
3753 - }
3754 - else {
3755 - if(ri->priority < min_priority_not_collected)
3756 - min_priority_not_collected = ri->priority;
3757 - }
3758 - }
3759 -
3760 - if (ri->first_time_s && ri->first_time_s < min_first_time_t)
3761 - min_first_time_t = ri->first_time_s;
3762 -
3763 - if (ri->last_time_s && ri->last_time_s > max_last_time_t)
3764 - max_last_time_t = ri->last_time_s;
3765 - }
3766 - dfe_done(ri);
3767 -
3768 - if(min_priority_collected != LONG_MAX)
3769 - // use the collected priority
3770 - min_priority = min_priority_collected;
3771 - else
3772 - // use the non-collected priority
3773 - min_priority = min_priority_not_collected;
3774 - }
3775 -
3776 - {
3777 - bool previous_hidden = rrd_flag_check(rc, RRD_FLAG_HIDDEN);
3778 - if (hidden != previous_hidden) {
3779 - if (hidden && !rrd_flag_check(rc, RRD_FLAG_HIDDEN))
3780 - rrd_flag_set(rc, RRD_FLAG_HIDDEN);
3781 - else if (!hidden && rrd_flag_check(rc, RRD_FLAG_HIDDEN))
3782 - rrd_flag_clear(rc, RRD_FLAG_HIDDEN);
3783 - }
3784 -
3785 - bool previous_live_retention = rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION);
3786 - if (live_retention != previous_live_retention) {
3787 - if (live_retention && !rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
3788 - rrd_flag_set(rc, RRD_FLAG_LIVE_RETENTION);
3789 - else if (!live_retention && rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
3790 - rrd_flag_clear(rc, RRD_FLAG_LIVE_RETENTION);
3791 - }
3792 - }
3793 -
3794 - rrdcontext_lock(rc);
3795 - rc->pp.executions++;
3796 -
3797 - if(unlikely(!instances_active)) {
3798 - // we had some instances, but they are gone now...
3799 -
3800 - if(rc->first_time_s) {
3801 - rc->first_time_s = 0;
3802 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
3803 - }
3804 -
3805 - if(rc->last_time_s) {
3806 - rc->last_time_s = 0;
3807 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
3808 - }
3809 -
3810 - rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3811 - }
3812 - else {
3813 - // we have some active instances...
3814 -
3815 - if (unlikely(min_first_time_t == LONG_MAX))
3816 - min_first_time_t = 0;
3817 -
3818 - if (unlikely(min_first_time_t == 0 && max_last_time_t == 0)) {
3819 - if(rc->first_time_s) {
3820 - rc->first_time_s = 0;
3821 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
3822 - }
3823 -
3824 - if(rc->last_time_s) {
3825 - rc->last_time_s = 0;
3826 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
3827 - }
3828 -
3829 - rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3830 - }
3831 - else {
3832 - rrd_flag_clear(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3833 -
3834 - if (unlikely(rc->first_time_s != min_first_time_t)) {
3835 - rc->first_time_s = min_first_time_t;
3836 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
3837 - }
3838 -
3839 - if (rc->last_time_s != max_last_time_t) {
3840 - rc->last_time_s = max_last_time_t;
3841 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
3842 - }
3843 -
3844 - if(likely(currently_collected))
3845 - rrd_flag_set_collected(rc);
3846 - else
3847 - rrd_flag_set_archived(rc);
3848 - }
3849 -
3850 - if (min_priority != LONG_MAX && rc->priority != min_priority) {
3851 - rc->priority = min_priority;
3852 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
3853 - }
3854 - }
3855 -
3856 - if(unlikely(rrd_flag_is_updated(rc) && rc->rrdhost->rrdctx_hub_queue)) {
3857 - if(check_if_cloud_version_changed_unsafe(rc, false)) {
3858 - rc->version = rrdcontext_get_next_version(rc);
3859 - dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_hub_queue,
3860 - string2str(rc->id), rc, sizeof(*rc));
3861 - }
3862 - }
3863 -
3864 - rrd_flag_unset_updated(rc);
3865 - rrdcontext_unlock(rc);
3866 -}
3867 -
3868 -static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function __maybe_unused, RRD_FLAGS flags __maybe_unused) {
3869 - if(unlikely(!rc->rrdhost->rrdctx_post_processing_queue)) return;
3870 -
3871 - if(!rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_PP)) {
3872 - dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue,
3873 - string2str(rc->id),
3874 - rc,
3875 - sizeof(*rc));
3876 -
3877 -#if(defined(NETDATA_INTERNAL_CHECKS) && defined(LOG_POST_PROCESSING_QUEUE_INSERTIONS))
3878 - {
3879 - BUFFER *wb_flags = buffer_create(1000);
3880 - rrd_flags_to_buffer(flags, wb_flags);
3881 -
3882 - BUFFER *wb_reasons = buffer_create(1000);
3883 - rrd_reasons_to_buffer(flags, wb_reasons);
3884 -
3885 - internal_error(true, "RRDCONTEXT: '%s' update triggered by function %s(), due to flags: %s, reasons: %s",
3886 - string2str(rc->id), function,
3887 - buffer_tostring(wb_flags),
3888 - buffer_tostring(wb_reasons));
3889 -
3890 - buffer_free(wb_reasons);
3891 - buffer_free(wb_flags);
3892 - }
3893 -#endif
3894 - }
3895 -}
3896 -
3897 -static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc) {
3898 - if(unlikely(!rc->rrdhost->rrdctx_post_processing_queue)) return;
3899 - dictionary_del((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue, string2str(rc->id));
3900 -}
3901 -
3902 -static void rrdcontext_post_process_queued_contexts(RRDHOST *host) {
3903 - if(unlikely(!host->rrdctx_post_processing_queue)) return;
3904 -
3905 - RRDCONTEXT *rc;
3906 - dfe_start_reentrant((DICTIONARY *)host->rrdctx_post_processing_queue, rc) {
3907 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
3908 -
3909 - rrdcontext_dequeue_from_post_processing(rc);
3910 - rrdcontext_post_process_updates(rc, false, RRD_FLAG_NONE, true);
3911 - }
3912 - dfe_done(rc);
3913 -}
3914 -
3915 -// ----------------------------------------------------------------------------
3916 -// dispatching contexts to cloud
3917 -
3918 -static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc) {
3919 - time_t now = now_realtime_sec();
3920 - uint64_t version = MAX(rc->version, rc->hub.version);
3921 - version = MAX((uint64_t)now, version);
3922 - version++;
3923 - return version;
3924 -}
3925 -
3926 -static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused) {
3927 -
3928 - // save it, so that we know the last version we sent to hub
3929 - rc->version = rc->hub.version = rrdcontext_get_next_version(rc);
3930 - rc->hub.id = string2str(rc->id);
3931 - rc->hub.title = string2str(rc->title);
3932 - rc->hub.units = string2str(rc->units);
3933 - rc->hub.family = string2str(rc->family);
3934 - rc->hub.chart_type = rrdset_type_name(rc->chart_type);
3935 - rc->hub.priority = rc->priority;
3936 - rc->hub.first_time_s = rc->first_time_s;
3937 - rc->hub.last_time_s = rrd_flag_is_collected(rc) ? 0 : rc->last_time_s;
3938 - rc->hub.deleted = rrd_flag_is_deleted(rc) ? true : false;
3939 -
3940 -#ifdef ENABLE_ACLK
3941 - struct context_updated message = {
3942 - .id = rc->hub.id,
3943 - .version = rc->hub.version,
3944 - .title = rc->hub.title,
3945 - .units = rc->hub.units,
3946 - .family = rc->hub.family,
3947 - .chart_type = rc->hub.chart_type,
3948 - .priority = rc->hub.priority,
3949 - .first_entry = rc->hub.first_time_s,
3950 - .last_entry = rc->hub.last_time_s,
3951 - .deleted = rc->hub.deleted,
3952 - };
3953 -
3954 - if(likely(!rrd_flag_check(rc, RRD_FLAG_HIDDEN))) {
3955 - if (snapshot) {
3956 - if (!rc->hub.deleted)
3957 - contexts_snapshot_add_ctx_update(bundle, &message);
3958 - }
3959 - else
3960 - contexts_updated_add_ctx_update(bundle, &message);
3961 - }
3962 -#endif
3963 -
3964 - // store it to SQL
3965 -
3966 - if(rrd_flag_is_deleted(rc))
3967 - rrdcontext_delete_from_sql_unsafe(rc);
3968 -
3969 - else if (ctx_store_context(&rc->rrdhost->host_uuid, &rc->hub) != 0)
3970 - error("RRDCONTEXT: failed to save context '%s' version %"PRIu64" to SQL.", rc->hub.id, rc->hub.version);
3971 -}
3972 -
3973 -static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused) {
3974 - bool id_changed = false,
3975 - title_changed = false,
3976 - units_changed = false,
3977 - family_changed = false,
3978 - chart_type_changed = false,
3979 - priority_changed = false,
3980 - first_time_changed = false,
3981 - last_time_changed = false,
3982 - deleted_changed = false;
3983 -
3984 - RRD_FLAGS flags = rrd_flags_get(rc);
3985 -
3986 - if(unlikely(string2str(rc->id) != rc->hub.id))
3987 - id_changed = true;
3988 -
3989 - if(unlikely(string2str(rc->title) != rc->hub.title))
3990 - title_changed = true;
3991 -
3992 - if(unlikely(string2str(rc->units) != rc->hub.units))
3993 - units_changed = true;
3994 -
3995 - if(unlikely(string2str(rc->family) != rc->hub.family))
3996 - family_changed = true;
3997 -
3998 - if(unlikely(rrdset_type_name(rc->chart_type) != rc->hub.chart_type))
3999 - chart_type_changed = true;
4000 -
4001 - if(unlikely(rc->priority != rc->hub.priority))
4002 - priority_changed = true;
4003 -
4004 - if(unlikely((uint64_t)rc->first_time_s != rc->hub.first_time_s))
4005 - first_time_changed = true;
4006 -
4007 - if(unlikely((uint64_t)((flags & RRD_FLAG_COLLECTED) ? 0 : rc->last_time_s) != rc->hub.last_time_s))
4008 - last_time_changed = true;
4009 -
4010 - if(unlikely(((flags & RRD_FLAG_DELETED) ? true : false) != rc->hub.deleted))
4011 - deleted_changed = true;
4012 -
4013 - if(unlikely(id_changed || title_changed || units_changed || family_changed || chart_type_changed || priority_changed || first_time_changed || last_time_changed || deleted_changed)) {
4014 -
4015 - internal_error(LOG_TRANSITIONS,
4016 - "RRDCONTEXT: %s NEW VERSION '%s'%s of host '%s', version %"PRIu64", title '%s'%s, units '%s'%s, family '%s'%s, chart type '%s'%s, priority %u%s, first_time_t %ld%s, last_time_t %ld%s, deleted '%s'%s, (queued for %llu ms, expected %llu ms)",
4017 - sending?"SENDING":"QUEUE",
4018 - string2str(rc->id), id_changed ? " (CHANGED)" : "",
4019 - rrdhost_hostname(rc->rrdhost),
4020 - rc->version,
4021 - string2str(rc->title), title_changed ? " (CHANGED)" : "",
4022 - string2str(rc->units), units_changed ? " (CHANGED)" : "",
4023 - string2str(rc->family), family_changed ? " (CHANGED)" : "",
4024 - rrdset_type_name(rc->chart_type), chart_type_changed ? " (CHANGED)" : "",
4025 - rc->priority, priority_changed ? " (CHANGED)" : "",
4026 - rc->first_time_s, first_time_changed ? " (CHANGED)" : "",
4027 - (flags & RRD_FLAG_COLLECTED) ? 0 : rc->last_time_s, last_time_changed ? " (CHANGED)" : "",
4028 - (flags & RRD_FLAG_DELETED) ? "true" : "false", deleted_changed ? " (CHANGED)" : "",
4029 - sending ? (now_realtime_usec() - rc->queue.queued_ut) / USEC_PER_MS : 0,
4030 - sending ? (rc->queue.scheduled_dispatch_ut - rc->queue.queued_ut) / USEC_PER_MS : 0
4031 - );
4032 -
4033 - return true;
4034 - }
4035 -
4036 - return false;
4037 -}
4038 -
4039 -static inline usec_t rrdcontext_calculate_queued_dispatch_time_ut(RRDCONTEXT *rc, usec_t now_ut) {
4040 -
4041 - if(likely(rc->queue.delay_calc_ut >= rc->queue.queued_ut))
4042 - return rc->queue.scheduled_dispatch_ut;
4043 -
4044 - RRD_FLAGS flags = rc->queue.queued_flags;
4045 -
4046 - usec_t delay = LONG_MAX;
4047 - int i;
4048 - struct rrdcontext_reason *reason;
4049 - for(i = 0, reason = &rrdcontext_reasons[i]; reason->name ; reason = &rrdcontext_reasons[++i]) {
4050 - if(unlikely(flags & reason->flag)) {
4051 - if(reason->delay_ut < delay)
4052 - delay = reason->delay_ut;
4053 - }
4054 - }
4055 -
4056 - if(unlikely(delay == LONG_MAX)) {
4057 - internal_error(true, "RRDCONTEXT: '%s', cannot find minimum delay of flags %x", string2str(rc->id), (unsigned int)flags);
4058 - delay = 60 * USEC_PER_SEC;
4059 - }
4060 -
4061 - rc->queue.delay_calc_ut = now_ut;
4062 - usec_t dispatch_ut = rc->queue.scheduled_dispatch_ut = rc->queue.queued_ut + delay;
4063 - return dispatch_ut;
4064 -}
4065 -
4066 -static void rrdcontext_dequeue_from_hub_queue(RRDCONTEXT *rc) {
4067 - dictionary_del((DICTIONARY *)rc->rrdhost->rrdctx_hub_queue, string2str(rc->id));
4068 -}
4069 -
4070 -static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now_ut) {
4071 -
4072 - // check if we have received a streaming command for this host
4073 - if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS) || !aclk_connected || !host->rrdctx_hub_queue)
4074 - return;
4075 -
4076 - // check if there are queued items to send
4077 - if(!dictionary_entries((DICTIONARY *)host->rrdctx_hub_queue))
4078 - return;
4079 -
4080 - if(!host->node_id)
4081 - return;
4082 -
4083 - size_t messages_added = 0;
4084 - contexts_updated_t bundle = NULL;
4085 -
4086 - RRDCONTEXT *rc;
4087 - dfe_start_reentrant((DICTIONARY *)host->rrdctx_hub_queue, rc) {
4088 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
4089 -
4090 - if(unlikely(messages_added >= MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST))
4091 - break;
4092 -
4093 - worker_is_busy(WORKER_JOB_QUEUED);
4094 - usec_t dispatch_ut = rrdcontext_calculate_queued_dispatch_time_ut(rc, now_ut);
4095 - char *claim_id = get_agent_claimid();
4096 -
4097 - if(unlikely(now_ut >= dispatch_ut) && claim_id) {
4098 - worker_is_busy(WORKER_JOB_CHECK);
4099 -
4100 - rrdcontext_lock(rc);
4101 -
4102 - if(check_if_cloud_version_changed_unsafe(rc, true)) {
4103 - worker_is_busy(WORKER_JOB_SEND);
4104 -
4105 -#ifdef ENABLE_ACLK
4106 - if(!bundle) {
4107 - // prepare the bundle to send the messages
4108 - char uuid[UUID_STR_LEN];
4109 - uuid_unparse_lower(*host->node_id, uuid);
4110 -
4111 - bundle = contexts_updated_new(claim_id, uuid, 0, now_ut);
4112 - }
4113 -#endif
4114 - // update the hub data of the context, give a new version, pack the message
4115 - // and save an update to SQL
4116 - rrdcontext_message_send_unsafe(rc, false, bundle);
4117 - messages_added++;
4118 -
4119 - rc->queue.dispatches++;
4120 - rc->queue.dequeued_ut = now_ut;
4121 - }
4122 - else
4123 - rc->version = rc->hub.version;
4124 -
4125 - // remove it from the queue
4126 - worker_is_busy(WORKER_JOB_DEQUEUE);
4127 - rrdcontext_dequeue_from_hub_queue(rc);
4128 -
4129 - if(unlikely(rrdcontext_should_be_deleted(rc))) {
4130 - // this is a deleted context - delete it forever...
4131 -
4132 - worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
4133 -
4134 - rrdcontext_dequeue_from_post_processing(rc);
4135 - rrdcontext_delete_from_sql_unsafe(rc);
4136 -
4137 - STRING *id = string_dup(rc->id);
4138 - rrdcontext_unlock(rc);
4139 -
4140 - // delete it from the master dictionary
4141 - if(!dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)))
4142 - error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.",
4143 - string2str(id), rrdhost_hostname(host));
4144 -
4145 - string_freez(id);
4146 - }
4147 - else
4148 - rrdcontext_unlock(rc);
4149 - }
4150 - freez(claim_id);
4151 - }
4152 - dfe_done(rc);
4153 -
4154 -#ifdef ENABLE_ACLK
4155 - if(service_running(SERVICE_CONTEXT) && bundle) {
4156 - // we have a bundle to send messages
4157 -
4158 - // update the version hash
4159 - contexts_updated_update_version_hash(bundle, rrdcontext_version_hash(host));
4160 -
4161 - // send it
4162 - aclk_send_contexts_updated(bundle);
4163 - }
4164 - else if(bundle)
4165 - contexts_updated_delete(bundle);
4166 -#endif
4167 -
4168 -}
4169 -
4170 -// ----------------------------------------------------------------------------
4171 -// worker thread
4172 -
4173 -static void rrdcontext_main_cleanup(void *ptr) {
4174 - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
4175 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
4176 -
4177 - // custom code
4178 - worker_unregister();
4179 -
4180 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
4181 -}
4182 -
4183 -void *rrdcontext_main(void *ptr) {
4184 - netdata_thread_cleanup_push(rrdcontext_main_cleanup, ptr);
4185 -
4186 - worker_register("RRDCONTEXT");
4187 - worker_register_job_name(WORKER_JOB_HOSTS, "hosts");
4188 - worker_register_job_name(WORKER_JOB_CHECK, "dedup checks");
4189 - worker_register_job_name(WORKER_JOB_SEND, "sent contexts");
4190 - worker_register_job_name(WORKER_JOB_DEQUEUE, "deduplicated contexts");
4191 - worker_register_job_name(WORKER_JOB_RETENTION, "metrics retention");
4192 - worker_register_job_name(WORKER_JOB_QUEUED, "queued contexts");
4193 - worker_register_job_name(WORKER_JOB_CLEANUP, "cleanups");
4194 - worker_register_job_name(WORKER_JOB_CLEANUP_DELETE, "deletes");
4195 - worker_register_job_name(WORKER_JOB_PP_METRIC, "check metrics");
4196 - worker_register_job_name(WORKER_JOB_PP_INSTANCE, "check instances");
4197 - worker_register_job_name(WORKER_JOB_PP_CONTEXT, "check contexts");
4198 -
4199 - worker_register_job_custom_metric(WORKER_JOB_HUB_QUEUE_SIZE, "hub queue size", "contexts", WORKER_METRIC_ABSOLUTE);
4200 - worker_register_job_custom_metric(WORKER_JOB_PP_QUEUE_SIZE, "post processing queue size", "contexts", WORKER_METRIC_ABSOLUTE);
4201 -
4202 - heartbeat_t hb;
4203 - heartbeat_init(&hb);
4204 - usec_t step = RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC;
4205 -
4206 - while (service_running(SERVICE_CONTEXT)) {
4207 - worker_is_idle();
4208 - heartbeat_next(&hb, step);
4209 -
4210 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
4211 -
4212 - usec_t now_ut = now_realtime_usec();
4213 -
4214 - if(rrdcontext_next_db_rotation_ut && now_ut > rrdcontext_next_db_rotation_ut) {
4215 - rrdcontext_recalculate_retention_all_hosts();
4216 - rrdcontext_garbage_collect_for_all_hosts();
4217 - rrdcontext_next_db_rotation_ut = 0;
4218 - }
4219 -
4220 - size_t hub_queued_contexts_for_all_hosts = 0;
4221 - size_t pp_queued_contexts_for_all_hosts = 0;
4222 -
4223 - rrd_rdlock();
4224 - RRDHOST *host;
4225 - rrdhost_foreach_read(host) {
4226 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
4227 -
4228 - worker_is_busy(WORKER_JOB_HOSTS);
4229 -
4230 - if(host->rrdctx_post_processing_queue) {
4231 - pp_queued_contexts_for_all_hosts +=
4232 - dictionary_entries((DICTIONARY *)host->rrdctx_post_processing_queue);
4233 - rrdcontext_post_process_queued_contexts(host);
4234 - }
4235 -
4236 - if(host->rrdctx_hub_queue) {
4237 - hub_queued_contexts_for_all_hosts += dictionary_entries((DICTIONARY *)host->rrdctx_hub_queue);
4238 - rrdcontext_dispatch_queued_contexts_to_hub(host, now_ut);
4239 - }
4240 - }
4241 - rrd_unlock();
4242 -
4243 - worker_set_metric(WORKER_JOB_HUB_QUEUE_SIZE, (NETDATA_DOUBLE)hub_queued_contexts_for_all_hosts);
4244 - worker_set_metric(WORKER_JOB_PP_QUEUE_SIZE, (NETDATA_DOUBLE)pp_queued_contexts_for_all_hosts);
4245 - }
4246 -
4247 - netdata_thread_cleanup_pop(1);
4248 - return NULL;
4249 -}
database/rrdhost.c
+5 -5
@@ -229,7 +229,8 @@ static void rrdhost_initialize_rrdpush_sender(RRDHOST *host,
229 rrdpush_destinations_init(host);
230
231 host->rrdpush_send_api_key = strdupz(rrdpush_api_key);
232 - host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL, SIMPLE_PATTERN_EXACT);
232 + host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL,
233 + SIMPLE_PATTERN_EXACT, true);
234
235 rrdhost_option_set(host, RRDHOST_OPTION_SENDER_ENABLED);
236 }
@@ -1239,11 +1240,10 @@ void rrdhost_free_all(void) {
1240
1241 void rrd_finalize_collection_for_all_hosts(void) {
1242 RRDHOST *host;
1242 - rrd_wrlock();
1243 - rrdhost_foreach_read(host) {
1243 + dfe_start_reentrant(rrdhost_root_index, host) {
1244 rrdhost_finalize_collection(host);
1245 }
1246 - rrd_unlock();
1246 + dfe_done(host);
1247 }
1248
1249 // ----------------------------------------------------------------------------
@@ -1441,7 +1441,7 @@ void rrdhost_finalize_collection(RRDHOST *host) {
1441 info("RRD: 'host:%s' stopping data collection...", rrdhost_hostname(host));
1442
1443 RRDSET *st;
1444 - rrdset_foreach_write(st, host)
1444 + rrdset_foreach_read(st, host)
1445 rrdset_finalize_collection(st, true);
1446 rrdset_foreach_done(st);
1447 }
database/rrdlabels.c
+13 -4
@@ -832,6 +832,7 @@ void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src) {
832 // returns true when there are keys in the dictionary matching a simple pattern
833
834 struct simple_pattern_match_name_value {
835 + size_t searches;
836 SIMPLE_PATTERN *pattern;
837 char equal;
838 };
@@ -842,6 +843,7 @@ static int simple_pattern_match_name_only_callback(const DICTIONARY_ITEM *item,
843 (void)value;
844
845 // we return -1 to stop the walkthrough on first match
846 + t->searches++;
847 if(simple_pattern_matches(t->pattern, name)) return -1;
848
849 return 0;
@@ -853,6 +855,7 @@ static int simple_pattern_match_name_and_value_callback(const DICTIONARY_ITEM *i
855 RRDLABEL *lb = (RRDLABEL *)value;
856
857 // we return -1 to stop the walkthrough on first match
858 + t->searches++;
859 if(simple_pattern_matches(t->pattern, name)) return -1;
860
861 size_t len = RRDLABELS_MAX_NAME_LENGTH + RRDLABELS_MAX_VALUE_LENGTH + 2; // +1 for =, +1 for \0
@@ -871,28 +874,34 @@ static int simple_pattern_match_name_and_value_callback(const DICTIONARY_ITEM *i
874 // terminate it
875 *dst = '\0';
876
874 - if(simple_pattern_matches(t->pattern, tmp)) return -1;
877 + t->searches++;
878 + if(simple_pattern_matches_length_extract(t->pattern, tmp, dst - tmp, NULL, 0))
879 + return -1;
880
881 return 0;
882 }
883
879 -bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal) {
884 +bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches) {
885 if (!labels) return false;
886
887 struct simple_pattern_match_name_value t = {
888 + .searches = 0,
889 .pattern = pattern,
890 .equal = equal
891 };
892
893 int ret = dictionary_walkthrough_read(labels, equal?simple_pattern_match_name_and_value_callback:simple_pattern_match_name_only_callback, &t);
894
895 + if(searches)
896 + *searches = t.searches;
897 +
898 return (ret == -1)?true:false;
899 }
900
901 bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt) {
902 if (!labels) return false;
903
895 - SIMPLE_PATTERN *pattern = simple_pattern_create(simple_pattern_txt, " ,|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
904 + SIMPLE_PATTERN *pattern = simple_pattern_create(simple_pattern_txt, " ,|\t\r\n\f\v", SIMPLE_PATTERN_EXACT, true);
905 char equal = '\0';
906
907 const char *s;
@@ -903,7 +912,7 @@ bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_patte
912 }
913 }
914
906 - bool ret = rrdlabels_match_simple_pattern_parsed(labels, pattern, equal);
915 + bool ret = rrdlabels_match_simple_pattern_parsed(labels, pattern, equal, NULL);
916
917 simple_pattern_free(pattern);
918
exporting/check_filters.c
+1 -1
@@ -65,7 +65,7 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st)
65
66 if(unlikely(!(*flags & RRDSET_FLAG_EXPORTING_SEND))) {
67 // we have not checked this chart
68 - if(simple_pattern_matches(instance->config.charts_pattern, rrdset_id(st)) || simple_pattern_matches(instance->config.charts_pattern, rrdset_name(st)))
68 + if(simple_pattern_matches_string(instance->config.charts_pattern, st->id) || simple_pattern_matches_string(instance->config.charts_pattern, st->name))
69 *flags |= RRDSET_FLAG_EXPORTING_SEND;
70 else {
71 *flags |= RRDSET_FLAG_EXPORTING_IGNORE;
exporting/prometheus/prometheus.c
+3 -3
@@ -9,9 +9,9 @@
9
10 static int is_matches_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN *filter) {
11 if (instance->config.options & EXPORTING_OPTION_SEND_NAMES) {
12 - return simple_pattern_matches(filter, rrdset_name(st));
12 + return simple_pattern_matches_string(filter, st->name);
13 }
14 - return simple_pattern_matches(filter, rrdset_id(st));
14 + return simple_pattern_matches_string(filter, st->id);
15 }
16
17 /**
@@ -514,7 +514,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
514 int allhosts,
515 PROMETHEUS_OUTPUT_OPTIONS output_options)
516 {
517 - SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT);
517 + SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT, true);
518
519 char hostname[PROMETHEUS_ELEMENT_MAX + 1];
520 prometheus_label_copy(hostname, rrdhost_hostname(host), PROMETHEUS_ELEMENT_MAX);
exporting/read_config.c
+8 -6
@@ -264,11 +264,11 @@ struct engine *read_exporting_config()
264 prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
265
266 prometheus_exporter_instance->config.charts_pattern = simple_pattern_create(
267 - prometheus_config_get("send charts matching", "*"),
268 - NULL,
269 - SIMPLE_PATTERN_EXACT);
267 + prometheus_config_get("send charts matching", "*"),
268 + NULL,
269 + SIMPLE_PATTERN_EXACT, true);
270 prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create(
271 - prometheus_config_get("send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
271 + prometheus_config_get("send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT, true);
272
273 prometheus_exporter_instance->config.prefix = prometheus_config_get("prefix", global_exporting_prefix);
274
@@ -369,10 +369,12 @@ struct engine *read_exporting_config()
369 tmp_instance->config.timeoutms = exporter_get_number(instance_name, "timeout ms", 10000);
370
371 tmp_instance->config.charts_pattern =
372 - simple_pattern_create(exporter_get(instance_name, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
372 + simple_pattern_create(exporter_get(instance_name, "send charts matching", "*"), NULL,
373 + SIMPLE_PATTERN_EXACT,
374 + true);
375
376 tmp_instance->config.hosts_pattern = simple_pattern_create(
375 - exporter_get(instance_name, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
377 + exporter_get(instance_name, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT, true);
378
379 char *data_source = exporter_get(instance_name, "data source", "average");
380
health/health.c
+17 -23
@@ -357,13 +357,11 @@ static void health_reload_host(RRDHOST *host) {
357 void health_reload(void) {
358 sql_refresh_hashes();
359
360 - rrd_rdlock();
361 -
360 RRDHOST *host;
363 - rrdhost_foreach_read(host)
361 + dfe_start_reentrant(rrdhost_root_index, host){
362 health_reload_host(host);
365 -
366 - rrd_unlock();
363 + }
364 + dfe_done(host);
365
366 #ifdef ENABLE_ACLK
367 if (netdata_cloud_setting) {
@@ -781,7 +779,8 @@ static void initialize_health(RRDHOST *host)
779 else
780 host->health_log.max = (unsigned int)n;
781
784 - conf_enabled_alarms = simple_pattern_create(config_get(CONFIG_SECTION_HEALTH, "enabled alarms", "*"), NULL, SIMPLE_PATTERN_EXACT);
782 + conf_enabled_alarms = simple_pattern_create(config_get(CONFIG_SECTION_HEALTH, "enabled alarms", "*"), NULL,
783 + SIMPLE_PATTERN_EXACT, true);
784
785 netdata_rwlock_init(&host->health_log.alarm_log_rwlock);
786
@@ -803,7 +802,7 @@ static void initialize_health(RRDHOST *host)
802
803 // link the loaded alarms to their charts
804 RRDSET *st;
806 - rrdset_foreach_write(st, host) {
805 + rrdset_foreach_reentrant(st, host) {
806 if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))
807 continue;
808
@@ -838,11 +837,11 @@ static SILENCE_TYPE check_silenced(RRDCALC *rc, const char *host, SILENCERS *sil
837
838 for (s = silencers->silencers; s!=NULL; s=s->next){
839 if (
841 - (!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches(s->alarms_pattern, rrdcalc_name(rc)))) &&
842 - (!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches(s->contexts_pattern, rrdset_context(rc->rrdset)))) &&
843 - (!s->hosts_pattern || (host && s->hosts_pattern && simple_pattern_matches(s->hosts_pattern,host))) &&
844 - (!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches(s->charts_pattern, rrdcalc_chart_name(rc)))) &&
845 - (!s->families_pattern || (rc->rrdset && rc->rrdset->family && s->families_pattern && simple_pattern_matches(s->families_pattern, rrdset_family(rc->rrdset))))
840 + (!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches_string(s->alarms_pattern, rc->name))) &&
841 + (!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches_string(s->contexts_pattern, rc->rrdset->context))) &&
842 + (!s->hosts_pattern || (host && s->hosts_pattern && simple_pattern_matches(s->hosts_pattern, host))) &&
843 + (!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches_string(s->charts_pattern, rc->chart))) &&
844 + (!s->families_pattern || (rc->rrdset && rc->rrdset->family && s->families_pattern && simple_pattern_matches_string(s->families_pattern, rc->rrdset->family)))
845 ) {
846 debug(D_HEALTH, "Alarm matches command API silence entry %s:%s:%s:%s:%s", s->alarms,s->charts, s->contexts, s->hosts, s->families);
847 if (unlikely(silencers->stype == STYPE_NONE)) {
@@ -1015,9 +1014,7 @@ void *health_main(void *ptr) {
1014 #endif
1015
1016 worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
1018 - rrd_rdlock();
1019 -
1020 - rrdhost_foreach_read(host) {
1017 + dfe_start_reentrant(rrdhost_root_index, host) {
1018
1019 if(unlikely(!service_running(SERVICE_HEALTH)))
1020 break;
@@ -1025,11 +1022,8 @@ void *health_main(void *ptr) {
1022 if (unlikely(!host->health.health_enabled))
1023 continue;
1024
1028 - if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))) {
1029 - rrd_unlock();
1025 + if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
1026 initialize_health(host);
1031 - rrd_rdlock();
1032 - }
1027
1028 health_execute_delayed_initializations(host);
1029
@@ -1494,9 +1488,8 @@ void *health_main(void *ptr) {
1488 }
1489 break;
1490 }
1497 - } //for each host
1498 -
1499 - rrd_unlock();
1491 + }
1492 + dfe_done(host);
1493
1494 // wait for all notifications to finish before allowing health to be cleaned up
1495 ALARM_ENTRY *ae;
@@ -1509,7 +1502,7 @@ void *health_main(void *ptr) {
1502
1503 #ifdef ENABLE_ACLK
1504 if (netdata_cloud_setting && unlikely(aclk_alert_reloaded) && loop > (marked_aclk_reload_loop + 2)) {
1512 - rrdhost_foreach_read(host) {
1505 + dfe_start_reentrant(rrdhost_root_index, host) {
1506 if(unlikely(!service_running(SERVICE_HEALTH)))
1507 break;
1508
@@ -1518,6 +1511,7 @@ void *health_main(void *ptr) {
1511
1512 sql_queue_removed_alerts_to_aclk(host);
1513 }
1514 + dfe_done(host);
1515 aclk_alert_reloaded = 0;
1516 marked_aclk_reload_loop = 0;
1517 }
health/health_config.c
+19 -13
@@ -251,7 +251,7 @@ static SIMPLE_PATTERN *health_pattern_from_foreach(const char *s) {
251
252 if(convert) {
253 dimension_remove_pipe_comma(convert);
254 - val = simple_pattern_create(convert, NULL, SIMPLE_PATTERN_EXACT);
254 + val = simple_pattern_create(convert, NULL, SIMPLE_PATTERN_EXACT, true);
255 freez(convert);
256 }
257
@@ -679,9 +679,9 @@ static int health_readfile(const char *filename, void *data) {
679 else if(hash == hash_os && !strcasecmp(key, HEALTH_OS_KEY)) {
680 char *os_match = value;
681 if (alert_cfg) alert_cfg->os = string_strdupz(value);
682 - SIMPLE_PATTERN *os_pattern = simple_pattern_create(os_match, NULL, SIMPLE_PATTERN_EXACT);
682 + SIMPLE_PATTERN *os_pattern = simple_pattern_create(os_match, NULL, SIMPLE_PATTERN_EXACT, true);
683
684 - if(!simple_pattern_matches(os_pattern, rrdhost_os(host))) {
684 + if(!simple_pattern_matches_string(os_pattern, host->os)) {
685 if(rc)
686 debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: host O/S does not match '%s'", rrdhost_hostname(host), rrdcalc_name(rc), line, filename, os_match);
687
@@ -696,9 +696,9 @@ static int health_readfile(const char *filename, void *data) {
696 else if(hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) {
697 char *host_match = value;
698 if (alert_cfg) alert_cfg->host = string_strdupz(value);
699 - SIMPLE_PATTERN *host_pattern = simple_pattern_create(host_match, NULL, SIMPLE_PATTERN_EXACT);
699 + SIMPLE_PATTERN *host_pattern = simple_pattern_create(host_match, NULL, SIMPLE_PATTERN_EXACT, true);
700
701 - if(!simple_pattern_matches(host_pattern, rrdhost_hostname(host))) {
701 + if(!simple_pattern_matches_string(host_pattern, host->hostname)) {
702 if(rc)
703 debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: hostname does not match '%s'", rrdhost_hostname(host), rrdcalc_name(rc), line, filename, host_match);
704
@@ -918,7 +918,8 @@ static int health_readfile(const char *filename, void *data) {
918 rc->host_labels = string_strdupz(tmp);
919 freez(tmp);
920 }
921 - rc->host_labels_pattern = simple_pattern_create(rrdcalc_host_labels(rc), NULL, SIMPLE_PATTERN_EXACT);
921 + rc->host_labels_pattern = simple_pattern_create(rrdcalc_host_labels(rc), NULL, SIMPLE_PATTERN_EXACT,
922 + true);
923 }
924 else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) {
925 alert_cfg->plugin = string_strdupz(value);
@@ -926,7 +927,7 @@ static int health_readfile(const char *filename, void *data) {
927 simple_pattern_free(rc->plugin_pattern);
928
929 rc->plugin_match = string_strdupz(value);
929 - rc->plugin_pattern = simple_pattern_create(rrdcalc_plugin_match(rc), NULL, SIMPLE_PATTERN_EXACT);
930 + rc->plugin_pattern = simple_pattern_create(rrdcalc_plugin_match(rc), NULL, SIMPLE_PATTERN_EXACT, true);
931 }
932 else if(hash == hash_module && !strcasecmp(key, HEALTH_MODULE_KEY)) {
933 alert_cfg->module = string_strdupz(value);
@@ -934,7 +935,7 @@ static int health_readfile(const char *filename, void *data) {
935 simple_pattern_free(rc->module_pattern);
936
937 rc->module_match = string_strdupz(value);
937 - rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT);
938 + rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT, true);
939 }
940 else {
941 error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.",
@@ -998,7 +999,8 @@ static int health_readfile(const char *filename, void *data) {
999 simple_pattern_free(rt->family_pattern);
1000
1001 rt->family_match = string_strdupz(value);
1001 - rt->family_pattern = simple_pattern_create(rrdcalctemplate_family_match(rt), NULL, SIMPLE_PATTERN_EXACT);
1002 + rt->family_pattern = simple_pattern_create(rrdcalctemplate_family_match(rt), NULL, SIMPLE_PATTERN_EXACT,
1003 + true);
1004 }
1005 else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) {
1006 alert_cfg->plugin = string_strdupz(value);
@@ -1006,7 +1008,8 @@ static int health_readfile(const char *filename, void *data) {
1008 simple_pattern_free(rt->plugin_pattern);
1009
1010 rt->plugin_match = string_strdupz(value);
1009 - rt->plugin_pattern = simple_pattern_create(rrdcalctemplate_plugin_match(rt), NULL, SIMPLE_PATTERN_EXACT);
1011 + rt->plugin_pattern = simple_pattern_create(rrdcalctemplate_plugin_match(rt), NULL, SIMPLE_PATTERN_EXACT,
1012 + true);
1013 }
1014 else if(hash == hash_module && !strcasecmp(key, HEALTH_MODULE_KEY)) {
1015 alert_cfg->module = string_strdupz(value);
@@ -1014,7 +1017,8 @@ static int health_readfile(const char *filename, void *data) {
1017 simple_pattern_free(rt->module_pattern);
1018
1019 rt->module_match = string_strdupz(value);
1017 - rt->module_pattern = simple_pattern_create(rrdcalctemplate_module_match(rt), NULL, SIMPLE_PATTERN_EXACT);
1020 + rt->module_pattern = simple_pattern_create(rrdcalctemplate_module_match(rt), NULL, SIMPLE_PATTERN_EXACT,
1021 + true);
1022 }
1023 else if(hash == hash_charts && !strcasecmp(key, HEALTH_CHARTS_KEY)) {
1024 alert_cfg->charts = string_strdupz(value);
@@ -1022,7 +1026,8 @@ static int health_readfile(const char *filename, void *data) {
1026 simple_pattern_free(rt->charts_pattern);
1027
1028 rt->charts_match = string_strdupz(value);
1025 - rt->charts_pattern = simple_pattern_create(rrdcalctemplate_charts_match(rt), NULL, SIMPLE_PATTERN_EXACT);
1029 + rt->charts_pattern = simple_pattern_create(rrdcalctemplate_charts_match(rt), NULL, SIMPLE_PATTERN_EXACT,
1030 + true);
1031 }
1032 else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1033 alert_cfg->lookup = string_strdupz(value);
@@ -1181,7 +1186,8 @@ static int health_readfile(const char *filename, void *data) {
1186 rt->host_labels = string_strdupz(tmp);
1187 freez(tmp);
1188 }
1184 - rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL, SIMPLE_PATTERN_EXACT);
1189 + rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL,
1190 + SIMPLE_PATTERN_EXACT, true);
1191 }
1192 else {
1193 error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",
libnetdata/buffer/buffer.c
+9 -5
@@ -304,11 +304,13 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
304
305 // ----------------------------------------------------------------------------
306
307 -void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth, bool add_anonymous_object) {
307 +void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
308 + bool add_anonymous_object, bool minify) {
309 strncpyz(wb->json.key_quote, key_quote, BUFFER_QUOTE_MAX_SIZE);
310 strncpyz(wb->json.value_quote, value_quote, BUFFER_QUOTE_MAX_SIZE);
311
311 - wb->json.depth = depth - 1;
312 + wb->json.minify = minify;
313 + wb->json.depth = (int8_t)(depth - 1);
314 _buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
315
316 if(add_anonymous_object)
@@ -331,7 +333,9 @@ void buffer_json_finalize(BUFFER *wb) {
333 break;
334 }
335 }
334 - buffer_fast_strcat(wb, "\n", 1);
336 +
337 + if(!wb->json.minify)
338 + buffer_fast_strcat(wb, "\n", 1);
339 }
340
341 // ----------------------------------------------------------------------------
@@ -481,13 +485,13 @@ int buffer_unittest(void) {
485
486 buffer_flush(wb);
487
484 - buffer_json_initialize(wb, "\"", "\"", 0, true);
488 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
489 buffer_json_finalize(wb);
490 errors += buffer_expect(wb, "{\n}\n");
491
492 buffer_flush(wb);
493
490 - buffer_json_initialize(wb, "\"", "\"", 0, true);
494 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
495 buffer_json_member_add_string(wb, "hello", "world");
496 buffer_json_member_add_string(wb, "alpha", "this: \" is a double quote");
497 buffer_json_member_add_object(wb, "object1");
libnetdata/buffer/buffer.h
+42 -8
@@ -7,7 +7,7 @@
7
8 #define WEB_DATA_LENGTH_INCREASE_STEP 1024
9
10 -#define BUFFER_JSON_MAX_DEPTH 32
10 +#define BUFFER_JSON_MAX_DEPTH 32 // max is 255
11
12 extern const char hex_digits[16];
13 extern const char base64_digits[64];
@@ -71,7 +71,8 @@ typedef struct web_buffer {
71 struct {
72 char key_quote[BUFFER_QUOTE_MAX_SIZE + 1];
73 char value_quote[BUFFER_QUOTE_MAX_SIZE + 1];
74 - int depth;
74 + int8_t depth;
75 + bool minify;
76 BUFFER_JSON_NODE stack[BUFFER_JSON_MAX_DEPTH];
77 } json;
78 } BUFFER;
@@ -132,7 +133,9 @@ static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
133 buffer_increase(buffer, needed_free_size + 1);
134 }
135
135 -void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth, bool add_anonymous_object);
136 +void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
137 + bool add_anonymous_object, bool minify);
138 +
139 void buffer_json_finalize(BUFFER *wb);
140
141 static inline void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_TYPE type) {
@@ -563,10 +566,12 @@ static inline void buffer_print_spaces(BUFFER *wb, size_t spaces) {
566
567 static inline void buffer_print_json_comma_newline_spacing(BUFFER *wb) {
568 if(wb->json.stack[wb->json.depth].count)
566 - buffer_fast_strcat(wb, ",\n", 2);
567 - else
568 - buffer_fast_strcat(wb, "\n", 1);
569 + buffer_fast_strcat(wb, ",", 1);
570
571 + if(wb->json.minify)
572 + return;
573 +
574 + buffer_fast_strcat(wb, "\n", 1);
575 buffer_print_spaces(wb, wb->json.depth + 1);
576 }
577
@@ -610,8 +615,10 @@ static inline void buffer_json_object_close(BUFFER *wb) {
615 assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
616 assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_OBJECT && "BUFFER JSON: an object is not open to close it");
617 #endif
613 - buffer_fast_strcat(wb, "\n", 1);
614 - buffer_print_spaces(wb, wb->json.depth);
618 + if(!wb->json.minify) {
619 + buffer_fast_strcat(wb, "\n", 1);
620 + buffer_print_spaces(wb, wb->json.depth);
621 + }
622 buffer_fast_strcat(wb, "}", 1);
623 _buffer_json_depth_pop(wb);
624 }
@@ -717,6 +724,23 @@ static inline void buffer_json_add_array_item_uint64(BUFFER *wb, uint64_t value)
724 wb->json.stack[wb->json.depth].count++;
725 }
726
727 +static inline void buffer_json_add_array_item_time_t(BUFFER *wb, time_t value) {
728 + if(wb->json.stack[wb->json.depth].count)
729 + buffer_fast_strcat(wb, ",", 1);
730 +
731 + buffer_print_int64(wb, value);
732 + wb->json.stack[wb->json.depth].count++;
733 +}
734 +
735 +static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value) {
736 + if(wb->json.stack[wb->json.depth].count)
737 + buffer_fast_strcat(wb, ",", 1);
738 +
739 + buffer_print_int64(wb, value);
740 + buffer_fast_strcat(wb, "000", 3);
741 + wb->json.stack[wb->json.depth].count++;
742 +}
743 +
744 static inline void buffer_json_add_array_item_object(BUFFER *wb) {
745 if(wb->json.stack[wb->json.depth].count)
746 buffer_fast_strcat(wb, ",", 1);
@@ -736,6 +760,16 @@ static inline void buffer_json_member_add_time_t(BUFFER *wb, const char *key, ti
760 wb->json.stack[wb->json.depth].count++;
761 }
762
763 +static inline void buffer_json_member_add_time_t2ms(BUFFER *wb, const char *key, time_t value) {
764 + buffer_print_json_comma_newline_spacing(wb);
765 + buffer_print_json_key(wb, key);
766 + buffer_fast_strcat(wb, ":", 1);
767 + buffer_print_int64(wb, value);
768 + buffer_fast_strcat(wb, "000", 3);
769 +
770 + wb->json.stack[wb->json.depth].count++;
771 +}
772 +
773 static inline void buffer_json_member_add_uint64(BUFFER *wb, const char *key, uint64_t value) {
774 buffer_print_json_comma_newline_spacing(wb);
775 buffer_print_json_key(wb, key);
libnetdata/dictionary/dictionary.c
+4 -5
@@ -344,7 +344,7 @@ size_t dictionary_version(DICTIONARY *dict) {
344 if(unlikely(!dict)) return 0;
345
346 // this is required for views to return the right number
347 - garbage_collect_pending_deletes(dict);
347 + // garbage_collect_pending_deletes(dict);
348
349 return __atomic_load_n(&dict->version, __ATOMIC_RELAXED);
350 }
@@ -352,11 +352,10 @@ size_t dictionary_entries(DICTIONARY *dict) {
352 if(unlikely(!dict)) return 0;
353
354 // this is required for views to return the right number
355 - garbage_collect_pending_deletes(dict);
355 + // garbage_collect_pending_deletes(dict);
356
357 long int entries = __atomic_load_n(&dict->entries, __ATOMIC_RELAXED);
358 - if(entries < 0)
359 - fatal("DICTIONARY: entries is negative: %ld", entries);
358 + internal_fatal(entries < 0, "DICTIONARY: entries is negative: %ld", entries);
359
360 return entries;
361 }
@@ -3505,7 +3504,7 @@ size_t dictionary_unittest_views(void) {
3504
3505 fprintf(stderr, "\nPASS 2: Deleting master item:\n");
3506 dictionary_del(master, "KEY 1");
3508 - dictionary_version(view);
3507 + garbage_collect_pending_deletes(view);
3508 errors += unittest_check_dictionary("master", master, 0, 0, 1, 1, 0);
3509 errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0);
3510 errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true);
libnetdata/health/health.c
+5 -5
@@ -81,19 +81,19 @@ SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value)
81
82 if (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
83 silencer->alarms = strdupz(value);
84 - silencer->alarms_pattern = simple_pattern_create(silencer->alarms, NULL, SIMPLE_PATTERN_EXACT);
84 + silencer->alarms_pattern = simple_pattern_create(silencer->alarms, NULL, SIMPLE_PATTERN_EXACT, true);
85 } else if (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) {
86 silencer->charts = strdupz(value);
87 - silencer->charts_pattern = simple_pattern_create(silencer->charts, NULL, SIMPLE_PATTERN_EXACT);
87 + silencer->charts_pattern = simple_pattern_create(silencer->charts, NULL, SIMPLE_PATTERN_EXACT, true);
88 } else if (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) {
89 silencer->contexts = strdupz(value);
90 - silencer->contexts_pattern = simple_pattern_create(silencer->contexts, NULL, SIMPLE_PATTERN_EXACT);
90 + silencer->contexts_pattern = simple_pattern_create(silencer->contexts, NULL, SIMPLE_PATTERN_EXACT, true);
91 } else if (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) {
92 silencer->hosts = strdupz(value);
93 - silencer->hosts_pattern = simple_pattern_create(silencer->hosts, NULL, SIMPLE_PATTERN_EXACT);
93 + silencer->hosts_pattern = simple_pattern_create(silencer->hosts, NULL, SIMPLE_PATTERN_EXACT, true);
94 } else if (hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) {
95 silencer->families = strdupz(value);
96 - silencer->families_pattern = simple_pattern_create(silencer->families, NULL, SIMPLE_PATTERN_EXACT);
96 + silencer->families_pattern = simple_pattern_create(silencer->families, NULL, SIMPLE_PATTERN_EXACT, true);
97 }
98
99 return silencer;
libnetdata/simple_pattern/simple_pattern.c
+98 -42
@@ -4,17 +4,20 @@
4
5 struct simple_pattern {
6 const char *match;
7 - size_t len;
7 + uint32_t len;
8
9 SIMPLE_PREFIX_MODE mode;
10 - char negative;
10 + bool negative;
11 + bool case_sensitive;
12
13 struct simple_pattern *child;
13 -
14 struct simple_pattern *next;
15 };
16
17 -static inline struct simple_pattern *parse_pattern(char *str, SIMPLE_PREFIX_MODE default_mode) {
17 +static struct simple_pattern *parse_pattern(char *str, SIMPLE_PREFIX_MODE default_mode, size_t count) {
18 + if(unlikely(count >= 1000))
19 + return NULL;
20 +
21 // fprintf(stderr, "PARSING PATTERN: '%s'\n", str);
22
23 SIMPLE_PREFIX_MODE mode;
@@ -31,7 +34,7 @@ static inline struct simple_pattern *parse_pattern(char *str, SIMPLE_PREFIX_MODE
34 // do we have an asterisk in the middle?
35 if(*c == '*' && c[1] != '\0') {
36 // yes, we have
34 - child = parse_pattern(c, default_mode);
37 + child = parse_pattern(c, default_mode, count + 1);
38 c[1] = '\0';
39 }
40
@@ -70,12 +73,12 @@ static inline struct simple_pattern *parse_pattern(char *str, SIMPLE_PREFIX_MODE
73 return m;
74 }
75
73 -SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode) {
76 +SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode, bool case_sensitive) {
77 struct simple_pattern *root = NULL, *last = NULL;
78
79 if(unlikely(!list || !*list)) return root;
80
78 - int isseparator[256] = {
81 + char isseparator[256] = {
82 [' '] = 1 // space
83 , ['\t'] = 1 // tab
84 , ['\r'] = 1 // carriage return
@@ -96,14 +99,14 @@ SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators,
99 buf[0] = '\0';
100 char *c = buf;
101
99 - char negative = 0;
102 + bool negative = false;
103
104 // skip all spaces
105 while(isseparator[(unsigned char)*s])
106 s++;
107
108 if(*s == '!') {
106 - negative = 1;
109 + negative = true;
110 s++;
111 }
112
@@ -137,8 +140,9 @@ SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators,
140 continue;
141
142 // fprintf(stderr, "FOUND PATTERN: '%s'\n", buf);
140 - struct simple_pattern *m = parse_pattern(buf, default_mode);
143 + struct simple_pattern *m = parse_pattern(buf, default_mode, 0);
144 m->negative = negative;
145 + m->case_sensitive = case_sensitive;
146
147 // link it at the end
148 if(unlikely(!root))
@@ -174,70 +178,102 @@ static inline char *add_wildcarded(const char *matched, size_t matched_size, cha
178 return wildcarded;
179 }
180
177 -static inline int match_pattern(struct simple_pattern *m, const char *str, size_t len, char *wildcarded, size_t *wildcarded_size) {
181 +static inline int sp_strcmp(const char *s1, const char *s2, bool case_sensitive) {
182 + if(case_sensitive)
183 + return strcmp(s1, s2);
184 +
185 + return strcasecmp(s1, s2);
186 +}
187 +
188 +static inline int sp_strncmp(const char *s1, const char *s2, size_t n, bool case_sensitive) {
189 + if(case_sensitive)
190 + return strncmp(s1, s2, n);
191 +
192 + return strncasecmp(s1, s2, n);
193 +}
194 +
195 +static inline char *sp_strstr(const char *haystack, const char *needle, bool case_sensitive) {
196 + if(case_sensitive)
197 + return strstr(haystack, needle);
198 +
199 + return strcasestr(haystack, needle);
200 +}
201 +
202 +static inline bool match_pattern(struct simple_pattern *m, const char *str, size_t len, char *wildcarded, size_t *wildcarded_size) {
203 char *s;
204
180 - if(m->len <= len) {
205 + bool loop = true;
206 + while(loop && m->len <= len) {
207 + loop = false;
208 +
209 switch(m->mode) {
210 + default:
211 + case SIMPLE_PATTERN_EXACT:
212 + if(unlikely(sp_strcmp(str, m->match, m->case_sensitive) == 0)) {
213 + if(!m->child) return true;
214 + return false;
215 + }
216 + break;
217 +
218 case SIMPLE_PATTERN_SUBSTRING:
183 - if(!m->len) return 1;
184 - if((s = strstr(str, m->match))) {
219 + if(!m->len) return true;
220 + if((s = sp_strstr(str, m->match, m->case_sensitive))) {
221 wildcarded = add_wildcarded(str, s - str, wildcarded, wildcarded_size);
222 if(!m->child) {
187 - wildcarded = add_wildcarded(&s[m->len], len - (&s[m->len] - str), wildcarded, wildcarded_size);
188 - return 1;
223 + add_wildcarded(&s[m->len], len - (&s[m->len] - str), wildcarded, wildcarded_size);
224 + return true;
225 + }
226 +
227 + // instead of recursion
228 + {
229 + len = len - (s - str) - m->len;
230 + str = &s[m->len];
231 + m = m->child;
232 + loop = true;
233 + // return match_pattern(m->child, &s[m->len], len - (s - str) - m->len, wildcarded, wildcarded_size);
234 }
190 - return match_pattern(m->child, &s[m->len], len - (s - str) - m->len, wildcarded, wildcarded_size);
235 }
236 break;
237
238 case SIMPLE_PATTERN_PREFIX:
195 - if(unlikely(strncmp(str, m->match, m->len) == 0)) {
239 + if(unlikely(sp_strncmp(str, m->match, m->len, m->case_sensitive) == 0)) {
240 if(!m->child) {
197 - wildcarded = add_wildcarded(&str[m->len], len - m->len, wildcarded, wildcarded_size);
198 - return 1;
241 + add_wildcarded(&str[m->len], len - m->len, wildcarded, wildcarded_size);
242 + return true;
243 + }
244 + // instead of recursion
245 + {
246 + len = len - m->len;
247 + str = &str[m->len];
248 + m = m->child;
249 + loop = true;
250 + // return match_pattern(m->child, &str[m->len], len - m->len, wildcarded, wildcarded_size);
251 }
200 - return match_pattern(m->child, &str[m->len], len - m->len, wildcarded, wildcarded_size);
252 }
253 break;
254
255 case SIMPLE_PATTERN_SUFFIX:
205 - if(unlikely(strcmp(&str[len - m->len], m->match) == 0)) {
206 - wildcarded = add_wildcarded(str, len - m->len, wildcarded, wildcarded_size);
207 - if(!m->child) return 1;
208 - return 0;
209 - }
210 - break;
211 -
212 - case SIMPLE_PATTERN_EXACT:
213 - default:
214 - if(unlikely(strcmp(str, m->match) == 0)) {
215 - if(!m->child) return 1;
216 - return 0;
256 + if(unlikely(sp_strcmp(&str[len - m->len], m->match, m->case_sensitive) == 0)) {
257 + add_wildcarded(str, len - m->len, wildcarded, wildcarded_size);
258 + if(!m->child) return true;
259 + return false;
260 }
261 break;
262 }
263 }
264
222 - return 0;
265 + return false;
266 }
267
225 -int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) {
268 +static inline int simple_pattern_matches_extract_with_length(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
269 struct simple_pattern *m, *root = (struct simple_pattern *)list;
270
228 - if(unlikely(!root || !str || !*str)) return 0;
229 -
230 - size_t len = strlen(str);
271 for(m = root; m ; m = m->next) {
272 char *ws = wildcarded;
273 size_t wss = wildcarded_size;
274 if(unlikely(ws)) *ws = '\0';
275
276 if (match_pattern(m, str, len, ws, &wss)) {
237 -
238 - //if(ws && wss)
239 - // fprintf(stderr, "FINAL WILDCARDED '%s' of length %zu\n", ws, strlen(ws));
240 -
277 if (m->negative) return 0;
278 return 1;
279 }
@@ -246,6 +282,26 @@ int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *
282 return 0;
283 }
284
285 +int simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size) {
286 + if(!list || !str || buffer_strlen(str)) return 0;
287 + return simple_pattern_matches_extract_with_length(list, buffer_tostring(str), buffer_strlen(str), wildcarded, wildcarded_size);
288 +}
289 +
290 +int simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, STRING *str, char *wildcarded, size_t wildcarded_size) {
291 + if(!list || !str) return 0;
292 + return simple_pattern_matches_extract_with_length(list, string2str(str), string_strlen(str), wildcarded, wildcarded_size);
293 +}
294 +
295 +int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) {
296 + if(!list || !str || !*str) return 0;
297 + return simple_pattern_matches_extract_with_length(list, str, strlen(str), wildcarded, wildcarded_size);
298 +}
299 +
300 +int simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
301 + if(!list || !str || !*str || !len) return 0;
302 + return simple_pattern_matches_extract_with_length(list, str, len, wildcarded, wildcarded_size);
303 +}
304 +
305 static inline void free_pattern(struct simple_pattern *m) {
306 if(!m) return;
307
libnetdata/simple_pattern/simple_pattern.h
+14 -2
@@ -6,7 +6,7 @@
6 #include "../libnetdata.h"
7
8
9 -typedef enum {
9 +typedef enum __attribute__ ((__packed__)) {
10 SIMPLE_PATTERN_EXACT,
11 SIMPLE_PATTERN_PREFIX,
12 SIMPLE_PATTERN_SUFFIX,
@@ -18,13 +18,20 @@ typedef void SIMPLE_PATTERN;
18 // create a simple_pattern from the string given
19 // default_mode is used in cases where EXACT matches, without an asterisk,
20 // should be considered PREFIX matches.
21 -SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode);
21 +SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode, bool case_sensitive);
22
23 // test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*'
24 int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size);
25
26 +struct netdata_string;
27 +int simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, struct netdata_string *str, char *wildcarded, size_t wildcarded_size);
28 +int simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size);
29 +int simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size);
30 +
31 // test if string str is matched from the pattern
32 #define simple_pattern_matches(list, str) simple_pattern_matches_extract(list, str, NULL, 0)
33 +#define simple_pattern_matches_string(list, str) simple_pattern_matches_string_extract(list, str, NULL, 0)
34 +#define simple_pattern_matches_buffer(list, str) simple_pattern_matches_buffer_extract(list, str, NULL, 0)
35
36 // free a simple_pattern that was created with simple_pattern_create()
37 // list can be NULL, in which case, this does nothing.
@@ -37,6 +44,11 @@ char *simple_pattern_iterate(SIMPLE_PATTERN **p);
44 // Auxiliary function to create a pattern
45 char *simple_pattern_trim_around_equal(char *src);
46
47 +#define SIMPLE_PATTERN_DEFAULT_WEB_SEPARATORS ",|\t\r\n\f\v"
48 +
49 #define is_valid_sp(x) ((x) && *(x) && !((x)[0] == '*' && (x)[1] == '\0'))
50
51 +#define string_to_simple_pattern(str) (is_valid_sp(str) ? simple_pattern_create(str, SIMPLE_PATTERN_DEFAULT_WEB_SEPARATORS, SIMPLE_PATTERN_EXACT, true) : NULL)
52 +#define string_to_simple_pattern_nocase(str) (is_valid_sp(str) ? simple_pattern_create(str, SIMPLE_PATTERN_DEFAULT_WEB_SEPARATORS, SIMPLE_PATTERN_EXACT, false) : NULL)
53 +
54 #endif //NETDATA_SIMPLE_PATTERN_H
ml/Config.cc
+2 -2
@@ -101,12 +101,12 @@ void nml_config_load(nml_config_t *cfg) {
101 cfg->dimension_anomaly_score_threshold = dimension_anomaly_rate_threshold;
102
103 cfg->hosts_to_skip = config_get(config_section_ml, "hosts to skip from training", "!*");
104 - cfg->sp_host_to_skip = simple_pattern_create(cfg->hosts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
104 + cfg->sp_host_to_skip = simple_pattern_create(cfg->hosts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT, true);
105
106 // Always exclude anomaly_detection charts from training.
107 cfg->charts_to_skip = "anomaly_detection.* ";
108 cfg->charts_to_skip += config_get(config_section_ml, "charts to skip from training", "netdata.*");
109 - cfg->sp_charts_to_skip = simple_pattern_create(cfg->charts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
109 + cfg->sp_charts_to_skip = simple_pattern_create(cfg->charts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT, true);
110
111 cfg->stream_anomaly_detection_charts = config_get_boolean(config_section_ml, "stream anomaly detection charts", true);
112 }
ml/ad_charts.cc
+1 -1
@@ -276,7 +276,7 @@ void nml_update_host_and_detection_rate_charts(nml_host_t *host, collected_numbe
276 0, /* timeout */
277 0, /* tier */
278 QUERY_SOURCE_ML,
279 - STORAGE_PRIORITY_BEST_EFFORT
279 + STORAGE_PRIORITY_SYNCHRONOUS
280 );
281
282 if (R) {
ml/nml.cc
+4 -5
@@ -1118,17 +1118,16 @@ void *nml_detect_main(void *arg) {
1118 worker_is_idle();
1119 heartbeat_next(&hb, USEC_PER_SEC);
1120
1121 - rrd_rdlock();
1121 + void *rhp;
1122 + dfe_start_reentrant(rrdhost_root_index, rhp) {
1123 + RRDHOST *rh = (RRDHOST *) rhp;
1124
1123 - RRDHOST *rh;
1124 - rrdhost_foreach_read(rh) {
1125 if (!rh->ml_host)
1126 continue;
1127
1128 nml_host_detect_once(reinterpret_cast<nml_host_t *>(rh->ml_host));
1129 }
1130 -
1131 - rrd_unlock();
1130 + dfe_done(rhp);
1131 }
1132
1133 return NULL;
streaming/replication.c
+27 -13
@@ -969,11 +969,12 @@ struct replication_sort_entry {
969
970 // the global variables for the replication thread
971 static struct replication_thread {
972 + ARAL *aral_rse;
973 +
974 SPINLOCK spinlock;
975
976 struct {
977 size_t pending; // number of requests pending in the queue
976 - Word_t unique_id; // the last unique id we gave to a request (auto-increment, starting from 1)
978
979 // statistics
980 size_t added; // number of requests added to the queue
@@ -992,6 +993,7 @@ static struct replication_thread {
993 } unsafe; // protected from replication_recursive_lock()
994
995 struct {
996 + Word_t unique_id; // the last unique id we gave to a request (auto-increment, starting from 1)
997 size_t executed; // the number of replication requests executed
998 size_t latest_first_time; // the 'after' timestamp of the last request we executed
999 size_t memory; // the total memory allocated by replication
@@ -1005,10 +1007,10 @@ static struct replication_thread {
1007 } main_thread; // access is allowed only by the main thread
1008
1009 } replication_globals = {
1010 + .aral_rse = NULL,
1011 .spinlock = NETDATA_SPINLOCK_INITIALIZER,
1012 .unsafe = {
1013 .pending = 0,
1011 - .unique_id = 0,
1014
1015 .added = 0,
1016 .removed = 0,
@@ -1025,6 +1027,7 @@ static struct replication_thread {
1027 },
1028 },
1029 .atomic = {
1030 + .unique_id = 0,
1031 .executed = 0,
1032 .latest_first_time = 0,
1033 .memory = 0,
@@ -1088,17 +1091,15 @@ void replication_set_next_point_in_time(time_t after, size_t unique_id) {
1091 // ----------------------------------------------------------------------------
1092 // replication sort entry management
1093
1091 -static struct replication_sort_entry *replication_sort_entry_create_unsafe(struct replication_request *rq) {
1092 - fatal_when_replication_is_not_locked_for_me();
1093 -
1094 - struct replication_sort_entry *rse = mallocz(sizeof(struct replication_sort_entry));
1094 +static struct replication_sort_entry *replication_sort_entry_create(struct replication_request *rq) {
1095 + struct replication_sort_entry *rse = aral_mallocz(replication_globals.aral_rse);
1096 __atomic_add_fetch(&replication_globals.atomic.memory, sizeof(struct replication_sort_entry), __ATOMIC_RELAXED);
1097
1098 rrdpush_sender_pending_replication_requests_plus_one(rq->sender);
1099
1100 // copy the request
1101 rse->rq = rq;
1101 - rse->unique_id = ++replication_globals.unsafe.unique_id;
1102 + rse->unique_id = __atomic_add_fetch(&replication_globals.atomic.unique_id, 1, __ATOMIC_SEQ_CST);
1103
1104 // save the unique id into the request, to be able to delete it later
1105 rq->unique_id = rse->unique_id;
@@ -1109,26 +1110,30 @@ static struct replication_sort_entry *replication_sort_entry_create_unsafe(struc
1110 }
1111
1112 static void replication_sort_entry_destroy(struct replication_sort_entry *rse) {
1112 - freez(rse);
1113 + aral_freez(replication_globals.aral_rse, rse);
1114 __atomic_sub_fetch(&replication_globals.atomic.memory, sizeof(struct replication_sort_entry), __ATOMIC_RELAXED);
1115 }
1116
1117 static void replication_sort_entry_add(struct replication_request *rq) {
1117 - replication_recursive_lock();
1118 -
1118 if(rrdpush_sender_replication_buffer_full_get(rq->sender)) {
1119 rq->indexed_in_judy = false;
1120 rq->not_indexed_buffer_full = true;
1121 rq->not_indexed_preprocessing = false;
1122 + replication_recursive_lock();
1123 replication_globals.unsafe.pending_no_room++;
1124 replication_recursive_unlock();
1125 return;
1126 }
1127
1128 - if(rq->not_indexed_buffer_full)
1129 - replication_globals.unsafe.pending_no_room--;
1128 + // cache this, because it will be changed
1129 + bool decrement_no_room = rq->not_indexed_buffer_full;
1130
1131 - struct replication_sort_entry *rse = replication_sort_entry_create_unsafe(rq);
1131 + struct replication_sort_entry *rse = replication_sort_entry_create(rq);
1132 +
1133 + replication_recursive_lock();
1134 +
1135 + if(decrement_no_room)
1136 + replication_globals.unsafe.pending_no_room--;
1137
1138 // if(rq->after < (time_t)replication_globals.protected.queue.after &&
1139 // rq->sender->buffer_used_percentage <= MAX_SENDER_BUFFER_PERCENTAGE_ALLOWED &&
@@ -1827,12 +1832,21 @@ static void replication_main_cleanup(void *ptr) {
1832 replication_globals.main_thread.threads_ptrs = NULL;
1833 __atomic_sub_fetch(&replication_buffers_allocated, threads * sizeof(netdata_thread_t *), __ATOMIC_RELAXED);
1834
1835 + aral_destroy(replication_globals.aral_rse);
1836 + replication_globals.aral_rse = NULL;
1837 +
1838 // custom code
1839 worker_unregister();
1840
1841 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
1842 }
1843
1844 +void replication_initialize(void) {
1845 + replication_globals.aral_rse = aral_create("rse", sizeof(struct replication_sort_entry),
1846 + 0, 65536, aral_by_size_statistics(),
1847 + NULL, NULL, false, false);
1848 +}
1849 +
1850 void *replication_thread_main(void *ptr __maybe_unused) {
1851 replication_initialize_workers(true);
1852
streaming/rrdpush.c
+4 -4
@@ -190,8 +190,8 @@ static inline bool should_send_chart_matching(RRDSET *st, RRDSET_FLAGS flags) {
190 else
191 rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_IGNORE);
192 }
193 - else if(simple_pattern_matches(host->rrdpush_send_charts_matching, rrdset_id(st)) ||
194 - simple_pattern_matches(host->rrdpush_send_charts_matching, rrdset_name(st)))
193 + else if(simple_pattern_matches_string(host->rrdpush_send_charts_matching, st->id) ||
194 + simple_pattern_matches_string(host->rrdpush_send_charts_matching, st->name))
195
196 rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_SEND);
197 else
@@ -941,7 +941,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
941 {
942 SIMPLE_PATTERN *key_allow_from = simple_pattern_create(
943 appconfig_get(&stream_config, rpt->key, "allow from", "*"),
944 - NULL, SIMPLE_PATTERN_EXACT);
944 + NULL, SIMPLE_PATTERN_EXACT, true);
945
946 if(key_allow_from) {
947 if(!simple_pattern_matches(key_allow_from, w->client_ip)) {
@@ -988,7 +988,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
988 {
989 SIMPLE_PATTERN *machine_allow_from = simple_pattern_create(
990 appconfig_get(&stream_config, rpt->machine_guid, "allow from", "*"),
991 - NULL, SIMPLE_PATTERN_EXACT);
991 + NULL, SIMPLE_PATTERN_EXACT, true);
992
993 if(machine_allow_from) {
994 if(!simple_pattern_matches(machine_allow_from, w->client_ip)) {
web/api/exporters/shell/allmetrics_shell.c
+4 -4
@@ -24,12 +24,12 @@ static inline size_t shell_name_copy(char *d, const char *s, size_t usable) {
24
25 void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb) {
26 analytics_log_shell();
27 - SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT);
27 + SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT, true);
28
29 // for each chart
30 RRDSET *st;
31 rrdset_foreach_read(st, host) {
32 - if (filter && !simple_pattern_matches(filter, rrdset_name(st)))
32 + if (filter && !simple_pattern_matches_string(filter, st->name))
33 continue;
34
35 NETDATA_DOUBLE total = 0.0;
@@ -97,7 +97,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
97
98 void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb) {
99 analytics_log_json();
100 - SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT);
100 + SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT, true);
101
102 buffer_strcat(wb, "{");
103
@@ -107,7 +107,7 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
107 // for each chart
108 RRDSET *st;
109 rrdset_foreach_read(st, host) {
110 - if (filter && !(simple_pattern_matches(filter, rrdset_id(st)) || simple_pattern_matches(filter, rrdset_name(st))))
110 + if (filter && !(simple_pattern_matches_string(filter, st->id) || simple_pattern_matches_string(filter, st->name)))
111 continue;
112
113 if(rrdset_is_available_for_viewers(st)) {
web/api/formatters/json_wrapper.c
+332 -140
@@ -86,7 +86,7 @@ static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, R
86
87 QUERY_METRIC *qm = query_metric(qt, c);
88 QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
89 - buffer_json_add_array_item_string(wb, string2str(qi->id_fqdn));
89 + buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria));
90 i++;
91 }
92 buffer_json_array_close(wb);
@@ -94,23 +94,97 @@ static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, R
94 return i;
95 }
96
97 +struct summary_total_counts {
98 + size_t selected;
99 + size_t excluded;
100 + size_t queried;
101 + size_t failed;
102 +};
103 +
104 +static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, struct query_metrics_counts *metrics) {
105 + if(unlikely(!totals || !metrics))
106 + return;
107 +
108 + if(metrics->selected) {
109 + totals->selected++;
110 +
111 + if(metrics->queried)
112 + totals->queried++;
113 +
114 + else if(metrics->failed)
115 + totals->failed++;
116 + }
117 + else
118 + totals->excluded++;
119 +}
120 +
121 +static inline void query_target_total_counts(BUFFER *wb, const char *key, struct summary_total_counts *totals) {
122 + if(!totals->selected && !totals->queried && !totals->failed && !totals->excluded)
123 + return;
124 +
125 + buffer_json_member_add_object(wb, key);
126 +
127 + if(totals->selected)
128 + buffer_json_member_add_uint64(wb, "sl", totals->selected);
129 +
130 + if(totals->excluded)
131 + buffer_json_member_add_uint64(wb, "ex", totals->excluded);
132 +
133 + if(totals->queried)
134 + buffer_json_member_add_uint64(wb, "qr", totals->queried);
135 +
136 + if(totals->failed)
137 + buffer_json_member_add_uint64(wb, "fl", totals->failed);
138 +
139 + buffer_json_object_close(wb);
140 +}
141 +
142 static inline void query_target_metric_counts(BUFFER *wb, struct query_metrics_counts *metrics) {
143 + if(!metrics->selected && !metrics->queried && !metrics->failed && !metrics->excluded)
144 + return;
145 +
146 buffer_json_member_add_object(wb, "ds");
99 - buffer_json_member_add_uint64(wb, "sl", metrics->selected);
100 - buffer_json_member_add_uint64(wb, "ex", metrics->excluded);
101 - buffer_json_member_add_uint64(wb, "qr", metrics->queried);
102 - buffer_json_member_add_uint64(wb, "fl", metrics->failed);
147 +
148 + if(metrics->selected)
149 + buffer_json_member_add_uint64(wb, "sl", metrics->selected);
150 +
151 + if(metrics->excluded)
152 + buffer_json_member_add_uint64(wb, "ex", metrics->excluded);
153 +
154 + if(metrics->queried)
155 + buffer_json_member_add_uint64(wb, "qr", metrics->queried);
156 +
157 + if(metrics->failed)
158 + buffer_json_member_add_uint64(wb, "fl", metrics->failed);
159 +
160 buffer_json_object_close(wb);
161 }
162
163 static inline void query_target_instance_counts(BUFFER *wb, struct query_instances_counts *instances) {
164 + if(!instances->selected && !instances->queried && !instances->failed && !instances->excluded)
165 + return;
166 +
167 buffer_json_member_add_object(wb, "is");
108 - buffer_json_member_add_uint64(wb, "sl", instances->selected);
109 - buffer_json_member_add_uint64(wb, "ex", instances->excluded);
168 +
169 + if(instances->selected)
170 + buffer_json_member_add_uint64(wb, "sl", instances->selected);
171 +
172 + if(instances->excluded)
173 + buffer_json_member_add_uint64(wb, "ex", instances->excluded);
174 +
175 + if(instances->queried)
176 + buffer_json_member_add_uint64(wb, "qr", instances->queried);
177 +
178 + if(instances->failed)
179 + buffer_json_member_add_uint64(wb, "fl", instances->failed);
180 +
181 buffer_json_object_close(wb);
182 }
183
184 static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_counts *alerts, const char *name, bool array) {
185 + if(!alerts->clear && !alerts->other && !alerts->critical && !alerts->warning)
186 + return;
187 +
188 if(array)
189 buffer_json_add_array_item_object(wb);
190 else
@@ -118,44 +192,81 @@ static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_co
192
193 if(name)
194 buffer_json_member_add_string(wb, "nm", name);
121 - buffer_json_member_add_uint64(wb, "cl", alerts->clear);
122 - buffer_json_member_add_uint64(wb, "wr", alerts->warning);
123 - buffer_json_member_add_uint64(wb, "cr", alerts->critical);
124 - buffer_json_member_add_uint64(wb, "ot", alerts->other);
195 +
196 + if(alerts->clear)
197 + buffer_json_member_add_uint64(wb, "cl", alerts->clear);
198 +
199 + if(alerts->warning)
200 + buffer_json_member_add_uint64(wb, "wr", alerts->warning);
201 +
202 + if(alerts->critical)
203 + buffer_json_member_add_uint64(wb, "cr", alerts->critical);
204 +
205 + if(alerts->other)
206 + buffer_json_member_add_uint64(wb, "ot", alerts->other);
207 +
208 buffer_json_object_close(wb);
209 }
210
211 static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, struct query_data_statistics *d) {
212 + if(!d->group_points)
213 + return;
214 +
215 buffer_json_member_add_object(wb, "sts");
130 -// buffer_json_member_add_double(wb, "min", d->min);
131 -// buffer_json_member_add_double(wb, "max", d->max);
132 -// buffer_json_member_add_double(wb, "sum", d->sum);
133 - buffer_json_member_add_double(wb, "avg", (d->count) ? d->sum / (NETDATA_DOUBLE)d->count : 0.0);
134 -// buffer_json_member_add_double(wb, "vol", d->volume);
135 - buffer_json_member_add_double(wb, "con", (qt->query_stats.volume > 0) ? d->volume * 100.0 / qt->query_stats.volume : 0.0);
216 + if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
217 + buffer_json_member_add_uint64(wb, "cnt", d->group_points);
218 +
219 + if(d->sum != 0.0)
220 + buffer_json_member_add_double(wb, "sum", d->sum);
221 +
222 + if(d->volume != 0.0)
223 + buffer_json_member_add_double(wb, "vol", d->volume);
224 +
225 + if(d->anomaly_sum != 0.0)
226 + buffer_json_member_add_double(wb, "ars", d->anomaly_sum);
227 + }
228 + else {
229 +// buffer_json_member_add_double(wb, "min", d->min);
230 +// buffer_json_member_add_double(wb, "max", d->max);
231 +
232 + NETDATA_DOUBLE avg = (d->group_points) ? d->sum / (NETDATA_DOUBLE)d->group_points : 0.0;
233 + if(avg != 0.0)
234 + buffer_json_member_add_double(wb, "avg", avg);
235 +
236 + NETDATA_DOUBLE arp = (d->group_points) ? d->anomaly_sum / (NETDATA_DOUBLE)d->group_points : 0.0;
237 + if(arp != 0.0)
238 + buffer_json_member_add_double(wb, "arp", arp);
239 +
240 + NETDATA_DOUBLE con = (qt->query_stats.volume > 0) ? d->volume * 100.0 / qt->query_stats.volume : 0.0;
241 + if(con != 0.0)
242 + buffer_json_member_add_double(wb, "con", con);
243 + }
244 buffer_json_object_close(wb);
245 }
246
139 -static void query_target_summary_hosts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
247 +static void query_target_summary_nodes_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) {
248 buffer_json_member_add_array(wb, key);
141 - for (long c = 0; c < (long) qt->hosts.used; c++) {
142 - QUERY_HOST *qh = query_host(qt, c);
143 - RRDHOST *host = qh->host;
249 + for (size_t c = 0; c < qt->nodes.used; c++) {
250 + QUERY_NODE *qn = query_node(qt, c);
251 + RRDHOST *host = qn->rrdhost;
252 buffer_json_add_array_item_object(wb);
253 + buffer_json_member_add_uint64(wb, "ni", qn->slot);
254 buffer_json_member_add_string(wb, "mg", host->machine_guid);
146 - if(qh->node_id[0])
147 - buffer_json_member_add_string(wb, "nd", qh->node_id);
255 + if(qn->node_id[0])
256 + buffer_json_member_add_string(wb, "nd", qn->node_id);
257 buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
149 - query_target_instance_counts(wb, &qh->instances);
150 - query_target_metric_counts(wb, &qh->metrics);
151 - query_target_alerts_counts(wb, &qh->alerts, NULL, false);
152 - query_target_data_statistics(wb, qt, &qh->query_stats);
258 + query_target_instance_counts(wb, &qn->instances);
259 + query_target_metric_counts(wb, &qn->metrics);
260 + query_target_alerts_counts(wb, &qn->alerts, NULL, false);
261 + query_target_data_statistics(wb, qt, &qn->query_stats);
262 buffer_json_object_close(wb);
263 +
264 + aggregate_into_summary_totals(totals, &qn->metrics);
265 }
266 buffer_json_array_close(wb);
267 }
268
158 -static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
269 +static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) {
270 buffer_json_member_add_array(wb, key);
271 DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
272
@@ -164,11 +275,7 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
275 struct query_instances_counts instances;
276 struct query_metrics_counts metrics;
277 struct query_alerts_counts alerts;
167 - } x = {
168 - .instances = (struct query_instances_counts){ 0 },
169 - .metrics = (struct query_metrics_counts){ 0 },
170 - .alerts = (struct query_alerts_counts){ 0 },
171 - }, *z;
278 + } x = { 0 }, *z;
279
280 for (long c = 0; c < (long) qt->contexts.used; c++) {
281 QUERY_CONTEXT *qc = query_context(qt, c);
@@ -177,6 +284,8 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
284
285 z->instances.selected += qc->instances.selected;
286 z->instances.excluded += qc->instances.selected;
287 + z->instances.queried += qc->instances.queried;
288 + z->instances.failed += qc->instances.failed;
289
290 z->metrics.selected += qc->metrics.selected;
291 z->metrics.excluded += qc->metrics.excluded;
@@ -199,6 +308,8 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
308 query_target_alerts_counts(wb, &z->alerts, NULL, false);
309 query_target_data_statistics(wb, qt, &z->query_stats);
310 buffer_json_object_close(wb);
311 +
312 + aggregate_into_summary_totals(totals, &z->metrics);
313 }
314 dfe_done(z);
315 buffer_json_array_close(wb);
@@ -233,28 +344,36 @@ static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, cons
344 buffer_json_array_close(wb);
345 }
346
236 -static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
347 +static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) {
348 buffer_json_member_add_array(wb, key);
349 for (long c = 0; c < (long) qt->instances.used; c++) {
350 QUERY_INSTANCE *qi = query_instance(qt, c);
240 - QUERY_HOST *qh = query_host(qt, qi->query_host_id);
351 +// QUERY_HOST *qh = query_host(qt, qi->query_host_id);
352
353 buffer_json_add_array_item_object(wb);
243 - buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn));
244 - buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn));
245 - buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria));
246 - buffer_json_member_add_string(wb, "mg", qh->host->machine_guid);
247 - if(qh->node_id[0])
248 - buffer_json_member_add_string(wb, "nd", qh->node_id);
354 + buffer_json_member_add_string(wb, "id", rrdinstance_acquired_id(qi->ria));
355 +
356 + if(!rrdinstance_acquired_id_and_name_are_same(qi->ria))
357 + buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(qi->ria));
358 +
359 + buffer_json_member_add_uint64(wb, "ni", qi->query_host_id);
360 +// buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn));
361 +// buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn));
362 +// buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria));
363 +// buffer_json_member_add_string(wb, "mg", qh->host->machine_guid);
364 +// if(qh->node_id[0])
365 +// buffer_json_member_add_string(wb, "nd", qh->node_id);
366 query_target_metric_counts(wb, &qi->metrics);
367 query_target_alerts_counts(wb, &qi->alerts, NULL, false);
368 query_target_data_statistics(wb, qt, &qi->query_stats);
369 buffer_json_object_close(wb);
370 +
371 + aggregate_into_summary_totals(totals, &qi->metrics);
372 }
373 buffer_json_array_close(wb);
374 }
375
257 -static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2) {
376 +static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *totals) {
377 char name[RRD_ID_LENGTH_MAX * 2 + 2];
378
379 buffer_json_member_add_array(wb, key);
@@ -264,7 +383,7 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
383 const char *name;
384 struct query_data_statistics query_stats;
385 struct query_metrics_counts metrics;
267 - } x, *z;
386 + } *z;
387 size_t q = 0;
388 for (long c = 0; c < (long) qt->dimensions.used; c++) {
389 QUERY_DIMENSION * qd = query_dimension(qt, c);
@@ -282,11 +401,11 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
401 rrdmetric_acquired_id(rma),
402 rrdmetric_acquired_name(rma));
403
285 - x.id = rrdmetric_acquired_id(rma);
286 - x.name = rrdmetric_acquired_name(rma);
287 - x.metrics = (struct query_metrics_counts){ 0 };
288 -
289 - z = dictionary_set(dict, name, &x, sizeof(x));
404 + z = dictionary_set(dict, name, NULL, sizeof(*z));
405 + if(!z->id)
406 + z->id = rrdmetric_acquired_id(rma);
407 + if(!z->name)
408 + z->name = rrdmetric_acquired_name(rma);
409
410 if(qm) {
411 z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0;
@@ -308,6 +427,8 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
427 query_target_metric_counts(wb, &z->metrics);
428 query_target_data_statistics(wb, qt, &z->query_stats);
429 buffer_json_object_close(wb);
430 +
431 + aggregate_into_summary_totals(totals, &z->metrics);
432 }
433 else {
434 buffer_json_add_array_item_array(wb);
@@ -359,6 +480,7 @@ static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL
480 struct rrdlabels_key_value_dict_entry x = {
481 .key = name,
482 .value = value,
483 + .query_stats = (struct query_data_statistics) { 0 },
484 .metrics = (struct query_metrics_counts){ 0 },
485 }, *z = dictionary_set(d->values, n, &x, sizeof(x));
486
@@ -382,7 +504,7 @@ static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL
504 return 1;
505 }
506
385 -static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2) {
507 +static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *key_totals, struct summary_total_counts *value_totals) {
508 buffer_json_member_add_array(wb, key);
509 struct rrdlabels_formatting_v2 t = {
510 .keys = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE),
@@ -401,6 +523,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
523 buffer_json_member_add_string(wb, "id", d_dfe.name);
524 query_target_metric_counts(wb, &d->metrics);
525 query_target_data_statistics(wb, qt, &d->query_stats);
526 + aggregate_into_summary_totals(key_totals, &d->metrics);
527 buffer_json_member_add_array(wb, "vl");
528 }
529 struct rrdlabels_key_value_dict_entry *z;
@@ -411,6 +534,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
534 query_target_metric_counts(wb, &z->metrics);
535 query_target_data_statistics(wb, qt, &z->query_stats);
536 buffer_json_object_close(wb);
537 + aggregate_into_summary_totals(value_totals, &z->metrics);
538 } else {
539 buffer_json_add_array_item_array(wb);
540 buffer_json_add_array_item_string(wb, z->key);
@@ -501,7 +625,7 @@ static inline void query_target_functions(BUFFER *wb, const char *key, RRDR *r)
625 buffer_json_array_close(wb);
626 }
627
504 -static inline long query_target_chart_labels_filter(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
628 +static inline long query_target_chart_labels_filter_v1(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
629 QUERY_TARGET *qt = r->internal.qt;
630 const long query_used = qt->query.used;
631 long c, i = 0;
@@ -608,7 +732,7 @@ static inline size_t rrdr_latest_values(BUFFER *wb, const char *key, RRDR *r, RR
732 return i;
733 }
734
611 -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value,
735 +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options,
736 RRDR_TIME_GROUPING group_method)
737 {
738 QUERY_TARGET *qt = r->internal.qt;
@@ -627,7 +751,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR
751 sq[0] = '"';
752 }
753
630 - buffer_json_initialize(wb, kq, sq, 0, true);
754 + buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
755
756 buffer_json_member_add_uint64(wb, "api", 1);
757 buffer_json_member_add_string(wb, "id", qt->id);
@@ -649,8 +773,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR
773
774 if (r->view.options & RRDR_OPTION_ALL_DIMENSIONS) {
775 query_target_summary_instances_v1(wb, qt, "full_chart_list");
652 - query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false);
653 - query_target_summary_labels_v12(wb, qt, "full_chart_labels", false);
776 + query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false, NULL);
777 + query_target_summary_labels_v12(wb, qt, "full_chart_labels", false, NULL, NULL);
778 }
779
780 query_target_functions(wb, "functions", r);
@@ -658,7 +782,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR
782 if (!qt->request.st && !jsonwrap_v1_chart_ids(wb, "chart_ids", r, options))
783 rows = 0;
784
661 - if (qt->instances.chart_label_key_pattern && !query_target_chart_labels_filter(wb, "chart_labels", r, options))
785 + if (qt->instances.chart_label_key_pattern && !query_target_chart_labels_filter_v1(wb, "chart_labels", r, options))
786 rows = 0;
787
788 if(!query_target_metrics_latest_values(wb, "latest_values", r, options))
@@ -674,17 +798,14 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR
798
799 buffer_json_member_add_array(wb, "db_points_per_tier");
800 for(size_t tier = 0; tier < storage_tiers ; tier++)
677 - buffer_json_add_array_item_uint64(wb, r->stats.tier_points_read[tier]);
801 + buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points);
802 buffer_json_array_close(wb);
803
804 if(options & RRDR_OPTION_SHOW_PLAN)
805 jsonwrap_query_plan(r, wb);
682 -
683 - buffer_sprintf(wb, ",\n %sresult%s:", kq, kq);
684 - if(string_value) buffer_strcat(wb, sq);
806 }
807
687 -static void rrdset_rrdcalc_entries(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) {
808 +static void rrdset_rrdcalc_entries_v2(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) {
809 RRDSET *st = rrdinstance_acquired_rrdset(ria);
810 if(st) {
811 netdata_rwlock_rdlock(&st->alerts.rwlock);
@@ -706,8 +827,11 @@ static void rrdset_rrdcalc_entries(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) {
827 }
828 }
829
709 -static void query_target_combined_units(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
710 - if(contexts == 1) {
830 +static void query_target_combined_units_v2(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
831 + if(query_target_has_percentage_units(qt)) {
832 + buffer_json_member_add_string(wb, "units", "%");
833 + }
834 + else if(contexts == 1) {
835 buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca));
836 }
837 else if(contexts > 1) {
@@ -734,33 +858,50 @@ static void query_target_combined_chart_type(BUFFER *wb, QUERY_TARGET *qt, size_
858 buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rrdcontext_acquired_chart_type(qt->contexts.array[0].rca)));
859 }
860
737 -static void rrdr_dimension_units_array(BUFFER *wb, RRDR *r) {
861 +static void rrdr_dimension_units_array_v2(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) {
862 if(!r->du)
863 return;
864
865 + bool percentage = query_target_has_percentage_units(r->internal.qt);
866 +
867 buffer_json_member_add_array(wb, "units");
742 - for(size_t c = 0; c < r->d ; c++)
743 - buffer_json_add_array_item_string(wb, string2str(r->du[c]));
868 + for(size_t c = 0; c < r->d ; c++) {
869 + if(!rrdr_dimension_should_be_exposed(r->od[c], options))
870 + continue;
871 +
872 + if(percentage)
873 + buffer_json_add_array_item_string(wb, "%");
874 + else
875 + buffer_json_add_array_item_string(wb, string2str(r->du[c]));
876 + }
877 buffer_json_array_close(wb);
878 }
879
747 -static void rrdr_dimension_priority_array(BUFFER *wb, RRDR *r) {
880 +static void rrdr_dimension_priority_array(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) {
881 if(!r->dp)
882 return;
883
884 buffer_json_member_add_array(wb, "priorities");
752 - for(size_t c = 0; c < r->d ; c++)
885 + for(size_t c = 0; c < r->d ; c++) {
886 + if(!rrdr_dimension_should_be_exposed(r->od[c], options))
887 + continue;
888 +
889 buffer_json_add_array_item_uint64(wb, r->dp[c]);
890 + }
891 buffer_json_array_close(wb);
892 }
893
757 -static void rrdr_dimension_grouped_array(BUFFER *wb, RRDR *r) {
894 +static void rrdr_dimension_grouped_array(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) {
895 if(!r->dgbc)
896 return;
897
898 buffer_json_member_add_array(wb, "grouped");
762 - for(size_t c = 0; c < r->d ;c++)
899 + for(size_t c = 0; c < r->d ;c++) {
900 + if(!rrdr_dimension_should_be_exposed(r->od[c], options))
901 + continue;
902 +
903 buffer_json_add_array_item_uint64(wb, r->dgbc[c]);
904 + }
905 buffer_json_array_close(wb);
906 }
907
@@ -795,7 +936,7 @@ static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
936
937 static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) {
938 QUERY_TARGET *qt = r->internal.qt;
798 - buffer_json_member_add_object(wb, "hosts");
939 + buffer_json_member_add_object(wb, "nodes");
940
941 time_t now_s = now_realtime_sec();
942 RRDHOST *last_host = NULL;
@@ -803,9 +944,9 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
944 RRDINSTANCE_ACQUIRED *last_ria = NULL;
945
946 size_t h = 0, c = 0, i = 0, m = 0, q = 0;
806 - for(; h < qt->hosts.used ; h++) {
807 - QUERY_HOST *qh = query_host(qt, h);
808 - RRDHOST *host = qh->host;
947 + for(; h < qt->nodes.used ; h++) {
948 + QUERY_NODE *qn = query_node(qt, h);
949 + RRDHOST *host = qn->rrdhost;
950
951 for( ;c < qt->contexts.used ;c++) {
952 QUERY_CONTEXT *qc = query_context(qt, c);
@@ -854,9 +995,9 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
995 }
996
997 buffer_json_member_add_object(wb, host->machine_guid);
857 - if(qh->node_id[0])
858 - buffer_json_member_add_string(wb, "nd", qh->node_id);
859 - buffer_json_member_add_uint64(wb, "idx", qh->slot);
998 + if(qn->node_id[0])
999 + buffer_json_member_add_string(wb, "nd", qn->node_id);
1000 + buffer_json_member_add_uint64(wb, "ni", qn->slot);
1001 buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
1002 buffer_json_member_add_object(wb, "contexts");
1003
@@ -889,7 +1030,6 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
1030 }
1031
1032 buffer_json_member_add_object(wb, rrdinstance_acquired_id(ria));
892 - buffer_json_member_add_uint64(wb, "idx", qi->slot);
1033 buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(ria));
1034 buffer_json_member_add_time_t(wb, "ue", rrdinstance_acquired_update_every(ria));
1035 DICTIONARY *labels = rrdinstance_acquired_labels(ria);
@@ -898,7 +1038,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
1038 rrdlabels_to_buffer_json_members(labels, wb);
1039 buffer_json_object_close(wb);
1040 }
901 - rrdset_rrdcalc_entries(wb, ria);
1041 + rrdset_rrdcalc_entries_v2(wb, ria);
1042 buffer_json_member_add_object(wb, "dimensions");
1043
1044 last_ria = ria;
@@ -949,7 +1089,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
1089 buffer_json_object_close(wb); // hosts
1090 }
1091
952 -void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value,
1092 +void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options,
1093 RRDR_TIME_GROUPING group_method)
1094 {
1095 QUERY_TARGET *qt = r->internal.qt;
@@ -964,7 +1104,7 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1104 sq[0] = '\'';
1105 }
1106
967 - buffer_json_initialize(wb, kq, sq, 0, true);
1107 + buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
1108
1109 buffer_json_member_add_uint64(wb, "api", 2);
1110
@@ -976,17 +1116,17 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1116 web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", qt->request.options);
1117
1118 buffer_json_member_add_object(wb, "scope");
979 - buffer_json_member_add_string(wb, "scope_hosts", qt->request.scope_hosts);
1119 + buffer_json_member_add_string(wb, "scope_nodes", qt->request.scope_nodes);
1120 buffer_json_member_add_string(wb, "scope_contexts", qt->request.scope_contexts);
1121 buffer_json_object_close(wb); // scope
1122
1123 buffer_json_member_add_object(wb, "selectors");
1124 if (qt->request.host)
985 - buffer_json_member_add_string(wb, "host", rrdhost_hostname(qt->request.host));
1125 + buffer_json_member_add_string(wb, "nodes", rrdhost_hostname(qt->request.host));
1126 else
987 - buffer_json_member_add_string(wb, "hosts", qt->request.hosts);
1127 + buffer_json_member_add_string(wb, "nodes", qt->request.nodes);
1128 buffer_json_member_add_string(wb, "contexts", qt->request.contexts);
989 - buffer_json_member_add_string(wb, "instances", qt->request.charts);
1129 + buffer_json_member_add_string(wb, "instances", qt->request.instances);
1130 buffer_json_member_add_string(wb, "dimensions", qt->request.dimensions);
1131 buffer_json_member_add_string(wb, "labels", qt->request.labels);
1132 buffer_json_member_add_string(wb, "alerts", qt->request.alerts);
@@ -1024,7 +1164,7 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1164 buffer_json_add_array_item_string(wb, qt->group_by.label_keys[l]);
1165 buffer_json_array_close(wb);
1166
1027 - buffer_json_member_add_string(wb, "group_by_aggregate",
1167 + buffer_json_member_add_string(wb, "aggregation",
1168 group_by_aggregate_function_to_string(
1169 qt->request.group_by_aggregate_function));
1170 buffer_json_object_close(wb); // dimensions
@@ -1036,18 +1176,39 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1176 buffer_json_object_close(wb); // request
1177 }
1178
1179 + buffer_json_member_add_object(wb, "versions");
1180 + buffer_json_member_add_uint64(wb, "contexts_hard_hash", qt->versions.contexts_hard_hash);
1181 + buffer_json_member_add_uint64(wb, "contexts_soft_hash", qt->versions.contexts_soft_hash);
1182 + buffer_json_object_close(wb);
1183 +
1184 size_t contexts;
1185 buffer_json_member_add_object(wb, "summary");
1186 + struct summary_total_counts
1187 + nodes_totals = { 0 },
1188 + contexts_totals = { 0 },
1189 + instances_totals = { 0 },
1190 + metrics_totals = { 0 },
1191 + label_key_totals = { 0 },
1192 + label_key_value_totals = { 0 };
1193 {
1042 - query_target_summary_hosts_v2(wb, qt, "hosts");
1043 - contexts = query_target_summary_contexts_v2(wb, qt, "contexts");
1044 - query_target_summary_instances_v2(wb, qt, "instances");
1045 - query_target_summary_dimensions_v12(wb, qt, "dimensions", true);
1046 - query_target_summary_labels_v12(wb, qt, "labels", true);
1194 + query_target_summary_nodes_v2(wb, qt, "nodes", &nodes_totals);
1195 + contexts = query_target_summary_contexts_v2(wb, qt, "contexts", &contexts_totals);
1196 + query_target_summary_instances_v2(wb, qt, "instances", &instances_totals);
1197 + query_target_summary_dimensions_v12(wb, qt, "dimensions", true, &metrics_totals);
1198 + query_target_summary_labels_v12(wb, qt, "labels", true, &label_key_totals, &label_key_value_totals);
1199 query_target_summary_alerts_v2(wb, qt, "alerts");
1200 }
1201 buffer_json_object_close(wb); // summary
1202
1203 + buffer_json_member_add_object(wb, "totals");
1204 + query_target_total_counts(wb, "nodes", &nodes_totals);
1205 + query_target_total_counts(wb, "contexts", &contexts_totals);
1206 + query_target_total_counts(wb, "instances", &instances_totals);
1207 + query_target_total_counts(wb, "dimensions", &metrics_totals);
1208 + query_target_total_counts(wb, "label_keys", &label_key_totals);
1209 + query_target_total_counts(wb, "label_key_values", &label_key_value_totals);
1210 + buffer_json_object_close(wb); // totals
1211 +
1212 if(options & RRDR_OPTION_SHOW_DETAILS) {
1213 buffer_json_member_add_object(wb, "detailed");
1214 query_target_detailed_objects_tree(wb, r, options);
@@ -1058,12 +1219,22 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1219
1220 buffer_json_member_add_object(wb, "db");
1221 {
1222 + buffer_json_member_add_uint64(wb, "tiers", storage_tiers);
1223 buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
1224 buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
1225 buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
1064 - buffer_json_member_add_array(wb, "points_per_tier");
1065 - for(size_t tier = 0; tier < storage_tiers ; tier++)
1066 - buffer_json_add_array_item_uint64(wb, r->stats.tier_points_read[tier]);
1226 +
1227 + buffer_json_member_add_array(wb, "tiers");
1228 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
1229 + buffer_json_add_array_item_object(wb);
1230 + buffer_json_member_add_uint64(wb, "tier", tier);
1231 + buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries);
1232 + buffer_json_member_add_uint64(wb, "points", qt->db.tiers[tier].points);
1233 + buffer_json_member_add_time_t(wb, "update_every", qt->db.tiers[tier].update_every);
1234 + buffer_json_member_add_time_t(wb, "first_entry", qt->db.tiers[tier].retention.first_time_s);
1235 + buffer_json_member_add_time_t(wb, "last_entry", qt->db.tiers[tier].retention.last_time_s);
1236 + buffer_json_object_close(wb);
1237 + }
1238 buffer_json_array_close(wb);
1239 }
1240 buffer_json_object_close(wb);
@@ -1078,77 +1249,98 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
1249 buffer_json_member_add_time_t(wb, "after", r->view.after);
1250 buffer_json_member_add_time_t(wb, "before", r->view.before);
1251 buffer_json_member_add_uint64(wb, "points", rows);
1081 - query_target_combined_units(wb, qt, contexts);
1252 + query_target_combined_units_v2(wb, qt, contexts);
1253 query_target_combined_chart_type(wb, qt, contexts);
1254 buffer_json_member_add_object(wb, "dimensions");
1255 {
1256 rrdr_dimension_ids(wb, "ids", r, options);
1257 rrdr_dimension_names(wb, "names", r, options);
1087 - rrdr_dimension_units_array(wb, r);
1088 - rrdr_dimension_priority_array(wb, r);
1089 - rrdr_dimension_grouped_array(wb, r);
1258 + rrdr_dimension_units_array_v2(wb, r, options);
1259 + rrdr_dimension_priority_array(wb, r, options);
1260 + rrdr_dimension_grouped_array(wb, r, options);
1261 size_t dims = rrdr_latest_values(wb, "view_latest_values", r, options);
1262 buffer_json_member_add_uint64(wb, "count", dims);
1263 }
1264 buffer_json_object_close(wb);
1265 }
1266 buffer_json_object_close(wb);
1096 -
1097 - buffer_sprintf(wb, ",\n %sresult%s:", kq, kq);
1098 - if(string_value) buffer_strcat(wb, sq);
1267 }
1268
1101 -void rrdr_json_wrapper_anomaly_rates(RRDR *r __maybe_unused, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, bool string_value) {
1102 -
1103 - char kq[2] = "", // key quote
1104 - sq[2] = ""; // string quote
1105 -
1106 - if( options & RRDR_OPTION_GOOGLE_JSON ) {
1107 - kq[0] = '\0';
1108 - sq[0] = '\'';
1109 - }
1110 - else {
1111 - kq[0] = '"';
1112 - sq[0] = '"';
1113 - }
1269 +static void annotations_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) {
1270 + const size_t dims = r->d, rows = r->rows;
1271 + size_t next_d_idx = 0;
1272 + for(size_t d = 0; d < dims ; d++) {
1273 + if(!rrdr_dimension_should_be_exposed(r->od[d], options))
1274 + continue;
1275
1115 - if(string_value) buffer_strcat(wb, sq);
1276 + size_t d_idx = next_d_idx++;
1277
1117 - buffer_sprintf(wb, ",\n %sanomaly_rates%s: ", kq, kq);
1118 -}
1278 + size_t t = 0;
1279 + while(t < rows) {
1280
1120 -void rrdr_json_wrapper_group_by_count(RRDR *r __maybe_unused, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, bool string_value) {
1281 + // find the beginning
1282 + time_t started = 0;
1283 + for(; t < rows ;t++) {
1284 + RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
1285 + if(o & flags) {
1286 + started = r->t[t];
1287 + break;
1288 + }
1289 + }
1290
1122 - char kq[2] = "", // key quote
1123 - sq[2] = ""; // string quote
1291 + if(started) {
1292 + time_t ended = 0;
1293 + for(; t < rows ;t++) {
1294 + RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
1295 + if(!(o & flags)) {
1296 + ended = r->t[t];
1297 + break;
1298 + }
1299 + }
1300
1125 - if( options & RRDR_OPTION_GOOGLE_JSON ) {
1126 - kq[0] = '\0';
1127 - sq[0] = '\'';
1128 - }
1129 - else {
1130 - kq[0] = '"';
1131 - sq[0] = '"';
1301 + if(!ended)
1302 + ended = r->t[rows - 1];
1303 +
1304 + buffer_json_add_array_item_object(wb);
1305 + buffer_json_member_add_string(wb, "t", type);
1306 + // buffer_json_member_add_string(wb, "d", string2str(r->dn[d]));
1307 + buffer_json_member_add_uint64(wb, "d", d_idx);
1308 + if(started == ended) {
1309 + if(options & RRDR_OPTION_MILLISECONDS)
1310 + buffer_json_member_add_time_t2ms(wb, "x", started);
1311 + else
1312 + buffer_json_member_add_time_t(wb, "x", started);
1313 + }
1314 + else {
1315 + buffer_json_member_add_array(wb, "x");
1316 + if(options & RRDR_OPTION_MILLISECONDS) {
1317 + buffer_json_add_array_item_time_t2ms(wb, started);
1318 + buffer_json_add_array_item_time_t2ms(wb, ended);
1319 + }
1320 + else {
1321 + buffer_json_add_array_item_time_t(wb, started);
1322 + buffer_json_add_array_item_time_t(wb, ended);
1323 + }
1324 + buffer_json_array_close(wb);
1325 + }
1326 + buffer_json_object_close(wb);
1327 + }
1328 + }
1329 }
1133 -
1134 - if(string_value) buffer_strcat(wb, sq);
1135 -
1136 - buffer_sprintf(wb, ",\n %sgroup_by_count%s: ", kq, kq);
1330 }
1331
1139 -void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, bool string_value) {
1140 - QUERY_TARGET *qt = r->internal.qt;
1332 +void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
1333 + buffer_json_member_add_array(wb, "annotations");
1334
1142 - char sq[2] = ""; // string quote
1335 + annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap
1336 + annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow
1337 + annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial
1338
1144 - if( options & RRDR_OPTION_GOOGLE_JSON ) {
1145 - sq[0] = '\'';
1146 - }
1147 - else {
1148 - sq[0] = '"';
1149 - }
1339 + buffer_json_array_close(wb); // annotations
1340 +}
1341
1151 - if(string_value) buffer_strcat(wb, sq);
1342 +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options __maybe_unused) {
1343 + QUERY_TARGET *qt = r->internal.qt;
1344
1345 buffer_json_member_add_double(wb, "min", r->view.min);
1346 buffer_json_member_add_double(wb, "max", r->view.max);
web/api/formatters/json_wrapper.h
+6 -7
@@ -6,14 +6,13 @@
6 #include "rrd2json.h"
7 #include "web/api/queries/query.h"
8
9 -typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value, RRDR_TIME_GROUPING group_method);
10 -typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
9 +typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method);
10 +typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
11
12 -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value, RRDR_TIME_GROUPING group_method);
13 -void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
14 -void rrdr_json_wrapper_group_by_count(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
15 -void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value);
12 +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method);
13 +void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
14 +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
15
17 -void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, bool string_value, RRDR_TIME_GROUPING group_method);
16 +void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method);
17
18 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
web/api/formatters/rrd2json.c
+149 -88
@@ -3,6 +3,14 @@
3 #include "web/api/web_api_v1.h"
4 #include "database/storage_engine.h"
5
6 +inline bool query_target_has_percentage_units(struct query_target *qt) {
7 + if(qt->request.options & RRDR_OPTION_PERCENTAGE ||
8 + qt->request.time_group_method == RRDR_GROUPING_CV)
9 + return true;
10 +
11 + return false;
12 +}
13 +
14 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
15 rrdset2json(st, wb, NULL, NULL, 0);
16 }
@@ -11,47 +19,36 @@ const char *rrdr_format_to_string(DATASOURCE_FORMAT format) {
19 switch(format) {
20 case DATASOURCE_JSON:
21 return DATASOURCE_FORMAT_JSON;
14 - break;
22
23 case DATASOURCE_DATATABLE_JSON:
24 return DATASOURCE_FORMAT_DATATABLE_JSON;
18 - break;
25
26 case DATASOURCE_DATATABLE_JSONP:
27 return DATASOURCE_FORMAT_DATATABLE_JSONP;
22 - break;
28
29 case DATASOURCE_JSONP:
30 return DATASOURCE_FORMAT_JSONP;
26 - break;
31
32 case DATASOURCE_SSV:
33 return DATASOURCE_FORMAT_SSV;
30 - break;
34
35 case DATASOURCE_CSV:
36 return DATASOURCE_FORMAT_CSV;
34 - break;
37
38 case DATASOURCE_TSV:
39 return DATASOURCE_FORMAT_TSV;
38 - break;
40
41 case DATASOURCE_HTML:
42 return DATASOURCE_FORMAT_HTML;
42 - break;
43
44 case DATASOURCE_JS_ARRAY:
45 return DATASOURCE_FORMAT_JS_ARRAY;
46 - break;
46
47 case DATASOURCE_SSV_COMMA:
48 return DATASOURCE_FORMAT_SSV_COMMA;
50 - break;
49
50 default:
51 return "unknown";
54 - break;
52 }
53 }
54
@@ -109,7 +106,7 @@ int rrdset2value_api_v1(
106
107 if(db_points_per_tier) {
108 for(size_t t = 0; t < storage_tiers ;t++)
112 - db_points_per_tier[t] += r->stats.tier_points_read[t];
109 + db_points_per_tier[t] += r->internal.qt->db.tiers[t].points;
110 }
111
112 if(result_points_generated)
@@ -150,6 +147,7 @@ struct group_by_entry {
147 STRING *id;
148 STRING *name;
149 STRING *units;
150 + RRDR_DIMENSION_FLAGS od;
151 };
152
153 static int group_by_label_is_space(char c) {
@@ -159,7 +157,7 @@ static int group_by_label_is_space(char c) {
157 return 0;
158 }
159
162 -RRDR *data_query_group_by(RRDR *r) {
160 +static RRDR *data_query_group_by(RRDR *r) {
161 QUERY_TARGET *qt = r->internal.qt;
162 RRDR_OPTIONS options = qt->request.options;
163 size_t rows = rrdr_rows(r);
@@ -189,7 +187,7 @@ RRDR *data_query_group_by(RRDR *r) {
187
188 QUERY_METRIC *qm = query_metric(qt, c);
189 QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
192 - QUERY_HOST *qh = query_host(qt, qm->link.query_host_id);
190 + QUERY_NODE *qn = query_node(qt, qm->link.query_host_id);
191
192 if(qi != last_qi) {
193 priority = 0;
@@ -207,7 +205,7 @@ RRDR *data_query_group_by(RRDR *r) {
205 }
206 if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
207 buffer_fast_strcat(key, "|", 1);
210 - buffer_strcat(key, string2str(qi->id_fqdn));
208 + buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
209 }
210 if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
211 DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
@@ -218,10 +216,17 @@ RRDR *data_query_group_by(RRDR *r) {
216 }
217 if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
218 buffer_fast_strcat(key, "|", 1);
221 - buffer_strcat(key, qh->host->machine_guid);
219 + buffer_strcat(key, qn->rrdhost->machine_guid);
220 + }
221 +
222 + // append the units
223 + if(query_target_has_percentage_units(qt)) {
224 + buffer_fast_strcat(key, "|%", 2);
225 + }
226 + else {
227 + buffer_fast_strcat(key, "|", 1);
228 + buffer_strcat(key, rrdinstance_acquired_units(qi->ria));
229 }
223 - buffer_fast_strcat(key, "|", 1);
224 - buffer_strcat(key, rrdinstance_acquired_units(qi->ria));
230
231 // lookup the key in the dictionary
232
@@ -245,7 +250,7 @@ RRDR *data_query_group_by(RRDR *r) {
250 if(qt->request.group_by & RRDR_GROUP_BY_NODE)
251 buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
252 else
248 - buffer_strcat(key, string2str(qi->id_fqdn));
253 + buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
254 }
255 if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
256 DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
@@ -259,7 +264,7 @@ RRDR *data_query_group_by(RRDR *r) {
264 if(buffer_strlen(key) != 0)
265 buffer_fast_strcat(key, ",", 1);
266
262 - buffer_strcat(key, qh->host->machine_guid);
267 + buffer_strcat(key, qn->rrdhost->machine_guid);
268 }
269 entries[pos].id = string_strdupz(buffer_tostring(key));
270
@@ -276,7 +281,7 @@ RRDR *data_query_group_by(RRDR *r) {
281 if(qt->request.group_by & RRDR_GROUP_BY_NODE)
282 buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
283 else
279 - buffer_strcat(key, string2str(qi->name_fqdn));
284 + buffer_strcat(key, string2str(query_instance_name_fqdn(qt, qi)));
285 }
286 if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
287 DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
@@ -290,7 +295,7 @@ RRDR *data_query_group_by(RRDR *r) {
295 if(buffer_strlen(key) != 0)
296 buffer_fast_strcat(key, ",", 1);
297
293 - buffer_strcat(key, rrdhost_hostname(qh->host));
298 + buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
299 }
300 entries[pos].name = string_strdupz(buffer_tostring(key));
301
@@ -312,7 +317,12 @@ RRDR *data_query_group_by(RRDR *r) {
317 qm->grouped_as.id = entries[pos].id;
318 qm->grouped_as.name = entries[pos].name;
319 qm->grouped_as.units = entries[pos].units;
315 - qm->status |= RRDR_DIMENSION_GROUPED;
320 +
321 + // copy the dimension flags decided by the query target
322 + // we need this, because if a dimension is explicitly selected
323 + // the query target adds to it the non-zero flag
324 + qm->status |= RRDR_DIMENSION_GROUPED | r->od[c];
325 + entries[pos].od |= RRDR_DIMENSION_GROUPED | r->od[c];
326 }
327
328 // check if we have multiple units
@@ -353,7 +363,7 @@ RRDR *data_query_group_by(RRDR *r) {
363
364 // initialize r2 (dimension options, names, and ids)
365 for(size_t c2 = 0; c2 < r2->d ; c2++) {
356 - r2->od[c2] = RRDR_DIMENSION_QUERIED;
366 + r2->od[c2] = entries[c2].od;
367 r2->di[c2] = entries[c2].id;
368 r2->dn[c2] = entries[c2].name;
369 r2->du[c2] = entries[c2].units;
@@ -380,23 +390,23 @@ RRDR *data_query_group_by(RRDR *r) {
390 // do the group_by
391 for(size_t i = 0; i != rows ;i++) {
392 size_t idx = i * r->d;
383 - size_t idx2 = i * r2->d;
393 + NETDATA_DOUBLE *cn_base = &r->v[ idx ];
394 + RRDR_VALUE_FLAGS *co_base = &r->o[ idx ];
395 + NETDATA_DOUBLE *ar_base = &r->ar[ idx ];
396
385 - NETDATA_DOUBLE *cn = &r->v[ idx ];
386 - RRDR_VALUE_FLAGS *co = &r->o[ idx ];
387 - NETDATA_DOUBLE *ar = &r->ar[ idx ];
388 -
389 - NETDATA_DOUBLE *cn2 = &r2->v[ idx2 ];
390 - RRDR_VALUE_FLAGS *co2 = &r2->o[ idx2 ];
391 - NETDATA_DOUBLE *ar2 = &r2->ar[ idx2 ];
392 - uint32_t *gbc2 = &r2->gbc[ idx2 ];
397 + size_t idx2 = i * r2->d;
398 + NETDATA_DOUBLE *cn2_base = &r2->v[ idx2 ];
399 + RRDR_VALUE_FLAGS *co2_base = &r2->o[ idx2 ];
400 + NETDATA_DOUBLE *ar2_base = &r2->ar[ idx2 ];
401 + uint32_t *gbc2_base = &r2->gbc[ idx2 ];
402
403 for(size_t c = 0; c < r->d ;c++) {
404 if (!rrdr_dimension_should_be_exposed(r->od[c], options))
405 continue;
406
398 - NETDATA_DOUBLE n = cn[c];
399 - RRDR_VALUE_FLAGS o = co[c];
407 + NETDATA_DOUBLE n = cn_base[c];
408 + RRDR_VALUE_FLAGS o = co_base[c];
409 + NETDATA_DOUBLE ar = ar_base[c];
410
411 if(o & RRDR_VALUE_EMPTY) {
412 if(options & RRDR_OPTION_NULL2ZERO)
@@ -411,35 +421,38 @@ RRDR *data_query_group_by(RRDR *r) {
421 QUERY_METRIC *qm = query_metric(qt, c);
422 size_t c2 = qm->grouped_as.slot;
423
424 + NETDATA_DOUBLE *cn2 = &cn2_base[c2];
425 + RRDR_VALUE_FLAGS *co2 = &co2_base[c2];
426 + NETDATA_DOUBLE *ar2 = &ar2_base[c2];
427 + uint32_t *gbc2 = &gbc2_base[c2];
428 +
429 switch(qt->request.group_by_aggregate_function) {
430 default:
431 case RRDR_GROUP_BY_FUNCTION_AVERAGE:
432 case RRDR_GROUP_BY_FUNCTION_SUM:
433 case RRDR_GROUP_BY_FUNCTION_SUM_COUNT:
419 - cn2[c2] += n;
434 + *cn2 += n;
435 break;
436
437 case RRDR_GROUP_BY_FUNCTION_MIN:
423 - if(n < cn2[c2])
424 - cn2[c2] = n;
438 + if(n < *cn2)
439 + *cn2 = n;
440 break;
441
442 case RRDR_GROUP_BY_FUNCTION_MAX:
428 - if(n > cn2[c2])
429 - cn2[c2] = n;
443 + if(n > *cn2)
444 + *cn2 = n;
445 break;
446 }
447
433 - if(o & RRDR_VALUE_RESET)
434 - co2[c2] |= RRDR_VALUE_RESET;
435 -
436 - ar2[c2] += ar[c];
437 - gbc2[c2]++;
448 + *co2 |= (o & (RRDR_VALUE_RESET|RRDR_VALUE_PARTIAL));
449 + *ar2 += ar;
450 + (*gbc2)++;
451 }
452 }
453
454 // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
442 - size_t values = 0;
455 + size_t min_max_values = 0;
456 NETDATA_DOUBLE min = NAN, max = NAN;
457 for (size_t c2 = 0; c2 < r2->d; c2++) {
458 size_t non_zero = 0;
@@ -450,29 +463,33 @@ RRDR *data_query_group_by(RRDR *r) {
463 NETDATA_DOUBLE *cn2 = &r2->v[ idx2 ];
464 RRDR_VALUE_FLAGS *co2 = &r2->o[ idx2 ];
465 NETDATA_DOUBLE *ar2 = &r2->ar[ idx2 ];
453 - uint32_t *gbc2 = &r2->gbc[ idx2 ];
466 + uint32_t gbc2 = r2->gbc[ idx2 ];
467
455 - if(likely(*gbc2)) {
468 + if(likely(gbc2)) {
469 *co2 &= ~RRDR_VALUE_EMPTY;
470
458 - *ar2 /= *gbc2;
471 + if(gbc2 != r2->dgbc[c2])
472 + *co2 |= RRDR_VALUE_PARTIAL;
473
474 NETDATA_DOUBLE n;
475
476 if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
463 - n = *cn2 / *gbc2;
477 + n = *cn2 / gbc2;
478 }
479 else if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_AVERAGE) {
466 - n = *cn2 / *gbc2;
480 + *ar2 /= gbc2;
481 + n = *cn2 / gbc2;
482 *cn2 = n;
483 }
469 - else
484 + else {
485 + *ar2 /= gbc2;
486 n = *cn2;
487 + }
488
489 if(islessgreater(n, 0.0))
490 non_zero++;
491
475 - if(unlikely(!values++)) {
492 + if(unlikely(!min_max_values++)) {
493 min = n;
494 max = n;
495 }
@@ -508,6 +525,22 @@ cleanup:
525 return r2;
526 }
527
528 +static inline void buffer_json_member_add_key_only(BUFFER *wb, const char *key) {
529 + buffer_print_json_comma_newline_spacing(wb);
530 + buffer_print_json_key(wb, key);
531 + buffer_fast_strcat(wb, ":", 1);
532 + wb->json.stack[wb->json.depth].count++;
533 +}
534 +
535 +static inline void buffer_json_member_add_string_open(BUFFER *wb, const char *key) {
536 + buffer_json_member_add_key_only(wb, key);
537 + buffer_strcat(wb, wb->json.value_quote);
538 +}
539 +
540 +static inline void buffer_json_member_add_string_close(BUFFER *wb) {
541 + buffer_strcat(wb, wb->json.value_quote);
542 +}
543 +
544 int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) {
545 wrapper_begin_t wrapper_begin = rrdr_json_wrapper_begin;
546 wrapper_end_t wrapper_end = rrdr_json_wrapper_end;
@@ -547,9 +580,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
580 case DATASOURCE_SSV:
581 if(options & RRDR_OPTION_JSON_WRAP) {
582 wb->content_type = CT_APPLICATION_JSON;
550 - wrapper_begin(r, wb, format, options, true, group_method);
583 + wrapper_begin(r, wb, format, options, group_method);
584 + buffer_json_member_add_string_open(wb, "result");
585 rrdr2ssv(r, wb, options, "", " ", "");
552 - wrapper_end(r, wb, format, options, true);
586 + buffer_json_member_add_string_close(wb);
587 + wrapper_end(r, wb, format, options);
588 }
589 else {
590 wb->content_type = CT_TEXT_PLAIN;
@@ -560,9 +595,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
595 case DATASOURCE_SSV_COMMA:
596 if(options & RRDR_OPTION_JSON_WRAP) {
597 wb->content_type = CT_APPLICATION_JSON;
563 - wrapper_begin(r, wb, format, options, true, group_method);
598 + wrapper_begin(r, wb, format, options, group_method);
599 + buffer_json_member_add_string_open(wb, "result");
600 rrdr2ssv(r, wb, options, "", ",", "");
565 - wrapper_end(r, wb, format, options, true);
601 + buffer_json_member_add_string_close(wb);
602 + wrapper_end(r, wb, format, options);
603 }
604 else {
605 wb->content_type = CT_TEXT_PLAIN;
@@ -573,9 +610,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
610 case DATASOURCE_JS_ARRAY:
611 if(options & RRDR_OPTION_JSON_WRAP) {
612 wb->content_type = CT_APPLICATION_JSON;
576 - wrapper_begin(r, wb, format, options, false, group_method);
577 - rrdr2ssv(r, wb, options, "[", ",", "]");
578 - wrapper_end(r, wb, format, options, false);
613 + wrapper_begin(r, wb, format, options, group_method);
614 + buffer_json_member_add_array(wb, "result");
615 + rrdr2ssv(r, wb, options, "", ",", "");
616 + buffer_json_array_close(wb);
617 + wrapper_end(r, wb, format, options);
618 }
619 else {
620 wb->content_type = CT_APPLICATION_JSON;
@@ -586,9 +625,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
625 case DATASOURCE_CSV:
626 if(options & RRDR_OPTION_JSON_WRAP) {
627 wb->content_type = CT_APPLICATION_JSON;
589 - wrapper_begin(r, wb, format, options, true, group_method);
628 + wrapper_begin(r, wb, format, options, group_method);
629 + buffer_json_member_add_string_open(wb, "result");
630 rrdr2csv(r, wb, format, options, "", ",", "\\n", "");
591 - wrapper_end(r, wb, format, options, true);
631 + buffer_json_member_add_string_close(wb);
632 + wrapper_end(r, wb, format, options);
633 }
634 else {
635 wb->content_type = CT_TEXT_PLAIN;
@@ -599,9 +640,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
640 case DATASOURCE_CSV_MARKDOWN:
641 if(options & RRDR_OPTION_JSON_WRAP) {
642 wb->content_type = CT_APPLICATION_JSON;
602 - wrapper_begin(r, wb, format, options, true, group_method);
643 + wrapper_begin(r, wb, format, options, group_method);
644 + buffer_json_member_add_string_open(wb, "result");
645 rrdr2csv(r, wb, format, options, "", "|", "\\n", "");
604 - wrapper_end(r, wb, format, options, true);
646 + buffer_json_member_add_string_close(wb);
647 + wrapper_end(r, wb, format, options);
648 }
649 else {
650 wb->content_type = CT_TEXT_PLAIN;
@@ -612,11 +655,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
655 case DATASOURCE_CSV_JSON_ARRAY:
656 wb->content_type = CT_APPLICATION_JSON;
657 if(options & RRDR_OPTION_JSON_WRAP) {
615 - wrapper_begin(r, wb, format, options, false, group_method);
616 - buffer_strcat(wb, "[\n");
658 + wrapper_begin(r, wb, format, options, group_method);
659 + buffer_json_member_add_array(wb, "result");
660 rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
618 - buffer_strcat(wb, "\n]");
619 - wrapper_end(r, wb, format, options, false);
661 + buffer_json_array_close(wb);
662 + wrapper_end(r, wb, format, options);
663 }
664 else {
665 wb->content_type = CT_APPLICATION_JSON;
@@ -629,9 +672,11 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
672 case DATASOURCE_TSV:
673 if(options & RRDR_OPTION_JSON_WRAP) {
674 wb->content_type = CT_APPLICATION_JSON;
632 - wrapper_begin(r, wb, format, options, true, group_method);
675 + wrapper_begin(r, wb, format, options, group_method);
676 + buffer_json_member_add_string_open(wb, "result");
677 rrdr2csv(r, wb, format, options, "", "\t", "\\n", "");
634 - wrapper_end(r, wb, format, options, true);
678 + buffer_json_member_add_string_close(wb);
679 + wrapper_end(r, wb, format, options);
680 }
681 else {
682 wb->content_type = CT_TEXT_PLAIN;
@@ -642,11 +687,13 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
687 case DATASOURCE_HTML:
688 if(options & RRDR_OPTION_JSON_WRAP) {
689 wb->content_type = CT_APPLICATION_JSON;
645 - wrapper_begin(r, wb, format, options, true, group_method);
690 + wrapper_begin(r, wb, format, options, group_method);
691 + buffer_json_member_add_string_open(wb, "result");
692 buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
693 rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
694 buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
649 - wrapper_end(r, wb, format, options, true);
695 + buffer_json_member_add_string_close(wb);
696 + wrapper_end(r, wb, format, options);
697 }
698 else {
699 wb->content_type = CT_TEXT_HTML;
@@ -659,57 +706,71 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
706 case DATASOURCE_DATATABLE_JSONP:
707 wb->content_type = CT_APPLICATION_X_JAVASCRIPT;
708
662 - if(options & RRDR_OPTION_JSON_WRAP)
663 - wrapper_begin(r, wb, format, options, false, group_method);
709 + if(options & RRDR_OPTION_JSON_WRAP) {
710 + wrapper_begin(r, wb, format, options, group_method);
711 + buffer_json_member_add_key_only(wb, "result");
712 + }
713
714 rrdr2json(r, wb, options, 1);
715
716 if(options & RRDR_OPTION_JSON_WRAP)
668 - wrapper_end(r, wb, format, options, false);
717 + wrapper_end(r, wb, format, options);
718 +
719 break;
720
721 case DATASOURCE_DATATABLE_JSON:
722 wb->content_type = CT_APPLICATION_JSON;
723
674 - if(options & RRDR_OPTION_JSON_WRAP)
675 - wrapper_begin(r, wb, format, options, false, group_method);
724 + if(options & RRDR_OPTION_JSON_WRAP) {
725 + wrapper_begin(r, wb, format, options, group_method);
726 + buffer_json_member_add_key_only(wb, "result");
727 + }
728
729 rrdr2json(r, wb, options, 1);
730
731 if(options & RRDR_OPTION_JSON_WRAP)
680 - wrapper_end(r, wb, format, options, false);
732 + wrapper_end(r, wb, format, options);
733 +
734 break;
735
736 case DATASOURCE_JSONP:
737 wb->content_type = CT_APPLICATION_X_JAVASCRIPT;
685 - if(options & RRDR_OPTION_JSON_WRAP)
686 - wrapper_begin(r, wb, format, options, false, group_method);
738 + if(options & RRDR_OPTION_JSON_WRAP) {
739 + wrapper_begin(r, wb, format, options, group_method);
740 + buffer_json_member_add_key_only(wb, "result");
741 + }
742
743 rrdr2json(r, wb, options, 0);
744
745 if(options & RRDR_OPTION_JSON_WRAP)
691 - wrapper_end(r, wb, format, options, false);
746 + wrapper_end(r, wb, format, options);
747 +
748 break;
749
750 case DATASOURCE_JSON:
751 default:
752 wb->content_type = CT_APPLICATION_JSON;
753
698 - if(options & RRDR_OPTION_JSON_WRAP)
699 - wrapper_begin(r, wb, format, options, false, group_method);
754 + if(options & RRDR_OPTION_JSON_WRAP) {
755 + wrapper_begin(r, wb, format, options, group_method);
756 + buffer_json_member_add_key_only(wb, "result");
757 + }
758
759 rrdr2json(r, wb, options, 0);
760
761 if(options & RRDR_OPTION_JSON_WRAP) {
762 if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
705 - rrdr_json_wrapper_group_by_count(r, wb, format, options, 0);
706 - rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_GBC, 0);
763 + buffer_json_member_add_key_only(wb, "group_by_count");
764 + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_GBC, false);
765 }
766 if(options & RRDR_OPTION_RETURN_JWAR) {
709 - rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0);
710 - rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0);
767 + buffer_json_member_add_key_only(wb, "anomaly_rates");
768 + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, false);
769 + }
770 + if(options & RRDR_OPTION_JW_ANNOTATIONS) {
771 + rrdr_json_wrapper_annotations(r, wb, format, options);
772 }
712 - wrapper_end(r, wb, format, options, false);
773 + wrapper_end(r, wb, format, options);
774 }
775 break;
776 }
web/api/formatters/rrd2json.h
+3
@@ -57,6 +57,9 @@ const char *rrdr_format_to_string(DATASOURCE_FORMAT format);
57
58 int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp);
59
60 +struct query_target;
61 +bool query_target_has_percentage_units(struct query_target *qt);
62 +
63 int rrdset2value_api_v1(
64 RRDSET *st
65 , BUFFER *wb
web/api/formatters/value/value.c
+4 -6
@@ -4,9 +4,7 @@
4
5
6 inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) {
7 - QUERY_TARGET *qt = r->internal.qt;
8 - long c;
9 - const long used = qt->query.used;
7 + size_t c;
8
9 NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
10 RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
@@ -21,7 +19,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
19 int set_min_max = 0;
20 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
21 total = 0;
24 - for (c = 0; c < used; c++) {
22 + for (c = 0; c < r->d ; c++) {
23 if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
24 NETDATA_DOUBLE n = cn[c];
25
@@ -36,7 +34,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
34 }
35
36 // for each dimension
39 - for (c = 0; c < used; c++) {
37 + for (c = 0; c < r->d ; c++) {
38 if(!rrdr_dimension_should_be_exposed(r->od[c], options))
39 continue;
40
@@ -147,7 +145,7 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
145 };
146
147 for(size_t t = 0; t < storage_tiers ;t++)
150 - qv.storage_points_per_tier[t] = r->stats.tier_points_read[t];
148 + qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points;
149
150 long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
151 int all_values_are_null = 0;
web/api/queries/query.c
+62 -25
@@ -679,9 +679,6 @@ RRDR_GROUP_BY group_by_parse(char *s) {
679 group_by |= RRDR_GROUP_BY_LABEL;
680 }
681
682 - if(group_by == RRDR_GROUP_BY_NONE)
683 - group_by = RRDR_GROUP_BY_DIMENSION;
684 -
682 return group_by;
683 }
684
@@ -971,7 +968,8 @@ typedef struct query_point {
968 time_t end_time;
969 time_t start_time;
970 NETDATA_DOUBLE value;
974 - NETDATA_DOUBLE anomaly;
971 + size_t anomaly_outlier_points;
972 + size_t anomaly_all_points;
973 SN_FLAGS flags;
974 #ifdef NETDATA_INTERNAL_CHECKS
975 size_t id;
@@ -982,7 +980,8 @@ QUERY_POINT QUERY_POINT_EMPTY = {
980 .end_time = 0,
981 .start_time = 0,
982 .value = NAN,
985 - .anomaly = 0,
983 + .anomaly_outlier_points = 0,
984 + .anomaly_all_points = 0,
985 .flags = SN_FLAG_NONE,
986 #ifdef NETDATA_INTERNAL_CHECKS
987 .id = 0,
@@ -1022,7 +1021,8 @@ typedef struct query_engine_ops {
1021 NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
1022 size_t group_points_non_zero;
1023 size_t group_points_added;
1025 - NETDATA_DOUBLE group_anomaly_rate;
1024 + size_t group_anomaly_outlier_points;
1025 + size_t group_anomaly_all_points;
1026 RRDR_VALUE_FLAGS group_value_flags;
1027
1028 // statistics
@@ -1087,6 +1087,8 @@ static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) {
1087 qm->plan.array[p].expanded_after = after;
1088 qm->plan.array[p].expanded_before = before;
1089
1090 + ops->r->internal.qt->db.tiers[tier].queries++;
1091 +
1092 struct query_metric_tier *tier_ptr = &qm->tiers[tier];
1093 tier_ptr->eng->api.query_ops.init(
1094 tier_ptr->db_metric_handle,
@@ -1350,7 +1352,8 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1352 } \
1353 \
1354 (ops)->group_points_added++; \
1353 - (ops)->group_anomaly_rate += (point).anomaly; \
1355 + (ops)->group_anomaly_outlier_points = (point).anomaly_outlier_points; \
1356 + (ops)->group_anomaly_all_points = (point).anomaly_all_points; \
1357 } while(0)
1358
1359 static QUERY_ENGINE_OPS *rrd2rrdr_query_prep(RRDR *r, size_t dim_id_in_rrdr) {
@@ -1484,7 +1487,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1487
1488 new_point.start_time = sp.start_time_s;
1489 new_point.end_time = sp.end_time_s;
1487 - new_point.anomaly = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0;
1490 + new_point.anomaly_outlier_points = sp.anomaly_count;
1491 + new_point.anomaly_all_points = sp.count;
1492 query_point_set_id(new_point, ops->db_total_points_read);
1493
1494 // if(debug_this)
@@ -1495,7 +1499,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1499 if(likely(!storage_point_is_unset(sp) && !storage_point_is_gap(sp))) {
1500
1501 if(unlikely(use_anomaly_bit_as_value))
1498 - new_point.value = new_point.anomaly;
1502 + new_point.value = new_point.anomaly_all_points ? (NETDATA_DOUBLE)new_point.anomaly_outlier_points * 100.0 / (NETDATA_DOUBLE)new_point.anomaly_all_points : 0.0;
1503
1504 else {
1505 switch (ops->tier_query_fetch) {
@@ -1688,10 +1692,9 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1692 NETDATA_DOUBLE group_value = ops->grouping_flush(r, rrdr_value_options_ptr);
1693 r->v[rrdr_o_v_index] = group_value;
1694
1691 - // we only store uint8_t anomaly rates,
1692 - // so let's get double precision by storing
1693 - // anomaly rates in the range 0 - 200
1694 - r->ar[rrdr_o_v_index] = ops->group_anomaly_rate / (NETDATA_DOUBLE)ops->group_points_added;
1695 + NETDATA_DOUBLE group_ar = r->ar[rrdr_o_v_index] = ops->group_anomaly_all_points ?
1696 + (NETDATA_DOUBLE)ops->group_anomaly_outlier_points * 100.0 / (NETDATA_DOUBLE)ops->group_anomaly_all_points
1697 + : 0.0;
1698
1699 if(likely(points_added || dim_id_in_rrdr)) {
1700 // find the min/max across all dimensions
@@ -1720,15 +1723,17 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1723 qm->query_stats.max = stats_value;
1724 }
1725
1726 + qm->query_stats.anomaly_sum += group_ar;
1727 qm->query_stats.sum += stats_value;
1728 qm->query_stats.volume += stats_value * (NETDATA_DOUBLE)ops->view_update_every;
1725 - qm->query_stats.count++;
1729 + qm->query_stats.group_points++;
1730
1731 points_added++;
1732 ops->group_points_added = 0;
1733 ops->group_value_flags = RRDR_VALUE_NOTHING;
1734 ops->group_points_non_zero = 0;
1731 - ops->group_anomaly_rate = 0;
1735 + ops->group_anomaly_outlier_points = 0;
1736 + ops->group_anomaly_all_points = 0;
1737 }
1738 // the loop above increased "now" by query_granularity,
1739 // but the main loop will increase it too,
@@ -1741,7 +1746,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1746 r->stats.result_points_generated += points_added;
1747 r->stats.db_points_read += ops->db_total_points_read;
1748 for(size_t tr = 0; tr < storage_tiers ; tr++)
1744 - r->stats.tier_points_read[tr] += ops->db_points_read_per_tier[tr];
1749 + qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
1750
1751 r->view.min = min;
1752 r->view.max = max;
@@ -2283,11 +2288,12 @@ RRDR *rrd2rrdr_legacy(
2288 }
2289
2290 void query_target_merge_data_statistics(struct query_data_statistics *d, struct query_data_statistics *s) {
2286 - if(!d->count)
2291 + if(!d->group_points)
2292 *d = *s;
2293 else {
2289 - d->count += s->count;
2294 + d->group_points += s->group_points;
2295 d->sum += s->sum;
2296 + d->anomaly_sum += s->anomaly_sum;
2297 d->volume += s->volume;
2298
2299 if(s->min < d->min)
@@ -2361,7 +2367,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2367 if(qt->query.used)
2368 ops = onewayalloc_callocz(r->internal.owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
2369
2364 - size_t capacity = libuv_worker_threads * 2;
2370 + size_t capacity = libuv_worker_threads * 10;
2371 size_t max_queries_to_prepare = (qt->query.used > (capacity - 1)) ? (capacity - 1) : qt->query.used;
2372 size_t queries_prepared = 0;
2373 while(queries_prepared < max_queries_to_prepare) {
@@ -2375,7 +2381,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2381 QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
2382 QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
2383 QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id);
2378 - QUERY_HOST *qh = query_host(qt, qm->link.query_host_id);
2384 + QUERY_NODE *qn = query_node(qt, qm->link.query_host_id);
2385
2386 if(queries_prepared < max) {
2387 // preload another query
@@ -2390,30 +2396,30 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2396 r->grouping.reset(r);
2397
2398 if(ops[c]) {
2399 + rrd2rrdr_query_execute(r, c, ops[c]);
2400 +
2401 r->od[c] |= RRDR_DIMENSION_QUERIED;
2402 r->di[c] = rrdmetric_acquired_id_dup(qd->rma);
2403 r->dn[c] = rrdmetric_acquired_name_dup(qd->rma);
2404
2405 qi->metrics.queried++;
2406 qc->metrics.queried++;
2399 - qh->metrics.queried++;
2407 + qn->metrics.queried++;
2408
2409 qd->status |= QUERY_STATUS_QUERIED;
2410 qm->status |= RRDR_DIMENSION_QUERIED;
2411
2404 - rrd2rrdr_query_execute(r, c, ops[c]);
2405 -
2412 if(qt->request.version >= 2) {
2413 query_target_merge_data_statistics(&qi->query_stats, &qm->query_stats);
2414 query_target_merge_data_statistics(&qc->query_stats, &qm->query_stats);
2409 - query_target_merge_data_statistics(&qh->query_stats, &qm->query_stats);
2415 + query_target_merge_data_statistics(&qn->query_stats, &qm->query_stats);
2416 query_target_merge_data_statistics(&qt->query_stats, &qm->query_stats);
2417 }
2418 }
2419 else {
2420 qi->metrics.failed++;
2421 qc->metrics.failed++;
2416 - qh->metrics.failed++;
2422 + qn->metrics.failed++;
2423
2424 qd->status |= QUERY_STATUS_FAILED;
2425 qm->status |= RRDR_DIMENSION_FAILED;
@@ -2480,6 +2486,37 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2486 }
2487 }
2488
2489 + // update query instance counts in query host and query context
2490 + if(qt->request.version >= 2) {
2491 + size_t h = 0, c = 0, i = 0;
2492 + for(; h < qt->nodes.used ; h++) {
2493 + QUERY_NODE *qn = &qt->nodes.array[h];
2494 +
2495 + for(; c < qt->contexts.used ;c++) {
2496 + QUERY_CONTEXT *qc = &qt->contexts.array[c];
2497 +
2498 + if(!rrdcontext_acquired_belongs_to_host(qc->rca, qn->rrdhost))
2499 + break;
2500 +
2501 + for(; i < qt->instances.used ;i++) {
2502 + QUERY_INSTANCE *qi = &qt->instances.array[i];
2503 +
2504 + if(!rrdinstance_acquired_belongs_to_context(qi->ria, qc->rca))
2505 + break;
2506 +
2507 + if(qi->metrics.queried) {
2508 + qc->instances.queried++;
2509 + qn->instances.queried++;
2510 + }
2511 + else if(qi->metrics.failed) {
2512 + qc->instances.failed++;
2513 + qn->instances.failed++;
2514 + }
2515 + }
2516 + }
2517 + }
2518 + }
2519 +
2520 #ifdef NETDATA_INTERNAL_CHECKS
2521 if (dimensions_used) {
2522 if(r->internal.log)
web/api/queries/rrdr.h
+34 -31
@@ -18,43 +18,47 @@ typedef enum tier_query_fetch {
18 } TIER_QUERY_FETCH;
19
20 typedef enum rrdr_options {
21 - RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
22 - RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest)
23 - RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing
24 - RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum
25 - RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates
26 - RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates
27 - RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros
28 - RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array
29 - RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs
30 - RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result
31 - RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes
32 - RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total
33 - RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes
34 - RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
35 - RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs
36 - RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
37 - RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
38 - RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points
39 - RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number
40 - RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes
41 - RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap
42 - RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query
43 - RRDR_OPTION_ALL_DIMENSIONS = 0x00800000, // Return the full dimensions list
44 - RRDR_OPTION_SHOW_PLAN = 0x01000000, // Return the query plan in jsonwrap
45 - RRDR_OPTION_SHOW_DETAILS = 0x02000000, // v2 returns detailed object tree
46 - RRDR_OPTION_DEBUG = 0x04000000, // v2 returns request description
21 + RRDR_OPTION_NONZERO = (1 << 0), // don't output dimensions with just zero values
22 + RRDR_OPTION_REVERSED = (1 << 1), // output the rows in reverse order (oldest to newest)
23 + RRDR_OPTION_ABSOLUTE = (1 << 2), // values positive, for DATASOURCE_SSV before summing
24 + RRDR_OPTION_MIN2MAX = (1 << 3), // when adding dimensions, use max - min, instead of sum
25 + RRDR_OPTION_SECONDS = (1 << 4), // output seconds, instead of dates
26 + RRDR_OPTION_MILLISECONDS = (1 << 5), // output milliseconds, instead of dates
27 + RRDR_OPTION_NULL2ZERO = (1 << 6), // do not show nulls, convert them to zeros
28 + RRDR_OPTION_OBJECTSROWS = (1 << 7), // each row of values should be an object, not an array
29 + RRDR_OPTION_GOOGLE_JSON = (1 << 8), // comply with google JSON/JSONP specs
30 + RRDR_OPTION_JSON_WRAP = (1 << 9), // wrap the response in a JSON header with info about the result
31 + RRDR_OPTION_LABEL_QUOTES = (1 << 10), // in CSV output, wrap header labels in double quotes
32 + RRDR_OPTION_PERCENTAGE = (1 << 11), // give values as percentage of total
33 + RRDR_OPTION_NOT_ALIGNED = (1 << 12), // do not align charts for persistent timeframes
34 + RRDR_OPTION_DISPLAY_ABS = (1 << 13), // for badges, display the absolute value, but calculate colors with sign
35 + RRDR_OPTION_MATCH_IDS = (1 << 14), // when filtering dimensions, match only IDs
36 + RRDR_OPTION_MATCH_NAMES = (1 << 15), // when filtering dimensions, match only names
37 + RRDR_OPTION_NATURAL_POINTS = (1 << 16), // return the natural points of the database
38 + RRDR_OPTION_VIRTUAL_POINTS = (1 << 17), // return virtual points
39 + RRDR_OPTION_ANOMALY_BIT = (1 << 18), // Return the anomaly bit stored in each collected_number
40 + RRDR_OPTION_RETURN_RAW = (1 << 19), // Return raw data for aggregating across multiple nodes
41 + RRDR_OPTION_RETURN_JWAR = (1 << 20), // Return anomaly rates in jsonwrap
42 + RRDR_OPTION_SELECTED_TIER = (1 << 21), // Use the selected tier for the query
43 + RRDR_OPTION_ALL_DIMENSIONS = (1 << 22), // Return the full dimensions list
44 + RRDR_OPTION_SHOW_PLAN = (1 << 23), // Return the query plan in jsonwrap
45 + RRDR_OPTION_SHOW_DETAILS = (1 << 24), // v2 returns detailed object tree
46 + RRDR_OPTION_DEBUG = (1 << 25), // v2 returns request description
47 + RRDR_OPTION_MINIFY = (1 << 26), // remove JSON spaces and newlines from JSON output
48 + RRDR_OPTION_JW_ANNOTATIONS = (1 << 27), // add annotation array to the JSON output
49
50 // internal ones - not to be exposed to the API
49 - RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
50 - RRDR_OPTION_INTERNAL_GBC = 0x20000000, // internal use only, to let the formatters we want to render the group by count
51 - RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
51 + RRDR_OPTION_HEALTH_RSRVD1 = (1 << 28), // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
52 + RRDR_OPTION_INTERNAL_ANNOTATIONS = (1 << 29), // internal use only, to let the formatters know we want to render the annotations
53 + RRDR_OPTION_INTERNAL_AR = (1 << 30), // internal use only, to let the formatters know we want to render the anomaly rate
54 + RRDR_OPTION_INTERNAL_GBC = (1 << 31), // internal use only, to let the formatters know we want to render the group by count
55 } RRDR_OPTIONS;
56
57 typedef enum __attribute__ ((__packed__)) rrdr_value_flag {
58 RRDR_VALUE_NOTHING = 0, // no flag set (a good default)
59 RRDR_VALUE_EMPTY = (1 << 0), // the database value is empty
60 RRDR_VALUE_RESET = (1 << 1), // the database value is marked as reset (overflown)
61 + RRDR_VALUE_PARTIAL = (1 << 2), // the database provides partial data about this point (used in group-by)
62 } RRDR_VALUE_FLAGS;
63
64 typedef enum __attribute__ ((__packed__)) rrdr_dimension_flag {
@@ -87,7 +91,7 @@ typedef struct rrdresult {
91 STRING **dn; // array of d dimension names
92 STRING **du; // array of d dimension units
93 uint32_t *dgbc; // array of d dimension units - NOT ALLOCATED when RRDR is created
90 - uint32_t *dp; // array of d dimension units - NOT ALLOCATED when RRDR is created
94 + uint32_t *dp; // array of d dimension priority - NOT ALLOCATED when RRDR is created
95
96 time_t *t; // array of n timestamps
97 NETDATA_DOUBLE *v; // array n x d values
@@ -109,7 +113,6 @@ typedef struct rrdresult {
113 struct {
114 size_t db_points_read;
115 size_t result_points_generated;
112 - size_t tier_points_read[RRD_STORAGE_TIERS];
116 } stats;
117
118 struct {
web/api/queries/weights.c
+2 -2
@@ -118,7 +118,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
118 size_t examined_dimensions __maybe_unused, usec_t duration,
119 WEIGHTS_STATS *stats) {
120
121 - buffer_json_initialize(wb, "\"", "\"", 0, true);
121 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
122 buffer_json_member_add_time_t(wb, "after", after);
123 buffer_json_member_add_time_t(wb, "before", before);
124 buffer_json_member_add_time_t(wb, "duration", before - after);
@@ -501,7 +501,7 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
501 stats->result_points += r->stats.result_points_generated;
502 stats->db_points += r->stats.db_points_read;
503 for(size_t tr = 0; tr < storage_tiers ; tr++)
504 - stats->db_points_per_tier[tr] += r->stats.tier_points_read[tr];
504 + stats->db_points_per_tier[tr] += r->internal.qt->db.tiers[tr].points;
505
506 if(r->d != 1) {
507 error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu", r->internal.qt->id, r->d);
web/api/web_api.c
+32
@@ -25,3 +25,35 @@ int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url, st
25 buffer_strcat_htmlescape(w->response.data, url);
26 return HTTP_RESP_NOT_FOUND;
27 }
28 +
29 +RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {
30 + RRDCONTEXT_TO_JSON_OPTIONS options = RRDCONTEXT_OPTION_NONE;
31 + char *tok;
32 +
33 + while(o && *o && (tok = mystrsep(&o, ", |"))) {
34 + if(!*tok) continue;
35 +
36 + if(!strcmp(tok, "full") || !strcmp(tok, "all"))
37 + options |= RRDCONTEXT_OPTIONS_ALL;
38 + else if(!strcmp(tok, "charts") || !strcmp(tok, "instances"))
39 + options |= RRDCONTEXT_OPTION_SHOW_INSTANCES;
40 + else if(!strcmp(tok, "dimensions") || !strcmp(tok, "metrics"))
41 + options |= RRDCONTEXT_OPTION_SHOW_METRICS;
42 + else if(!strcmp(tok, "queue"))
43 + options |= RRDCONTEXT_OPTION_SHOW_QUEUED;
44 + else if(!strcmp(tok, "flags"))
45 + options |= RRDCONTEXT_OPTION_SHOW_FLAGS;
46 + else if(!strcmp(tok, "uuids"))
47 + options |= RRDCONTEXT_OPTION_SHOW_UUIDS;
48 + else if(!strcmp(tok, "deleted"))
49 + options |= RRDCONTEXT_OPTION_SHOW_DELETED;
50 + else if(!strcmp(tok, "labels"))
51 + options |= RRDCONTEXT_OPTION_SHOW_LABELS;
52 + else if(!strcmp(tok, "deepscan"))
53 + options |= RRDCONTEXT_OPTION_DEEPSCAN;
54 + else if(!strcmp(tok, "hidden"))
55 + options |= RRDCONTEXT_OPTION_SHOW_HIDDEN;
56 + }
57 +
58 + return options;
59 +}
web/api/web_api_v1.c
+18 -46
@@ -44,6 +44,8 @@ static struct {
44 , {"plan" , 0 , RRDR_OPTION_SHOW_PLAN}
45 , {"details" , 0 , RRDR_OPTION_SHOW_DETAILS}
46 , {"debug" , 0 , RRDR_OPTION_DEBUG}
47 + , {"minify" , 0 , RRDR_OPTION_MINIFY}
48 + , {"annotations" , 0 , RRDR_OPTION_JW_ANNOTATIONS}
49 , {NULL , 0 , 0}
50 };
51
@@ -403,38 +405,6 @@ inline int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_c
405 return web_client_api_request_single_chart(host, w, url, health_api_v1_chart_variables2json);
406 }
407
406 -static RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {
407 - RRDCONTEXT_TO_JSON_OPTIONS options = RRDCONTEXT_OPTION_NONE;
408 - char *tok;
409 -
410 - while(o && *o && (tok = mystrsep(&o, ", |"))) {
411 - if(!*tok) continue;
412 -
413 - if(!strcmp(tok, "full") || !strcmp(tok, "all"))
414 - options |= RRDCONTEXT_OPTIONS_ALL;
415 - else if(!strcmp(tok, "charts") || !strcmp(tok, "instances"))
416 - options |= RRDCONTEXT_OPTION_SHOW_INSTANCES;
417 - else if(!strcmp(tok, "dimensions") || !strcmp(tok, "metrics"))
418 - options |= RRDCONTEXT_OPTION_SHOW_METRICS;
419 - else if(!strcmp(tok, "queue"))
420 - options |= RRDCONTEXT_OPTION_SHOW_QUEUED;
421 - else if(!strcmp(tok, "flags"))
422 - options |= RRDCONTEXT_OPTION_SHOW_FLAGS;
423 - else if(!strcmp(tok, "uuids"))
424 - options |= RRDCONTEXT_OPTION_SHOW_UUIDS;
425 - else if(!strcmp(tok, "deleted"))
426 - options |= RRDCONTEXT_OPTION_SHOW_DELETED;
427 - else if(!strcmp(tok, "labels"))
428 - options |= RRDCONTEXT_OPTION_SHOW_LABELS;
429 - else if(!strcmp(tok, "deepscan"))
430 - options |= RRDCONTEXT_OPTION_DEEPSCAN;
431 - else if(!strcmp(tok, "hidden"))
432 - options |= RRDCONTEXT_OPTION_SHOW_HIDDEN;
433 - }
434 -
435 - return options;
436 -}
437 -
408 static int web_client_api_request_v1_context(RRDHOST *host, struct web_client *w, char *url) {
409 char *context = NULL;
410 RRDCONTEXT_TO_JSON_OPTIONS options = RRDCONTEXT_OPTION_NONE;
@@ -478,13 +448,15 @@ static int web_client_api_request_v1_context(RRDHOST *host, struct web_client *w
448 SIMPLE_PATTERN *chart_dimensions_pattern = NULL;
449
450 if(chart_label_key)
481 - chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
451 + chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT, true);
452
453 if(chart_labels_filter)
484 - chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
454 + chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT,
455 + true);
456
457 if(dimensions) {
487 - chart_dimensions_pattern = simple_pattern_create(buffer_tostring(dimensions), ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
458 + chart_dimensions_pattern = simple_pattern_create(buffer_tostring(dimensions), ",|\t\r\n\f\v",
459 + SIMPLE_PATTERN_EXACT, true);
460 buffer_free(dimensions);
461 }
462
@@ -534,13 +506,15 @@ static int web_client_api_request_v1_contexts(RRDHOST *host, struct web_client *
506 SIMPLE_PATTERN *chart_dimensions_pattern = NULL;
507
508 if(chart_label_key)
537 - chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
509 + chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT, true);
510
511 if(chart_labels_filter)
540 - chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
512 + chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT,
513 + true);
514
515 if(dimensions) {
543 - chart_dimensions_pattern = simple_pattern_create(buffer_tostring(dimensions), ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
516 + chart_dimensions_pattern = simple_pattern_create(buffer_tostring(dimensions), ",|\t\r\n\f\v",
517 + SIMPLE_PATTERN_EXACT, true);
518 buffer_free(dimensions);
519 }
520
@@ -719,9 +693,9 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
693 .before = before,
694 .host = host,
695 .st = st,
722 - .hosts = NULL,
696 + .nodes = NULL,
697 .contexts = context,
724 - .charts = chart,
698 + .instances = chart,
699 .dimensions = (dimensions)?buffer_tostring(dimensions):NULL,
700 .timeout = timeout,
701 .points = points,
@@ -757,9 +731,6 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
731 }
732 }
733
760 - debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%u', format '%u', options '0x%08x'"
761 - , w->id, chart, (dimensions)?buffer_tostring(dimensions):"", after, before , points, group, format, options);
762 -
734 if(outFileName && *outFileName) {
735 buffer_sprintf(w->response.header, "Content-Disposition: attachment; filename=\"%s\"\r\n", outFileName);
736 debug(D_WEB_CLIENT, "%llu: generating outfilename header: '%s'", w->id, outFileName);
@@ -1114,7 +1085,7 @@ static void host_collectors(RRDHOST *host, BUFFER *wb) {
1085
1086 extern int aclk_connected;
1087 inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) {
1117 - buffer_json_initialize(wb, "\"", "\"", 0, true);
1088 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
1089
1090 buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));
1091 buffer_json_member_add_string(wb, "uid", host->machine_guid);
@@ -1369,7 +1340,8 @@ static int web_client_api_request_v1_weights_internal(RRDHOST *host, struct web_
1340 buffer_flush(wb);
1341 wb->content_type = CT_APPLICATION_JSON;
1342
1372 - SIMPLE_PATTERN *contexts = (contexts_str) ? simple_pattern_create(contexts_str, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
1343 + SIMPLE_PATTERN *contexts = (contexts_str) ? simple_pattern_create(contexts_str, ",|\t\r\n\f\v",
1344 + SIMPLE_PATTERN_EXACT, true) : NULL;
1345
1346 int ret = web_api_v1_weights(host, wb, method, format, group, group_options, baseline_after, baseline_before, after, before, points, options, contexts, tier, timeout);
1347
@@ -1425,7 +1397,7 @@ int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, cha
1397 wb->content_type = CT_APPLICATION_JSON;
1398 buffer_no_cacheable(wb);
1399
1428 - buffer_json_initialize(wb, "\"", "\"", 0, true);
1400 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
1401 host_functions2json(host, wb);
1402 buffer_json_finalize(wb);
1403
web/api/web_api_v2.c
+56 -13
@@ -2,10 +2,47 @@
2
3 #include "web_api_v2.h"
4
5 +static int web_client_api_request_v2_contexts_internal(RRDHOST *host __maybe_unused, struct web_client *w, char *url, CONTEXTS_V2_OPTIONS options) {
6 + struct api_v2_contexts_request req = { 0 };
7 + req.timings.received_ut = now_monotonic_usec();
8
6 -static inline int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
7 - debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
9 + while(url) {
10 + char *value = mystrsep(&url, "&");
11 + if(!value || !*value) continue;
12 +
13 + char *name = mystrsep(&value, "=");
14 + if(!name || !*name) continue;
15 + if(!value || !*value) continue;
16
17 + // name and value are now the parameters
18 + // they are not null and not empty
19 +
20 + if(!strcmp(name, "scope_nodes")) req.scope_nodes = value;
21 + else if((options & (CONTEXTS_V2_NODES | CONTEXTS_V2_CONTEXTS)) && !strcmp(name, "nodes")) req.nodes = value;
22 + else if((options & CONTEXTS_V2_CONTEXTS) && !strcmp(name, "scope_contexts")) req.scope_contexts = value;
23 + else if((options & CONTEXTS_V2_CONTEXTS) && !strcmp(name, "contexts")) req.contexts = value;
24 + else if((options & CONTEXTS_V2_SEARCH) && !strcmp(name, "q")) req.q = value;
25 + }
26 +
27 + options |= CONTEXTS_V2_DEBUG;
28 +
29 + buffer_flush(w->response.data);
30 + return rrdcontext_to_json_v2(w->response.data, &req, options);
31 +}
32 +
33 +static int web_client_api_request_v2_q(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
34 + return web_client_api_request_v2_contexts_internal(host, w, url, CONTEXTS_V2_SEARCH | CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_NODES);
35 +}
36 +
37 +static int web_client_api_request_v2_contexts(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
38 + return web_client_api_request_v2_contexts_internal(host, w, url, CONTEXTS_V2_CONTEXTS);
39 +}
40 +
41 +static int web_client_api_request_v2_nodes(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
42 + return web_client_api_request_v2_contexts_internal(host, w, url, CONTEXTS_V2_NODES);
43 +}
44 +
45 +static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
46 usec_t received_ut = now_monotonic_usec();
47
48 int ret = HTTP_RESP_BAD_REQUEST;
@@ -21,9 +58,9 @@ static inline int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, s
58
59 time_t last_timestamp_in_data = 0, google_timestamp = 0;
60
24 - char *scope_hosts = NULL;
61 + char *scope_nodes = NULL;
62 char *scope_contexts = NULL;
26 - char *hosts = NULL;
63 + char *nodes = NULL;
64 char *contexts = NULL;
65 char *instances = NULL;
66 char *dimensions = NULL;
@@ -55,9 +92,9 @@ static inline int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, s
92 // name and value are now the parameters
93 // they are not null and not empty
94
58 - if(!strcmp(name, "scope_hosts")) scope_hosts = value;
95 + if(!strcmp(name, "scope_nodes")) scope_nodes = value;
96 else if(!strcmp(name, "scope_contexts")) scope_contexts = value;
60 - else if(!strcmp(name, "hosts")) hosts = value;
97 + else if(!strcmp(name, "nodes")) nodes = value;
98 else if(!strcmp(name, "contexts")) contexts = value;
99 else if(!strcmp(name, "instances")) instances = value;
100 else if(!strcmp(name, "dimensions")) dimensions = value;
@@ -69,7 +106,7 @@ static inline int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, s
106 else if(!strcmp(name, "timeout")) timeout_str = value;
107 else if(!strcmp(name, "group_by")) group_by = group_by_parse(value);
108 else if(!strcmp(name, "group_by_label")) group_by_label = value;
72 - else if(!strcmp(name, "group_by_aggregate")) group_by_aggregate = group_by_aggregate_function_parse(value);
109 + else if(!strcmp(name, "aggregation")) group_by_aggregate = group_by_aggregate_function_parse(value);
110 else if(!strcmp(name, "format")) format = web_client_api_request_v1_data_format(value);
111 else if(!strcmp(name, "options")) options |= web_client_api_request_v1_data_options(value);
112 else if(!strcmp(name, "time_group")) time_group = time_grouping_parse(value, RRDR_GROUPING_AVERAGE);
@@ -119,15 +156,18 @@ static inline int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, s
156 fix_google_param(responseHandler);
157 fix_google_param(outFileName);
158
159 + if(group_by_label && *group_by_label)
160 + group_by |= RRDR_GROUP_BY_LABEL;
161 +
162 + if(group_by == RRDR_GROUP_BY_NONE)
163 + group_by = RRDR_GROUP_BY_DIMENSION;
164 +
165 if(group_by & ~(RRDR_GROUP_BY_DIMENSION))
166 options |= RRDR_OPTION_ABSOLUTE;
167
168 if(options & RRDR_OPTION_SHOW_PLAN)
169 options |= RRDR_OPTION_DEBUG;
170
128 - if(group_by_label && *group_by_label)
129 - group_by |= RRDR_GROUP_BY_LABEL;
130 -
171 if(tier_str && *tier_str) {
172 tier = str2ul(tier_str);
173 if(tier < storage_tiers)
@@ -144,15 +184,15 @@ static inline int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, s
184
185 QUERY_TARGET_REQUEST qtr = {
186 .version = 2,
147 - .scope_hosts = scope_hosts,
187 + .scope_nodes = scope_nodes,
188 .scope_contexts = scope_contexts,
189 .after = after,
190 .before = before,
191 .host = NULL,
192 .st = NULL,
153 - .hosts = hosts,
193 + .nodes = nodes,
194 .contexts = contexts,
155 - .charts = instances,
195 + .instances = instances,
196 .dimensions = dimensions,
197 .alerts = alerts,
198 .timeout = timeout,
@@ -251,6 +291,9 @@ cleanup:
291
292 static struct web_api_command api_commands_v2[] = {
293 {"data", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_data},
294 + {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_nodes},
295 + {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_contexts},
296 + {"q", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_q},
297
298 // terminator
299 {NULL, 0, WEB_CLIENT_ACL_NONE, NULL},